-
-
Notifications
You must be signed in to change notification settings - Fork 2k
Expand file tree
/
Copy pathindex.js
More file actions
1274 lines (1179 loc) · 49.6 KB
/
Copy pathindex.js
File metadata and controls
1274 lines (1179 loc) · 49.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
const path = require('path')
const fs = require('fs/promises')
// Sentinel attributes / strings used to make HTML and markdown post-processing
// idempotent across re-runs (e.g. when scripts/verify-llms-output.js is invoked
// against an already-processed outDir).
const MD_ALT_MARKER = 'data-llms-md-alt'
const BODY_DIRECTIVE_MARKER = 'data-llms-directive'
const MD_DIRECTIVE_LINE = '> For the complete documentation index, see [llms.txt](/llms.txt).'
// Matches the opening tag of every `<span class="katex-mathml">` (KaTeX
// always emits this exact class with no additional classes on the wrapper
// itself; the mathml subtree is the off-screen MathML rendering used by
// screen readers). The `[^>]*` allow for any other attributes (rare but
// possible if upstream rehype plugins add them). The capture group is
// reused to splice the marker into the same position in the tag.
const KATEX_MATHML_OPEN_TAG_RE =
/<span\b((?:[^>"']|"[^"]*"|'[^']*')*?\bclass="katex-mathml"(?:[^>"']|"[^"]*"|'[^']*')*?)>/g
// Visible-but-screen-reader-only directive injected at the top of every
// `<body>` in the build. AFDocs scans `<body>` (excluding nav/script/style)
// for a mention of `llms.txt`, so this satisfies `llms-txt-directive-html`.
// `.sr-only` is defined in src/scss/commons/_utils.scss and loaded site-wide
// via src/scss/custom.scss, so no styling work is needed at the page level.
function buildBodyDirective(siteUrl) {
// The href is intentionally site-relative (`/llms.txt`) so it works on the
// production host and any preview/staging deployment alike. The siteUrl
// argument is accepted for symmetry with the head-link injector and to
// allow a future absolute variant without touching call sites.
void siteUrl
return (
`<div ${BODY_DIRECTIVE_MARKER} class="sr-only">` +
'For AI agents: a documentation index is available at ' +
'<a href="/llms.txt">/llms.txt</a>. A markdown version of this page is ' +
'available at the same URL with .md appended (or via Accept: text/markdown).' +
`</div>`
)
}
/**
* Build the URL prefix the upstream `docusaurus-plugin-llms` uses when it
* stamps absolute links into the generated `llms*.txt` indexes and `.md`
* files. The upstream concatenates `siteConfig.url` with `siteConfig.baseUrl`
* (trimmed of any trailing slash), so when the site is deployed under a
* non-root base path (e.g. `DEST=/staging/` in CI) the emitted URLs look like
* `https://docs.metamask.io/staging/...`.
*
* Both `rewriteLlmsIndexes` (find-and-replace match keys) and
* `injectAlternateLinks` (`<link rel="alternate">` hrefs) must use the same
* prefix; otherwise the rewrites silently no-op and the alternate links 404.
*
* Mirrors `node_modules/docusaurus-plugin-llms/lib/index.js`.
*/
function resolveSiteUrl(siteConfig) {
const url = (siteConfig && siteConfig.url) || ''
const rawBase = (siteConfig && siteConfig.baseUrl) || ''
const base = rawBase.endsWith('/') ? rawBase.slice(0, -1) : rawBase
return `${url}${base}`
}
/**
* Vercel sets several deployment URL env vars on every build:
*
* - `VERCEL_URL` — the deployment-specific host
* (e.g. `metamask-docs-f8nytczy3-consensys.vercel.app`). Changes on
* every deploy.
* - `VERCEL_BRANCH_URL` — the stable branch alias
* (e.g. `metamask-docs-git-my-branch-consensys.vercel.app`). Same URL
* across rebuilds of the same branch — this is the URL collaborators
* and CI tools normally test against.
* - `VERCEL_ENV` — `production`, `preview`, or `development`.
*
* On preview/development deployments we want the curated `static/llms.txt`,
* the generated `llms-*.txt` index files, and the per-page `.md` link
* bodies to point at the same host AFDocs is being pointed at — otherwise
* AFDocs sees the absolute URLs as cross-origin and silently degrades
* checks like `llms-txt-links-markdown` and `llms-txt-coverage` to
* "skipped: all links external".
*
* `VERCEL_BRANCH_URL` is preferred because it matches the URL most reviewers
* paste into `npx afdocs check`. `VERCEL_URL` is the fallback when the
* branch URL isn't available (e.g. on `vercel dev`, where only the
* deployment-specific URL is set). Returns null on production deployments
* and outside Vercel so the canonical `siteConfig.url` is preserved.
*/
function resolvePreviewSiteUrl() {
const env = process.env.VERCEL_ENV
const host = process.env.VERCEL_BRANCH_URL || process.env.VERCEL_URL
if (!host) return null
// VERCEL_ENV is "preview" for branch deploys (PRs, branch URLs) and
// "development" when running `vercel dev`. Both should rewrite to the
// ephemeral host. "production" deploys keep the canonical URL.
if (env !== 'preview' && env !== 'development') return null
return `https://${host}`
}
// Source path prefixes that Docusaurus strips when constructing public URLs.
// Mirrors `pathTransformation.ignorePaths` we pass to `docusaurus-plugin-llms`.
const URL_STRIPPED_PREFIXES = ['src/pages/']
// Path prefix → all-pages bucket filename. Generated files live alongside the
// existing per-product `llms-<product>.txt` files in the build root and are
// referenced from `static/llms.txt` so the AFDocs walker descends one level
// into them and discovers every sitemap URL (fixes `llms-txt-coverage`).
const ALL_PAGES_BUCKETS = [
{
prefix: '/metamask-connect/',
filename: 'llms-all-metamask-connect.txt',
title: 'All MetaMask Connect pages',
},
{
prefix: '/embedded-wallets/',
filename: 'llms-all-embedded-wallets.txt',
title: 'All Embedded Wallets pages',
},
{
prefix: '/smart-accounts-kit/',
filename: 'llms-all-smart-accounts-kit.txt',
title: 'All Smart Accounts Kit pages',
},
{ prefix: '/snaps/', filename: 'llms-all-snaps.txt', title: 'All Snaps pages' },
{ prefix: '/tutorials/', filename: 'llms-all-tutorials.txt', title: 'All Tutorials pages' },
]
const ALL_PAGES_MISC_FILENAME = 'llms-all-misc.txt'
const ALL_PAGES_MISC_TITLE = 'All other documentation pages'
// docusaurus-plugin-llms is wrapped (rather than registered separately) because
// Docusaurus 3.x runs `postBuild` hooks concurrently via `Promise.all`. As two
// independent plugins, the injector below would race against the generator and
// find an empty outDir. Wrapping guarantees the generator's postBuild completes
// before we normalize/rewrite/inject.
const upstreamLlmsPlugin = require('docusaurus-plugin-llms').default
/**
* Docusaurus plugin that:
*
* 1. Delegates per-section `llms-<product>.txt` and per-page `.md` generation
* to `docusaurus-plugin-llms` (the upstream walks raw source MDX).
*
* 2. Normalizes the per-page `.md` files so their location mirrors the public
* URL of the corresponding HTML page (e.g. `embedded-wallets/README.md` ->
* `embedded-wallets.md`, `src/pages/tutorials/foo.md` -> `tutorials/foo.md`).
*
* 3. Replaces every per-page `.md` body with markdown derived from the
* rendered HTML `<article>` (node-html-markdown). This is the only way
* to get true HTML/`.md` parity for pages assembled from MDX `import`
* partials, which the upstream plugin cannot resolve. Fixes
* `markdown-content-parity`.
*
* 4. Prepends a `> For the complete documentation index, see [llms.txt](/llms.txt).`
* blockquote to every per-page `.md`. Fixes `llms-txt-directive-md`.
*
* 5. Rewrites every `build/llms*.txt` URL so it points at the normalized
* `.md` location, eliminating stale "links not in sitemap" warnings.
*
* 6. Prunes link-index `llms-*.txt` files of any link whose target URL is
* absent from the build's sitemap (defensive — only operates on
* non-`-full` link indexes to avoid corrupting embedded content).
*
* 7. Generates `llms-all-<product>.txt` files from the build's sitemap so
* `llms-txt-coverage` reaches 100% deterministically. The static root
* `llms.txt` links to these files; the AFDocs walker descends one level
* and picks up every sitemap URL.
*
* 8. Injects `<link rel="alternate" type="text/markdown">` into `<head>` and
* a hidden `<div>` directive at the top of `<body>` of every doc page,
* satisfying `llms-txt-directive-html` and the existing markdown-alt
* discovery hook for HTML clients.
*
* Together these steps satisfy every Agent Score check in the
* content-discoverability and observability categories.
*/
async function postProcessLlmsOutput(outDir, siteUrl) {
if (!siteUrl) {
throw new Error(
'[llms-html-injector] postProcessLlmsOutput requires a siteUrl. ' +
'Pass the value produced by resolveSiteUrl(siteConfig) so the URL prefix ' +
'(including any non-root baseUrl such as /staging/) matches what ' +
'docusaurus-plugin-llms emitted into the generated files.'
)
}
const renames = await normalizeMarkdownLayout(outDir)
console.log(
`[llms-html-injector] Normalized ${renames.size} .md file(s) into URL-aligned positions`
)
const regen = await regenerateMdFromHtml(outDir)
console.log(
`[llms-html-injector] Regenerated ${regen.regenerated} per-page .md from rendered HTML, ` +
`created ${regen.created} new .md for pages without an upstream-emitted sibling ` +
`(skipped ${regen.skipped}: unreadable HTML or empty <article>)`
)
const directiveCount = await prependMdDirective(outDir)
console.log(`[llms-html-injector] Prepended llms.txt directive to ${directiveCount} .md file(s)`)
const rewriteCount = await rewriteLlmsIndexes(outDir, renames, siteUrl)
console.log(`[llms-html-injector] Updated URLs in ${rewriteCount} llms*.txt file(s)`)
// Normalize the sitemap *before* reading its URLs: normalizeSitemap drops
// `<loc>`s shadowed by a vercel.json redirect (they 3XX on the deployed
// site) and strips trailing slashes from dotted-segment routes. Reading the
// sitemap afterward ensures pruneStaleLlmsLinks and generateAllPagesIndex
// operate on the same 200-only URL set the deployed sitemap exposes, so the
// generated llms-all-*.txt indexes never point agents at redirecting pages.
const sitemap = await normalizeSitemap(outDir, siteUrl)
if (sitemap.skipped) {
console.warn(
'[llms-html-injector] No sitemap.xml found in outDir; skipping sitemap normalization.'
)
} else {
console.log(
`[llms-html-injector] Sitemap: stripped trailing slash from ${sitemap.rewritten} dotted-segment <loc>(s) ` +
`and dropped ${sitemap.dropped} URL(s) shadowed by a vercel.json redirect`
)
}
const sitemapUrls = await readSitemapUrls(outDir)
if (sitemapUrls) {
const stale = await pruneStaleLlmsLinks(outDir, sitemapUrls, siteUrl)
console.log(
`[llms-html-injector] Pruned ${stale.removed} stale link(s) from ${stale.fileCount} link-index file(s)`
)
const all = await generateAllPagesIndex(outDir, siteUrl, sitemapUrls)
console.log(
`[llms-html-injector] Generated ${all.fileCount} llms-all-*.txt index file(s) covering ${all.urlCount} sitemap URL(s)`
)
} else {
console.warn(
'[llms-html-injector] No sitemap.xml found in outDir; skipping pruneStaleLlmsLinks and generateAllPagesIndex.'
)
}
const { injected, missing, skippedRoot, bodyInjected, katexMarked, canonicalFixed } =
await injectAlternateLinks(outDir, siteUrl)
console.log(
`[llms-html-injector] Injected markdown alternate link into ${injected} HTML pages ` +
`(skipped ${missing} pages with no matching .md sibling, ` +
`${skippedRoot} root index.html page(s) by design)`
)
console.log(
`[llms-html-injector] Injected sr-only llms.txt body directive into ${bodyInjected} HTML pages`
)
console.log(
`[llms-html-injector] Marked ${katexMarked} KaTeX MathML span(s) as data-markdown-ignore`
)
console.log(
`[llms-html-injector] Stripped trailing slash from canonical/og:url on ${canonicalFixed} dotted-route page(s)`
)
// On Vercel preview/development deployments, rewrite every absolute URL in
// generated llms*.txt files and per-page .md files from the canonical
// production host to the preview's auto-assigned host. The HTML alternate
// link injected above already uses a path-only href, so HTML files don't
// need this pass — only text artifacts that the llmstxt.org spec requires
// to carry absolute URLs.
const previewSiteUrl = resolvePreviewSiteUrl()
if (previewSiteUrl && previewSiteUrl !== siteUrl) {
const rewriteStats = await rewriteHostInBuildArtifacts(outDir, siteUrl, previewSiteUrl)
const hostSource = process.env.VERCEL_BRANCH_URL ? 'VERCEL_BRANCH_URL' : 'VERCEL_URL'
console.log(
`[llms-html-injector] Vercel preview detected (${hostSource}=${process.env.VERCEL_BRANCH_URL || process.env.VERCEL_URL}); ` +
`rewrote ${siteUrl} -> ${previewSiteUrl} in ${rewriteStats.txtFiles} llms*.txt file(s) ` +
`and ${rewriteStats.mdFiles} per-page .md file(s)`
)
}
}
module.exports = function llmsHtmlInjectorPlugin(context, options = {}) {
const inner = upstreamLlmsPlugin(context, options)
const siteUrl = resolveSiteUrl(context && context.siteConfig)
return {
...inner,
name: 'llms-html-injector',
async postBuild(props) {
if (typeof inner.postBuild === 'function') {
await inner.postBuild.call(inner, props)
}
await postProcessLlmsOutput(props.outDir, siteUrl)
},
}
}
// Exported for scripts/verify-llms-output.js, which invokes the generator
// directly and then runs only the post-processing stage.
module.exports.postProcessLlmsOutput = postProcessLlmsOutput
module.exports.resolveSiteUrl = resolveSiteUrl
/**
* Walk every `.md` file under `outDir`, skipping the llms*.txt index files,
* and move it to its URL-aligned target path if different. Returns a map of
* old-relative-path -> new-relative-path (using forward slashes, no leading `/`).
*/
async function normalizeMarkdownLayout(outDir) {
const renames = new Map()
const allMd = await collectMarkdownFiles(outDir)
const dirsToPrune = new Set()
for (const absFrom of allMd) {
const relFrom = toPosix(path.relative(outDir, absFrom))
const relTo = urlAlignedRelative(relFrom)
if (!relTo || relTo === relFrom) continue
const absTo = path.join(outDir, relTo)
let targetExists = true
try {
await fs.access(absTo)
} catch {
targetExists = false
}
if (targetExists) {
console.warn(
`[llms-html-injector] Target ${relTo} already exists; dropping duplicate source ${relFrom}`
)
await fs.unlink(absFrom)
renames.set(relFrom, relTo)
dirsToPrune.add(path.dirname(absFrom))
continue
}
await fs.mkdir(path.dirname(absTo), { recursive: true })
await fs.rename(absFrom, absTo)
renames.set(relFrom, relTo)
dirsToPrune.add(path.dirname(absFrom))
}
await pruneEmptyDirs(outDir, dirsToPrune)
return renames
}
async function pruneEmptyDirs(outDir, dirs) {
const root = path.resolve(outDir)
for (const start of dirs) {
let current = path.resolve(start)
while (current.startsWith(root) && current !== root) {
let entries
try {
entries = await fs.readdir(current)
} catch {
break
}
if (entries.length > 0) break
try {
await fs.rmdir(current)
} catch {
break
}
current = path.dirname(current)
}
}
}
async function collectMarkdownFiles(outDir, acc = []) {
const entries = await fs.readdir(outDir, { withFileTypes: true })
for (const entry of entries) {
const full = path.join(outDir, entry.name)
if (entry.isDirectory()) {
await collectMarkdownFiles(full, acc)
} else if (
entry.name.endsWith('.md') &&
!/^llms.*\.txt$/.test(entry.name) &&
!/^llms.*\.md$/.test(entry.name)
) {
acc.push(full)
}
}
return acc
}
function urlAlignedRelative(relFrom) {
let p = relFrom
for (const prefix of URL_STRIPPED_PREFIXES) {
if (p.startsWith(prefix)) {
p = p.slice(prefix.length)
break
}
}
if (p === 'README.md' || p === 'index.md') return null
p = p.replace(/\/(README|index)\.md$/, '.md')
return p
}
async function rewriteLlmsIndexes(outDir, renames, siteUrl) {
if (renames.size === 0) return 0
const entries = await fs.readdir(outDir, { withFileTypes: true })
let count = 0
for (const entry of entries) {
if (!entry.isFile()) continue
if (!/^llms.*\.txt$/.test(entry.name)) continue
const abs = path.join(outDir, entry.name)
let text = await fs.readFile(abs, 'utf8')
let changed = false
for (const [from, to] of renames) {
const fromUrl = `${siteUrl}/${from}`
const toUrl = `${siteUrl}/${to}`
if (text.includes(fromUrl)) {
text = text.split(fromUrl).join(toUrl)
changed = true
}
}
if (changed) {
await fs.writeFile(abs, text, 'utf8')
count++
}
}
return count
}
/**
* Walk `build/**\/index.html`, and for each page that has a sibling `.md` at
* the URL-aligned location, inject a `<link rel="alternate" type="text/markdown" href="...">`
* into the `<head>`. Independently, inject a hidden `<div>` directive into the
* `<body>` of every page so HTML-only agents can discover `/llms.txt`.
*
* Also marks every `<span class="katex-mathml">` with `data-markdown-ignore`.
* KaTeX renders each math expression three times in the DOM (`mrow` text +
* `<annotation>` LaTeX source + the visual `.katex-html`), which AFDocs'
* `markdown-content-parity` HTML extractor concatenates into segments like
* `retrievePubKey()retrievePubKey()retrievePubKey()` that can never match
* the single `$retrievePubKey()$` in the regenerated `.md`. Marking the
* MathML subtree as `data-markdown-ignore` makes AFDocs strip it before
* extracting text, leaving only the visual rendering — invisible to the
* markdown side via `htmlToMarkdown`'s own `.katex` replacement, but
* structurally aligned with what the parity check measures.
*/
async function injectAlternateLinks(outDir, siteUrl) {
let injected = 0
let missing = 0
let skippedRoot = 0
let bodyInjected = 0
let katexMarked = 0
let canonicalFixed = 0
await visit(outDir)
return { injected, missing, skippedRoot, bodyInjected, katexMarked, canonicalFixed }
async function visit(dir) {
const entries = await fs.readdir(dir, { withFileTypes: true })
for (const entry of entries) {
const full = path.join(dir, entry.name)
if (entry.isDirectory()) {
await visit(full)
} else if (entry.name === 'index.html') {
const result = await injectOne(full)
if (result.altLink === true) injected++
else if (result.altLink === false) missing++
else if (result.altLink === 'root') skippedRoot++
if (result.bodyDirective) bodyInjected++
if (result.canonicalFixed) canonicalFixed++
katexMarked += result.katexMarked
}
}
}
async function injectOne(htmlAbs) {
const relDir = toPosix(path.relative(outDir, path.dirname(htmlAbs)))
let html
try {
html = await fs.readFile(htmlAbs, 'utf8')
} catch {
return { altLink: false, bodyDirective: false, katexMarked: 0 }
}
let changed = false
let altLinkResult
if (!relDir || relDir === '.') {
altLinkResult = 'root'
} else {
const mdRel = `${relDir}.md`
const mdAbs = path.join(outDir, mdRel)
let mdExists = true
try {
await fs.access(mdAbs)
} catch {
mdExists = false
}
if (!mdExists) {
altLinkResult = false
} else if (html.includes(MD_ALT_MARKER) || !html.includes('</head>')) {
altLinkResult = null
} else {
// Use a path-only href (`/foo/bar.md`) rather than an absolute URL.
// Browsers and AFDocs both resolve relative hrefs against the page's
// origin, so the same HTML works correctly on production
// (docs.metamask.io), Vercel preview deployments
// (`*.vercel.app`), and `localhost` without any host-rewrite step
// touching HTML files. The `siteUrl` parameter is retained for
// signature compatibility but no longer baked into the href.
void siteUrl
const href = `/${mdRel}`
const tag = `<link ${MD_ALT_MARKER} rel="alternate" type="text/markdown" href="${href}">`
html = html.replace('</head>', `${tag}</head>`)
altLinkResult = true
changed = true
}
}
let bodyDirectiveAdded = false
if (!html.includes(BODY_DIRECTIVE_MARKER)) {
const directive = buildBodyDirective(siteUrl)
// Insert just after the opening <body...> tag so the directive is at the
// very top of the document body, well before navigation/header markup.
// Use a regex on the first <body...> tag rather than a naive string
// replace to handle attributes/whitespace variations.
const bodyTagMatch = html.match(/<body\b[^>]*>/i)
if (bodyTagMatch) {
const insertAt = bodyTagMatch.index + bodyTagMatch[0].length
html = html.slice(0, insertAt) + directive + html.slice(insertAt)
bodyDirectiveAdded = true
changed = true
}
}
// Mark every KaTeX MathML wrapper with data-markdown-ignore. The marker
// is idempotent: we skip any opening tag that already carries the
// attribute (which a previous run may have added) so re-invoking the
// postBuild stage against an already-processed outDir does not append
// duplicate attributes.
let katexMarkedHere = 0
html = html.replace(KATEX_MATHML_OPEN_TAG_RE, (match, attrs) => {
if (/\bdata-markdown-ignore\b/.test(attrs)) return match
katexMarkedHere++
return `<span${attrs} data-markdown-ignore>`
})
if (katexMarkedHere > 0) changed = true
// Normalize trailing slashes in canonical/og:url metadata for "dotted"
// routes. Docusaurus (`trailingSlash: true`) always emits a trailing-slash
// canonical, but Vercel (`trailingSlash: true`) treats any final path
// segment containing a dot (e.g. `use-web3.js`, version `1.0.0`,
// `android-v7.1.1-to-v7.1.2`) as a file and 308-strips the slash. That
// makes the canonical point to a redirect. Stripping the slash here aligns
// the canonical/og:url with the URL Vercel actually serves with a 200.
let canonicalFixedHere = false
if (relDir && relDir !== '.' && lastSegmentHasDot(relDir)) {
const beforeCanonical = html
html = stripDottedTrailingSlashInMetadata(html)
if (html !== beforeCanonical) {
canonicalFixedHere = true
changed = true
}
}
if (changed) {
await fs.writeFile(htmlAbs, html, 'utf8')
}
return {
altLink: altLinkResult,
bodyDirective: bodyDirectiveAdded,
katexMarked: katexMarkedHere,
canonicalFixed: canonicalFixedHere,
}
}
}
/**
* Return true when the final non-empty segment of a route/path contains a dot.
* These are the routes Vercel treats as files and serves without a trailing
* slash (308-stripping any trailing-slash variant).
*/
function lastSegmentHasDot(routePath) {
const segments = routePath.split('/').filter(Boolean)
const last = segments[segments.length - 1] || ''
return last.includes('.')
}
/**
* Strip the trailing slash from the `<link rel="canonical">` href and the
* `og:url` content in a page's HTML. Only called for dotted routes, where the
* emitted absolute URL is guaranteed to be this page's own trailing-slash URL.
*/
function stripDottedTrailingSlashInMetadata(html) {
// <link rel="canonical" href="...://.../use-web3.js/">
html = html.replace(/(<link\b[^>]*\brel="canonical"[^>]*\bhref=")([^"]+?)\/("[^>]*>)/i, '$1$2$3')
// <meta property="og:url" content="...://.../use-web3.js/"> (property first)
html = html.replace(
/(<meta\b[^>]*\bproperty="og:url"[^>]*\bcontent=")([^"]+?)\/("[^>]*>)/i,
'$1$2$3'
)
// <meta content="..." property="og:url"> (content first)
html = html.replace(
/(<meta\b[^>]*\bcontent=")([^"]+?)\/("[^>]*\bproperty="og:url"[^>]*>)/i,
'$1$2$3'
)
// <link rel="alternate" href="...://.../changelog/1.5.0/" hreflang="en"> (and
// hreflang="x-default"). Docusaurus emits these self-referencing hreflang
// alternates with a trailing slash too, so they hit the same 308 as canonical
// on dotted routes. Global flag covers both the locale and x-default tags. The
// markdown alternate (`type="text/markdown"`, path-only href without a
// trailing slash) has no `hreflang` attribute and is left untouched.
html = html.replace(
/(<link\b[^>]*\brel="alternate"[^>]*\bhref=")([^"]+?)\/("[^>]*\bhreflang="[^"]*"[^>]*>)/gi,
'$1$2$3'
)
return html
}
/**
* Generate (or regenerate) every per-page `.md` from its rendered HTML
* `<article>`. This achieves true HTML/MD parity AND fills coverage gaps:
*
* - The upstream `docusaurus-plugin-llms` reads raw source MDX and
* cannot resolve `import` partials, leaving pages built from imported
* components nearly empty in the upstream `.md` (e.g.
* `docs.infura.io/reference/.../eth_sendrawtransaction`). Regenerating from
* the rendered HTML resolves every partial and component.
*
* - Some pages exist in the build with NO upstream-emitted `.md` at all
* (e.g. Docusaurus-versioned docs: gator's `current` version publishes
* under `/smart-accounts-kit/development/...` while the upstream only
* emits a single `.md` per source file at the version-less path). The
* ghost URLs fail AFDocs' `markdown-url-support`, `content-negotiation`,
* and `llms-txt-directive-md` checks. Walking `build/**\/index.html`
* and creating a `.md` sibling for any page that lacks one closes
* that gap deterministically.
*
* Algorithm:
* 1. Walk every `build/**\/index.html` (skipping the root index.html,
* which has no meaningful `.md` companion).
* 2. Compute the URL-aligned `.md` path (`<dir>.md`).
* 3. Read the HTML, parse with cheerio, select `<article>`, strip
* `data-markdown-ignore` plus nav/script/style/aside.
* 4. Convert the remaining HTML to markdown via node-html-markdown
* (GFM-aware: tables, strikethrough, line breaks).
* 5. Write the `.md`, creating the directory if needed.
*
* Frontmatter is intentionally not preserved: the upstream-generated `.md`
* frontmatter mirrors the source MDX frontmatter, but the rendered HTML
* already inlines the resolved title/description/etc. into the article body
* via Docusaurus components, so the new `.md` is self-contained.
*/
async function regenerateMdFromHtml(outDir) {
const mdConverter = makeMarkdownConverter()
const cheerio = require('cheerio')
let regenerated = 0
let created = 0
let skipped = 0
await visit(outDir)
return { regenerated, created, skipped }
// Sequential is fine: ~2.5k pages * ~10ms parse + convert is well under a
// minute. Parallelism would add complexity without measurable benefit at
// this scale and risk hitting the open-file-descriptor limit.
async function visit(dir) {
const entries = await fs.readdir(dir, { withFileTypes: true })
for (const entry of entries) {
const full = path.join(dir, entry.name)
if (entry.isDirectory()) {
await visit(full)
} else if (entry.name === 'index.html') {
await processOne(full, dir)
}
}
}
async function processOne(htmlAbs, dir) {
const relDir = toPosix(path.relative(outDir, dir))
// Skip the site root: `<outDir>/index.html` would map to a top-level
// `.md` that has no canonical URL counterpart in the markdown space.
if (!relDir || relDir === '.') return
let html
try {
html = await fs.readFile(htmlAbs, 'utf8')
} catch {
skipped++
return
}
const newBody = htmlToMarkdown(html, cheerio, mdConverter)
if (!newBody) {
skipped++
return
}
const mdAbs = path.join(outDir, `${relDir}.md`)
let existed = true
try {
await fs.access(mdAbs)
} catch {
existed = false
}
if (!existed) {
await fs.mkdir(path.dirname(mdAbs), { recursive: true })
}
await fs.writeFile(mdAbs, newBody, 'utf8')
if (existed) regenerated++
else created++
}
}
function htmlToMarkdown(html, cheerio, mdConverter) {
const $ = cheerio.load(html)
let $article = $('article').first()
if ($article.length === 0) {
$article = $('main').first()
}
if ($article.length === 0) return null
// KaTeX renders each math expression as a `.katex` container holding BOTH a
// `.katex-mathml` (MathML for screen readers, with a `<math>` element whose
// child `<mrow>` and child `<annotation>` both produce the same text) AND a
// `.katex-html` (visual rendering, marked `aria-hidden="true"`). If we
// simply pass the container through to the markdown converter, the
// resulting markdown contains every math expression two or three times
// (mrow text + annotation text + visual text), which AFDocs' content-parity
// check counts as "missing from markdown" relative to the rendered HTML's
// deduplicated visual text.
//
// Replace each `.katex` with a `<code>` element holding the **visual**
// text extracted from `.katex-html`. NHM converts `<code>` to a single
// backtick span (`` `n` ``) in the .md, and AFDocs' markdown stripper
// protects backtick spans then restores their bare content as plain text.
// The HTML side (post-`injectAlternateLinks`) marks `.katex-mathml` with
// `data-markdown-ignore` so AFDocs only sees the `.katex-html` rendering
// — the exact same string we embed here. Both sides thus normalize to
// identical text, so prose sentences with inline math (`\sigma` → σ,
// `a_1` → a1, etc.) substring-match cleanly. Using the LaTeX annotation
// source instead would re-introduce the mismatch that broke parity on
// `embedded-wallets/infrastructure/sss-architecture` (52% missing).
$article.find('.katex').each((_, el) => {
const $el = $(el)
const visual = $el.find('.katex-html').first().text().trim()
const annotation = $el.find('annotation[encoding="application/x-tex"]').first().text().trim()
const inner = visual || annotation
if (!inner) {
$el.remove()
return
}
const $code = $('<code></code>')
$code.text(inner)
$el.replaceWith($code)
})
// Strip elements that are intentionally human-only (chrome wrappers tagged
// with data-markdown-ignore in src/theme/DocItem/Layout/index.jsx) plus
// structural noise the converter would otherwise leave as empty markdown.
// `.katex` elements are gone by this point so the aria-hidden selector
// only catches non-math hidden content (e.g. anchor link icons).
$article.find('[data-markdown-ignore]').remove()
$article.find('nav, script, style, aside, [aria-hidden="true"]').remove()
// The body directive injected by injectAlternateLinks() lives outside
// <article>, so it is not present here. No removal needed.
// Flatten Docusaurus chrome inside <details> elements. Docusaurus wraps
// collapsible content like this:
//
// <details class="details_lb9f alert ..." data-collapsed="true">
// <summary>Title</summary>
// <div>
// <div class="collapsibleContent_i85q">
// <div>
// <p>real content</p>
// </div>
// </div>
// </div>
// </details>
//
// The custom NHM translator below preserves <details>/<summary> as raw
// HTML wrappers and recurses into children. Without this flattening, the
// emitted .md would carry every wrapper <div>, which AFDocs'
// markdown-content-parity normalizer collapses to text (`<x>` becomes
// `x`), smushing classnames like `collapsibleContent_i85q` into adjacent
// prose and breaking substring matches against the clean HTML segments.
//
// Unwrap iteratively so arbitrary depths of nested wrapper <div>s flatten.
// We only unwrap direct children of <details> that are <div>s with either
// no class or the Docusaurus collapsible-content class — never <div>s
// that the author intentionally placed (which always carry meaningful
// class names).
$article.find('details').each((_, el) => {
const $det = $(el)
Object.keys($det.attr() || {}).forEach(attr => $det.removeAttr(attr))
$det.children('summary').each((__, sEl) => {
const $sum = $(sEl)
Object.keys($sum.attr() || {}).forEach(attr => $sum.removeAttr(attr))
})
let didUnwrap = true
while (didUnwrap) {
didUnwrap = false
$det.children('div').each((__, dEl) => {
const $d = $(dEl)
const cls = $d.attr('class') || ''
if (!cls.trim() || /\bcollapsibleContent[_-]/.test(cls)) {
$d.replaceWith($d.contents())
didUnwrap = true
}
})
}
})
const contentHtml = ($article.html() || '').trim()
if (!contentHtml) return null
const md = mdConverter.translate(contentHtml).trim()
if (!md) return null
return md + '\n'
}
function makeMarkdownConverter() {
const { NodeHtmlMarkdown } = require('node-html-markdown')
// node-html-markdown is GFM-aware by default (tables, strikethrough, line
// breaks), and ATX-only for headings, so no plugin equivalent of
// turndown-plugin-gfm is required. The remaining style options mirror the
// previous turndown configuration: fenced code blocks with ```, `-`
// bullets (NHM defaults to `*`), `_` for emphasis, `**` for strong,
// inline links over reference style.
const options = {
codeFence: '```',
codeBlockStyle: 'fenced',
bulletMarker: '-',
emDelimiter: '_',
strongDelimiter: '**',
useInlineLinks: true,
// Override both default escapes to a narrower set. Character sequences
// that have no ambiguous meaning in our doc body — square brackets in
// prose like "(string) [optional]", parentheses, hash symbols
// mid-sentence, asterisks adjacent to letters etc. — shouldn't get
// backslash-escaped. NHM's defaults would insert `\[`, `\]`, `\#`,
// `\>`, `\*`, `\_`, etc., none of which the AFDocs
// `markdown-content-parity` normalizer strips. The result is segments
// like `callGasLimit: (string) [optional] - description` failing to
// match `callGasLimit: (string) \[optional\] - description` even
// though the prose is identical. Underscores inside identifiers are
// also escaped by default (`my_var` -> `my\_var`), which breaks the
// same substring check.
//
// `globalEscape` is narrowed to backticks ONLY. A literal backtick in
// a text node (i.e. outside any `<code>`/`<pre>` — those elements set
// `noEscape: true` in NHM's default translators, so this regex is not
// applied to their contents) is the one character we cannot leave raw:
// unescaped, it would open an unintended inline code span in the
// emitted markdown, splicing surrounding prose into a `<code>` block
// when the markdown is rendered. The other characters NHM escapes by
// default (`\\`, `*`, `_`, `~`, `[`, `]`) are intentionally left raw
// for the parity-check reasons above. `lineStartEscape` is reduced
// to the sequences that would actually produce broken markdown if
// left raw at the start of a line: a leading `#`/`>` (could create a
// heading or blockquote) or a leading `-`/`*`/`+` followed by
// whitespace (could create a list item). A leading `\d+.<space>`
// escape was previously included, but NHM applies these escapes to
// the text content of headings BEFORE prefixing `### ` etc.: a
// heading like `<h3>1. Set up the project</h3>` becomes
// `### 1\. Set up the project`, and AFDocs'
// markdown-content-parity stripper does not consume the literal `\`,
// so the segment fails to match the HTML side `1. Set up the
// project`. Body lists are emitted by NHM's `<ol>`/`<li>`
// translators directly (which insert their own `1. ` markers), so
// dropping this escape only affects free-standing `1. foo` text at
// line start — which our docs do not produce outside list contexts.
// NHM's TS signature documents the replacement as a string, but it
// passes the tuple straight through to `String.prototype.replace()`,
// which accepts a function at runtime; the function form is the only
// way to express the per-branch backslash positioning below.
globalEscape: [/`/g, '\\$&'],
lineStartEscape: [
/^([#>])|^([-*+])(?=\s)/gm,
(match, hashOrGt, bullet) => {
if (hashOrGt !== undefined) return '\\' + hashOrGt
if (bullet !== undefined) return '\\' + bullet
return match
},
],
}
const customTranslators = {
// Keep `<details>` and `<summary>` as raw HTML wrappers (GFM renders
// them natively in `.md`), but let NHM convert their children to
// markdown so the body of a collapsible block is plain markdown — not
// raw HTML chrome.
//
// The blank line between `</summary>` and the first content element is
// load-bearing: GFM-flavoured markdown parsers only treat inline-HTML
// children as markdown when there's a blank line separator. The
// `postfix: '</summary>\n\n'` plus NHM's own `\n\n` around block
// children produce that separator.
//
// The previous translator returned `content: node.outerHTML`, which
// embedded Docusaurus class wrappers (`details_lb9f`,
// `collapsibleContent_i85q`, etc.) verbatim into the .md. AFDocs'
// markdown-content-parity normalizer collapses `<x>` to its content,
// jamming those class names into adjacent prose and breaking
// substring matches against the rendered HTML's clean text segments.
// The cheerio pre-pass in `htmlToMarkdown` flattens those wrappers
// before NHM ever sees the tree.
details: {
prefix: '<details>\n',
postfix: '\n</details>',
surroundingNewlines: 2,
noEscape: true,
},
summary: {
prefix: '<summary>',
postfix: '</summary>\n\n',
noEscape: true,
},
}
return new NodeHtmlMarkdown(options, customTranslators)
}
/**
* Prepend a single-line blockquote pointing to `/llms.txt` to every per-page
* `.md`. Idempotent: skips files that already contain the directive (so
* scripts/verify-llms-output.js can be re-run safely against an existing
* outDir).
*/
async function prependMdDirective(outDir) {
const mdFiles = await collectMarkdownFiles(outDir)
let updated = 0
for (const mdAbs of mdFiles) {
const text = await fs.readFile(mdAbs, 'utf8')
if (text.includes(MD_DIRECTIVE_LINE)) continue
// If a file still has YAML frontmatter (i.e. `regenerateMdFromHtml`
// skipped it because no HTML sibling was found), insert the directive
// after the closing `---` so frontmatter parsers still see the document
// as well-formed.
let next
const fmMatch = text.match(/^---\r?\n[\s\S]*?\r?\n---\r?\n/)
if (fmMatch) {
const idx = fmMatch[0].length
next = text.slice(0, idx) + '\n' + MD_DIRECTIVE_LINE + '\n\n' + text.slice(idx)
} else {
next = MD_DIRECTIVE_LINE + '\n\n' + text
}
await fs.writeFile(mdAbs, next, 'utf8')
updated++
}
return updated
}
/**
* Convert a `vercel.json` redirect `source` into an anchored RegExp matching a
* URL pathname (trailing slash stripped). Mirrors Vercel's path-to-regexp
* subset we use: `:name*` / `:name+` match across segments, `:name` matches a
* single segment.
*/
function redirectSourceToRegExp(source) {
let s = source.replace(/\/+$/, '')
// Escape regex metacharacters, but leave `:`/`*`/`+` for param handling.
s = s.replace(/[.^${}()|[\]\\?]/g, '\\$&')
// `:name*` or `:name+` -> match across one or more segments (`.*`).
s = s.replace(/:\w+[*+]/g, '.*')
// `:name` -> match a single path segment.
s = s.replace(/:\w+/g, '[^/]+')
return new RegExp(`^${s}$`)
}
/**
* Load the list of redirect matchers from `vercel.json` (located in the repo
* root, one level above the build `outDir`). Returns an empty array if the file
* is missing or unparseable so sitemap normalization degrades gracefully.
*/
async function loadRedirectMatchers(outDir) {
const vercelPath = path.join(outDir, '..', 'vercel.json')
let raw
try {
raw = await fs.readFile(vercelPath, 'utf8')
} catch {
return []
}
let parsed
try {
parsed = JSON.parse(raw)
} catch {
return []
}
const redirects = Array.isArray(parsed.redirects) ? parsed.redirects : []
const matchers = []
for (const r of redirects) {
if (!r || typeof r.source !== 'string') continue
try {
matchers.push(redirectSourceToRegExp(r.source))
} catch {
// Ignore sources we can't compile.
}
}
return matchers
}
/**
* Normalize `build/sitemap.xml` so every listed URL returns 200 (not a 3XX):
*
* - Strip the trailing slash from `<loc>`s whose final path segment contains
* a dot (Vercel serves these without a trailing slash; see
* `stripDottedTrailingSlashInMetadata`).
* - Drop entire `<url>` entries whose pathname is shadowed by a `vercel.json`
* redirect (e.g. `.../eth_newpendingtransactionfilter/`, `.../tron/web/`),
* since those 3XX-redirect on the deployed site and should not be indexed.
*/
async function normalizeSitemap(outDir, siteUrl) {