fix(docs): make the docs site readable to agents that fetch it (#34)

lukaisailovic · web-flow · commit 92521ee4b091 · 2026-06-25T18:59:58.000+02:00
Agents pointed at the docs hit a client-rendered SPA: a wrong URL guess
(e.g. /docs) returns a 200 shell whose only text is "OpenIslands", with no
pointer to the plain-text docs that already exist (/llms.txt, /llms-full.txt,
per-page .md). And once on a .md page, its internal links pointed back at the
JS-rendered HTML routes, bouncing the agent out of markdown.

- __root.tsx: signpost the plain-text docs from the root (so a JS-less fetch
  of any path, incl. the SPA fallback shell, finds them): a
  rel="alternate" text/markdown link plus a &lt;noscript&gt; listing
  /llms-full.txt, /llms.txt, the append-.md trick, and /start.md.
- source.ts: linkToMarkdownSiblings() rewrites internal absolute doc links to
  their .md sibling (skips the root, asset files, and external links).
- {$}.md route: rewrite cross-links to .md and append a footer pointing at
  /llms-full.txt and /llms.txt.
- llms.txt route: rewrite the index links to .md and prepend a note telling
  the agent the .md suffix works for any page (per the llms.txt convention of
  highlighting markdown versions).

Content negotiation (Accept: text/markdown on the same URL) is intentionally
not done — it needs a request-time server, and the docs deploy is assets-only
static. The .md suffix is the static-compatible equivalent.
diff --git a/apps/docs/src/lib/source.ts b/apps/docs/src/lib/source.ts
@@ -36,3 +36,16 @@ export async function getLLMText(page: (typeof source)["$inferPage"]) {
 
 ${processed}`;
 }
+
+const INTERNAL_DOC_LINK = /\]\((\/[^)\s#]*)(#[^)\s]*)?\)/g;
+
+// An agent that opens one page's .md and follows a link should land on the next page's
+// .md, not the JS-rendered HTML. Rewrite internal absolute doc links to their .md sibling
+// (the same /foo → /foo.md mapping the site already serves), leaving the site root and
+// asset files (anything with an extension) and external links untouched.
+export function linkToMarkdownSiblings(md: string): string {
+  return md.replace(INTERNAL_DOC_LINK, (match, path: string, anchor = "") => {
+    if (path === "/" || /\.[a-z0-9]+$/i.test(path)) return match;
+    return `](${path}.md${anchor})`;
+  });
+}
diff --git a/apps/docs/src/routes/__root.tsx b/apps/docs/src/routes/__root.tsx
@@ -19,6 +19,15 @@ export const Route = createRootRoute({
     links: [
       { rel: "stylesheet", href: appCss },
       { rel: "icon", href: "/favicon.svg", type: "image/svg+xml" },
+      // Agent/LLM discovery: every page advertises the plain-text docs. Pairs with the
+      // <noscript> below so a JS-less fetch of ANY url (incl. wrong guesses → the SPA
+      // shell) finds /llms.txt instead of an empty "OpenIslands" page.
+      {
+        rel: "alternate",
+        type: "text/markdown",
+        href: "/llms.txt",
+        title: "OpenIslands docs as plain text (for LLMs)",
+      },
     ],
   }),
   component: RootComponent,
@@ -58,6 +67,29 @@ function RootComponent() {
         <HeadContent />
       </head>
       <body className="flex flex-col min-h-screen">
+        {/* Rendered into the static shell that serves every unmatched path. A JS-less
+            client (WebFetch, crawlers, agents) sees this pointer home instead of a blank
+            page; real JS clients never paint it. */}
+        <noscript>
+          <p>
+            These docs render with JavaScript. Reading as an LLM or agent? Plain-text docs:
+          </p>
+          <ul>
+            <li>
+              <a href="/llms-full.txt">/llms-full.txt</a> — every page, one file
+            </li>
+            <li>
+              <a href="/llms.txt">/llms.txt</a> — page index
+            </li>
+            <li>
+              Any page as Markdown: append <code>.md</code> (e.g.{" "}
+              <a href="/introduction.md">/introduction.md</a>)
+            </li>
+            <li>
+              <a href="/start.md">/start.md</a> — agent onboarding
+            </li>
+          </ul>
+        </noscript>
         <RootProvider theme={{ defaultTheme: "dark", enabled: false }} search={{ SearchDialog }}>
           <Outlet />
         </RootProvider>
diff --git a/apps/docs/src/routes/llms[.]txt.ts b/apps/docs/src/routes/llms[.]txt.ts
@@ -1,12 +1,19 @@
-import { source } from "@/lib/source";
+import { linkToMarkdownSiblings, source } from "@/lib/source";
 import { createFileRoute } from "@tanstack/react-router";
 import { llms } from "fumadocs-core/source";
 
+// llms.txt is the agent's entry point, so it points straight at the markdown: links
+// below are rewritten to .md, and the note tells an agent the .md suffix works for any
+// page it constructs itself (per the llms.txt convention of highlighting markdown).
+const MARKDOWN_NOTE = `> These docs are markdown-first for agents. Each link below points to a page's raw \`.md\` — append \`.md\` to any page URL yourself, too. For every page concatenated into one file, fetch [/llms-full.txt](/llms-full.txt).
+
+`;
+
 export const Route = createFileRoute("/llms.txt")({
   server: {
     handlers: {
       GET() {
-        return new Response(llms(source).index());
+        return new Response(MARKDOWN_NOTE + linkToMarkdownSiblings(llms(source).index()));
       },
     },
   },
diff --git a/apps/docs/src/routes/{$}[.]md.ts b/apps/docs/src/routes/{$}[.]md.ts
@@ -1,5 +1,12 @@
 import { createFileRoute, notFound } from "@tanstack/react-router";
-import { getLLMText, markdownPathToSlugs, source } from "@/lib/source";
+import { getLLMText, linkToMarkdownSiblings, markdownPathToSlugs, source } from "@/lib/source";
+
+const MARKDOWN_FOOTER = `
+
+---
+
+*This is one page of the OpenIslands docs. Every page in one file: [/llms-full.txt](/llms-full.txt). Page index: [/llms.txt](/llms.txt). Links above point to \`.md\` siblings — append \`.md\` to any page URL for its raw markdown.*
+`;
 
 export const Route = createFileRoute("/{$}.md")({
   server: {
@@ -9,7 +16,8 @@ export const Route = createFileRoute("/{$}.md")({
         const page = source.getPage(slugs);
         if (!page) throw notFound();
 
-        return new Response(await getLLMText(page), {
+        const markdown = linkToMarkdownSiblings(await getLLMText(page)) + MARKDOWN_FOOTER;
+        return new Response(markdown, {
           headers: {
             "Content-Type": "text/markdown",
           },
