Add markdown and llm.txt to docs (#6408)

* add markdown and llm.txt to docs

* rename markdown-clean to markdown, sync cache-control headers, add edge injection

* revert sst-env.d.ts

* guard entry.body instead of non-null assertion

* use consistent newlines in markdown template literals

* rename utils/ to util/, add edge injection comments

* strip tsdoc HTML artifacts from markdown output

Author: vimtor

www/src/pages/docs/[...slug].md.ts

@@ -0,0 +1,40 @@
+import { getCollection, getEntry } from "astro:content";
+import type { APIRoute } from "astro";
+import { cleanMarkdown } from "../../util/markdown";
+
+export async function getStaticPaths() {
+  const docs = await getCollection("docs");
+  return docs
+    .filter(
+      (doc) =>
+        doc.id.startsWith("docs/") &&
+        doc.id !== "docs/index.mdx"
+    )
+    .map((doc) => ({
+      params: { slug: doc.id.replace(/^docs\//, "").replace(/\.mdx?$/, "") },
+    }));
+}
+
+export const GET: APIRoute = async ({ params }) => {
+  const slug = params.slug!;
+  const entry = await getEntry("docs", `docs/${slug}`);
+  if (!entry?.body) return new Response("Not found", { status: 404 });
+
+  const cleaned = cleanMarkdown(entry.body);
+  const markdown = `# ${entry.data.title}
+
+${entry.data.description || ""}
+
+Source: https://sst.dev/docs/${slug}
+
+---
+
+${cleaned}`;
+
+  return new Response(markdown, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      "Cache-Control": "public,max-age=0,s-maxage=86400,stale-while-revalidate=86400",
+    },
+  });
+};

www/src/pages/docs/index.md.ts

@@ -0,0 +1,26 @@
+import { getEntry } from "astro:content";
+import type { APIRoute } from "astro";
+import { cleanMarkdown } from "../../util/markdown";
+
+export const GET: APIRoute = async () => {
+  const entry = await getEntry("docs", "docs");
+  if (!entry?.body) return new Response("Not found", { status: 404 });
+
+  const cleaned = cleanMarkdown(entry.body);
+  const markdown = `# ${entry.data.title}
+
+${entry.data.description || ""}
+
+Source: https://sst.dev/docs
+
+---
+
+${cleaned}`;
+
+  return new Response(markdown, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      "Cache-Control": "public,max-age=0,s-maxage=86400,stale-while-revalidate=86400",
+    },
+  });
+};

www/src/pages/llms-full.txt.ts

@@ -0,0 +1,36 @@
+import { getCollection } from "astro:content";
+import type { APIRoute } from "astro";
+import { cleanMarkdown } from "../util/markdown";
+
+export const GET: APIRoute = async () => {
+  const docs = await getCollection("docs");
+  const filtered = docs
+    .filter((doc) => doc.id.startsWith("docs/"))
+    .sort((a, b) => a.id.localeCompare(b.id));
+
+  const pages = filtered.map((doc) => {
+    const slug = doc.id.replace(/\.mdx?$/, "");
+    const cleaned = cleanMarkdown(doc.body || "");
+    return `## ${doc.data.title}
+
+${doc.data.description || ""}
+
+https://sst.dev/${slug}
+
+${cleaned}`;
+  });
+
+  const body = `# SST Documentation
+
+> The complete SST documentation for building full-stack applications on AWS and Cloudflare.
+
+${pages.join("\n\n---\n\n")}
+`;
+
+  return new Response(body, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public,max-age=0,s-maxage=86400,stale-while-revalidate=86400",
+    },
+  });
+};

www/src/pages/llms.txt.ts

@@ -0,0 +1,33 @@
+import { getCollection } from "astro:content";
+import type { APIRoute } from "astro";
+
+export const GET: APIRoute = async () => {
+  const docs = await getCollection("docs");
+  const filtered = docs
+    .filter((doc) => doc.id.startsWith("docs/"))
+    .sort((a, b) => a.id.localeCompare(b.id));
+
+  const links = filtered
+    .map((doc) => {
+      const slug = doc.id.replace(/\.mdx?$/, "");
+      const description = doc.data.description || "";
+      return `- [${doc.data.title}](https://sst.dev/${slug})${description ? `: ${description}` : ""}`;
+    })
+    .join("\n");
+
+  const body = `# SST
+
+> SST is a framework for building full-stack apps on your own infrastructure with support for AWS, Cloudflare, and 150+ providers.
+
+## Docs
+
+${links}
+`;
+
+  return new Response(body, {
+    headers: {
+      "Content-Type": "text/plain; charset=utf-8",
+      "Cache-Control": "public,max-age=0,s-maxage=86400,stale-while-revalidate=86400",
+    },
+  });
+};

www/src/util/markdown.ts

@@ -0,0 +1,55 @@
+export function cleanMarkdown(source: string): string {
+  return (
+    source
+      // Remove import statements
+      .replace(/^import\s+.*$/gm, "")
+      // Remove export statements
+      .replace(/^export\s+.*$/gm, "")
+      // Remove JSX comments {/* ... */} (single and multiline)
+      .replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
+      // Remove <Image ... /> self-closing tags
+      .replace(/<Image\s+[^>]*\/>/g, "")
+      // Remove <VideoAside ... /> self-closing tags
+      .replace(/<VideoAside\s+[^>]*\/>/g, "")
+      // Remove <LinkCard ... /> self-closing tags
+      .replace(/<LinkCard\s+[^>]*\/>/g, "")
+      // Remove <Icon ... /> self-closing tags
+      .replace(/<Icon\s+[^>]*\/>/g, "")
+      // Remove tsdoc component tags (opening and closing)
+      .replace(/<\/?(?:Section|Segment|InlineSection)(?:\s+[^>]*)?>/g, "")
+      // Convert <NestedTitle ...>content</NestedTitle> to just content
+      .replace(/<NestedTitle[^>]*>([\s\S]*?)<\/NestedTitle>/g, "$1")
+      // Remove <div class="tsdoc"> and </div>
+      .replace(/<div\s+class="tsdoc">/g, "")
+      .replace(/^<\/div>\s*$/gm, "")
+      // Merge adjacent <code> tags into one (for type expressions like Input<string>)
+      .replace(/<\/code><code class="[^"]*">/g, "")
+      // Convert innermost <code class="...">content</code> → `content`
+      .replace(/<code class="[^"]*">([^<]*)<\/code>/g, "`$1`")
+      // Strip remaining outer <code class="..."> and </code> from nested structures
+      .replace(/<code class="[^"]*">/g, "")
+      // Convert plain <code>content</code> → `content`
+      .replace(/<code>([^<]*)<\/code>/g, "`$1`")
+      .replace(/<\/code>/g, "")
+      // Remove <p> and </p> wrapper tags
+      .replace(/<\/?p>/g, "")
+      // Decode common HTML entities
+      .replace(/&lt;/g, "<")
+      .replace(/&gt;/g, ">")
+      .replace(/&amp;/g, "&")
+      .replace(/&ldquo;/g, '"')
+      .replace(/&rdquo;/g, '"')
+      .replace(/&#123;/g, "{")
+      .replace(/&#125;/g, "}")
+      .replace(/&lcub;/g, "{")
+      .replace(/&rcub;/g, "}")
+      // Convert <Tabs>/<TabItem> to labeled sections
+      .replace(/<Tabs>/g, "")
+      .replace(/<\/Tabs>/g, "")
+      .replace(/<TabItem\s+label="([^"]*)">/g, "**$1**\n")
+      .replace(/<\/TabItem>/g, "")
+      // Collapse 3+ consecutive blank lines to 2
+      .replace(/\n{3,}/g, "\n\n")
+      .trim()
+  );
+}

www/sst.config.ts

@@ -185,6 +185,28 @@ export default $config({
               ],
             }
           : domain,
+      edge: {
+        // Rewrite /docs/* to .md when Accept: text/markdown (for AI agents)
+        viewerRequest: {
+          injection: [
+            `var uri = event.request.uri;`,
+            `var accept = (event.request.headers['accept'] || {}).value || '';`,
+            `if (uri.startsWith('/docs') && accept.includes('text/markdown') && !/\\.[a-z0-9]+$/i.test(uri)) {`,
+            `  event.request.uri = (uri === '/docs' || uri === '/docs/')`,
+            `    ? '/docs/index.md'`,
+            `    : uri.replace(/\\/$/, '') + '.md';`,
+            `}`,
+          ].join("\n"),
+        },
+        // Fix Content-Type on .md responses (S3 serves them as octet-stream)
+        viewerResponse: {
+          injection: [
+            `if (event.request.uri.endsWith('.md')) {`,
+            `  event.response.headers['content-type'] = { value: 'text/markdown; charset=utf-8' };`,
+            `}`,
+          ].join("\n"),
+        },
+      },
       transform: {
         cdn: (args) => {
           args.origins = $output(args.origins).apply((origins) => [