feat: serve markdown source files for AI agent access (#2932)

Co-authored-by: Jo Franchetti <jofranchetti@gmail.com>

Author: donjo

_config.ts

@@ -16,6 +16,8 @@ import title from "https://deno.land/x/lume_markdown_plugins@v0.7.0/title.ts";
 import toc from "https://deno.land/x/lume_markdown_plugins@v0.7.0/toc.ts";
 // See note below about GFM CSS
 // import { CSS as GFM_CSS } from "https://jsr.io/@deno/gfm/0.11.0/style.ts";
+import { walk } from "@std/fs";
+import { dirname } from "@std/path";
 import { log } from "lume/core/utils/log.ts";
 import anchor from "npm:markdown-it-anchor@9";
 import admonitionPlugin from "./markdown-it/admonition.ts";
@@ -28,6 +30,7 @@ import createRoutingMiddleware from "./middleware/functionRoutes.ts";
 import createGAMiddleware from "./middleware/googleAnalytics.ts";
 import redirectsMiddleware from "./middleware/redirects.ts";
 import createLlmsFilesMiddleware from "./middleware/llmsFiles.ts";
+import createMarkdownSourceMiddleware from "./middleware/markdownSource.ts";
 import { toFileAndInMemory } from "./utils/redirects.ts";
 import { cliNow } from "./timeUtils.ts";
 
@@ -73,6 +76,7 @@ const site = lume(
     server: {
       middlewares: [
         redirectsMiddleware,
+        createMarkdownSourceMiddleware({ root: "_site" }),
         createRoutingMiddleware(),
         createGAMiddleware({
           addr: { transport: "tcp", hostname: "localhost", port: 3000 },
@@ -230,6 +234,49 @@ site.addEventListener("afterBuild", async () => {
       log.error("Error generating LLMs files:" + error);
     }
   }
+
+  // Copy source .md files to _site so AI agents can request them directly.
+  // Excludes "reference/" (dynamically generated, no static .md source files).
+  const contentDirs = [
+    "runtime",
+    "deploy",
+    "sandbox",
+    "subhosting",
+    "examples",
+  ];
+  let mdCopied = 0;
+  let mdErrors = false;
+  for (const dir of contentDirs) {
+    // Skip directories that don't exist in this build
+    try {
+      await Deno.stat(dir);
+    } catch (error) {
+      if (!(error instanceof Deno.errors.NotFound)) {
+        log.error(`Error accessing content directory ${dir}: ${error}`);
+      }
+      continue;
+    }
+    try {
+      for await (
+        const entry of walk(dir, { exts: [".md"], includeDirs: false })
+      ) {
+        const destPath = site.dest(entry.path);
+        await Deno.mkdir(dirname(destPath), { recursive: true });
+        await Deno.copyFile(entry.path, destPath);
+        mdCopied++;
+      }
+    } catch (error) {
+      log.error(`Error copying markdown files from ${dir}: ${error}`);
+      mdErrors = true;
+    }
+  }
+  if (mdErrors) {
+    log.warn(
+      `Copied ${mdCopied} source markdown files to _site (some directories had errors, see above)`,
+    );
+  } else {
+    log.info(`Copied ${mdCopied} source markdown files to _site`);
+  }
 });
 
 site.copy("reference_gen/gen/deno/page.css", "/api/deno/page.css");

_includes/layout.tsx

@@ -51,6 +51,15 @@ export default function Layout(data: Lume.Data) {
           type="font/woff2"
           crossOrigin="anonymous"
         />
+        {data.page?.sourcePath?.endsWith(".md") && data.url !== "/" && (
+          <link
+            rel="alternate"
+            type="text/markdown"
+            href={data.page.sourcePath.endsWith("/index.md")
+              ? `/${data.page.sourcePath}`
+              : `${data.url.replace(/\/$/, "")}.md`}
+          />
+        )}
         <link rel="me" href="https://fosstodon.org/@deno_land" />
         <data.comp.OpenGraph
           title={data.title}

middleware/markdownSource.ts

@@ -0,0 +1,98 @@
+import { join } from "@std/path";
+import { log } from "lume/core/utils/log.ts";
+import type { RequestHandler } from "lume/core/server.ts";
+
+interface MarkdownSourceMiddlewareOptions {
+  /** Directory where the built static assets live. */
+  root?: string;
+}
+
+/** Reads a markdown file and returns a Response, or null if not found. Throws on other errors. */
+async function serveMarkdownFile(
+  filePath: string,
+  absoluteRoot: string,
+): Promise<Response | null> {
+  if (!filePath.startsWith(absoluteRoot + "/")) {
+    return new Response("Forbidden", { status: 403 });
+  }
+  try {
+    const file = await Deno.readFile(filePath);
+    return new Response(file, {
+      headers: new Headers({
+        "content-type": "text/markdown; charset=utf-8",
+        "cache-control": "public, max-age=300",
+      }),
+    });
+  } catch (error) {
+    if (error instanceof Deno.errors.NotFound) {
+      return null;
+    }
+    throw error;
+  }
+}
+
+export default function createMarkdownSourceMiddleware(
+  { root = "_site" }: MarkdownSourceMiddlewareOptions = {},
+): RequestHandler {
+  const absoluteRoot = root.startsWith("/") ? root : join(Deno.cwd(), root);
+
+  return async function markdownSourceMiddleware(req, next) {
+    const pathname = new URL(req.url).pathname;
+
+    // Case A: Direct .md URL request (e.g. /runtime/getting_started/installation.md)
+    if (pathname.endsWith(".md")) {
+      const filePath = join(absoluteRoot, pathname.slice(1));
+      try {
+        const response = await serveMarkdownFile(filePath, absoluteRoot);
+        if (response) return response;
+        // Fallback: /foo/bar.md → /foo/bar/index.md (for index.md source files)
+        const indexPath = filePath.replace(/\.md$/, "/index.md");
+        const indexResponse = await serveMarkdownFile(indexPath, absoluteRoot);
+        return indexResponse ?? next(req);
+      } catch (error) {
+        log.error(`Failed serving ${pathname} from ${filePath}: ${error}`);
+        return new Response("Internal Server Error", { status: 500 });
+      }
+    }
+
+    // Case B: Accept: text/markdown header — content negotiation
+    // Tools like Claude Code send this header when they prefer markdown over HTML
+    const acceptHeader = req.headers.get("accept") ?? "";
+    if (
+      acceptHeader.includes("text/markdown") &&
+      !acceptHeader.includes("text/html")
+    ) {
+      // Convert the HTML path to its .md equivalent
+      // e.g. /runtime/getting_started/installation/ -> /runtime/getting_started/installation.md
+      // Also try the index.md variant for directory-style URLs (e.g. /runtime/contributing/)
+      const mdPath = pathname.replace(/\/$/, "") + ".md";
+      const mdIndexPath = pathname.replace(/\/$/, "") + "/index.md";
+      const filePath = join(absoluteRoot, mdPath.slice(1));
+      const indexFilePath = join(absoluteRoot, mdIndexPath.slice(1));
+      try {
+        const response = (await serveMarkdownFile(filePath, absoluteRoot)) ??
+          (await serveMarkdownFile(indexFilePath, absoluteRoot));
+        if (response) {
+          // Add Vary: Accept so caches don't serve this markdown to requests that want HTML
+          response.headers.set("vary", "Accept");
+          return response;
+        }
+        // No .md source for this page (e.g. generated API reference) — serve HTML normally.
+        // Add Vary: Accept so HTTP caches don't serve this HTML to markdown-only requests.
+        const htmlResponse = await next(req);
+        const headers = new Headers(htmlResponse.headers);
+        headers.set("vary", "Accept");
+        return new Response(htmlResponse.body, {
+          status: htmlResponse.status,
+          statusText: htmlResponse.statusText,
+          headers,
+        });
+      } catch (error) {
+        log.error(`Failed serving markdown for ${pathname}: ${error}`);
+        return new Response("Internal Server Error", { status: 500 });
+      }
+    }
+
+    return next(req);
+  };
+}

server.ts

@@ -8,10 +8,12 @@ import redirectsMiddleware from "./middleware/redirects.ts";
 import createRoutingMiddleware from "./middleware/functionRoutes.ts";
 import expires from "lume/middlewares/expires.ts";
 import createLlmsFilesMiddleware from "./middleware/llmsFiles.ts";
+import createMarkdownSourceMiddleware from "./middleware/markdownSource.ts";
 
 export const server = new Server({ root: "_site" });
 
 server.use(redirectsMiddleware);
+server.use(createMarkdownSourceMiddleware({ root: "_site" }));
 server.use(createLlmsFilesMiddleware({ root: "_site" }));
 server.use(NotFoundMiddleware({ root: "_site", page404: "./404/" }));
 server.use(createRoutingMiddleware());