Improve LLM visibility

Author: DarthSim

.gitignore

@@ -18,3 +18,6 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# Local Netlify folder
+.netlify

docs/intro.mdx

@@ -2,7 +2,7 @@
 slug: '/'
 title: ''
 description: imgproxy is a fast and secure standalone server for resizing and converting remote images
-displayed_sidebar: tutorialSidebar
+displayed_sidebar: main
 ---
 
 <h1>

docusaurus.config.ts

@@ -3,6 +3,7 @@ import {
   Options as PresetClassicOptions,
   ThemeConfig as PresetClassicThemeConfig,
 } from "@docusaurus/preset-classic";
+import { join } from "node:path";
 
 import badgeRemarkPlugin from "./src/remark/badge";
 import codeAnchorRemarkPlugin from "./src/remark/code-anchor";
@@ -28,6 +29,12 @@ const config: Config = {
   baseUrl: "/",
 
   onBrokenLinks: "throw",
+  // Anchors for configuration options are generated dynamically,
+  // so Docusaurus can't know them in advance.
+  // It'd be nice to be able to verify anchors, but for now,
+  // let's just ignore broken anchors instead flooding the build
+  // output with warnings.
+  onBrokenAnchors: "ignore",
 
   i18n: {
     defaultLocale: "en",
@@ -92,6 +99,8 @@ const config: Config = {
     },
   ],
 
+  plugins: [join(__dirname, "src/plugins/llms.ts")],
+
   presets: [
     [
       "classic",

netlify/edge-functions/llm-middleware.ts

@@ -0,0 +1,176 @@
+import type { Config, Context } from "@netlify/edge-functions";
+import { extname } from "path";
+
+const LLMS_REWRITES = new Set(["/llms.txt", "/llms-full.txt"]);
+
+export default async function handler(request: Request, context: Context) {
+  try {
+    // Only handle GET and HEAD requests
+    if (request.method !== "GET" && request.method !== "HEAD") return;
+
+    // Skip our own Algolia crawler — it follows rel="alternate" links and
+    // would otherwise index the .md variants.
+    const userAgent = request.headers.get("user-agent") || "";
+    if (/algolia/i.test(userAgent)) return;
+
+    const url = new URL(request.url);
+    const { pathname } = url;
+
+    // Respond with index.md for llms.txt and llms-full.txt,
+    // as index.md is well suited for this
+    if (LLMS_REWRITES.has(pathname)) {
+      return buildTarget("/index.md", url);
+    }
+
+    const ext = extname(pathname);
+    if (ext === ".html" || ext === ".md") {
+      // For direct requests to .html or .md files,
+      // add a link header pointing to the alternate format.
+      return modifyHeaders(await context.next(), (headers) => {
+        addAlternateLink(headers, url);
+      });
+    } else if (ext) {
+      // Skip other requests with file extensions,
+      // as they are static assets that shouldn't have alternate links.
+      return;
+    }
+
+    // For other requests, check if the client prefers Markdown over HTML.
+    // If so, try to serve the corresponding Markdown file
+    // (e.g., /foo -> /foo/index.md).
+    // If the Markdown file doesn't exist (404),
+    // continue with the normal request handling.
+    if (prefersMarkdown(request.headers.get("accept"))) {
+      const target = buildTarget(joinIndex(pathname), url);
+      const response = await fetch(target);
+      if (response.status !== 404) return finalize(response, url);
+    }
+
+    // For all other cases, proceed with the normal request handling.
+    return finalize(await context.next(), url);
+  } catch (error) {
+    console.error("Error in LLM middleware:", error);
+    // In case of any error, proceed with the normal request handling
+    return context.next();
+  }
+}
+
+function buildTarget(pathname: string, base: URL): URL {
+  const target = new URL(pathname, base);
+  target.search = base.search;
+  return target;
+}
+
+function joinIndex(pathname: string): string {
+  return pathname.replace(/\/?$/, "/") + "index.md";
+}
+
+function prefersMarkdown(accept: string | null): boolean {
+  if (!accept) return false;
+
+  let markdownQ = -1;
+  let htmlQ = -1;
+  let textQ = -1;
+  let anyQ = -1;
+
+  for (const part of accept.split(",")) {
+    const segments = part.trim().split(";");
+    const type = segments[0].trim().toLowerCase();
+    if (!type) continue;
+
+    let q = 1;
+    for (let i = 1; i < segments.length; i++) {
+      const param = segments[i].trim();
+      if (!param.startsWith("q=")) continue;
+      const value = Number.parseFloat(param.slice(2));
+      if (!Number.isNaN(value)) q = value;
+    }
+
+    if (type === "text/markdown") {
+      if (q > markdownQ) markdownQ = q;
+    } else if (type === "text/html") {
+      if (q > htmlQ) htmlQ = q;
+    } else if (type === "text/*") {
+      if (q > textQ) textQ = q;
+    } else if (type === "*/*") {
+      if (q > anyQ) anyQ = q;
+    }
+  }
+
+  if (htmlQ < 0) htmlQ = textQ > 0 ? textQ : anyQ;
+
+  return markdownQ > 0 && markdownQ >= htmlQ;
+}
+
+function finalize(response: Response, url: URL): Response {
+  return modifyHeaders(response, (headers) => {
+    appendVary(headers, "Accept");
+    addAlternateLink(headers, new URL(response.url, url));
+  });
+}
+
+function modifyHeaders(
+  response: Response,
+  fn: (headers: Headers) => void,
+): Response {
+  const headers = new Headers(response.headers);
+
+  fn(headers);
+
+  return new Response(response.body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  });
+}
+
+function appendVary(headers: Headers, value: string) {
+  const existing = headers.get("vary");
+  if (!existing) {
+    headers.set("vary", value);
+    return;
+  }
+  const tokens = existing.split(",").map((s) => s.trim());
+  if (tokens.some((t) => t.toLowerCase() === value.toLowerCase())) return;
+  headers.set("vary", `${existing}, ${value}`);
+}
+
+function addAlternateLink(headers: Headers, url: URL) {
+  let alternatePath: string | null = null;
+  let alternateType = "text/markdown";
+
+  const ext = extname(url.pathname);
+  if (ext === ".html") {
+    alternatePath = url.pathname.replace(/\.html$/, ".md");
+  } else if (ext === ".md") {
+    alternatePath = url.pathname.replace(/\.md$/, ".html");
+    alternateType = "text/html";
+  } else if (ext === "") {
+    alternatePath = joinIndex(url.pathname);
+  }
+
+  if (!alternatePath) return;
+
+  const alternateUrl = buildTarget(alternatePath, url);
+
+  const link = `<${alternateUrl}>; rel="alternate"; type="${alternateType}"`;
+  headers.set("link", link);
+}
+
+export const config: Config = {
+  path: "/*",
+  excludedPath: [
+    "/**/*.js",
+    "/**/*.css",
+    "/**/*.png",
+    "/**/*.jpg",
+    "/**/*.jpeg",
+    "/**/*.svg",
+    "/**/*.ico",
+    "/**/*.xml",
+    "/robots.txt",
+    "/404.html",
+    "/_redirects",
+    "/.nojekyll",
+  ],
+};

package.json

@@ -33,6 +33,7 @@
     "@docusaurus/types": "^3.9.2",
     "@eslint/js": "^9.39.1",
     "@evilmartians/lefthook": "^2.0.4",
+    "@netlify/edge-functions": "^3.0.6",
     "@types/mdast": "4.0.4",
     "eslint": "^9.39.1",
     "eslint-config-prettier": "^10.1.8",

pnpm-lock.yaml

@@ -1,7 +1,7 @@
 import { SidebarsConfig } from "@docusaurus/plugin-content-docs";
 
 const sidebars: SidebarsConfig = {
-  tutorialSidebar: [
+  main: [
     "getting_started",
     {
       type: "link",