Add AI agent detection and automatic markdown rewrites

apps/docs/lib/ai-agent-detection.ts

@@ -0,0 +1,168 @@
+/**
+ * AI Agent Detection Utility
+ *
+ * Multi-signal detection for AI agents/bots. Used to serve markdown
+ * responses when agents request docs pages.
+ *
+ * Three detection layers:
+ * 1. Known UA patterns (definitive) — curated from https://bots.fyi/?tags=ai_assistant
+ * 2. Signature-Agent header (definitive) — catches ChatGPT agent (RFC 9421)
+ * 3. Missing browser fingerprint heuristic — catches unknown bots
+ *
+ * Optimizes for recall over precision: serving markdown to a non-AI bot
+ * is low-harm; missing an AI agent means a worse experience.
+ *
+ * Last reviewed: 2026-03-20 against bots.fyi + official vendor docs
+ */
+
+// Layer 1: Known AI agent UA substrings (lowercase).
+const AI_AGENT_UA_PATTERNS = [
+  // Anthropic — https://support.claude.com/en/articles/8896518
+  'claudebot',
+  'claude-searchbot',
+  'claude-user',
+  'anthropic-ai',
+  'claude-web',
+
+  // OpenAI — https://platform.openai.com/docs/bots
+  'chatgpt',
+  'gptbot',
+  'oai-searchbot',
+  'openai',
+
+  // Google AI
+  'gemini',
+  'bard',
+  'google-cloudvertexbot',
+  'google-extended',
+
+  // Meta
+  'meta-externalagent',
+  'meta-externalfetcher',
+  'meta-webindexer',
+
+  // Search/Research AI
+  'perplexity',
+  'youbot',
+  'you.com',
+  'deepseekbot',
+
+  // Coding assistants
+  'cursor',
+  'github-copilot',
+  'codeium',
+  'tabnine',
+  'sourcegraph',
+
+  // Other AI agents / data scrapers (low-harm to serve markdown)
+  'cohere-ai',
+  'bytespider',
+  'amazonbot',
+  'ai2bot',
+  'diffbot',
+  'omgili',
+  'omgilibot',
+];
+
+// Layer 2: Known AI service URLs in Signature-Agent header (RFC 9421).
+const SIGNATURE_AGENT_DOMAINS = ['chatgpt.com'];
+
+// Layer 3: Traditional bot exclusion list — bots that should NOT trigger
+// the heuristic layer (they're search engine crawlers, social previews, or
+// monitoring tools, not AI agents).
+const TRADITIONAL_BOT_PATTERNS = [
+  'googlebot',
+  'bingbot',
+  'yandexbot',
+  'baiduspider',
+  'duckduckbot',
+  'slurp',
+  'msnbot',
+  'facebot',
+  'twitterbot',
+  'linkedinbot',
+  'whatsapp',
+  'telegrambot',
+  'pingdom',
+  'uptimerobot',
+  'newrelic',
+  'datadog',
+  'statuspage',
+  'site24x7',
+  'applebot',
+];
+
+// Broad regex for bot-like UA strings (used only in Layer 3 heuristic).
+const BOT_LIKE_REGEX = /bot|agent|fetch|crawl|spider|search/i;
+
+export type DetectionMethod = 'ua-match' | 'signature-agent' | 'heuristic';
+
+export interface DetectionResult {
+  detected: boolean;
+  method: DetectionMethod | null;
+}
+
+/**
+ * Detects AI agents from HTTP request headers.
+ *
+ * Returns both whether the agent was detected and which signal triggered,
+ * so callers can log the detection method for accuracy tracking.
+ */
+export function isAIAgent(request: {
+  headers: { get(name: string): string | null };
+}): DetectionResult {
+  const userAgent = request.headers.get('user-agent');
+
+  // Layer 1: Known UA pattern match
+  if (userAgent) {
+    const lowerUA = userAgent.toLowerCase();
+    if (AI_AGENT_UA_PATTERNS.some((pattern) => lowerUA.includes(pattern))) {
+      return { detected: true, method: 'ua-match' };
+    }
+  }
+
+  // Layer 2: Signature-Agent header (RFC 9421, used by ChatGPT agent)
+  const signatureAgent = request.headers.get('signature-agent');
+  if (signatureAgent) {
+    const lowerSig = signatureAgent.toLowerCase();
+    if (SIGNATURE_AGENT_DOMAINS.some((domain) => lowerSig.includes(domain))) {
+      return { detected: true, method: 'signature-agent' };
+    }
+  }
+
+  // Layer 3: Missing browser fingerprint heuristic
+  // Real browsers (Chrome 76+, Firefox 90+, Safari 16.4+) send sec-fetch-mode
+  // on navigation requests. Its absence signals a programmatic client.
+  const secFetchMode = request.headers.get('sec-fetch-mode');
+  if (!secFetchMode && userAgent && BOT_LIKE_REGEX.test(userAgent)) {
+    const lowerUA = userAgent.toLowerCase();
+    const isTraditionalBot = TRADITIONAL_BOT_PATTERNS.some((pattern) =>
+      lowerUA.includes(pattern),
+    );
+    if (!isTraditionalBot) {
+      return { detected: true, method: 'heuristic' };
+    }
+  }
+
+  return { detected: false, method: null };
+}
+
+/**
+ * Generates a markdown response for AI agents that hit non-existent URLs.
+ */
+export function generateAgentNotFoundResponse(requestedPath: string): string {
+  return `# Page Not Found
+
+The URL \`${requestedPath}\` does not exist in the documentation.
+
+## How to find the correct page
+
+1. **Browse the sitemap**: [/sitemap.md](/sitemap.md) — A structured index of all pages with URLs, content types, and descriptions
+2. **Browse the full index**: [/llms.txt](/llms.txt) — Complete documentation index
+
+## Tips for requesting documentation
+
+- For markdown responses, append \`.md\` to URLs (e.g., \`/docs/getting-started.md\`)
+- Use \`Accept: text/markdown\` header for content negotiation
+`;
+}

apps/docs/lib/geistdocs/md-tracking.ts

@@ -1,4 +1,5 @@
 import { siteId } from "@/geistdocs";
+import type { DetectionMethod } from "@/lib/ai-agent-detection";
 
 const PLATFORM_URL = "https://geistdocs.com/md-tracking";
 
@@ -8,7 +9,9 @@ type TrackMdRequestParams = {
   referer: string | null;
   acceptHeader: string | null;
   /** How the markdown was requested: 'md-url' for direct .md URLs, 'header-negotiated' for Accept header */
-  requestType?: "md-url" | "header-negotiated";
+  requestType?: "md-url" | "header-negotiated" | "agent-rewrite";
+  /** Which detection method identified the AI agent */
+  detectionMethod?: DetectionMethod | null;
 };
 
 /**
@@ -21,6 +24,7 @@ export async function trackMdRequest({
   referer,
   acceptHeader,
   requestType,
+  detectionMethod,
 }: TrackMdRequestParams): Promise<void> {
   try {
     const response = await fetch(PLATFORM_URL, {
@@ -35,6 +39,7 @@ export async function trackMdRequest({
         referer,
         acceptHeader,
         requestType,
+        detectionMethod,
       }),
     });
 

apps/docs/proxy.ts

@@ -8,6 +8,7 @@ import {
 } from "next/server";
 import { rootFlags } from "@/flags";
 import { i18n } from "@/lib/geistdocs/i18n";
+import { isAIAgent } from "@/lib/ai-agent-detection";
 import { trackMdRequest } from "@/lib/geistdocs/md-tracking";
 
 const { rewrite: rewriteLLM } = rewritePath(
@@ -63,6 +64,35 @@ const proxy = async (request: NextRequest, context: NextFetchEvent) => {
     }
   }
 
+  // AI agent detection — rewrite docs pages to markdown for agents
+  // so they always get structured content without needing .md URLs or Accept headers
+  if (
+    (pathname === "/docs" || pathname.startsWith("/docs/")) &&
+    !pathname.includes("/llms.mdx/")
+  ) {
+    const agentResult = isAIAgent(request);
+    if (agentResult.detected && !isMarkdownPreferred(request)) {
+      const result =
+        pathname === "/docs"
+          ? `/${i18n.defaultLanguage}/llms.mdx`
+          : rewriteLLM(pathname);
+
+      if (result) {
+        context.waitUntil(
+          trackMdRequest({
+            path: pathname,
+            userAgent: request.headers.get("user-agent"),
+            referer: request.headers.get("referer"),
+            acceptHeader: request.headers.get("accept"),
+            requestType: "agent-rewrite",
+            detectionMethod: agentResult.method,
+          })
+        );
+        return NextResponse.rewrite(new URL(result, request.nextUrl));
+      }
+    }
+  }
+
   // Handle Accept header content negotiation and track the request
   if (isMarkdownPreferred(request)) {
     const result = rewriteLLM(pathname);