docs: improve agentdocsspec compliance (#451)

marc0olo · web-flow · commit eb31bd9fe02e · 2026-03-19T16:19:08.000-04:00
* fix(docs): add cache headers for markdown endpoints and llms.txt

The agentdocsspec checker flagged missing cache headers on .md and .txt
files. Add Cache-Control: public, max-age=300 to match the HTML pages.

* fix(docs): move llms.txt directive earlier in page for agent discovery

The agentdocsspec checker flagged the llms.txt directive as "buried deep
(past 50%)" because the rehype plugin injected it into the Starlight
content area, after the header, nav, and sidebar.

Move the directive to the Banner component (top of &lt;main&gt;) and add a
&lt;link rel="help"&gt; in &lt;head&gt;. Remove the now-unused rehype plugin.

* feat(docs): include version in llms.txt title for versioned builds

When built with a versioned base path (e.g. /0.2/), the llms.txt title
now reads "ICP CLI Documentation (v0.2)" instead of the generic title.

* feat(docs): add IC skills registry link to llms.txt

Point agents to the skills discovery endpoint so they can find
and install specialized IC development skills (icp-cli, canister
security, ICRC ledger, etc.) alongside the documentation.
diff --git a/docs-site/.ic-assets.json5 b/docs-site/.ic-assets.json5
@@ -20,6 +20,13 @@
       "Cache-Control": "public, max-age=300"
     }
   },
+  {
+    // Markdown endpoints and llms.txt — short cache so updates are visible quickly
+    "match": "**/*.{md,txt}",
+    "headers": {
+      "Cache-Control": "public, max-age=300"
+    }
+  },
   {
     // Version list — very short cache so version switcher sees updates quickly
     "match": "versions.json",
diff --git a/docs-site/astro.config.mjs b/docs-site/astro.config.mjs
@@ -2,7 +2,6 @@ import { defineConfig } from 'astro/config';
 import starlight from '@astrojs/starlight';
 import rehypeExternalLinks from 'rehype-external-links';
 import rehypeRewriteLinks from './plugins/rehype-rewrite-links.mjs';
-import rehypeAgentSignaling from './plugins/rehype-agent-signaling.mjs';
 import agentDocs from './plugins/astro-agent-docs.mjs';
 
 // https://astro.build/config
@@ -17,8 +16,6 @@ export default defineConfig({
       rehypeRewriteLinks,
       // Open external links in new tab
       [rehypeExternalLinks, { target: '_blank', rel: ['noopener', 'noreferrer'] }],
-      // Inject hidden llms.txt directive for agent discovery
-      rehypeAgentSignaling,
     ],
   },
   integrations: [
@@ -33,6 +30,17 @@ export default defineConfig({
         Banner: './src/components/Banner.astro',
       },
       head: [
+        {
+          // Agent-friendly docs: surface llms.txt directive early in <head>
+          // so crawlers find it before the content area (agentdocsspec.com)
+          tag: 'link',
+          attrs: {
+            rel: 'help',
+            href: `${process.env.PUBLIC_BASE_PATH || '/'}llms.txt`,
+            type: 'text/plain',
+            title: 'LLM-friendly documentation index',
+          },
+        },
         {
           tag: 'script',
           attrs: {},
diff --git a/docs-site/plugins/astro-agent-docs.mjs b/docs-site/plugins/astro-agent-docs.mjs
@@ -66,11 +66,19 @@ function findSection(filePath) {
 function generateLlmsTxt(pages, siteUrl, basePath) {
   const base = (siteUrl + basePath).replace(/\/$/, "");
 
+  // Extract version from base path (e.g., "/0.2/" → "0.2", "/main/" → "main")
+  const version = basePath.replace(/^\/|\/$/g, "") || null;
+  const title = version
+    ? `# ICP CLI Documentation (v${version})`
+    : "# ICP CLI Documentation";
+
   const lines = [
-    "# ICP CLI Documentation",
+    title,
     "",
     "> Command-line tool for developing and deploying applications on the Internet Computer Protocol (ICP).",
     "",
+    "Agent skills for IC development: https://skills.internetcomputer.org/.well-known/skills/index.json",
+    "",
   ];
 
   // Root index page
diff --git a/docs-site/plugins/rehype-agent-signaling.mjs b/docs-site/plugins/rehype-agent-signaling.mjs
diff --git a/docs-site/src/components/Banner.astro b/docs-site/src/components/Banner.astro
@@ -3,8 +3,12 @@
 // Overrides Starlight's default Banner component so we don't need
 // banner frontmatter in each file.
 const content = 'Feedback welcome! Report issues on <a href="https://github.com/dfinity/icp-cli/issues" target="_blank" rel="noopener noreferrer">GitHub</a>, ask questions on the <a href="https://forum.dfinity.org/t/icp-cli-announcements-and-feedback-discussion/60410/1" target="_blank" rel="noopener noreferrer">Forum</a>, or chat with us on <a href="https://discord.internetcomputer.org" target="_blank" rel="noopener noreferrer">Discord</a>.';
+const llmsTxtPath = `${import.meta.env.BASE_URL}llms.txt`;
 ---
 
+<blockquote class="agent-signaling" data-pagefind-ignore>
+	<p>For AI agents: Documentation index at <a href={llmsTxtPath}>{llmsTxtPath}</a></p>
+</blockquote>
 <div class="sl-banner" data-pagefind-ignore set:html={content} />
 
 <style>

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,13 @@`
`20`	`20`	`"Cache-Control": "public, max-age=300"`
`21`	`21`	`}`
`22`	`22`	`},`
	`23`	`+ {`
	`24`	`+ // Markdown endpoints and llms.txt — short cache so updates are visible quickly`
	`25`	`+ "match": "*/.{md,txt}",`
	`26`	`+ "headers": {`
	`27`	`+ "Cache-Control": "public, max-age=300"`
	`28`	`+ }`
	`29`	`+ },`
`23`	`30`	`{`
`24`	`31`	`// Version list — very short cache so version switcher sees updates quickly`
`25`	`32`	`"match": "versions.json",`