docs: more fixes for afdocs

reference/index-syntax.md

@@ -101,7 +101,7 @@ fun borrow_first(input: &Vs): &u64 {
     &input.vs[0].v[0]
     // translates to `vector::borrow(&vector::borrow(&input.vs, 0).v, 0)`
 }
-````
+```
 
 ### Index Functions Take Flexible Arguments
 

site/serve-with-rewrites.js

@@ -0,0 +1,254 @@
+/*
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+*/
+
+/**
+ * Local server that serves the Docusaurus build with proper headers
+ * for markdown files, llms.txt, and content negotiation.
+ *
+ * Usage: node serve-with-rewrites.js [port]
+ */
+
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+const url = require('url');
+
+const PORT = process.argv[2] || 3001;
+const BUILD_DIR = path.join(__dirname, 'build');
+const BUILD_ROOT = path.resolve(BUILD_DIR);
+const BUILD_ROOT_WITH_SEP = BUILD_ROOT.endsWith(path.sep) ? BUILD_ROOT : BUILD_ROOT + path.sep;
+const CANONICAL_BUILD_ROOT = fs.realpathSync(BUILD_ROOT);
+const CANONICAL_BUILD_ROOT_WITH_SEP = CANONICAL_BUILD_ROOT.endsWith(path.sep)
+  ? CANONICAL_BUILD_ROOT
+  : CANONICAL_BUILD_ROOT + path.sep;
+
+const MIME_TYPES = {
+  '.html': 'text/html; charset=utf-8',
+  '.js': 'text/javascript',
+  '.css': 'text/css',
+  '.json': 'application/json',
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.svg': 'image/svg+xml',
+  '.ico': 'image/x-icon',
+  '.md': 'text/markdown; charset=utf-8',
+  '.txt': 'text/plain; charset=utf-8',
+  '.ebnf': 'text/plain; charset=utf-8',
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2',
+  '.pdf': 'application/pdf',
+};
+
+function getContentType(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  return MIME_TYPES[ext] || 'application/octet-stream';
+}
+
+function getCacheControl(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext === '.html' || ext === '.txt' || ext === '.md' || ext === '.ebnf') {
+    return 'public, max-age=0, must-revalidate';
+  }
+  return 'public, max-age=3600';
+}
+
+/**
+ * Checks whether the request Accept header includes text/markdown.
+ */
+function acceptsMarkdown(req) {
+  const accept = req.headers['accept'] || '';
+  return accept.includes('text/markdown');
+}
+
+function isCanonicalPathUnderBuildRoot(targetPath) {
+  try {
+    const realTarget = fs.realpathSync.native ? fs.realpathSync.native(targetPath) : fs.realpathSync(targetPath);
+    const realBuildRoot = fs.realpathSync.native ? fs.realpathSync.native(BUILD_ROOT) : fs.realpathSync(BUILD_ROOT);
+    const realBuildRootWithSep = realBuildRoot.endsWith(path.sep) ? realBuildRoot : realBuildRoot + path.sep;
+    return realTarget === realBuildRoot || realTarget.startsWith(realBuildRootWithSep);
+  } catch (e) {
+    return false;
+  }
+}
+
+function isSafeRequestPathname(pathname) {
+  if (typeof pathname !== 'string' || pathname.length === 0) return false;
+  if (!pathname.startsWith('/')) return false;
+  if (pathname.includes('\0') || pathname.includes('\\')) return false;
+
+  const segments = pathname.split('/');
+  for (const segment of segments) {
+    if (segment === '.' || segment === '..') return false;
+  }
+
+  return true;
+}
+
+function resolveUnderBuildDir(relativePath) {
+  let decodedPath;
+  try {
+    decodedPath = decodeURIComponent(relativePath || '/');
+  } catch (e) {
+    return null;
+  }
+
+  const normalizedPath = decodedPath.replace(/\\/g, '/');
+  const resolved = path.resolve(BUILD_ROOT, '.' + normalizedPath);
+  if (resolved !== BUILD_ROOT && !resolved.startsWith(BUILD_ROOT_WITH_SEP)) {
+    return null;
+  }
+
+  let canonicalResolved;
+  try {
+    canonicalResolved = fs.realpathSync(resolved);
+  } catch (e) {
+    return null;
+  }
+
+  if (
+    canonicalResolved !== CANONICAL_BUILD_ROOT &&
+    !canonicalResolved.startsWith(CANONICAL_BUILD_ROOT_WITH_SEP)
+  ) {
+    return null;
+  }
+
+  return canonicalResolved;
+}
+
+/**
+ * Tries to resolve a markdown file for the given pathname.
+ * Maps e.g. "/" -> "index.md", "/foreword" -> "foreword.md",
+ * "/move-basics/module" -> "move-basics/module.md".
+ */
+function resolveMarkdownFile(pathname) {
+  const clean = pathname.replace(/\/+$/, '') || '/';
+
+  // Explicit allowlist validation to prevent path traversal and make
+  // user-controlled path constraints obvious to static analysis.
+  if (
+    !clean.startsWith('/') ||
+    clean.includes('\0') ||
+    /(^|\/)\.\.(\/|$)/.test(clean) ||
+    !/^\/[A-Za-z0-9._/-]*$/.test(clean)
+  ) {
+    return null;
+  }
+
+  if (clean === '/') {
+    const candidate = resolveUnderBuildDir('/index.md');
+    if (candidate && fs.existsSync(candidate)) return candidate;
+    return null;
+  }
+
+  // Try <path>.md first, then <path>/index.md
+  const asMd = resolveUnderBuildDir(clean + '.md');
+  if (asMd && fs.existsSync(asMd)) return asMd;
+
+  const asIndex = resolveUnderBuildDir(path.join(clean, 'index.md'));
+  if (asIndex && fs.existsSync(asIndex)) return asIndex;
+
+  return null;
+}
+
+const server = http.createServer((req, res) => {
+  const parsedUrl = url.parse(req.url);
+  let pathname = parsedUrl.pathname;
+
+  if (!isSafeRequestPathname(pathname)) {
+    res.writeHead(400, { 'Content-Type': 'text/plain' });
+    res.end('400 Bad Request');
+    return;
+  }
+
+  // Content negotiation: serve markdown when Accept: text/markdown
+  if (acceptsMarkdown(req)) {
+    const mdFile = resolveMarkdownFile(pathname);
+    if (mdFile) {
+      const content = fs.readFileSync(mdFile);
+      res.writeHead(200, {
+        'Content-Type': 'text/markdown; charset=utf-8',
+        'Content-Disposition': 'inline',
+        'Cache-Control': 'public, max-age=0, must-revalidate',
+      });
+      res.end(content);
+      return;
+    }
+  }
+
+  // Resolve file path
+  let filePath = resolveUnderBuildDir(pathname);
+  if (!filePath) {
+    res.writeHead(403, { 'Content-Type': 'text/plain' });
+    res.end('403 Forbidden');
+    return;
+  }
+
+  if (!fs.existsSync(filePath)) {
+    // Try index.html for directory-style routes
+    const indexPath = resolveUnderBuildDir(path.join(pathname, 'index.html'));
+    if (indexPath && fs.existsSync(indexPath)) {
+      filePath = indexPath;
+    } else {
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('404 Not Found');
+      return;
+    }
+  } else {
+    let canonicalFilePath;
+    try {
+      canonicalFilePath = fs.realpathSync(filePath);
+    } catch (e) {
+      res.writeHead(404, { 'Content-Type': 'text/plain' });
+      res.end('404 Not Found');
+      return;
+    }
+
+    if (
+      canonicalFilePath !== CANONICAL_BUILD_ROOT &&
+      !canonicalFilePath.startsWith(CANONICAL_BUILD_ROOT_WITH_SEP)
+    ) {
+      res.writeHead(403, { 'Content-Type': 'text/plain' });
+      res.end('403 Forbidden');
+      return;
+    }
+
+    filePath = canonicalFilePath;
+
+    if (fs.statSync(filePath).isDirectory()) {
+      const directoryIndexPath = resolveUnderBuildDir(path.join(pathname, 'index.html'));
+      if (!directoryIndexPath) {
+        res.writeHead(403, { 'Content-Type': 'text/plain' });
+        res.end('403 Forbidden');
+        return;
+      }
+      filePath = directoryIndexPath;
+    }
+  }
+
+  if (!fs.existsSync(filePath)) {
+    res.writeHead(404, { 'Content-Type': 'text/plain' });
+    res.end('404 Not Found');
+    return;
+  }
+
+  if (!isCanonicalPathUnderBuildRoot(filePath)) {
+    res.writeHead(403, { 'Content-Type': 'text/plain' });
+    res.end('403 Forbidden');
+    return;
+  }
+
+  const content = fs.readFileSync(filePath);
+  res.writeHead(200, {
+    'Content-Type': getContentType(filePath),
+    'Content-Disposition': 'inline',
+    'Cache-Control': getCacheControl(filePath),
+  });
+  res.end(content);
+});
+
+server.listen(PORT, () => {
+  console.log(`Serving build at http://localhost:${PORT}/`);
+});

site/src/plugins/llms-txt.ts

@@ -96,8 +96,20 @@ function extractAnchor(fileContent: string, anchor: string): string | null {
   return fileContent.slice(contentStart, endIdx).trimEnd();
 }
 
+function stripHtmlComments(content: string): string {
+  let current = content;
+  let previous: string;
+  do {
+    previous = current;
+    current = current.replace(/<!--[\s\S]*?-->/g, '');
+  } while (current !== previous);
+
+  return current.replace(/\n{3,}/g, '\n\n');
+}
+
 function resolveCodeIncludes(content: string): string {
-  return content.replace(
+  const stripped = stripHtmlComments(content);
+  return stripped.replace(
     /^(```\w*)\s+(?:title="[^"]*"\s+)?file=(\S+?)(?:\s+anchor=(\S+))?\s*\n[\s\S]*?^```/gm,
     (match, fence, filePath, anchor) => {
       const absPath = path.join(ROOT, filePath);
@@ -190,17 +202,10 @@ function generateFiles(outDir: string) {
   // llms.txt
   const llmsTxt = `# The Move Book
 
-> A comprehensive guide to the Move programming language on Sui.
+> A comprehensive guide to the Move programming language on Sui. Additional resources: [EBNF grammar](${siteUrl}/move.ebnf), [Move semantics](${siteUrl}/move-semantics.md), [full book for large-context models](${siteUrl}/llms-full.txt).
 
 The Move Book covers the Move language fundamentals, Sui object model, advanced programmability patterns, and testing. The Move Reference provides formal language specification.
 
-## Move Language
-
-- [Move language syntax (EBNF grammar)](${siteUrl}/move.ebnf)
-- [Move semantics](${siteUrl}/move-semantics.md)
-- [Best practices](${siteUrl}/guides/code-quality-checklist.md)
-- [Full book content for large-context models](${siteUrl}/llms-full.txt)
-
 ## Book
 
 ${bookDocs.map(buildIndexLine).join('\n')}
@@ -209,11 +214,20 @@ ${bookDocs.map(buildIndexLine).join('\n')}
 
 ${refDocs.map(buildIndexLine).join('\n')}
 
+## Site
+
+- [Search](${siteUrl}/search.md)
+
 `;
 
   fs.writeFileSync(path.join(outDir, 'llms.txt'), llmsTxt);
   console.log('[llms-txt] Built llms.txt');
 
+  // Generate search.md for the custom search page
+  const searchMd = llmsTxtDirective + '# Search\n\nSearch across all Move Book and Move Reference documentation.\n';
+  fs.writeFileSync(path.join(outDir, 'search.md'), searchMd);
+  console.log('[llms-txt] Generated search.md');
+
   // llms-full.txt
   const fullTxt = [
     '# The Move Book\n',
@@ -274,6 +288,39 @@ export default function pluginLlmsTxt(): Plugin {
               },
             });
 
+            // Content negotiation: serve markdown when Accept: text/markdown
+            middlewares.unshift({
+              name: 'content-negotiation',
+              middleware: (req: any, res: any, next: any) => {
+                const accept = req.headers['accept'] || '';
+                if (!accept.includes('text/markdown')) return next();
+
+                const url = (req.url?.split('?')[0] || '').replace(/\/+$/, '') || '/';
+                let mdPath: string | null = null;
+
+                if (url === '/') {
+                  const candidate = path.join(staticDir, 'index.md');
+                  if (fs.existsSync(candidate)) mdPath = candidate;
+                } else {
+                  const asMd = path.join(staticDir, url + '.md');
+                  if (fs.existsSync(asMd)) mdPath = asMd;
+                  if (!mdPath) {
+                    const asIndex = path.join(staticDir, url, 'index.md');
+                    if (fs.existsSync(asIndex)) mdPath = asIndex;
+                  }
+                }
+
+                if (mdPath) {
+                  res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
+                  res.setHeader('Content-Disposition', 'inline');
+                  res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate');
+                  res.end(fs.readFileSync(mdPath, 'utf-8'));
+                  return;
+                }
+                next();
+              },
+            });
+
             // Return 404 for non-existent pages (prevent SPA fallback)
             middlewares.push({
               name: 'proper-404',