docs: add llms.txt, page descriptions, and edit links

.github/workflows/docs-deploy.yml

@@ -23,6 +23,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6.0.0
         with:
           node-version-file: docs/.nvmrc

.github/workflows/docs-test.yml

@@ -15,6 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6.0.0
         with:
           node-version-file: docs/.nvmrc

docs/docs/configuring.md

@@ -1,5 +1,6 @@
 ---
 title: Configuration
+description: "Juno's configuration options, settable by flag, environment variable, or YAML file, with defaults."
 ---
 
 # Configuring Juno

docs/docs/faq.md

@@ -1,5 +1,6 @@
 ---
 title: FAQ
+description: "Answers to common Juno questions: requirements, updating, databases, and frequent startup errors."
 ---
 
 # Frequently Asked Questions

docs/docs/hardware-requirements.md

@@ -1,5 +1,6 @@
 ---
 title: Hardware Requirements
+description: "CPU, memory and storage needed to run Juno, for validators and dApps or as a high-traffic RPC provider."
 ---
 
 # Hardware Requirements

docs/docs/intro.md

@@ -1,6 +1,7 @@
 ---
 slug: /
 title: Introduction
+description: "What Juno is and where to start: hardware, running a node, snapshots, configuration, and the JSON-RPC interface."
 ---
 
 # Welcome to Juno

docs/docs/json-rpc.md

@@ -1,5 +1,6 @@
 ---
 title: JSON-RPC
+description: "Call Juno over JSON-RPC: supported spec versions, endpoint paths, and example requests."
 ---
 
 # JSON-RPC Interface

docs/docs/monitoring.md

@@ -1,5 +1,6 @@
 ---
 title: Monitoring
+description: "Monitor a Juno node with Prometheus metrics, Grafana, and the /live and /ready readiness endpoints."
 ---
 
 # Metrics Monitoring

docs/docs/plugins.md

@@ -1,5 +1,6 @@
 ---
 title: Plugins
+description: "Extend Juno with Go plugins that receive new and reverted blocks through the JunoPlugin interface."
 ---
 
 Juno supports plugins that satisfy the `JunoPlugin` interface, enabling developers to extend and customize Juno's behaviour and functionality by dynamically loading external plugins during runtime.

docs/docs/running-juno.md

@@ -1,5 +1,6 @@
 ---
 title: Installation
+description: "Start a Juno node with Docker, a prebuilt binary, or a source build, on mainnet or Sepolia."
 ---
 
 # Running Juno

docs/docs/running-on-gcp.md

@@ -1,5 +1,6 @@
 ---
 title: GCP
+description: "Run a Juno node on Google Cloud Platform."
 ---
 
 # Running Juno on GCP

docs/docs/running-on-kubernetes.md

@@ -1,5 +1,6 @@
 ---
 title: Kubernetes
+description: "Deploy Juno on Kubernetes with the official Helm chart."
 ---
 
 # Running Juno on Kubernetes

docs/docs/running-p2p.md

@@ -1,5 +1,6 @@
 ---
 title: Running a Juno P2P Node
+description: "Run Juno's experimental peer-to-peer synchronisation."
 ---
 
 # Running a Juno P2P Node

docs/docs/sequencer.md

@@ -1,5 +1,6 @@
 ---
 title: Juno Sequencer
+description: "Run Juno in experimental sequencer mode to build blocks on a custom network."
 ---
 Juno can now operate as a **standalone sequencer**. When Juno is run in this experimental mode, users are able to submit transactions to the Juno client, which stores them in a mempool. Every _N_ seconds, Juno will attempt to build a new block using any transactions that are present.
 

docs/docs/snapshots.md

@@ -1,5 +1,6 @@
 ---
 title: Snapshot Sync
+description: "Download a database snapshot and start from a recent block instead of syncing from genesis, with sizes and commands per network."
 ---
 
 # Sync from a Snapshot

docs/docs/staking-validator.md

@@ -1,5 +1,6 @@
 ---
 title: Starknet Staking
+description: "Prerequisites and setup for using a Juno node as a Starknet staking validator."
 ---
 
 # Staking with Juno

docs/docs/tuning.md

@@ -1,5 +1,6 @@
 ---
 title: Performance Tuning
+description: "Tune Juno's database and compilation settings for faster sync and lower resource use."
 ---
 
 It is important for full nodes to scale accordingly to the hardware where they are being executed. To unlock this, the following are a list of configurations users can update based on their hardware specs to maximize the performance of their Juno node.

docs/docs/updating.md

@@ -1,5 +1,6 @@
 ---
 title: Updating
+description: "Update a Juno node to a new release with Docker, a prebuilt binary, or from source."
 ---
 
 # Updating Juno

docs/docs/websocket.md

@@ -1,5 +1,6 @@
 ---
 title: WebSockets
+description: "Subscribe to new blocks, events and transaction status over Juno's WebSocket interface."
 ---
 
 # WebSocket Interface

docs/docusaurus.config.js

@@ -47,6 +47,10 @@ const config = {
         docs: {
           sidebarPath: require.resolve("./sidebars.js"),
           routeBasePath: "/",
+          // Edit links point at docs/ (next) so fixes land where they flow forward.
+          editUrl: "https://github.com/NethermindEth/juno/edit/main/docs/",
+          editCurrentVersion: true,
+          showLastUpdateTime: true,
         },
         blog: false,
         theme: {
@@ -57,6 +61,7 @@ const config = {
   ],
 
   plugins: [
+    require.resolve("./plugins/llms-txt.js"),
     [
       "@easyops-cn/docusaurus-search-local",
       {

docs/plugins/llms-txt.js

@@ -0,0 +1,151 @@
+// Emits /llms.txt, /llms-full.txt and a raw .md route beside every page, on every build,
+// generated from the version the site root serves (versions.json[0]), never from next.
+
+const fs = require("fs");
+const path = require("path");
+
+// Strip MDX plumbing (imports, mdx-code-block fences) so agents get plain markdown.
+// dir is the page's directory, used to inline any local partial it renders.
+function cleanBody(body, dir) {
+  // Inline local partials: `import X from "./_p.md"` + `<X />` becomes the
+  // partial's content, so a page's real table isn't left as a dangling tag.
+  if (dir) {
+    const partials = {};
+    for (const m of body.matchAll(/^import\s+(\w+)\s+from\s+["'](\.\/[\w./-]+\.md)["']/gm)) {
+      const f = path.join(dir, m[2]);
+      if (fs.existsSync(f)) partials[m[1]] = readPage(f).body;
+    }
+    for (const [name, pbody] of Object.entries(partials)) {
+      body = body.replace(new RegExp(`<${name}\\s*/>`, "g"), "\n" + pbody.trim() + "\n");
+    }
+  }
+  // Tabs are presentation only; unwrap TabItems to "### <label>" and drop the wrappers.
+  body = body
+    .replace(/<TabItem\b[^>]*\b(?:label|value)="([^"]+)"[^>]*>/g, "\n### $1\n")
+    .replace(/<\/?Tabs\b[^>]*>/g, "")
+    .replace(/<\/TabItem>/g, "");
+  // GuideCards become plain markdown links so their routing survives.
+  body = body.replace(/<GuideCard\b([\s\S]*?)\/>/g, (m, props) => {
+    const g = (k) => (props.match(new RegExp(`${k}="([^"]+)"`)) || [])[1];
+    if (!g("href") || !g("title")) return m; // keep a broken card visible rather than dropping it
+    return `- [${g("title")}](${g("href")})${g("description") ? ": " + g("description") : ""}`;
+  });
+  const lines = [];
+  let inMdx = false;
+  let inFence = false; // never strip anything inside a real code fence
+  for (const l of body.split("\n")) {
+    const t = l.trim();
+    if (t === "```mdx-code-block") { inMdx = true; continue; }
+    if (inMdx && t === "```") { inMdx = false; continue; }
+    if (!inMdx && /^(```|~~~)/.test(t)) inFence = !inFence;
+    if (!inFence && /^import\s.+\sfrom\s["']/.test(l)) continue;
+    lines.push(l);
+  }
+  return lines.join("\n").replace(/\n{3,}/g, "\n\n");
+}
+
+// Read one page's frontmatter and body.
+function readPage(file) {
+  const raw = fs.readFileSync(file, "utf8");
+  const fm = raw.match(/^---\n([\s\S]*?)\n---\n/);
+  const field = (name) => {
+    const v = ((fm ? fm[1] : "").match(new RegExp(`^${name}:\\s*(.+)$`, "m")) || [])[1];
+    return v && v.replace(/^"(.*)"$/, "$1"); // values may be quoted
+  };
+  return {
+    id: path.basename(file, ".md"),
+    title: field("title") || path.basename(file, ".md"),
+    description: field("description") || "",
+    slug: field("slug"),
+    raw,
+    body: fm ? raw.slice(fm[0].length) : raw,
+  };
+}
+
+// Sidebar JSON: strings are doc ids, categories carry labels, html items are skipped.
+function sidebarSections(sidebarFile) {
+  const sidebars = Object.values(JSON.parse(fs.readFileSync(sidebarFile, "utf8")));
+  if (sidebars.length !== 1) {
+    throw new Error(`[llms-txt] expected one sidebar, found ${sidebars.length}; update the plugin`);
+  }
+  const sidebar = sidebars[0];
+  const sections = [];
+  let current = { label: null, ids: [] };
+  for (const item of sidebar) {
+    if (typeof item === "string") {
+      current.ids.push(item);
+    } else if (item.type === "category") {
+      if (current.ids.length) sections.push(current);
+      const ids = item.items.filter((i) => typeof i === "string");
+      sections.push({ label: item.label, ids });
+      current = { label: null, ids: [] };
+    }
+  }
+  if (current.ids.length) sections.push(current);
+  return sections;
+}
+
+module.exports = function llmsTxtPlugin() {
+  return {
+    name: "llms-txt",
+    async postBuild({ siteConfig, siteDir, outDir }) {
+      const versions = JSON.parse(
+        fs.readFileSync(path.join(siteDir, "versions.json"), "utf8"),
+      );
+      const version = versions[0];
+      const docsDir = path.join(siteDir, "versioned_docs", `version-${version}`);
+      const sidebarFile = path.join(
+        siteDir, "versioned_sidebars", `version-${version}-sidebars.json`,
+      );
+      const site = siteConfig.url;
+
+      // Disk decides what exists; the sidebar only orders and labels it.
+      const onDisk = fs.readdirSync(docsDir)
+        .filter((f) => f.endsWith(".md") && !f.startsWith("_"))
+        .map((f) => f.slice(0, -3));
+      const sections = sidebarSections(sidebarFile);
+      const listed = new Set(sections.flatMap((s) => s.ids));
+      const unlisted = onDisk.filter((id) => !listed.has(id));
+      if (unlisted.length) sections.push({ label: "Other pages", ids: unlisted.sort() });
+
+      const index = [];
+      const full = [];
+      for (const section of sections) {
+        if (section.label) index.push(`\n- ${section.label}\n`);
+        for (const id of section.ids) {
+          const file = path.join(docsDir, `${id}.md`);
+          if (!fs.existsSync(file)) {
+            console.warn(`[llms-txt] sidebar lists "${id}" but ${file} does not exist; skipping`);
+            continue;
+          }
+          const page = readPage(file);
+          const route = page.slug === "/" ? "" : (page.slug || `/${id}`);
+          const url = `${site}${route || "/"}`;
+          const note = page.description ? `: ${page.description}` : "";
+          index.push(`  - [${page.title}](${url})${note}`);
+          const body = cleanBody(page.body, docsDir).trim();
+          full.push(`# ${page.title}\n\nSource: ${url}\n\n${body}\n`);
+          // Raw markdown sits beside its HTML route; the front page becomes /index.md.
+          const clean = body.startsWith("# ")
+            ? `${body}\n`
+            : `# ${page.title}\n\n${page.description ? page.description + "\n\n" : ""}${body}\n`;
+          const target = path.join(outDir, `${route === "" ? "index" : route.slice(1)}.md`);
+          fs.mkdirSync(path.dirname(target), { recursive: true });
+          fs.writeFileSync(target, clean);
+          if (route === "") fs.writeFileSync(path.join(outDir, `${id}.md`), clean);
+        }
+      }
+
+      const header =
+        "Append `.md` to any docs page URL below for its raw Markdown " +
+        `(e.g. \`${site}/configuring.md\`; the front page is \`${site}/index.md\`), ` +
+        `or fetch all current pages at \`${site}/llms-full.txt\`.\n\n` +
+        `# Juno Docs\n\n> ${siteConfig.title}: ${siteConfig.tagline}. ` +
+        `Documentation for Juno ${version}, a Starknet full node written in Go.\n`;
+
+      fs.writeFileSync(path.join(outDir, "llms.txt"), header + "\n" + index.join("\n") + "\n");
+      fs.writeFileSync(path.join(outDir, "llms-full.txt"), full.join("\n---\n\n"));
+      console.log(`[llms-txt] wrote llms.txt, llms-full.txt and raw .md routes for ${version}`);
+    },
+  };
+};

docs/static/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://juno.nethermind.io/sitemap.xml

docs/versioned_docs/version-0.16.0/configuring.md

@@ -1,5 +1,6 @@
 ---
 title: Configuration
+description: "Juno's configuration options, settable by flag, environment variable, or YAML file, with defaults."
 ---
 
 # Configuring Juno

docs/versioned_docs/version-0.16.0/faq.md

@@ -1,5 +1,6 @@
 ---
 title: FAQ
+description: "Answers to common Juno questions: requirements, updating, databases, and frequent startup errors."
 ---
 
 # Frequently Asked Questions

docs/versioned_docs/version-0.16.0/hardware-requirements.md

@@ -1,5 +1,6 @@
 ---
 title: Hardware Requirements
+description: "CPU, memory and storage needed to run Juno, for validators and dApps or as a high-traffic RPC provider."
 ---
 
 # Hardware Requirements

docs/versioned_docs/version-0.16.0/intro.md

@@ -1,6 +1,7 @@
 ---
 slug: /
 title: Introduction
+description: "What Juno is and where to start: hardware, running a node, snapshots, configuration, and the JSON-RPC interface."
 ---
 
 # Welcome to Juno

docs/versioned_docs/version-0.16.0/json-rpc.md

@@ -1,5 +1,6 @@
 ---
 title: JSON-RPC
+description: "Call Juno over JSON-RPC: supported spec versions, endpoint paths, and example requests."
 ---
 
 # JSON-RPC Interface

docs/versioned_docs/version-0.16.0/monitoring.md

@@ -1,5 +1,6 @@
 ---
 title: Monitoring
+description: "Monitor a Juno node with Prometheus metrics, Grafana, and the /live and /ready readiness endpoints."
 ---
 
 # Metrics Monitoring

docs/versioned_docs/version-0.16.0/plugins.md

@@ -1,5 +1,6 @@
 ---
 title: Plugins
+description: "Extend Juno with Go plugins that receive new and reverted blocks through the JunoPlugin interface."
 ---
 
 Juno supports plugins that satisfy the `JunoPlugin` interface, enabling developers to extend and customize Juno's behaviour and functionality by dynamically loading external plugins during runtime.

docs/versioned_docs/version-0.16.0/running-juno.md

@@ -1,5 +1,6 @@
 ---
 title: Installation
+description: "Start a Juno node with Docker, a prebuilt binary, or a source build, on mainnet or Sepolia."
 ---
 
 # Running Juno

docs/versioned_docs/version-0.16.0/running-on-gcp.md

@@ -1,5 +1,6 @@
 ---
 title: GCP
+description: "Run a Juno node on Google Cloud Platform."
 ---
 
 # Running Juno on GCP

docs/versioned_docs/version-0.16.0/running-on-kubernetes.md

@@ -1,5 +1,6 @@
 ---
 title: Kubernetes
+description: "Deploy Juno on Kubernetes with the official Helm chart."
 ---
 
 # Running Juno on Kubernetes

docs/versioned_docs/version-0.16.0/running-p2p.md

@@ -1,5 +1,6 @@
 ---
 title: Running a Juno P2P Node
+description: "Run Juno's experimental peer-to-peer synchronisation."
 ---
 
 # Running a Juno P2P Node

docs/versioned_docs/version-0.16.0/sequencer.md

@@ -1,5 +1,6 @@
 ---
 title: Juno Sequencer
+description: "Run Juno in experimental sequencer mode to build blocks on a custom network."
 ---
 Juno can now operate as a **standalone sequencer**. When Juno is run in this experimental mode, users are able to submit transactions to the Juno client, which stores them in a mempool. Every _N_ seconds, Juno will attempt to build a new block using any transactions that are present.