diff --git a/.changeset/clean-routes-report.md b/.changeset/clean-routes-report.md new file mode 100644 index 00000000..6877464f --- /dev/null +++ b/.changeset/clean-routes-report.md @@ -0,0 +1,6 @@ +--- +"@cloudflare/nimbus-docs": patch +"@cloudflare/create-nimbus-docs": patch +--- + +Allow user-owned Astro pages and scaffolded Markdown and `llms.txt` endpoints to use native rendering semantics while retaining entrypoint-aware checks for active Nimbus contracts and composing with unrelated integration routes. These dynamic endpoints now resolve their payloads when rendered on request. Endpoint helpers now live at `@cloudflare/nimbus-docs/agent-endpoints`; the existing `@cloudflare/nimbus-docs/publication` entrypoint remains supported. diff --git a/apps/www/registry/feature-route-contract.test.ts b/apps/www/registry/feature-route-contract.test.ts index b5594b8d..73c298a3 100644 --- a/apps/www/registry/feature-route-contract.test.ts +++ b/apps/www/registry/feature-route-contract.test.ts @@ -47,7 +47,7 @@ test("collection recipes guard disabled table-of-contents configuration", async } }); -test("changelog serves and links its expanded source artifact", async () => { +test("changelog serves and links its expanded source version", async () => { const source = await feature("changelog"); assert.match(source, /surface: "source"/); assert.match(source, /sourcePath[\s\S]*index\.mdx/); diff --git a/apps/www/registry/features/ai-native.md b/apps/www/registry/features/ai-native.md index b48d236d..32dcf200 100644 --- a/apps/www/registry/features/ai-native.md +++ b/apps/www/registry/features/ai-native.md @@ -2,15 +2,15 @@ { "name": "ai-native", "type": "registry:feature", - "title": "Publish Markdown", - "description": "Add per-page Markdown versions, llms.txt indexes, llms-full.txt, robots.txt, and an AgentDirective to a Nimbus docs site.", + "title": "Markdown and llms.txt endpoints", + "description": "Add alternate Markdown/MDX versions, llms.txt indexes, llms-full.txt, robots.txt, and an AgentDirective to a Nimbus docs site.", "markers": ["src/pages/llms.txt.ts", "src/pages/llms-full.txt.ts", "src/pages/[...slug]/index.md.ts"] } --- -# Publish Markdown +# Markdown and llms.txt endpoints -You are helping the user publish Markdown versions and `llms.txt` indexes from an existing Nimbus docs site. These files are deterministic build output on every deployment provider. +You are helping the user add alternate Markdown/MDX versions and `llms.txt` indexes to an existing Nimbus docs site. Their generated content is deterministic on every deployment provider. Read this entire file before making changes. The target project should already depend on `nimbus-docs` and use the starter-style routes/layouts. @@ -32,32 +32,48 @@ Then wire the layout/page props: - `src/layouts/DocsLayout.astro` accepts `markdownUrl` and forwards it to `BaseLayout`. - `src/pages/[...slug].astro` computes `markdownUrl` for docs entries and passes it to `DocsLayout`. -Do not add an `ai` config block. Do not add an MCP server. This feature is build-time/static only. +Do not add an `ai` config block or an MCP server. Nimbus prepares the endpoint payloads at build time, while each endpoint may be prerendered or rendered on request. ## Reference implementation -Keep all five Markdown routes prerendered and use the prepared helpers from `@cloudflare/nimbus-docs/build`. +Keep all five endpoints prerendered and use the route helpers from `@cloudflare/nimbus-docs/agent-endpoints`. ```ts title="src/pages/[...slug]/index.md.ts" import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: "docs", surface: "markdown" }); +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getMarkdownStaticPaths({ + collection: "docs", + surface: "markdown", + }); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: "docs", + surface: "markdown", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` @@ -65,17 +81,21 @@ export async function GET({ props }: { props: SlugProps }) { Create `src/pages/[...slug]/index.mdx.ts` from the same code, changing `surface: "markdown"` to `surface: "source"`. ```ts title="src/pages/llms.txt.ts" -import { getPreparedLlmsArtifact } from "@cloudflare/nimbus-docs/build"; +import { getLlmsPayload } from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ - scope: "site", - surface: "index", - }); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET(context: { request: Request }) { + const payload = await getLlmsPayload( + { + scope: "site", + surface: "index", + }, + context, + ); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` @@ -84,23 +104,43 @@ Create `src/pages/llms-full.txt.ts` from the same code, changing `surface: "inde ```ts title="src/pages/[section]/llms.txt.ts" import { - getPreparedLlmsArtifact, - getPreparedLlmsStaticPaths, - type PreparedLlmsReference, -} from "@cloudflare/nimbus-docs/build"; + getLlmsPayload, + getLlmsStaticPaths, + type LlmsEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; interface SectionProps { - artifact: PreparedLlmsReference; + reference: LlmsEndpointReference; } -export const getStaticPaths = () => getPreparedLlmsStaticPaths(); +interface SectionContext { + params: { section?: string }; + props: Partial; + request: Request; +} -export async function GET({ props }: { props: SectionProps }) { - const artifact = await getPreparedLlmsArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export const getStaticPaths = async () => + getLlmsStaticPaths(); + +export async function GET({ params, props, request }: SectionContext) { + const reference = + props.reference ?? + (params.section + ? ({ + scope: "section", + surface: "index", + section: params.section, + } satisfies LlmsEndpointReference) + : null); + if (!reference) return new Response("Not found", { status: 404 }); + const payload = await getLlmsPayload(reference, { + request, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` @@ -116,7 +156,7 @@ Run the user's package manager build command (`pnpm build`, `npm run build`, etc - `dist/robots.txt` exists and includes a `Sitemap:` line. - `dist//index.md` exists for docs entries. - `dist//index.mdx` exists for authored docs entries. -- Section indexes such as `dist/
/llms.txt` list their Markdown versions. +- Section indexes such as `dist/
/llms.txt` list their alternate Markdown versions. - HTML pages include `` for docs entries. - HTML pages include the hidden `[data-ai-agent-directive]` block for docs entries. diff --git a/apps/www/registry/features/api-reference.md b/apps/www/registry/features/api-reference.md index 663493c9..c9704ede 100644 --- a/apps/www/registry/features/api-reference.md +++ b/apps/www/registry/features/api-reference.md @@ -3,7 +3,7 @@ "name": "api-reference", "type": "registry:feature", "title": "OpenAPI reference", - "description": "Mount an OpenAPI (Swagger) spec as a routed reference collection with generated pages, per-page Markdown versions, and llms.txt coverage from one spec file. For hand-authored API docs written as MDX, use `new-collection` instead.", + "description": "Mount an OpenAPI (Swagger) spec as a routed reference collection with generated pages, alternate Markdown versions, and llms.txt indexes from one spec file. For hand-authored API docs written as MDX, use `new-collection` instead.", "markers": ["src/pages/api/[...slug].astro"] } --- @@ -12,8 +12,8 @@ You are helping the user mount an **OpenAPI (Swagger) spec** as a first-class reference collection on a Nimbus docs site. One spec file in, and the user -gets: a routed page per operation/schema/tag under `/api`, a clean Markdown -version of every page, and automatic `llms.txt` and `llms-full.txt` coverage. +gets: a routed page per operation/schema/tag under `/api`, an alternate Markdown +version of every page, and automatic `llms.txt` indexes and `llms-full.txt`. The render is Nimbus's own — the spec is parsed once per build and projected into a stable view-model. There is no third-party reference renderer. diff --git a/apps/www/registry/features/changelog.md b/apps/www/registry/features/changelog.md index bd9efd71..d22e4ec8 100644 --- a/apps/www/registry/features/changelog.md +++ b/apps/www/registry/features/changelog.md @@ -954,10 +954,10 @@ export async function GET() { import { entryRouteKey, withBase } from "@cloudflare/nimbus-docs"; import { getEntry } from "astro:content"; import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; import { config } from "virtual:nimbus/config"; export const prerender = true; @@ -967,16 +967,29 @@ const absoluteUrl = (path: string) => new URL(withBase(path, import.meta.env.BASE_URL), config.site).href; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: COLLECTION, surface: "markdown" }) +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getMarkdownStaticPaths({ collection: COLLECTION, surface: "markdown" }) .then((paths) => paths.filter((path) => path.params.slug !== undefined)); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - const entry = await getEntry(COLLECTION, props.artifact.id); +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: COLLECTION, + surface: "markdown", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response(null, { status: 404 }); + const entry = await getEntry(COLLECTION, payload.id); if (!entry) return new Response(null, { status: 404 }); const data = (entry.data ?? {}) as Record; const title = String(data.title); @@ -1018,7 +1031,7 @@ export async function GET({ props }: { props: SlugProps }) { "", `# ${title}`, "", - artifact.content, + payload.content, "", `Source: ${absoluteUrl(sourcePath)}`, "", @@ -1034,25 +1047,38 @@ export async function GET({ props }: { props: SlugProps }) { ```ts import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: "changelog", surface: "source" }) +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getMarkdownStaticPaths({ collection: "changelog", surface: "source" }) .then((paths) => paths.filter((path) => path.params.slug !== undefined)); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: "changelog", + surface: "source", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` diff --git a/apps/www/registry/features/new-collection.md b/apps/www/registry/features/new-collection.md index bd493e94..1d05ac50 100644 --- a/apps/www/registry/features/new-collection.md +++ b/apps/www/registry/features/new-collection.md @@ -31,7 +31,7 @@ plain doc tree. **For an OpenAPI spec, this is also the wrong recipe.** This recipe makes a tree of hand-authored MDX pages. If the user wants their API reference *generated from an OpenAPI/Swagger document* — pages per operation and schema, -Markdown versions and `llms.txt` coverage — use `nimbus-docs add api-reference`. Use this +alternate Markdown versions and `llms.txt` indexes — use `nimbus-docs add api-reference`. Use this recipe for `api` only when they're writing the API docs by hand. **This recipe owns the whole setup of a non-version collection.** You @@ -115,7 +115,7 @@ URL convention — a `docs-v1` collection always mounts at `/v1/`, never at `/docs-v1/`. For every other collection, the URL prefix must match the collection name. -Per-page Markdown versions and the collection's `llms.txt` index use that identity as their mount prefix. +Alternate Markdown versions and the collection's `llms.txt` index use that identity as their mount prefix. ### Q4. Add a starter entry? @@ -288,33 +288,46 @@ Write `src/pages//[...slug]/index.md.ts`: */ import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; const COLLECTION = ""; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: COLLECTION, surface: "markdown" }); +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getMarkdownStaticPaths({ collection: COLLECTION, surface: "markdown" }); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: COLLECTION, + surface: "markdown", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` Substitute `` in the `COLLECTION` constant. -To serve the expanded source URL referenced by the prepared markdown, +To serve the expanded source URL referenced by the Markdown payload, mirror this route at `src/pages//[...slug]/index.mdx.ts` with `surface: "source"`. diff --git a/apps/www/registry/features/new-version.md b/apps/www/registry/features/new-version.md index fb56e67b..6bc5c1e1 100644 --- a/apps/www/registry/features/new-version.md +++ b/apps/www/registry/features/new-version.md @@ -454,26 +454,39 @@ Write `src/pages//[...slug]/index.md.ts`: ```ts import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; const COLLECTION = "docs-"; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: COLLECTION, surface: "markdown" }); +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getMarkdownStaticPaths({ collection: COLLECTION, surface: "markdown" }); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: COLLECTION, + surface: "markdown", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` @@ -482,7 +495,7 @@ Substitute the user's version slug for every `` token in the snippet above (the directory name in the file path, plus the `COLLECTION` constant value at the top). -The prepared artifact includes the version frontmatter so agents can pin a +The endpoint payload includes the version frontmatter so agents can pin a version. To also serve the expanded source form, mirror this route with `surface: "source"` at `src/pages//[...slug]/index.mdx.ts`. diff --git a/apps/www/src/content/docs/ai/docs-for-agents.mdx b/apps/www/src/content/docs/ai/docs-for-agents.mdx index d8469516..48329ad0 100644 --- a/apps/www/src/content/docs/ai/docs-for-agents.mdx +++ b/apps/www/src/content/docs/ai/docs-for-agents.mdx @@ -1,20 +1,20 @@ --- title: Docs for agents -description: Make documentation available to AI tools through Markdown versions, llms.txt indexes, and structured metadata. +description: Make documentation available to AI tools through alternate Markdown/MDX versions, llms.txt indexes, and structured metadata. aiGenerated: true sidebar: order: 1 --- -Nimbus publishes documentation in formats that AI tools can read without parsing the HTML site. +Nimbus serves documentation through endpoints that AI tools can read without parsing the HTML site. ## What ships -- **Two Markdown versions for every discoverable authored page** — `//index.md` contains clean Markdown with components converted to text, while `//index.mdx` retains imports, JSX, and directives. Nimbus resolves links and reusable snippets during the build. The `.md` version links to its `.mdx` source. Hidden documentation versions do not emit either file. -- **A site index at `/llms.txt`** — names top-level pages and section indexes. Each section index links its discoverable pages to their Markdown versions. -- **The full documentation at `/llms-full.txt`** — discoverable pages in one Markdown file, excluding non-current prose versions. AI tools and indexing systems can fetch the current site in one request. Nimbus generates it during the build so rebuilds are byte-identical. +- **Alternate Markdown/MDX versions for every discoverable authored page** — `//index.md` contains clean Markdown with components converted to text, while `//index.mdx` retains imports, JSX, and directives. Nimbus resolves links and reusable snippets during the build. The `.md` version links to its `.mdx` source. Hidden documentation versions do not emit either file. +- **A site `llms.txt` index** — names top-level pages and section indexes. Each section index links its discoverable pages to their alternate Markdown versions. +- **`llms-full.txt`** — combines discoverable pages into one Markdown document, excluding non-current prose versions. AI tools and indexing systems can fetch the current site in one request. Nimbus generates it during the build so rebuilds are byte-identical. - **Per-section indexes at `/
/llms.txt`** — each top-level group and collection (e.g. a `blog` or a docs version) gets its own index. -- **Version labels** — on versioned sites, every Markdown version's frontmatter carries a `version:` key (resolved from your `versions` manifest) so agents can pin a version. Unversioned sites emit nothing. +- **Version labels** — on versioned sites, every alternate Markdown version's frontmatter carries a `version:` key (resolved from your `versions` manifest) so agents can pin a version. Unversioned sites emit nothing. - **JSON-LD in every page head** — structured data so search and agent tools recognize what they're looking at. - **`/robots.txt` and a sitemap** — wired automatically when `site` is set. @@ -32,17 +32,17 @@ A way to build documentation sites on top of Astro. Indexing is multi-collection by default and schema-tolerant — every prepared collection registered in `src/content.config.ts` (except `partials` and reserved names) is walked, reading `title` and `description` when present. There is no per-page opt-in; all discoverable pages are listed. Hidden versions are excluded, and `/llms-full.txt` omits non-current prose versions. -Markdown versions and `llms.txt` files are public build outputs. Request-time authorization cannot protect them. Keep private material outside public content collections. Nimbus omits excluded entries and fails the build when it cannot determine whether content is public. +Alternate Markdown/MDX versions and `llms.txt` indexes use public agent-endpoint assets. Request-time authorization cannot make private material safe here. Keep private material outside public content collections. Nimbus omits excluded entries and fails the build when it cannot determine whether content is public. To keep a page **out** of the `llms.txt` indexes, set `draft: true` (excluded entirely) or `noindex: true`. A page that should never be agent-readable belongs outside the content collection altogether. ### Custom content loaders -Nimbus's collection factories prepare Markdown automatically. If a custom loader retains Markdown bodies for per-page Markdown versions or `llms-full.txt`, wrap it with `withNimbusMarkdown()` from `@cloudflare/nimbus-docs/content`. See [Other collections](/writing/pages-and-routing#other-collections) for a complete example. +Nimbus's collection factories prepare Markdown automatically. If a custom loader retains Markdown bodies for alternate Markdown versions or `llms-full.txt`, wrap it with `withNimbusMarkdown()` from `@cloudflare/nimbus-docs/content`. See [Other collections](/writing/pages-and-routing#other-collections) for a complete example. ## Route setup -New projects include these routes. To add or update them in an existing project, follow [Publish Markdown](/ai/publish-markdown). +New projects include these routes. To add or update them in an existing project, follow [Markdown and llms.txt endpoints](/ai/publish-markdown). ## AGENT.md diff --git a/apps/www/src/content/docs/ai/publish-markdown.mdx b/apps/www/src/content/docs/ai/publish-markdown.mdx index e2119efc..57065de5 100644 --- a/apps/www/src/content/docs/ai/publish-markdown.mdx +++ b/apps/www/src/content/docs/ai/publish-markdown.mdx @@ -1,37 +1,53 @@ --- -title: Publish Markdown -description: Add per-page Markdown versions, llms.txt indexes, and the full documentation file. +title: Markdown and llms.txt endpoints +description: Add alternate Markdown/MDX versions, llms.txt indexes, and llms-full.txt. aiGenerated: true sidebar: order: 2 --- -Add the five routes that publish Markdown versions of each page, site and section indexes, and the full documentation in one file. The Nimbus integration must already be present in `astro.config.ts`. +Add the five endpoints that serve alternate Markdown/MDX versions, site and section `llms.txt` indexes, and `llms-full.txt`. The Nimbus integration must already be present in `astro.config.ts`. -## 1. Publish per-page Markdown +## 1. Add alternate page versions Create the clean Markdown version: ```ts title="src/pages/[...slug]/index.md.ts" import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: "docs", surface: "markdown" }); +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getMarkdownStaticPaths({ + collection: "docs", + surface: "markdown", + }); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: "docs", + surface: "markdown", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` @@ -40,98 +56,142 @@ Create the MDX source version: ```ts title="src/pages/[...slug]/index.mdx.ts" import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; +} + +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: "docs", surface: "source" }); +export const getStaticPaths = async () => + getMarkdownStaticPaths({ + collection: "docs", + surface: "source", + }); -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET({ params, props, request }: SlugContext) { + const payload = await getMarkdownPayload({ + collection: "docs", + surface: "source", + slug: params.slug, + reference: props.reference, + context: { request }, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` -These root catch-all routes publish only the `docs` collection. For every additional authored collection, create the same `.md.ts` and `.mdx.ts` routes below its URL prefix and change `collection` to its literal collection key. For example, a `blog` collection at `/blog` uses `src/pages/blog/[...slug]/index.md.ts`; a `docs-v1` collection at `/v1` uses `src/pages/v1/[...slug]/index.md.ts`. Mirror each Markdown route with an MDX source route. +These root catch-all endpoints serve only the `docs` collection. For every additional authored collection, create the same `.md.ts` and `.mdx.ts` endpoints below its URL prefix and change `collection` to its literal collection key. For example, a `blog` collection at `/blog` uses `src/pages/blog/[...slug]/index.md.ts`; a `docs-v1` collection at `/v1` uses `src/pages/v1/[...slug]/index.md.ts`. Mirror each alternate Markdown endpoint with an MDX source endpoint. -## 2. Publish the site index +## 2. Add the site index Create the site-wide discovery index: ```ts title="src/pages/llms.txt.ts" -import { getPreparedLlmsArtifact } from "@cloudflare/nimbus-docs/build"; +import { getLlmsPayload } from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ - scope: "site", - surface: "index", - }); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET(context: { request: Request }) { + const payload = await getLlmsPayload( + { + scope: "site", + surface: "index", + }, + context, + ); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` -## 3. Publish the full documentation +## 3. Add llms-full.txt Create the file containing all discoverable documentation: ```ts title="src/pages/llms-full.txt.ts" -import { getPreparedLlmsArtifact } from "@cloudflare/nimbus-docs/build"; +import { getLlmsPayload } from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ - scope: "site", - surface: "full", - }); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export async function GET(context: { request: Request }) { + const payload = await getLlmsPayload( + { + scope: "site", + surface: "full", + }, + context, + ); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` -## 4. Publish section indexes +## 4. Add section indexes Create the route for paths such as `/writing/llms.txt`: ```ts title="src/pages/[section]/llms.txt.ts" import { - getPreparedLlmsArtifact, - getPreparedLlmsStaticPaths, - type PreparedLlmsReference, -} from "@cloudflare/nimbus-docs/build"; + getLlmsPayload, + getLlmsStaticPaths, + type LlmsEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; export const prerender = true; interface SectionProps { - artifact: PreparedLlmsReference; + reference: LlmsEndpointReference; } -export const getStaticPaths = () => getPreparedLlmsStaticPaths(); +interface SectionContext { + params: { section?: string }; + props: Partial; + request: Request; +} -export async function GET({ props }: { props: SectionProps }) { - const artifact = await getPreparedLlmsArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export const getStaticPaths = async () => + getLlmsStaticPaths(); + +export async function GET({ params, props, request }: SectionContext) { + const reference = + props.reference ?? + (params.section + ? ({ + scope: "section", + surface: "index", + section: params.section, + } satisfies LlmsEndpointReference) + : null); + if (!reference) return new Response("Not found", { status: 404 }); + const payload = await getLlmsPayload(reference, { + request, + }); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, }); } ``` -Keep all five routes prerendered, including when HTML pages render on request. The `/build` helpers require the Nimbus Astro integration and must not enter a Worker bundle. +Keep all five endpoints prerendered unless you deliberately want them rendered on request. The `agent-endpoints` helpers support either mode and require the Nimbus Astro integration. ## Verify @@ -156,7 +216,7 @@ Each command must return content with a successful status. A `404` means the cor ## Direct rendering -`renderEntryAsMarkdown()` and `getEntryMarkdown()` remain available for entry bodies that do not contain ``. Passing a body with `` throws. Use the routes above for authored Markdown versions and `llms.txt` files. `getEntryMarkdown()` also loads API citation data and is intended for prerendered routes. +`renderEntryAsMarkdown()` and `getEntryMarkdown()` remain available for entry bodies that do not contain ``. Passing a body with `` throws. Use the endpoints above for alternate authored Markdown versions and `llms.txt` indexes. `getEntryMarkdown()` also loads API citation data and is intended for prerendered routes. ## Next steps diff --git a/apps/www/src/content/docs/api-reference.mdx b/apps/www/src/content/docs/api-reference.mdx index f30a4086..9fdfdf02 100644 --- a/apps/www/src/content/docs/api-reference.mdx +++ b/apps/www/src/content/docs/api-reference.mdx @@ -10,7 +10,7 @@ Nimbus generates API references from machine-readable specifications. Each suppo ## OpenAPI -Nimbus currently supports OpenAPI 3.x. Nimbus turns a local OpenAPI document into a content collection that powers reference pages, navigation, Markdown versions, search entries, `llms.txt` indexes, and stable links from prose documentation. +Nimbus currently supports OpenAPI 3.x. Nimbus turns a local OpenAPI document into a content collection that powers reference pages, navigation, alternate Markdown versions, search entries, `llms.txt` indexes, and stable links from prose documentation. The OpenAPI document remains the source of truth. You do not create one MDX file per operation or schema. @@ -30,7 +30,7 @@ The recipe tells the agent to inspect your project, ask for the specification pa 2. Declares the specification in `nimbus.config.ts`. 3. Registers a loader-backed Astro content collection. 4. Adds the HTML catch-all route and static Markdown route. -5. Connects the collection to search, Markdown versions, and `llms.txt`. +5. Connects the collection to search, alternate Markdown versions, and `llms.txt` indexes. ### How the pieces fit @@ -41,7 +41,7 @@ The recipe tells the agent to inspect your project, ask for the specification pa | `nimbus.config.ts` | Declares the specification, versions, routes, and rendering policy. | | `src/content.config.ts` | Registers the generated entries as an Astro content collection. | | `src/pages/api/[...slug].astro` | Renders API HTML through the user-owned API layout. | -| `src/pages/api/[...slug]/index.md.ts` | Publishes a clean Markdown version of every API page. | +| `src/pages/api/[...slug]/index.md.ts` | Serves an alternate Markdown version of every API page. | | `src/components/ui/api-*/` | Owns the reference's appearance and interaction design. | #### Why the specification is not in `src/content` @@ -246,7 +246,7 @@ rendering: { }, ``` -The recipe's API catch-all uses Nimbus runtime helpers to resolve either mode. Request rendering changes the canonical HTML pages only. Markdown versions, coordinate data, search records, sitemaps, and `llms.txt` indexes remain build products, and requests consume prepared content entries rather than parsing the OpenAPI document. +The recipe's API catch-all uses Nimbus runtime helpers to resolve either mode. Request rendering changes the canonical HTML pages only. Alternate Markdown versions, coordinate data, search records, sitemaps, and `llms.txt` indexes remain build products, and requests consume prepared content entries rather than parsing the OpenAPI document. #### Prepared code examples diff --git a/apps/www/src/content/docs/configuration.mdx b/apps/www/src/content/docs/configuration.mdx index 775f642d..d99d21c6 100644 --- a/apps/www/src/content/docs/configuration.mdx +++ b/apps/www/src/content/docs/configuration.mdx @@ -198,4 +198,4 @@ nimbus(config, { Each transform requires a non-empty `revision`. Increment it whenever its output changes so Nimbus invalidates the generated Markdown cache. -Request-rendered HTML consumes prepared entries and headings. Markdown versions, `llms-full.txt`, reusable-snippet expansion, and syntax highlighting remain build-time work, and their routes must stay prerendered. `llms.txt` files remain indexes of those Markdown versions. Run a production build after changing these options; development mode does not validate the final Worker bundle. +Request-rendered HTML consumes prepared entries and headings. Generating alternate Markdown/MDX versions, `llms.txt` indexes, `llms-full.txt`, reusable-snippet expansion, and syntax highlighting remains build-time work. Their endpoints can prerender or serve those payloads on request. Run a production build after changing these options; development mode does not validate the final Worker bundle. diff --git a/apps/www/src/content/docs/project-structure.mdx b/apps/www/src/content/docs/project-structure.mdx index 86a257f1..f32f536c 100644 --- a/apps/www/src/content/docs/project-structure.mdx +++ b/apps/www/src/content/docs/project-structure.mdx @@ -32,7 +32,7 @@ A scaffolded Nimbus project is a regular Astro project. Everything visible lives - **`src/components/`** — UI components, yours to edit or replace. - **`src/content/docs/`** — your MDX content. The tree here is the site (see below). - **`src/layouts/`** — page shells (`BaseLayout`, `DocsLayout`). -- **`src/pages/`** — routes, including the docs catch-all, Markdown versions, and `llms.txt` files. +- **`src/pages/`** — routes, including the docs catch-all, alternate Markdown/MDX versions, and `llms.txt` indexes. - **`src/styles/globals.css`** — design tokens and Tailwind layers. - **`src/components.ts`** — the [MDX globals registry](/writing/markdown-and-mdx). - **`nimbus.json`** — a committed record of what your project is made of (see below). @@ -45,7 +45,7 @@ It is the machine half of a pair: your **behavior** config (versions, features, ## The framework boundary -The `nimbus-docs` package is the invisible half — the Astro integration, data helpers (`getSidebar`, `getPrevNext`, `getTOC`), content schemas, and routes for Markdown versions and `llms.txt` files. You import it; you don't fork it. +The `nimbus-docs` package is the invisible half — the Astro integration, data helpers (`getSidebar`, `getPrevNext`, `getTOC`), content schemas, and helpers for alternate Markdown/MDX versions and `llms.txt` indexes. You import it; you don't fork it. Everything else is yours. There is no upstream theme to override and no API to break — coding agents reason about the repo more easily when nothing important hides behind an import boundary. diff --git a/apps/www/src/content/docs/registry.mdx b/apps/www/src/content/docs/registry.mdx index b052f6ec..78382567 100644 --- a/apps/www/src/content/docs/registry.mdx +++ b/apps/www/src/content/docs/registry.mdx @@ -47,14 +47,14 @@ Features don't have an install graph — the recipe is self-contained markdown t A feature is an agent-handoff recipe — nothing copies as files. Install any with `npx @cloudflare/nimbus-docs add `; the CLI prints a brief your coding agent adapts to your project and applies. - `pagefind-search` — static full-text search and the Nimbus search dialog. -- `ai-native` — `llms.txt`, per-page Markdown versions, `robots.txt`, and an `AgentDirective`. +- `ai-native` — `llms.txt` indexes, alternate Markdown/MDX versions, `robots.txt`, and an `AgentDirective`. - `mermaid` — lazy, theme-aware Mermaid diagrams with a full-screen expand. - `404-page` — a brand-matched 404 page for your docs. - `lint-prose-textlint` — textlint with write-good, alex, and terminology rules. - `changelog` — a dated `/changelog` feed with tags, RSS, and permalinks. - `component-showcase` — a `/components` grid and per-component pages for your UI. - `new-collection` — a non-version content tree: blog, API, or glossary. -- `api-reference` — mount an OpenAPI spec as generated reference pages with Markdown versions, styled with the owned `api-layout` / `api-sidebar` / `api-field-row` / `api-code-rail` components. +- `api-reference` — mount an OpenAPI spec as generated reference pages with alternate Markdown versions, styled with the owned `api-layout` / `api-sidebar` / `api-field-row` / `api-code-rail` components. - `new-version` — wire a new docs version end-to-end, with a switcher. ## Content recipes diff --git a/apps/www/src/content/docs/writing/pages-and-routing.mdx b/apps/www/src/content/docs/writing/pages-and-routing.mdx index 6c22e9de..209ccc45 100644 --- a/apps/www/src/content/docs/writing/pages-and-routing.mdx +++ b/apps/www/src/content/docs/writing/pages-and-routing.mdx @@ -27,6 +27,32 @@ Slugs are lowercased and folder `index` files collapse to the directory URL. A d - styling.mdx +## Custom Astro routes + +Files you add under `src/pages/` use Astro's native routing and rendering semantics. In a server-output project, a page or endpoint renders on request by default; set Astro's `prerender` export when you want to make the choice explicit: + +```astro title="src/pages/status.astro" +--- +export const prerender = false; +--- + +

Service status

+``` + +```ts title="src/pages/api/ping.ts" +export const prerender = false; + +export function GET() { + return new Response("pong"); +} +``` + +Use `export const prerender = true` to emit a custom route as static output during the build. No Nimbus route registration or allowlist is required. + +This boundary is not limited to `prerender`. Dynamic segments, endpoint methods, redirects and responses, and other native Astro route behavior remain controlled by the route itself. Nimbus only intervenes when a route claims the exact pattern of a configured canonical content route, package-injected infrastructure, an active feature, or a published content entry. + +Nimbus's `rendering` configuration controls canonical content-collection routes, not other files under `src/pages/`. Files scaffolded for `llms.txt`, Markdown alternates, Open Graph images, and `robots.txt` belong to your project and may use Astro's native rendering behavior like any other route. + ## Page modes Every page renders inside `DocsLayout` by default — sidebar, table of contents, breadcrumbs, prev/next. Opt a page out of all chrome with `mode`: @@ -59,7 +85,7 @@ draft: true `src/content/docs/` is the primary collection, mounted at the site root. Additional collections (a `blog`, an `api`, a versioned `docs-v2`) mount under their own URL namespace. Register them in `src/content.config.ts` with the factories from `nimbus-docs/content`. -Nimbus's `docsCollection()`, `partialsCollection()`, and `componentsCollection()` factories prepare content for links, Markdown versions, `llms-full.txt`, and partial headings automatically. If you register a loader that stores Markdown bodies directly, wrap it with `withNimbusMarkdown()`: +Nimbus's `docsCollection()`, `partialsCollection()`, and `componentsCollection()` factories prepare content for links, alternate Markdown/MDX versions, `llms-full.txt`, and partial headings automatically. If you register a loader that stores Markdown bodies directly, wrap it with `withNimbusMarkdown()`: ```ts title="src/content.config.ts" import { defineCollection } from "astro:content"; @@ -75,7 +101,7 @@ export const collections = { }; ``` -The wrapper preserves the loader's methods and lifecycle. Use it when a loader stores entries with a Markdown `body`. Every general-purpose indexed collection must provide prepared Markdown for per-page versions and `llms-full.txt`; arbitrary data-only collections are not supported. `apiCollection()` is the purpose-built exception because Nimbus supplies its renderer. If a collection is not prepared, the build names it and tells you to wrap its loader. +The wrapper preserves the loader's methods and lifecycle. Use it when a loader stores entries with a Markdown `body`. Every general-purpose indexed collection must provide prepared Markdown for alternate Markdown/MDX versions and `llms-full.txt`; arbitrary data-only collections are not supported. `apiCollection()` is the purpose-built exception because Nimbus supplies its renderer. If a collection is not prepared, the build names it and tells you to wrap its loader. ## Custom route helpers diff --git a/apps/www/src/content/docs/writing/reusable-snippets.mdx b/apps/www/src/content/docs/writing/reusable-snippets.mdx index 0b1cc4e3..5e7ab5da 100644 --- a/apps/www/src/content/docs/writing/reusable-snippets.mdx +++ b/apps/www/src/content/docs/writing/reusable-snippets.mdx @@ -54,7 +54,7 @@ nimbus(config, { }); ``` -The resolver controls partial IDs in Markdown versions, `llms-full.txt`, and prepared headings. Change `revision` whenever its output rules change so Nimbus rebuilds the affected Markdown. +The resolver controls partial IDs in alternate Markdown versions, `llms-full.txt`, and prepared headings. Change `revision` whenever its output rules change so Nimbus rebuilds the affected Markdown. `Render.astro` is user-owned and resolves the canonical HTML partial separately. If you customize partial IDs, move the resolver function into a shared project file and call it from both `astro.config.ts` and `Render.astro` before `getVisibleEntry("partials", id)`. This keeps HTML, headings, and generated Markdown aligned. diff --git a/apps/www/src/pages/[...slug]/index.mdx.ts b/apps/www/src/pages/[...slug]/index.mdx.ts index af83161e..5e278fcc 100644 --- a/apps/www/src/pages/[...slug]/index.mdx.ts +++ b/apps/www/src/pages/[...slug]/index.mdx.ts @@ -2,8 +2,8 @@ * Per-page `//index.mdx` — the raw authored source for every * indexable entry of the primary `docs` collection that has a string body. * - * Markdown versions: `index.md` is generated Markdown for reading, while - * `index.mdx` is prepared source with imports, JSX, and directives intact. The + * Alternate Markdown/MDX versions: `index.md` is generated Markdown for reading, while + * `index.mdx` is expanded source with imports, JSX, and directives intact. The * body is served verbatim; only the canonical frontmatter block (shared * with the `.md` version) is framework-shaped. * diff --git a/packages/create-nimbus-docs/test/generate-templates.test.ts b/packages/create-nimbus-docs/test/generate-templates.test.ts index 927d6589..c931ad09 100644 --- a/packages/create-nimbus-docs/test/generate-templates.test.ts +++ b/packages/create-nimbus-docs/test/generate-templates.test.ts @@ -15,6 +15,27 @@ import path from "node:path"; import { test } from "node:test"; import { generateTemplates, variantNames } from "../scripts/copy-template.mjs"; +import { STARTER_ROUTE_INVENTORY } from "../../nimbus-docs/src/_internal/route-ownership.js"; + +function routeEntrypoints(root: string): string[] { + const pages = path.join(root, "src/pages"); + const routes: string[] = []; + const walk = (directory: string) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) { + walk(absolute); + } else if ( + !entry.name.startsWith("_") && + /\.(?:astro|mdx?|[cm]?[jt]sx?)$/.test(entry.name) + ) { + routes.push(path.relative(path.join(root, "src"), absolute).replaceAll("\\", "/")); + } + } + }; + walk(pages); + return routes.sort(); +} test("every generated variant ships the adapter marker and implicit build default", () => { const out = fs.mkdtempSync(path.join(os.tmpdir(), "nimbus-gen-")); @@ -33,6 +54,32 @@ test("every generated variant ships the adapter marker and implicit build defaul assert.doesNotMatch(cfg, /rendering:\s*\{/); const gitignore = fs.readFileSync(path.join(dir, "gitignore"), "utf8"); assert.match(gitignore, /^\.nimbus\/$/m); + + assert.deepEqual( + routeEntrypoints(dir), + STARTER_ROUTE_INVENTORY.map((route) => route.entrypoint).sort(), + `${path.basename(dir)} has an unclassified or missing route entrypoint`, + ); + assert.deepEqual( + STARTER_ROUTE_INVENTORY.filter( + (route) => route.role === "canonical", + ).map(({ pattern, entrypoint }) => ({ pattern, entrypoint })), + [{ pattern: "/[...slug]", entrypoint: "pages/[...slug].astro" }], + ); + assert.deepEqual( + STARTER_ROUTE_INVENTORY.filter( + (route) => route.role === "user-owned", + ).map(({ pattern, entrypoint }) => ({ pattern, entrypoint })), + STARTER_ROUTE_INVENTORY.filter( + (route) => route.role !== "canonical", + ).map(({ pattern, entrypoint }) => ({ pattern, entrypoint })), + ); + assert.deepEqual( + STARTER_ROUTE_INVENTORY.filter((route) => route.allowsContentShadow).map( + ({ pattern, entrypoint }) => ({ pattern, entrypoint }), + ), + [{ pattern: "/", entrypoint: "pages/index.astro" }], + ); } } finally { fs.rmSync(out, { recursive: true, force: true }); diff --git a/packages/nimbus-docs/package.json b/packages/nimbus-docs/package.json index 90dc75c5..a9ff94b5 100644 --- a/packages/nimbus-docs/package.json +++ b/packages/nimbus-docs/package.json @@ -37,6 +37,14 @@ "types": "./dist/runtime.d.ts", "import": "./dist/runtime.js" }, + "./agent-endpoints": { + "types": "./dist/agent-endpoints.d.ts", + "import": "./dist/agent-endpoints.js" + }, + "./publication": { + "types": "./dist/publication.d.ts", + "import": "./dist/publication.js" + }, "./build": { "types": "./dist/build.d.ts", "import": "./dist/build.js" diff --git a/packages/nimbus-docs/src/_internal/prepared-artifacts.ts b/packages/nimbus-docs/src/_internal/agent-endpoint-assets.ts similarity index 72% rename from packages/nimbus-docs/src/_internal/prepared-artifacts.ts rename to packages/nimbus-docs/src/_internal/agent-endpoint-assets.ts index 5da0f9f0..96b68bf4 100644 --- a/packages/nimbus-docs/src/_internal/prepared-artifacts.ts +++ b/packages/nimbus-docs/src/_internal/agent-endpoint-assets.ts @@ -3,6 +3,7 @@ import { lstat, mkdir, open, + copyFile, readFile, readdir, realpath, @@ -43,16 +44,18 @@ import { toBrowserHref, toRouteKey, withBase } from "./url.js"; import type { GeneratedMarkdownComponentTransform, GeneratedMarkdownPartialResolver, - PreparedLlmsArtifact, - PreparedLlmsReference, - PreparedMarkdownArtifact, - PreparedMarkdownReference, } from "../types.js"; +import type { + LlmsEndpointPayload, + LlmsEndpointReference, + MarkdownEndpointPayload, + MarkdownEndpointReference, +} from "../agent-endpoints.js"; -export const PREPARED_ARTIFACT_MANIFEST_VERSION = 4; -export const PREPARED_ARTIFACT_GENERATION = 1; +export const AGENT_ENDPOINT_ASSET_MANIFEST_VERSION = 4; +export const AGENT_ENDPOINT_ASSET_GENERATION = 1; -export interface PreparedMarkdownManifestArtifact extends PreparedMarkdownReference { +export interface MarkdownEndpointAsset extends MarkdownEndpointReference { digest: string; mediaType: string; path: string; @@ -60,28 +63,28 @@ export interface PreparedMarkdownManifestArtifact extends PreparedMarkdownRefere contentEnd: number; } -export type PreparedLlmsManifestArtifact = PreparedLlmsReference & { +export type LlmsEndpointAsset = LlmsEndpointReference & { digest: string; mediaType: string; path: string; }; -export interface PreparedArtifactManifest { +export interface AgentEndpointAssetManifest { version: 4; generation: number; base: string; audience: "public"; - markdownArtifacts: PreparedMarkdownManifestArtifact[]; - llmsArtifacts: PreparedLlmsManifestArtifact[]; + markdownAssets: MarkdownEndpointAsset[]; + llmsAssets: LlmsEndpointAsset[]; headings: PreparedHeadingRecord[]; } -export type PreparedArtifactPublicationDecision = +export type AgentEndpointVisibilityDecision = | { status: "include" } | { status: "exclude"; reason: string } | { status: "unknown"; reason: string }; -export interface BakePreparedArtifactsOptions { +export interface BakeAgentEndpointAssetsOptions { root: URL | string; base: string; site: string; @@ -98,9 +101,9 @@ export interface BakePreparedArtifactsOptions { citationIndex?: ReadonlyMap; componentMap?: Record; partialResolver?: GeneratedMarkdownPartialResolver; - decidePublic?: (entry: PreparedMarkdownEntry) => PreparedArtifactPublicationDecision; - apiEntries?: readonly PreparedLlmsApiEntry[]; - loadApiEntries?: () => Promise; + decidePublic?: (entry: PreparedMarkdownEntry) => AgentEndpointVisibilityDecision; + apiEntries?: readonly LlmsEndpointApiEntry[]; + loadApiEntries?: () => Promise; } export interface BakePreparedHeadingsOptions { @@ -110,14 +113,14 @@ export interface BakePreparedHeadingsOptions { partialResolver?: GeneratedMarkdownPartialResolver; } -export interface PreparedLlmsApiEntry { +export interface LlmsEndpointApiEntry { collection: string; id: string; data: Record; hidden?: boolean; } -interface ArtifactState { +interface AgentEndpointAssetState { version: 1; demands: Set; publications: Map>; @@ -129,36 +132,36 @@ interface ArtifactState { string, { mode: "build" | "dev"; - bake: () => Promise; + bake: () => Promise; bakeHeadings?: () => Promise; headingsBase?: string; bakedRevision?: number; invalidation: number; bakedInvalidation?: number; - inFlight?: Promise; - manifest?: PreparedArtifactManifest; - markdownArtifacts?: Map; - llmsArtifacts?: Map; + inFlight?: Promise; + manifest?: AgentEndpointAssetManifest; + markdownAssets?: Map; + llmsAssets?: Map; headings?: Map; } >; } const STATE_KEY = Symbol.for( - "@cloudflare/nimbus-docs/prepared-artifacts/v1", + "@cloudflare/nimbus-docs/agent-endpoint-assets/v1", ); const stateGlobal = globalThis as typeof globalThis & { - [STATE_KEY]?: ArtifactState; + [STATE_KEY]?: AgentEndpointAssetState; }; -const artifactState = (stateGlobal[STATE_KEY] ??= { +const agentEndpointAssetState = (stateGlobal[STATE_KEY] ??= { version: 1, demands: new Set(), publications: new Map(), readers: new Map(), roots: new Map(), }); -artifactState.publications ??= new Map(); -artifactState.readers ??= new Map(); +agentEndpointAssetState.publications ??= new Map(); +agentEndpointAssetState.readers ??= new Map(); function digest(value: string): string { return createHash("sha256").update(value).digest("hex"); @@ -168,11 +171,11 @@ function compare(a: string, b: string): number { return a < b ? -1 : a > b ? 1 : 0; } -function artifactKey(reference: PreparedMarkdownReference): string { +function markdownAssetKey(reference: MarkdownEndpointReference): string { return `${reference.collection}\0${reference.id}\0${reference.surface}`; } -function preparedLlmsKey(reference: PreparedLlmsReference): string { +function preparedLlmsKey(reference: LlmsEndpointReference): string { return reference.scope === "site" ? `${reference.scope}\0${reference.surface}` : `${reference.scope}\0${reference.section}\0${reference.surface}`; @@ -182,11 +185,25 @@ function headingKey(collection: string, id: string): string { return `${collection}\0${id}`; } -function artifactRoot(root: URL | string): string { - return path.join(preparedMarkdownRootKey(root), ".astro", "nimbus", "prepared-artifacts"); +function agentEndpointAssetRoot(root: URL | string): string { + return path.join( + preparedMarkdownRootKey(root), + ".astro", + "nimbus", + "agent-endpoint-assets", + ); } async function assertNoSymlink(root: string, target: string): Promise { + try { + if ((await lstat(root)).isSymbolicLink()) { + throw new Error( + `nimbus-docs: agent-endpoint asset path contains a symbolic link: ${root}.`, + ); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } const relative = path.relative(root, target); let current = root; for (const segment of relative.split(path.sep).filter(Boolean)) { @@ -194,7 +211,7 @@ async function assertNoSymlink(root: string, target: string): Promise { try { if ((await lstat(current)).isSymbolicLink()) { throw new Error( - `nimbus-docs: prepared artifact path contains a symbolic link: ${current}.`, + `nimbus-docs: agent-endpoint asset path contains a symbolic link: ${current}.`, ); } } catch (error) { @@ -203,7 +220,31 @@ async function assertNoSymlink(root: string, target: string): Promise { } } -async function writeArtifact(file: string, body: string): Promise { +function resolveContainedAssetPath(root: string, assetPath: string): string { + if ( + assetPath.length === 0 || + path.isAbsolute(assetPath) || + path.win32.isAbsolute(assetPath) + ) { + throw new Error( + `nimbus-docs: agent-endpoint asset path must be relative: ${assetPath}.`, + ); + } + const resolved = path.resolve(root, assetPath); + const relative = path.relative(root, resolved); + if ( + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + throw new Error( + `nimbus-docs: agent-endpoint asset path escapes its root: ${assetPath}.`, + ); + } + return resolved; +} + +async function writeAsset(file: string, body: string): Promise { let created = false; try { const handle = await open(file, "wx"); @@ -223,12 +264,12 @@ async function writeArtifact(file: string, body: string): Promise { const info = await lstat(file); if (!info.isFile() || info.isSymbolicLink()) { throw new Error( - `nimbus-docs: prepared artifact is not a regular file: ${file}.`, + `nimbus-docs: agent-endpoint asset is not a regular file: ${file}.`, ); } if ((await readFile(file, "utf8")) !== body) { throw new Error( - `nimbus-docs: content-addressed prepared artifact collision at ${file}.`, + `nimbus-docs: content-addressed agent-endpoint asset collision at ${file}.`, ); } return false; @@ -237,7 +278,7 @@ async function writeArtifact(file: string, body: string): Promise { async function writeManifest( directory: string, - manifest: PreparedArtifactManifest, + manifest: AgentEndpointAssetManifest, ): Promise { const temporary = path.join( directory, @@ -252,15 +293,15 @@ async function writeManifest( await rename(temporary, path.join(directory, "manifest.json")); } -function beginArtifactRead(root: string): () => void { - let readers = artifactState.readers.get(root); +function beginAssetRead(root: string): () => void { + let readers = agentEndpointAssetState.readers.get(root); if (!readers || readers.active === 0) { let resolve = () => {}; const idle = new Promise((done) => { resolve = done; }); readers = { active: 0, idle, resolve }; - artifactState.readers.set(root, readers); + agentEndpointAssetState.readers.set(root, readers); } readers.active += 1; return () => { @@ -270,28 +311,28 @@ function beginArtifactRead(root: string): () => void { }; } -async function cleanupArtifacts( +async function cleanupAssets( root: string, directory: string, - manifest: PreparedArtifactManifest, + manifest: AgentEndpointAssetManifest, ): Promise { - const pendingReaders = artifactState.readers.get(root)?.idle; + const pendingReaders = agentEndpointAssetState.readers.get(root)?.idle; if (pendingReaders) await pendingReaders; const retained = new Set( - [...manifest.markdownArtifacts, ...manifest.llmsArtifacts].map( - (artifact) => artifact.path, + [...manifest.markdownAssets, ...manifest.llmsAssets].map( + (asset) => asset.path, ), ); - const artifactDirectory = path.join(directory, "artifacts"); - const entries = await readdir(artifactDirectory, { withFileTypes: true }); + const assetDirectory = path.join(directory, "assets"); + const entries = await readdir(assetDirectory, { withFileTypes: true }); await Promise.all( entries .filter( (entry) => - !entry.isDirectory() && !retained.has(`artifacts/${entry.name}`), + !entry.isDirectory() && !retained.has(`assets/${entry.name}`), ) .map((entry) => - rm(path.join(artifactDirectory, entry.name), { force: true }), + rm(path.join(assetDirectory, entry.name), { force: true }), ), ); } @@ -299,13 +340,13 @@ async function cleanupArtifacts( async function publishManifest( root: string, directory: string, - manifest: PreparedArtifactManifest, - previous: PreparedArtifactManifest | undefined, + manifest: AgentEndpointAssetManifest, + previous: AgentEndpointAssetManifest | undefined, isFresh: () => boolean, - preparedArtifacts: ReadonlyArray<{ path: string; body: string }>, + endpointAssets: ReadonlyArray<{ path: string; body: string }>, onPublished: () => void | Promise, ): Promise { - const prior = artifactState.publications.get(root) ?? Promise.resolve(); + const prior = agentEndpointAssetState.publications.get(root) ?? Promise.resolve(); let published = false; const operation = prior.then(async () => { if (!isFresh()) return; @@ -313,29 +354,29 @@ async function publishManifest( const removeCreated = async () => { const retained = new Set( previous - ? [...previous.markdownArtifacts, ...previous.llmsArtifacts].map( - (artifact) => artifact.path, + ? [...previous.markdownAssets, ...previous.llmsAssets].map( + (asset) => asset.path, ) : [], ); await Promise.all( created - .filter((artifactPath) => !retained.has(artifactPath)) - .map((artifactPath) => - rm(path.join(directory, artifactPath), { force: true }), + .filter((assetPath) => !retained.has(assetPath)) + .map((assetPath) => + rm(path.join(directory, assetPath), { force: true }), ), ); }; try { const writes = await Promise.allSettled( - preparedArtifacts.map(async (artifact) => { + endpointAssets.map(async (asset) => { if ( - await writeArtifact( - path.join(directory, artifact.path), - artifact.body, + await writeAsset( + path.join(directory, asset.path), + asset.body, ) ) { - created.push(artifact.path); + created.push(asset.path); } }), ); @@ -368,12 +409,12 @@ async function publishManifest( () => undefined, () => undefined, ); - artifactState.publications.set(root, settled); + agentEndpointAssetState.publications.set(root, settled); try { await operation; } finally { - if (artifactState.publications.get(root) === settled) { - artifactState.publications.delete(root); + if (agentEndpointAssetState.publications.get(root) === settled) { + agentEndpointAssetState.publications.delete(root); } } return published; @@ -387,7 +428,7 @@ function assertPreparedCollection( const expected = preparedMarkdownCollectionCapability( name, collection.entries.values(), - { generation: PREPARED_ARTIFACT_GENERATION, base }, + { generation: AGENT_ENDPOINT_ASSET_GENERATION, base }, ); if ( collection.capability.generation !== expected.generation || @@ -396,13 +437,13 @@ function assertPreparedCollection( ) { throw new Error( `nimbus-docs: cannot bake collection "${name}" because its bodies were not prepared ` + - `for generation ${PREPARED_ARTIFACT_GENERATION} and base ${JSON.stringify(base)}. ` + + `for generation ${AGENT_ENDPOINT_ASSET_GENERATION} and base ${JSON.stringify(base)}. ` + "Use withNimbusMarkdown(loader) for custom body-retaining loaders.", ); } } -function defaultDecision(entry: PreparedMarkdownEntry): PreparedArtifactPublicationDecision { +function defaultDecision(entry: PreparedMarkdownEntry): AgentEndpointVisibilityDecision { if (entry.data.draft === true) return { status: "exclude", reason: "draft" }; if ( entry.data.visibility === undefined || @@ -431,7 +472,7 @@ function absoluteAssetUrl( return new URL(withBase(pathname, base), site).href; } -function withArtifactBase(base: string, pathname: string): string { +function withAssetBase(base: string, pathname: string): string { if (!pathname.startsWith("/") || pathname.startsWith("//")) return pathname; const prefix = base === "/" ? "" : base.replace(/\/+$/u, ""); return `${prefix}${pathname}`; @@ -439,7 +480,7 @@ function withArtifactBase(base: string, pathname: string): string { function entryVersion( entry: PreparedMarkdownEntry, - versions: BakePreparedArtifactsOptions["versions"], + versions: BakeAgentEndpointAssetsOptions["versions"], ): string | undefined { if (typeof entry.data.version === "string") return entry.data.version; if (!versions) return undefined; @@ -453,7 +494,7 @@ function entryVersion( function preparedMarkdownUrls( entry: PreparedMarkdownEntry, - options: BakePreparedArtifactsOptions, + options: BakeAgentEndpointAssetsOptions, ) { const route = entryRouteUrl( collectionMountPrefix(entry.collection, options.versions), @@ -467,7 +508,7 @@ function preparedMarkdownUrls( function frontmatter( entry: PreparedMarkdownEntry, - options: BakePreparedArtifactsOptions, + options: BakeAgentEndpointAssetsOptions, ): string[] { const title = typeof entry.data.title === "string" && entry.data.title.length > 0 @@ -498,25 +539,25 @@ function frontmatter( ]; } -interface PreparedMarkdownArtifactBody { +interface MarkdownEndpointPayloadBody { body: string; contentStart: number; contentEnd: number; } -function envelopedArtifact(prefix: string, content: string, suffix = "") { +function envelopedAsset(prefix: string, content: string, suffix = "") { return { body: `${prefix}${content}${suffix}`, contentStart: prefix.length, contentEnd: prefix.length + content.length, - } satisfies PreparedMarkdownArtifactBody; + } satisfies MarkdownEndpointPayloadBody; } -function markdownArtifact( +function markdownAsset( entry: PreparedMarkdownEntry, markdown: string, - options: BakePreparedArtifactsOptions, -): PreparedMarkdownArtifactBody { + options: BakeAgentEndpointAssetsOptions, +): MarkdownEndpointPayloadBody { const title = typeof entry.data.title === "string" && entry.data.title.length > 0 ? entry.data.title @@ -537,15 +578,15 @@ function markdownArtifact( `Source: ${absoluteUrl(options.site, options.base, urls.source)}`, "", ].join("\n"); - return envelopedArtifact(`${prefix}\n`, markdown, `\n${suffix}`); + return envelopedAsset(`${prefix}\n`, markdown, `\n${suffix}`); } -function sourceArtifact( +function sourceAsset( entry: PreparedMarkdownEntry, expanded: string, - options: BakePreparedArtifactsOptions, -): PreparedMarkdownArtifactBody { - return envelopedArtifact( + options: BakeAgentEndpointAssetsOptions, +): MarkdownEndpointPayloadBody { + return envelopedAsset( `${[...frontmatter(entry, options), ""].join("\n")}\n`, expanded, ); @@ -571,7 +612,7 @@ interface PreparedLlmsGroup { function preparedLlmsPage( entry: Pick, markdown: string, - options: BakePreparedArtifactsOptions, + options: BakeAgentEndpointAssetsOptions, ): PreparedLlmsPage { const route = entryRouteUrl( collectionMountPrefix(entry.collection, options.versions), @@ -597,7 +638,7 @@ function preparedLlmsPage( function groupPreparedLlmsPages( pages: readonly PreparedLlmsPage[], - options: BakePreparedArtifactsOptions, + options: BakeAgentEndpointAssetsOptions, ): { leaves: PreparedLlmsPage[]; groups: PreparedLlmsGroup[] } { const primary = new Map(); const secondary = new Map(); @@ -639,10 +680,10 @@ function groupPreparedLlmsPages( return { leaves, groups }; } -function siteIndexArtifact( +function siteIndexAsset( leaves: readonly PreparedLlmsPage[], groups: readonly PreparedLlmsGroup[], - options: BakePreparedArtifactsOptions, + options: BakeAgentEndpointAssetsOptions, ): string { const rows = [ ...leaves.map((page) => ({ @@ -670,9 +711,9 @@ function siteIndexArtifact( ].join("\n"); } -function sectionIndexArtifact( +function sectionIndexAsset( group: PreparedLlmsGroup, - options: BakePreparedArtifactsOptions, + options: BakeAgentEndpointAssetsOptions, ): string { return [ `# ${group.label}`, @@ -769,7 +810,7 @@ function assertLlmsRouteSafety( } function componentFingerprint( - componentMap: BakePreparedArtifactsOptions["componentMap"], + componentMap: BakeAgentEndpointAssetsOptions["componentMap"], ): string { return JSON.stringify( Object.entries(componentMap ?? {}) @@ -780,8 +821,12 @@ function componentFingerprint( export interface PreparedHeadingsPlugin { name: string; + enforce?: "pre" | "post"; resolveId(id: string): string | undefined; - load(id: string): Promise; + load( + this: { environment?: { name?: string } }, + id: string, + ): Promise | string | undefined; handleHotUpdate(context: { server: { moduleGraph: { @@ -794,6 +839,10 @@ export interface PreparedHeadingsPlugin { const HEADINGS_VIRTUAL_ID = "virtual:nimbus/headings"; const HEADINGS_RESOLVED_ID = `\0${HEADINGS_VIRTUAL_ID}`; +const ARTIFACTS_VIRTUAL_ID = "virtual:nimbus/agent-endpoint-assets"; +const ARTIFACTS_RESOLVED_ID = `\0${ARTIFACTS_VIRTUAL_ID}`; +const ASSET_LOADER_VIRTUAL_ID = "virtual:nimbus/agent-endpoint-asset-loader"; +const ASSET_LOADER_RESOLVED_ID = `\0${ASSET_LOADER_VIRTUAL_ID}`; export function preparedHeadingsPlugin( root: URL | string, @@ -806,7 +855,7 @@ export function preparedHeadingsPlugin( async load(id) { if (id !== HEADINGS_RESOLVED_ID) return undefined; const key = preparedMarkdownRootKey(root); - const configured = artifactState.roots.get(key); + const configured = agentEndpointAssetState.roots.get(key); if (!configured) { throw new Error( "nimbus-docs: prepared headings are available only during a configured Astro build or dev server.", @@ -814,7 +863,7 @@ export function preparedHeadingsPlugin( } const records = configured.bakeHeadings ? await configured.bakeHeadings() - : (await ensurePreparedArtifacts(root)).headings; + : (await ensureAgentEndpointAssets(root)).headings; return ( `export const generation = ${PREPARED_HEADINGS_GENERATION};\n` + `export const base = ${JSON.stringify(configured.headingsBase ?? records[0]?.base ?? "/")};\n` + @@ -829,16 +878,111 @@ export function preparedHeadingsPlugin( }; } -export function configurePreparedArtifactRoot( +export function agentEndpointAssetsRuntimePlugin( + root: URL | string, +): PreparedHeadingsPlugin { + return { + name: "nimbus-docs:agent-endpoint-assets-runtime", + resolveId(id) { + return id === ARTIFACTS_VIRTUAL_ID ? ARTIFACTS_RESOLVED_ID : undefined; + }, + async load(id) { + if (id !== ARTIFACTS_RESOLVED_ID) return undefined; + if (this.environment?.name === "ssr") { + registerAgentEndpointAssetDemand(root); + } + const manifest = await ensureAgentEndpointAssets(root); + return ( + `export const projectRoot = ${JSON.stringify(preparedMarkdownRootKey(root))};\n` + + `export const base = ${JSON.stringify(manifest.base)};\n` + + `export const markdownAssets = ${JSON.stringify(manifest.markdownAssets)};\n` + + `export const llmsAssets = ${JSON.stringify(manifest.llmsAssets)};\n` + ); + }, + handleHotUpdate(context) { + const module = + context.server.moduleGraph.getModuleById(ARTIFACTS_RESOLVED_ID); + if (module) context.server.moduleGraph.invalidateModule(module); + }, + }; +} + +export function agentEndpointAssetLoaderPlugin( + adapterName: () => string | null, +): PreparedHeadingsPlugin { + return { + name: "nimbus-docs:agent-endpoint-asset-loader", + enforce: "pre", + resolveId(id) { + return id === ASSET_LOADER_VIRTUAL_ID + ? ASSET_LOADER_RESOLVED_ID + : undefined; + }, + load(id) { + if (id !== ASSET_LOADER_RESOLVED_ID) return undefined; + if ( + adapterName() === "@astrojs/cloudflare" && + this.environment?.name === "ssr" + ) { + return ( + 'import { env } from "cloudflare:workers";\n' + + "export function fetchAgentEndpointAsset(path, request) {\n" + + " return env.ASSETS?.fetch(new Request(new URL(path, request.url))) ?? null;\n" + + "}\n" + ); + } + return "export function fetchAgentEndpointAsset() { return null; }\n"; + }, + handleHotUpdate() {}, + }; +} + +export async function removeAgentEndpointAssets( + outputRoot: string, +): Promise { + for (const directory of ["agent-endpoint-assets", "prepared-artifacts"]) { + const targetRoot = path.join(outputRoot, "_nimbus", directory); + await assertNoSymlink(outputRoot, targetRoot); + await rm(targetRoot, { recursive: true, force: true }); + } +} + +export async function stageAgentEndpointAssets( + root: URL | string, + outputRoot: string, +): Promise { + const projectRoot = preparedMarkdownRootKey(root); + const manifest = await ensureAgentEndpointAssets(projectRoot); + const sourceRoot = agentEndpointAssetRoot(projectRoot); + const targetRoot = path.join(outputRoot, "_nimbus", "agent-endpoint-assets"); + const assets = [...manifest.markdownAssets, ...manifest.llmsAssets]; + const copies = assets.map((asset) => ({ + source: resolveContainedAssetPath(sourceRoot, asset.path), + target: resolveContainedAssetPath(targetRoot, asset.path), + })); + await removeAgentEndpointAssets(outputRoot); + for (let index = 0; index < copies.length; index += 64) { + await Promise.all( + copies.slice(index, index + 64).map(async ({ source, target }) => { + await assertNoSymlink(projectRoot, source); + await mkdir(path.dirname(target), { recursive: true }); + await assertNoSymlink(outputRoot, target); + await copyFile(source, target); + }), + ); + } +} + +export function configureAgentEndpointAssetRoot( root: URL | string, mode: "build" | "dev", - bake: () => Promise, + bake: () => Promise, bakeHeadings?: () => Promise, headingsBase?: string, ): void { const key = preparedMarkdownRootKey(root); - artifactState.demands.delete(key); - artifactState.roots.set(key, { + agentEndpointAssetState.demands.delete(key); + agentEndpointAssetState.roots.set(key, { mode, bake, bakeHeadings, @@ -916,22 +1060,22 @@ export async function bakePreparedHeadings( } } -export function registerPreparedArtifactDemand(root: URL | string): void { - artifactState.demands.add(preparedMarkdownRootKey(root)); +export function registerAgentEndpointAssetDemand(root: URL | string): void { + agentEndpointAssetState.demands.add(preparedMarkdownRootKey(root)); } -export function isPreparedArtifactRequested(root: URL | string): boolean { - return artifactState.demands.has(preparedMarkdownRootKey(root)); +export function isAgentEndpointAssetRequested(root: URL | string): boolean { + return agentEndpointAssetState.demands.has(preparedMarkdownRootKey(root)); } -export async function ensurePreparedArtifacts( +export async function ensureAgentEndpointAssets( root: URL | string, -): Promise { +): Promise { const key = preparedMarkdownRootKey(root); - const configured = artifactState.roots.get(key); + const configured = agentEndpointAssetState.roots.get(key); if (!configured) { throw new Error( - "nimbus-docs: prepared artifact helpers are available only during a configured Astro build or dev server.", + "nimbus-docs: agent-endpoint assets are available only during a configured Astro build or dev server.", ); } while (true) { @@ -964,23 +1108,23 @@ export async function ensurePreparedArtifacts( configured.bakedRevision = revision; configured.bakedInvalidation = invalidation; configured.manifest = manifest; - configured.markdownArtifacts = new Map( - manifest.markdownArtifacts.map( + configured.markdownAssets = new Map( + manifest.markdownAssets.map( ( - artifact: PreparedMarkdownManifestArtifact, - ): [string, PreparedMarkdownManifestArtifact] => [ - artifactKey(artifact), - artifact, + asset: MarkdownEndpointAsset, + ): [string, MarkdownEndpointAsset] => [ + markdownAssetKey(asset), + asset, ], ), ); - configured.llmsArtifacts = new Map( - manifest.llmsArtifacts.map( + configured.llmsAssets = new Map( + manifest.llmsAssets.map( ( - artifact: PreparedLlmsManifestArtifact, - ): [string, PreparedLlmsManifestArtifact] => [ - preparedLlmsKey(artifact), - artifact, + asset: LlmsEndpointAsset, + ): [string, LlmsEndpointAsset] => [ + preparedLlmsKey(asset), + asset, ], ), ); @@ -1001,20 +1145,20 @@ export async function ensurePreparedArtifacts( } } -export function invalidatePreparedArtifacts(root: URL | string): void { - const configured = artifactState.roots.get(preparedMarkdownRootKey(root)); +export function invalidateAgentEndpointAssets(root: URL | string): void { + const configured = agentEndpointAssetState.roots.get(preparedMarkdownRootKey(root)); if (configured) configured.invalidation += 1; } -export async function bakePreparedArtifacts( - options: BakePreparedArtifactsOptions, -): Promise { +export async function bakeAgentEndpointAssets( + options: BakeAgentEndpointAssetsOptions, +): Promise { const root = preparedMarkdownRootKey(options.root); - const configuredAtStart = artifactState.roots.get(root); + const configuredAtStart = agentEndpointAssetState.roots.get(root); const previousManifest = configuredAtStart?.manifest; const invalidationAtStart = configuredAtStart?.invalidation; let snapshot: NonNullable>; - let apiEntries: PreparedLlmsApiEntry[]; + let apiEntries: LlmsEndpointApiEntry[]; while (true) { await waitForPreparedMarkdownTransactions(root); const candidateSnapshot = getPreparedMarkdownSnapshot(root); @@ -1051,7 +1195,7 @@ export async function bakePreparedArtifacts( } const decide = options.decidePublic ?? defaultDecision; - const decisions = new Map(); + const decisions = new Map(); const hiddenVersions = new Set(options.versions?.hidden ?? []); const decideEntry = (entry: PreparedMarkdownEntry) => { const version = entry.collection.startsWith("docs-") @@ -1106,7 +1250,7 @@ export async function bakePreparedArtifacts( return partial; }; - const records: Array = []; + const records: Array = []; const headingRecords: PreparedHeadingRecord[] = []; const preparedLlmsPages: PreparedLlmsPage[] = []; const llmsRoutePages: PreparedLlmsPage[] = []; @@ -1127,7 +1271,7 @@ export async function bakePreparedArtifacts( ? new Map( [...options.citationIndex].map(([coordinate, url]) => [ coordinate, - withArtifactBase(base, url), + withAssetBase(base, url), ]), ) : undefined; @@ -1190,14 +1334,14 @@ export async function bakePreparedArtifacts( }); } for (const surface of ["markdown", "source"] as const) { - const artifact = + const asset = surface === "markdown" - ? markdownArtifact(entry, markdown, options) - : sourceArtifact(entry, expanded, options); - const { body, contentStart, contentEnd } = artifact; + ? markdownAsset(entry, markdown, options) + : sourceAsset(entry, expanded, options); + const { body, contentStart, contentEnd } = asset; const fingerprint = digest( JSON.stringify({ - generation: PREPARED_ARTIFACT_GENERATION, + generation: AGENT_ENDPOINT_ASSET_GENERATION, base, audience: "public", collection: entry.collection, @@ -1219,7 +1363,7 @@ export async function bakePreparedArtifacts( surface === "markdown" ? "text/markdown; charset=utf-8" : "text/mdx; charset=utf-8", - path: `artifacts/${fingerprint}.${extension}`, + path: `assets/${fingerprint}.${extension}`, contentStart, contentEnd, body, @@ -1275,12 +1419,12 @@ export async function bakePreparedArtifacts( !versionSlugs.has(collectionLabel(page.collection, options.versions)), ); const llmsBodies: Array<{ - reference: PreparedLlmsReference; + reference: LlmsEndpointReference; body: string; }> = [ { reference: { scope: "site", surface: "index" }, - body: siteIndexArtifact(leaves, groups, options), + body: siteIndexAsset(leaves, groups, options), }, { reference: { scope: "site", surface: "full" }, @@ -1306,14 +1450,14 @@ export async function bakePreparedArtifacts( surface: "index" as const, section: group.slug, }, - body: sectionIndexArtifact(group, options), + body: sectionIndexAsset(group, options), })), ]; - const llmsRecords: Array = + const llmsRecords: Array = llmsBodies.map(({ reference, body }) => { const fingerprint = digest( JSON.stringify({ - generation: PREPARED_ARTIFACT_GENERATION, + generation: AGENT_ENDPOINT_ASSET_GENERATION, base, audience: "public", reference, @@ -1327,7 +1471,7 @@ export async function bakePreparedArtifacts( ...reference, digest: `sha256:${fingerprint}`, mediaType: "text/plain; charset=utf-8", - path: `artifacts/${fingerprint}.txt`, + path: `assets/${fingerprint}.txt`, body, }; }); @@ -1355,28 +1499,28 @@ export async function bakePreparedArtifacts( } if (hashes.has(record.digest)) { throw new Error( - `nimbus-docs: duplicate artifact digest ${record.digest}.`, + `nimbus-docs: duplicate asset digest ${record.digest}.`, ); } llmsIdentities.add(identity); hashes.add(record.digest); } - const directory = artifactRoot(root); + const directory = agentEndpointAssetRoot(root); await assertNoSymlink(root, directory); - await mkdir(path.join(directory, "artifacts"), { recursive: true }); - await assertNoSymlink(root, path.join(directory, "artifacts")); - const manifest: PreparedArtifactManifest = { - version: PREPARED_ARTIFACT_MANIFEST_VERSION, - generation: PREPARED_ARTIFACT_GENERATION, + await mkdir(path.join(directory, "assets"), { recursive: true }); + await assertNoSymlink(root, path.join(directory, "assets")); + const manifest: AgentEndpointAssetManifest = { + version: AGENT_ENDPOINT_ASSET_MANIFEST_VERSION, + generation: AGENT_ENDPOINT_ASSET_GENERATION, base, audience: "public", - markdownArtifacts: records.map(({ body: _body, ...record }) => record), - llmsArtifacts: llmsRecords.map(({ body: _body, ...record }) => record), + markdownAssets: records.map(({ body: _body, ...record }) => record), + llmsAssets: llmsRecords.map(({ body: _body, ...record }) => record), headings: headingRecords, }; const isFresh = () => { - const current = artifactState.roots.get(root); + const current = agentEndpointAssetState.roots.get(root); return ( (!configuredAtStart || current === configuredAtStart) && current?.invalidation === invalidationAtStart && @@ -1392,7 +1536,7 @@ export async function bakePreparedArtifacts( isFresh, [...records, ...llmsRecords], async () => { - const configured = artifactState.roots.get(root); + const configured = agentEndpointAssetState.roots.get(root); let installed = !configuredAtStart; if ( configured && @@ -1404,14 +1548,14 @@ export async function bakePreparedArtifacts( configured.bakedRevision = snapshot.revision; configured.bakedInvalidation = configured.invalidation; configured.manifest = manifest; - configured.markdownArtifacts = new Map( - manifest.markdownArtifacts.map((artifact) => [ - artifactKey(artifact), - artifact, + configured.markdownAssets = new Map( + manifest.markdownAssets.map((asset) => [ + markdownAssetKey(asset), + asset, ]), ); - configured.llmsArtifacts = new Map( - manifest.llmsArtifacts.map((artifact) => [preparedLlmsKey(artifact), artifact]), + configured.llmsAssets = new Map( + manifest.llmsAssets.map((asset) => [preparedLlmsKey(asset), asset]), ); configured.headings = new Map( manifest.headings.map((record): [string, PreparedHeadingRecord] => [ @@ -1421,7 +1565,7 @@ export async function bakePreparedArtifacts( ); } if (installed) { - await cleanupArtifacts(root, directory, manifest); + await cleanupAssets(root, directory, manifest); } }, )) @@ -1431,28 +1575,28 @@ export async function bakePreparedArtifacts( return manifest; } -export async function getPreparedArtifactManifest( +export async function getAgentEndpointAssetManifest( root: URL | string, -): Promise { - return ensurePreparedArtifacts(root); +): Promise { + return ensureAgentEndpointAssets(root); } -export async function readPreparedMarkdownArtifact( +export async function readMarkdownEndpointPayload( root: URL | string, - reference: PreparedMarkdownReference, -): Promise { + reference: MarkdownEndpointReference, +): Promise { const key = preparedMarkdownRootKey(root); - await ensurePreparedArtifacts(key); - const endRead = beginArtifactRead(key); + await ensureAgentEndpointAssets(key); + const endRead = beginAssetRead(key); try { - const configured = artifactState.roots.get(key); - const record = configured?.markdownArtifacts?.get(artifactKey(reference)); + const configured = agentEndpointAssetState.roots.get(key); + const record = configured?.markdownAssets?.get(markdownAssetKey(reference)); if (!record) { throw new Error( - `nimbus-docs: no prepared ${reference.surface} artifact for "${reference.collection}:${reference.id}".`, + `nimbus-docs: no ${reference.surface} agent-endpoint asset for "${reference.collection}:${reference.id}".`, ); } - const body = await readArtifactBody(key, record); + const body = await readAssetBody(key, record); if ( !Number.isSafeInteger(record.contentStart) || !Number.isSafeInteger(record.contentEnd) || @@ -1461,7 +1605,7 @@ export async function readPreparedMarkdownArtifact( record.contentEnd > body.length ) { throw new Error( - `nimbus-docs: prepared ${reference.surface} artifact for "${reference.collection}:${reference.id}" has invalid content bounds.`, + `nimbus-docs: ${reference.surface} agent-endpoint asset for "${reference.collection}:${reference.id}" has invalid content bounds.`, ); } return { @@ -1476,16 +1620,16 @@ export async function readPreparedMarkdownArtifact( } } -async function readArtifactBody( +async function readAssetBody( root: string, record: { path: string }, ): Promise { - const directory = artifactRoot(root); + const directory = agentEndpointAssetRoot(root); const resolved = path.resolve(directory, record.path); const relative = path.relative(directory, resolved); if (relative.startsWith("..") || path.isAbsolute(relative)) { throw new Error( - `nimbus-docs: prepared artifact path escapes its root: ${record.path}.`, + `nimbus-docs: agent-endpoint asset path escapes its root: ${record.path}.`, ); } await assertNoSymlink(root, resolved); @@ -1497,32 +1641,32 @@ async function readArtifactBody( path.isAbsolute(canonicalRelative) ) { throw new Error( - `nimbus-docs: prepared artifact path escapes its root: ${record.path}.`, + `nimbus-docs: agent-endpoint asset path escapes its root: ${record.path}.`, ); } return readFile(resolved, "utf8"); } -export async function readPreparedLlmsArtifact( +export async function readLlmsEndpointPayload( root: URL | string, - reference: PreparedLlmsReference, -): Promise { + reference: LlmsEndpointReference, +): Promise { const key = preparedMarkdownRootKey(root); - await ensurePreparedArtifacts(key); - const endRead = beginArtifactRead(key); + await ensureAgentEndpointAssets(key); + const endRead = beginAssetRead(key); try { - const configured = artifactState.roots.get(key); - const record = configured?.llmsArtifacts?.get(preparedLlmsKey(reference)); + const configured = agentEndpointAssetState.roots.get(key); + const record = configured?.llmsAssets?.get(preparedLlmsKey(reference)); if (!record) { const identity = reference.scope === "site" ? `${reference.scope} ${reference.surface}` : `${reference.scope} ${reference.section} ${reference.surface}`; throw new Error( - `nimbus-docs: no prepared llms.txt artifact for ${identity}.`, + `nimbus-docs: no llms.txt agent-endpoint asset for ${identity}.`, ); } - const body = await readArtifactBody(key, record); + const body = await readAssetBody(key, record); return { ...reference, digest: record.digest, diff --git a/packages/nimbus-docs/src/_internal/build-report.ts b/packages/nimbus-docs/src/_internal/build-report.ts index b790e0de..3fb5ac1e 100644 --- a/packages/nimbus-docs/src/_internal/build-report.ts +++ b/packages/nimbus-docs/src/_internal/build-report.ts @@ -1,19 +1,27 @@ -/** - * Prerender invariant reporter. From the routes Astro resolves at build, it - * asserts the bidirectional invariant — every public doc route is prerendered - * or explicitly request-rendered, and every on-demand route is *explained* — - * and produces the build summary line. An unexplained on-demand route is a - * build failure, not a warning. - * - * Astro's own internal routes are excluded by route provenance. Project and - * integration routes are explained only if they're declared feature routes. - */ - export interface ResolvedRouteLike { pattern: string; type: string; isPrerendered: boolean; origin: "internal" | "external" | "project"; + entrypoint?: string | null; +} + +export interface ManagedRouteDeclaration { + pattern: string; + entrypoint: string; + owner: "canonical" | "infrastructure"; + rendering: "build" | "request"; +} + +export interface FeatureRouteDeclaration { + pattern: string; + entrypoint: string; + feature: string; +} + +export interface UserRouteDeclaration { + pattern: string; + entrypoint: string; } export interface BuildReportInput { @@ -22,8 +30,10 @@ export interface BuildReportInput { routes: readonly ResolvedRouteLike[]; prerenderedPageCount: number; requestRenderedPageCount?: number; - declaredFeatureRoutes?: readonly string[]; - declaredRequestRoutes?: readonly string[]; + managedRoutes?: readonly ManagedRouteDeclaration[]; + featureRoutes?: readonly FeatureRouteDeclaration[]; + userExtensibleRoutes?: readonly UserRouteDeclaration[]; + contentRoutePatterns?: readonly string[]; serverFeatures?: readonly string[]; } @@ -31,78 +41,280 @@ export interface BuildReport { summaryLine: string; violations: string[]; onDemandDocRoutes: string[]; + customOnDemandRoutes: string[]; + customPrerenderedRoutes: string[]; + integrationOnDemandRoutes: string[]; + integrationPrerenderedRoutes: string[]; + nimbusRequestRoutes: string[]; + featureRoutes: string[]; fatal: string | null; } export function analyzeBuild(input: BuildReportInput): BuildReport { - const declaredFeatures = new Set(input.declaredFeatureRoutes ?? []); - const declaredRequests = new Set(input.declaredRequestRoutes ?? []); + const managedRoutes = input.managedRoutes ?? []; + const featureRoutes = input.featureRoutes ?? []; + const userExtensibleRoutes = input.userExtensibleRoutes ?? []; + const contentPatterns = new Set(input.contentRoutePatterns ?? []); + const managedByEntrypoint = groupBy(managedRoutes, (route) => route.entrypoint); + const managedByPattern = groupBy(managedRoutes, (route) => route.pattern); + const featuresByEntrypoint = groupBy(featureRoutes, (route) => route.entrypoint); + const featuresByPattern = groupBy(featureRoutes, (route) => route.pattern); + const userExtensibleIdentities = new Set( + userExtensibleRoutes.map((route) => routeLocationIdentity(route)), + ); + const activeDeclarations = [...managedRoutes, ...featureRoutes]; + for (const declarations of groupBy( + activeDeclarations, + routeLocationIdentity, + ).values()) { + if (new Set(declarations.map(routeIdentity)).size > 1) { + throw new Error( + `Nimbus route ${JSON.stringify(declarations[0]?.pattern)} at ${JSON.stringify(declarations[0]?.entrypoint)} matches multiple active declarations.`, + ); + } + } const routable = input.routes.filter( (r) => r.type === "page" || r.type === "endpoint", ); const reportable = routable.filter((r) => r.origin !== "internal"); - const nonInfraOnDemand = reportable.filter((r) => !r.isPrerendered); - const onDemandDocRoutes = nonInfraOnDemand.map((r) => r.pattern); - const violations = nonInfraOnDemand - .filter( - (r) => - !declaredFeatures.has(r.pattern) && - !declaredRequests.has(r.pattern), - ) - .map((r) => r.pattern); + const violations: string[] = []; + const metadataFailures: string[] = []; + const customOnDemandRoutes: string[] = []; + const customPrerenderedRoutes: string[] = []; + const integrationOnDemandRoutes: string[] = []; + const integrationPrerenderedRoutes: string[] = []; + const nimbusRequestRoutes: string[] = []; + const declaredFeatureRoutes: string[] = []; + const observedManagedRoutes = new Map(); + const observedFeatureRoutes = new Map(); + + for (const route of input.routes) { + const managedPattern = managedByPattern.get(route.pattern); + const featurePattern = featuresByPattern.get(route.pattern); + if (!route.entrypoint) { + if ( + managedPattern || + featurePattern || + (route.origin !== "internal" && + (route.type === "page" || route.type === "endpoint")) + ) { + addUnique(metadataFailures, route.pattern); + } else if ( + (route.type === "page" || route.type === "endpoint") && + contentPatterns.has(route.pattern) + ) { + addUnique(violations, route.pattern); + } + continue; + } + + const managedEntrypoint = managedByEntrypoint.get(route.entrypoint); + if (managedEntrypoint) { + const owner = managedEntrypoint.find( + (candidate) => candidate.pattern === route.pattern, + ); + if ( + !owner || + route.isPrerendered !== (owner.rendering === "build") + ) { + addUnique(violations, route.pattern); + } else { + increment(observedManagedRoutes, routeIdentity(owner)); + if (!route.isPrerendered) { + addUnique(nimbusRequestRoutes, route.pattern); + } + } + continue; + } + + const featureEntrypoint = featuresByEntrypoint.get(route.entrypoint); + if (featureEntrypoint) { + const owner = featureEntrypoint.find( + (candidate) => candidate.pattern === route.pattern, + ); + if (!owner || route.isPrerendered) { + addUnique(violations, route.pattern); + } else { + increment(observedFeatureRoutes, routeIdentity(owner)); + addUnique(declaredFeatureRoutes, route.pattern); + } + continue; + } + + if (managedPattern || featurePattern) { + addUnique(violations, route.pattern); + continue; + } + if (route.origin === "internal") continue; + if (route.type !== "page" && route.type !== "endpoint") { + continue; + } + if ( + contentPatterns.has(route.pattern) && + !userExtensibleIdentities.has(routeLocationIdentity(route)) + ) { + addUnique(violations, route.pattern); + } else if (route.origin === "external" && route.isPrerendered) { + addUnique(integrationPrerenderedRoutes, route.pattern); + } else if (route.origin === "external") { + addUnique(integrationOnDemandRoutes, route.pattern); + } else if (route.isPrerendered) { + addUnique(customPrerenderedRoutes, route.pattern); + } else { + addUnique(customOnDemandRoutes, route.pattern); + } + } + + for (const route of managedRoutes) { + if (observedManagedRoutes.get(routeIdentity(route)) !== 1) { + addUnique(violations, route.pattern); + } + } + for (const route of featureRoutes) { + if (observedFeatureRoutes.get(routeIdentity(route)) !== 1) { + addUnique(violations, route.pattern); + } + } + + const onDemandDocRoutes = [ + ...nimbusRequestRoutes, + ...customOnDemandRoutes, + ...integrationOnDemandRoutes, + ...declaredFeatureRoutes, + ]; const moved = input.requestRenderedPageCount ?? 0; - const fatal = - input.outputMode === "server" && reportable.length === 0 - ? "nimbus: prerender invariant CANNOT BE VERIFIED — astro:routes:resolved " + - "delivered no routable routes for this server build. This is a reporter " + - "malfunction (a server build always resolves the doc route plus Astro's " + - "infrastructure routes), not a clean pass. Failing the build." - : null; + let fatal: string | null = null; + if (input.outputMode === "server" && reportable.length === 0) { + fatal = + "nimbus: route ownership CANNOT BE VERIFIED — astro:routes:resolved " + + "delivered no project or integration routes for this server build. " + + "This is a reporter malfunction, not a clean pass. Failing the build."; + } else if (metadataFailures.length > 0) { + fatal = + "nimbus: route ownership CANNOT BE VERIFIED — Astro did not provide " + + `stable entrypoint metadata for: ${metadataFailures.join(", ")}. ` + + "Nimbus cannot safely infer route ownership from URL patterns."; + } return { - summaryLine: formatSummary(input, onDemandDocRoutes, moved), + summaryLine: formatSummary(input, { + moved, + customOnDemandRoutes, + customPrerenderedRoutes, + integrationOnDemandRoutes, + integrationPrerenderedRoutes, + nimbusRequestRoutes, + featureRoutes: declaredFeatureRoutes, + }), violations, onDemandDocRoutes, + customOnDemandRoutes, + customPrerenderedRoutes, + integrationOnDemandRoutes, + integrationPrerenderedRoutes, + nimbusRequestRoutes, + featureRoutes: declaredFeatureRoutes, fatal, }; } +interface SummaryRoutes { + moved: number; + customOnDemandRoutes: string[]; + customPrerenderedRoutes: string[]; + integrationOnDemandRoutes: string[]; + integrationPrerenderedRoutes: string[]; + nimbusRequestRoutes: string[]; + featureRoutes: string[]; +} + function formatSummary( input: BuildReportInput, - onDemandDocRoutes: string[], - moved: number, + routes: SummaryRoutes, ): string { const adapter = (input.adapterName ?? "none").replace(/^@astrojs\//, ""); const prerendered = input.prerenderedPageCount; + const customStatic = formatRouteList(routes.customPrerenderedRoutes); + const integrationStatic = formatRouteList( + routes.integrationPrerenderedRoutes, + ); if (input.outputMode === "static") { return ( `nimbus: output=static · adapter=${adapter} · ` + - `docs prerendered=${prerendered}/${prerendered} · on-demand routes=0` + `docs prerendered=${prerendered}/${prerendered} · ` + + `custom prerendered routes=${routes.customPrerenderedRoutes.length}${customStatic} · ` + + `integration prerendered routes=${routes.integrationPrerenderedRoutes.length}${integrationStatic}` ); } - const total = prerendered + moved; - const odList = onDemandDocRoutes.length - ? ` (${onDemandDocRoutes.join(", ")})` - : ""; + const total = prerendered + routes.moved; const features = input.serverFeatures?.length ? `[${input.serverFeatures.join(", ")}]` : "[]"; return ( `nimbus: output=server · adapter=${adapter} · ` + - `docs prerendered=${prerendered}/${total} (${moved} moved) · ` + - `on-demand routes=${onDemandDocRoutes.length}${odList} · ` + + `docs prerendered=${prerendered}/${total} (${routes.moved} moved) · ` + + `custom prerendered routes=${routes.customPrerenderedRoutes.length}${customStatic} · ` + + `integration prerendered routes=${routes.integrationPrerenderedRoutes.length}${integrationStatic} · ` + + `custom on-demand routes=${routes.customOnDemandRoutes.length}${formatRouteList(routes.customOnDemandRoutes)} · ` + + `integration on-demand routes=${routes.integrationOnDemandRoutes.length}${formatRouteList(routes.integrationOnDemandRoutes)} · ` + + `nimbus request routes=${routes.nimbusRequestRoutes.length}${formatRouteList(routes.nimbusRequestRoutes)} · ` + + `feature routes=${routes.featureRoutes.length}${formatRouteList(routes.featureRoutes)} · ` + `server features=${features}` ); } +function formatRouteList(routes: readonly string[]): string { + return routes.length ? ` (${routes.join(", ")})` : ""; +} + +function groupBy( + values: readonly T[], + key: (value: T) => string, +): Map { + const grouped = new Map(); + for (const value of values) { + const current = grouped.get(key(value)) ?? []; + current.push(value); + grouped.set(key(value), current); + } + return grouped; +} + +function addUnique(values: string[], value: string): void { + if (!values.includes(value)) values.push(value); +} + +function increment(counts: Map, key: string): void { + counts.set(key, (counts.get(key) ?? 0) + 1); +} + +function routeLocationIdentity(route: { + pattern: string; + entrypoint?: string | null; +}): string { + return `${route.pattern}\0${route.entrypoint ?? ""}`; +} + +function routeIdentity( + route: ManagedRouteDeclaration | FeatureRouteDeclaration, +): string { + return [ + routeLocationIdentity(route), + "owner" in route ? route.owner : "feature", + "rendering" in route ? route.rendering : "request", + "feature" in route ? route.feature : "", + ].join("\0"); +} + export function formatInvariantFailure(violations: readonly string[]): string { return ( - `nimbus: prerender invariant FAILED — ${violations.length} unexplained ` + - `on-demand route${violations.length === 1 ? "" : "s"}:\n` + + `nimbus: route ownership invariant FAILED — ${violations.length} route ` + + `violation${violations.length === 1 ? "" : "s"}:\n` + violations.map((p) => ` - ${p}`).join("\n") + - `\n\nEvery public doc route must be prerendered or selected by the Nimbus rendering policy. ` + - `Restore prerendering, configure the route's collection for request rendering, or ` + - `(for a server feature endpoint) declare it so the reporter can explain it.` + `\n\nNimbus-managed routes must match their declared entrypoint and rendering policy. ` + + `Feature routes must match the active feature's entrypoint. Custom project and unrelated ` + + `integration routes may use Astro's native prerender semantics when they do not impersonate ` + + `a managed route or collide with published content.` ); } diff --git a/packages/nimbus-docs/src/_internal/footprint.ts b/packages/nimbus-docs/src/_internal/footprint.ts index d4cc2c8f..74a08d75 100644 --- a/packages/nimbus-docs/src/_internal/footprint.ts +++ b/packages/nimbus-docs/src/_internal/footprint.ts @@ -23,13 +23,16 @@ export interface FeatureRecipe { env: EnvRequirement[]; /** npm package whose presence in the footprint means the feature is installed. */ dep: string; - /** - * On-demand route patterns this feature owns (e.g. `/mcp`). The prerender - * invariant treats these as *explained* on-demand routes; every other - * non-infra on-demand route is a violation. Empty/omitted for prerendered - * features. - */ - routes?: readonly string[]; + routes?: readonly FeatureRoute[]; +} + +export interface FeatureRoute { + pattern: string; + entrypoint: string; +} + +export interface OwnedFeatureRoute extends FeatureRoute { + feature: string; } // Populated by downstream feature slices (loaders, hosted MCP). Empty here: the @@ -43,7 +46,18 @@ export function deriveFootprint( return recipes.filter((recipe) => installedDeps.has(recipe.dep)); } -/** The on-demand routes a footprint's installed features declare (deduped). */ -export function footprintRoutes(footprint: readonly FeatureRecipe[]): string[] { - return [...new Set(footprint.flatMap((f) => f.routes ?? []))]; +export function footprintRoutes( + footprint: readonly FeatureRecipe[], +): OwnedFeatureRoute[] { + const routes = new Map(); + for (const feature of footprint) { + for (const route of feature.routes ?? []) { + const owned = { ...route, feature: feature.id }; + routes.set( + `${owned.feature}\0${owned.pattern}\0${owned.entrypoint}`, + owned, + ); + } + } + return [...routes.values()]; } diff --git a/packages/nimbus-docs/src/_internal/request-route-inventory.ts b/packages/nimbus-docs/src/_internal/request-route-inventory.ts index 6c817f69..e8246eba 100644 --- a/packages/nimbus-docs/src/_internal/request-route-inventory.ts +++ b/packages/nimbus-docs/src/_internal/request-route-inventory.ts @@ -1,6 +1,6 @@ import { collectionMountPrefix } from "./collection-mount.js"; import { - requestInventoryEntryUrl, + contentInventoryEntryUrl, requestInventoryVersionStatusKey, type RequestRouteInventoryEntry, } from "./request-route-url.js"; @@ -14,11 +14,17 @@ import { getVersionStatus, renderIndexedEntryMarkdown, } from "../runtime.js"; -import { getPreparedMarkdownArtifact } from "../build.js"; +import { readMarkdownEndpointPayload } from "./agent-endpoint-assets.js"; export const prerender = true; export async function GET() { + const projectRoot: unknown = import.meta.env.NIMBUS_PROJECT_ROOT; + if (typeof projectRoot !== "string" || projectRoot.length === 0) { + throw new Error( + "nimbus-docs: request route inventory requires the Nimbus Astro integration.", + ); + } const config = await loadNimbusConfig(); const requestCollections = new Set(await loadRequestRenderingCollections()); const apiCollections = new Set(await loadApiCollections()); @@ -32,6 +38,7 @@ export async function GET() { const collection = item.collection; const prefix = collectionMountPrefix(collection, versions); const data = (item.entry.data ?? {}) as Record; + const request = requestCollections.has(collection); const versionStatus = await getVersionStatus( requestInventoryVersionStatusKey( collection, @@ -46,12 +53,13 @@ export async function GET() { (data.searchable !== false && data.noindex !== true)); const route: RequestRouteInventoryEntry = { collection, - url: requestInventoryEntryUrl( + url: contentInventoryEntryUrl( prefix, item.entry.id, apiCollections.has(collection), + request, ), - request: requestCollections.has(collection), + request, discoverable, searchable, title: item.title, @@ -64,11 +72,14 @@ export async function GET() { route.content = apiCollections.has(collection) ? await renderIndexedEntryMarkdown(item, { base: import.meta.env.BASE_URL }) : ( - await getPreparedMarkdownArtifact({ - collection, - id: item.entry.id, - surface: "markdown", - }) + await readMarkdownEndpointPayload( + projectRoot, + { + collection, + id: item.entry.id, + surface: "markdown", + }, + ) ).content; } routes.push(route); diff --git a/packages/nimbus-docs/src/_internal/request-route-url.ts b/packages/nimbus-docs/src/_internal/request-route-url.ts index e2503b86..f558dc62 100644 --- a/packages/nimbus-docs/src/_internal/request-route-url.ts +++ b/packages/nimbus-docs/src/_internal/request-route-url.ts @@ -1,3 +1,5 @@ +import { entryRouteUrl } from "./astro-slug.js"; + export function requestInventoryEntryUrl( prefix: string, entryId: string, @@ -7,6 +9,17 @@ export function requestInventoryEntryUrl( return id === "" ? prefix || "/" : `${prefix}/${id}`; } +export function contentInventoryEntryUrl( + prefix: string, + entryId: string, + api: boolean, + request: boolean, +): string { + return request + ? requestInventoryEntryUrl(prefix, entryId, api) + : entryRouteUrl(prefix, entryId); +} + export function requestInventoryVersionStatusKey( collection: string, api: boolean, diff --git a/packages/nimbus-docs/src/_internal/route-ownership.ts b/packages/nimbus-docs/src/_internal/route-ownership.ts new file mode 100644 index 00000000..7eabba0a --- /dev/null +++ b/packages/nimbus-docs/src/_internal/route-ownership.ts @@ -0,0 +1,150 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { normalizeRouteComponent } from "./rendering-policy.js"; + +export type StarterRouteRole = "canonical" | "user-owned"; + +export interface StarterRouteDeclaration { + pattern: string; + entrypoint: string; + role: StarterRouteRole; + allowsContentShadow?: true; + publishesAgentEndpointAssets?: true; +} + +export const STARTER_ROUTE_INVENTORY: readonly StarterRouteDeclaration[] = [ + { + pattern: "/[...slug]", + entrypoint: "pages/[...slug].astro", + role: "canonical", + }, + { + pattern: "/", + entrypoint: "pages/index.astro", + role: "user-owned", + allowsContentShadow: true, + }, + { pattern: "/404", entrypoint: "pages/404.astro", role: "user-owned" }, + { + pattern: "/[...slug]/index.md", + entrypoint: "pages/[...slug]/index.md.ts", + role: "user-owned", + publishesAgentEndpointAssets: true, + }, + { + pattern: "/[...slug]/index.mdx", + entrypoint: "pages/[...slug]/index.mdx.ts", + role: "user-owned", + publishesAgentEndpointAssets: true, + }, + { + pattern: "/[section]/llms.txt", + entrypoint: "pages/[section]/llms.txt.ts", + role: "user-owned", + publishesAgentEndpointAssets: true, + }, + { + pattern: "/llms-full.txt", + entrypoint: "pages/llms-full.txt.ts", + role: "user-owned", + publishesAgentEndpointAssets: true, + }, + { + pattern: "/llms.txt", + entrypoint: "pages/llms.txt.ts", + role: "user-owned", + publishesAgentEndpointAssets: true, + }, + { + pattern: "/nimbus-api/coordinates.json", + entrypoint: "pages/nimbus-api/coordinates.json.ts", + role: "user-owned", + }, + { + pattern: "/og/[...slug]", + entrypoint: "pages/og/[...slug].ts", + role: "user-owned", + }, + { pattern: "/og.png", entrypoint: "pages/og.png.ts", role: "user-owned" }, + { + pattern: "/robots.txt", + entrypoint: "pages/robots.txt.ts", + role: "user-owned", + }, +]; + +export function normalizeRouteEntrypoint( + projectRoot: string, + entrypoint: unknown, +): string | null { + if ( + typeof entrypoint !== "string" || + entrypoint.length === 0 || + entrypoint !== entrypoint.trim() || + /[\0\r\n]/.test(entrypoint) + ) { + return null; + } + let component = entrypoint; + if (component.startsWith("file:")) { + try { + component = fileURLToPath(component); + } catch { + return null; + } + } else if ( + !path.isAbsolute(component) && + !path.win32.isAbsolute(component) && + /^[a-z][a-z\d+.-]*:/i.test(component) + ) { + return null; + } + const pathApi = + path.win32.isAbsolute(projectRoot) || path.win32.isAbsolute(component) + ? path.win32 + : path; + component = normalizeRouteComponent(component); + if (!component) return null; + const absolute = pathApi.isAbsolute(component) + ? component + : pathApi.resolve(projectRoot, component); + return normalizeRouteComponent( + pathApi.relative(projectRoot, absolute), + ); +} + +export function normalizeSourceRouteEntrypoint( + projectRoot: string, + srcDir: string, + entrypoint: unknown, +): string | null { + if (typeof entrypoint !== "string" || entrypoint.length === 0) return null; + if ( + entrypoint.startsWith("file:") || + path.isAbsolute(entrypoint) || + path.win32.isAbsolute(entrypoint) + ) { + return normalizeRouteEntrypoint(projectRoot, entrypoint); + } + const pathApi = path.win32.isAbsolute(srcDir) ? path.win32 : path; + const relative = normalizeRouteComponent(entrypoint).replace(/^src\//, ""); + return normalizeRouteEntrypoint(projectRoot, pathApi.join(srcDir, relative)); +} + +export function isRequiredCanonicalRouteComponent( + projectRoot: string, + srcDir: string, + component: string, +): boolean { + const normalized = normalizeRouteEntrypoint(projectRoot, component); + return STARTER_ROUTE_INVENTORY.some( + (route) => + route.role === "canonical" && + normalizeSourceRouteEntrypoint( + projectRoot, + srcDir, + route.entrypoint, + ) === normalized, + ); +} diff --git a/packages/nimbus-docs/src/_internal/transform.ts b/packages/nimbus-docs/src/_internal/transform.ts index e8f5908f..8ad0f455 100644 --- a/packages/nimbus-docs/src/_internal/transform.ts +++ b/packages/nimbus-docs/src/_internal/transform.ts @@ -287,7 +287,7 @@ export function renderEntryAsMarkdown( if (/])/.test(protectCode(markdown).markdown)) { throw new Error( "nimbus-docs: renderEntryAsMarkdown no longer expands partials at runtime. " + - "Use the prepared artifact helpers from @cloudflare/nimbus-docs/build.", + "Serve it with getMarkdownPayload from @cloudflare/nimbus-docs/agent-endpoints.", ); } diff --git a/packages/nimbus-docs/src/agent-endpoints.ts b/packages/nimbus-docs/src/agent-endpoints.ts new file mode 100644 index 00000000..405d5c26 --- /dev/null +++ b/packages/nimbus-docs/src/agent-endpoints.ts @@ -0,0 +1,263 @@ +import { entryRouteKey } from "./_internal/astro-slug.js"; +import type { + LlmsEndpointAsset, + MarkdownEndpointAsset, +} from "./_internal/agent-endpoint-assets.js"; +import { withBase } from "./_internal/url.js"; +export type MarkdownEndpointSurface = "markdown" | "source"; + +export interface MarkdownEndpointReference { + collection: string; + id: string; + surface: MarkdownEndpointSurface; +} + +export interface MarkdownEndpointPayload extends MarkdownEndpointReference { + digest: string; + mediaType: string; + body: string; + content: string; +} + +export type LlmsEndpointReference = + | { scope: "site"; surface: "index" | "full" } + | { scope: "section"; surface: "index"; section: string }; + +export type LlmsEndpointPayload = LlmsEndpointReference & { + digest: string; + mediaType: string; + body: string; +}; + +let agentEndpointAssetsModule: Promise< + typeof import("virtual:nimbus/agent-endpoint-assets") +> | null = null; +let agentEndpointAssetLoaderModule: Promise< + typeof import("virtual:nimbus/agent-endpoint-asset-loader") +> | null = null; +let markdownByIdentity: + | Map + | undefined; +let markdownByRoute: Map | undefined; +let llmsByIdentity: Map | undefined; + +interface AgentEndpointContext { + request?: Request; +} + +function agentEndpointAssetResponseError(url: URL, status: number): Error { + if (status === 404) { + return new Error( + `nimbus-docs: agent-endpoint asset not found at ${url.href}; verify client assets were deployed.`, + ); + } + return new Error( + `nimbus-docs: agent-endpoint asset at ${url.href} returned ${status}.`, + ); +} + +function loadAgentEndpointAssets() { + agentEndpointAssetsModule ??= import("virtual:nimbus/agent-endpoint-assets"); + return agentEndpointAssetsModule; +} + +function loadAgentEndpointAssetLoader() { + agentEndpointAssetLoaderModule ??= import( + "virtual:nimbus/agent-endpoint-asset-loader" + ); + return agentEndpointAssetLoaderModule; +} + +function markdownIdentity(reference: MarkdownEndpointReference): string { + return `${reference.collection}\0${reference.id}\0${reference.surface}`; +} + +function markdownRouteIdentity(options: { + collection: string; + surface: MarkdownEndpointSurface; + slug?: string; +}): string { + return `${options.collection}\0${options.surface}\0${options.slug ?? ""}`; +} + +function llmsIdentity(reference: LlmsEndpointReference): string { + return reference.scope === "site" + ? `${reference.scope}\0${reference.surface}` + : `${reference.scope}\0${reference.section}\0${reference.surface}`; +} + +async function readAssetBody( + assetPath: string, + context: AgentEndpointContext, +): Promise { + const assets = await loadAgentEndpointAssets(); + const publicPath = withBase( + `/_nimbus/agent-endpoint-assets/${assetPath}`, + assets.base, + ); + const request = context.request; + if (request) { + const assetUrl = new URL(publicPath, request.url); + const { fetchAgentEndpointAsset } = await loadAgentEndpointAssetLoader(); + const response = await fetchAgentEndpointAsset(publicPath, request); + if (response) { + if (!response.ok) { + throw agentEndpointAssetResponseError(assetUrl, response.status); + } + return response.text(); + } + } + try { + const [{ readFile }, path] = await Promise.all([ + import("node:fs/promises"), + import("node:path"), + ]); + return await readFile( + path.join( + assets.projectRoot, + ".astro", + "nimbus", + "agent-endpoint-assets", + assetPath, + ), + "utf8", + ); + } catch (error) { + if (!request) throw error; + } + const assetUrl = new URL(publicPath, request.url); + const response = await fetch(assetUrl); + if (!response.ok) { + throw agentEndpointAssetResponseError(assetUrl, response.status); + } + return response.text(); +} + +async function markdownIndexes() { + const { markdownAssets } = await loadAgentEndpointAssets(); + if (!markdownByIdentity || !markdownByRoute) { + markdownByIdentity = new Map(); + markdownByRoute = new Map(); + for (const asset of markdownAssets) { + markdownByIdentity.set(markdownIdentity(asset), asset); + markdownByRoute.set( + markdownRouteIdentity({ + collection: asset.collection, + surface: asset.surface, + slug: entryRouteKey(asset.id), + }), + asset, + ); + } + } + return { markdownAssets, markdownByIdentity, markdownByRoute }; +} + +async function llmsIndex() { + const { llmsAssets } = await loadAgentEndpointAssets(); + if (!llmsByIdentity) { + llmsByIdentity = new Map( + llmsAssets.map((asset) => [llmsIdentity(asset), asset]), + ); + } + return { llmsAssets, llmsByIdentity }; +} + +export async function getMarkdownStaticPaths(options: { + collection: string; + surface: MarkdownEndpointSurface; +}): Promise< + Array<{ + params: { slug: string | undefined }; + props: { reference: MarkdownEndpointReference }; + cacheKey: string; + }> +> { + const { markdownAssets } = await markdownIndexes(); + return markdownAssets + .filter( + (asset) => + asset.collection === options.collection && + asset.surface === options.surface, + ) + .map((asset) => ({ + params: { slug: entryRouteKey(asset.id) || undefined }, + props: { + reference: { + collection: asset.collection, + id: asset.id, + surface: asset.surface, + } satisfies MarkdownEndpointReference, + }, + cacheKey: asset.digest, + })); +} + +export async function getMarkdownPayload(options: { + collection: string; + surface: MarkdownEndpointSurface; + slug?: string; + reference?: MarkdownEndpointReference; + context?: AgentEndpointContext; +}): Promise { + const indexes = await markdownIndexes(); + const asset = options.reference + ? indexes.markdownByIdentity.get(markdownIdentity(options.reference)) + : indexes.markdownByRoute.get(markdownRouteIdentity(options)); + if (!asset) return null; + const body = await readAssetBody(asset.path, options.context ?? {}); + return { + collection: asset.collection, + id: asset.id, + surface: asset.surface, + digest: asset.digest, + mediaType: asset.mediaType, + body, + content: body.slice(asset.contentStart, asset.contentEnd), + }; +} + +export async function getLlmsPayload( + reference: LlmsEndpointReference, + context: AgentEndpointContext = {}, +): Promise { + const { llmsByIdentity } = await llmsIndex(); + const asset = llmsByIdentity.get(llmsIdentity(reference)); + if (!asset) return null; + return { + ...reference, + digest: asset.digest, + mediaType: asset.mediaType, + body: await readAssetBody(asset.path, context), + }; +} + +export async function getLlmsStaticPaths(): Promise< + Array<{ + params: { section: string }; + props: { reference: LlmsEndpointReference }; + cacheKey: string; + }> +> { + const { llmsAssets } = await llmsIndex(); + return llmsAssets + .filter( + ( + asset, + ): asset is Extract< + LlmsEndpointAsset, + { scope: "section" } + > => asset.scope === "section" && asset.surface === "index", + ) + .map((asset) => ({ + params: { section: asset.section }, + props: { + reference: { + scope: asset.scope, + surface: asset.surface, + section: asset.section, + } satisfies LlmsEndpointReference, + }, + cacheKey: asset.digest, + })); +} diff --git a/packages/nimbus-docs/src/build.ts b/packages/nimbus-docs/src/build.ts index a1cbf6ca..b49e9bfd 100644 --- a/packages/nimbus-docs/src/build.ts +++ b/packages/nimbus-docs/src/build.ts @@ -1,9 +1,9 @@ import { - getPreparedArtifactManifest, - registerPreparedArtifactDemand, - readPreparedLlmsArtifact, - readPreparedMarkdownArtifact, -} from "./_internal/prepared-artifacts.js"; + getAgentEndpointAssetManifest, + registerAgentEndpointAssetDemand, + readLlmsEndpointPayload, + readMarkdownEndpointPayload, +} from "./_internal/agent-endpoint-assets.js"; import type { PreparedLlmsArtifact, PreparedLlmsReference, @@ -27,7 +27,7 @@ const projectRoot: unknown = : undefined; if (typeof projectRoot === "string" && projectRoot.length > 0) { - registerPreparedArtifactDemand(projectRoot); + registerAgentEndpointAssetDemand(projectRoot); } function configuredRoot(): string { @@ -39,6 +39,7 @@ function configuredRoot(): string { return projectRoot; } +/** @deprecated Use `getMarkdownStaticPaths` from `@cloudflare/nimbus-docs/agent-endpoints`; route props use `reference` instead of `artifact`. */ export async function getPreparedMarkdownStaticPaths(options: { collection: string; surface: PreparedMarkdownSurface; @@ -49,8 +50,8 @@ export async function getPreparedMarkdownStaticPaths(options: { cacheKey: string; }> > { - const manifest = await getPreparedArtifactManifest(configuredRoot()); - return manifest.markdownArtifacts + const manifest = await getAgentEndpointAssetManifest(configuredRoot()); + return manifest.markdownAssets .filter( (artifact) => artifact.collection === options.collection && @@ -69,12 +70,14 @@ export async function getPreparedMarkdownStaticPaths(options: { })); } +/** @deprecated Use `getMarkdownPayload` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export function getPreparedMarkdownArtifact( reference: PreparedMarkdownReference, ): Promise { - return readPreparedMarkdownArtifact(configuredRoot(), reference); + return readMarkdownEndpointPayload(configuredRoot(), reference); } +/** @deprecated Use `getLlmsStaticPaths` from `@cloudflare/nimbus-docs/agent-endpoints`; route props use `reference` instead of `artifact`. */ export async function getPreparedLlmsStaticPaths(): Promise< Array<{ params: { section: string }; @@ -82,13 +85,13 @@ export async function getPreparedLlmsStaticPaths(): Promise< cacheKey: string; }> > { - const manifest = await getPreparedArtifactManifest(configuredRoot()); - return manifest.llmsArtifacts + const manifest = await getAgentEndpointAssetManifest(configuredRoot()); + return manifest.llmsAssets .filter( ( artifact, ): artifact is Extract< - (typeof manifest.llmsArtifacts)[number], + (typeof manifest.llmsAssets)[number], { scope: "section" } > => artifact.scope === "section", ) @@ -105,8 +108,9 @@ export async function getPreparedLlmsStaticPaths(): Promise< })); } +/** @deprecated Use `getLlmsPayload` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export function getPreparedLlmsArtifact( reference: PreparedLlmsReference, ): Promise { - return readPreparedLlmsArtifact(configuredRoot(), reference); + return readLlmsEndpointPayload(configuredRoot(), reference); } diff --git a/packages/nimbus-docs/src/check/structure.ts b/packages/nimbus-docs/src/check/structure.ts index 48822766..51d1b3aa 100644 --- a/packages/nimbus-docs/src/check/structure.ts +++ b/packages/nimbus-docs/src/check/structure.ts @@ -26,6 +26,7 @@ import { } from "../_internal/rendering-policy.js"; import { validateMdxContent } from "../_internal/validate-mdx-content.js"; import { validateNimbusConfig } from "../_internal/validate.js"; +import { isRequiredCanonicalRouteComponent } from "../_internal/route-ownership.js"; import { contentEntryUrl, enumerateEntriesByBase, @@ -89,9 +90,17 @@ async function checkRequestRendering( const versions = config.versions ? { others: config.versions.others ?? [] } : null; - const canonicalCollections = candidates.filter((collection) => - existsSync(canonicalCollectionRouteComponent(srcDir, collection, versions)), - ); + const canonicalCollections = candidates.filter((collection) => { + const component = canonicalCollectionRouteComponent( + srcDir, + collection, + versions, + ); + return ( + isRequiredCanonicalRouteComponent(cwd, srcDir, component) || + existsSync(component) + ); + }); const overrides = config.rendering.collections ?? {}; const unresolvedOverrides = Object.keys(overrides).filter( (collection) => !candidates.includes(collection), diff --git a/packages/nimbus-docs/src/cli/_registry.generated.ts b/packages/nimbus-docs/src/cli/_registry.generated.ts index 452ee19a..90b9178e 100644 --- a/packages/nimbus-docs/src/cli/_registry.generated.ts +++ b/packages/nimbus-docs/src/cli/_registry.generated.ts @@ -254,14 +254,14 @@ export const BUNDLED_INDEX: BundledIndex = { "ai-native": { "name": "ai-native", "type": "registry:feature", - "title": "Publish Markdown", - "description": "Add per-page Markdown versions, llms.txt indexes, llms-full.txt, robots.txt, and an AgentDirective to a Nimbus docs site." + "title": "Markdown and llms.txt endpoints", + "description": "Add alternate Markdown/MDX versions, llms.txt indexes, llms-full.txt, robots.txt, and an AgentDirective to a Nimbus docs site." }, "api-reference": { "name": "api-reference", "type": "registry:feature", "title": "OpenAPI reference", - "description": "Mount an OpenAPI (Swagger) spec as a routed reference collection with generated pages, per-page Markdown versions, and llms.txt coverage from one spec file. For hand-authored API docs written as MDX, use `new-collection` instead." + "description": "Mount an OpenAPI (Swagger) spec as a routed reference collection with generated pages, alternate Markdown versions, and llms.txt indexes from one spec file. For hand-authored API docs written as MDX, use `new-collection` instead." }, "changelog": { "name": "changelog", diff --git a/packages/nimbus-docs/src/integration.ts b/packages/nimbus-docs/src/integration.ts index 706be073..e5729c10 100644 --- a/packages/nimbus-docs/src/integration.ts +++ b/packages/nimbus-docs/src/integration.ts @@ -38,7 +38,9 @@ import { admonitionPlugin } from "./_internal/admonition-vite-plugin.js"; import { analyzeBuild, formatInvariantFailure, + type ManagedRouteDeclaration, type ResolvedRouteLike, + type UserRouteDeclaration, } from "./_internal/build-report.js"; import { deriveFootprint, footprintRoutes } from "./_internal/footprint.js"; import { readDependencyNames } from "./check/probe.js"; @@ -119,6 +121,13 @@ import { normalizeRouteComponent, routeComponentKeys, } from "./_internal/rendering-policy.js"; +import { collectionMountPrefix } from "./_internal/collection-mount.js"; +import { + isRequiredCanonicalRouteComponent, + normalizeRouteEntrypoint, + normalizeSourceRouteEntrypoint, + STARTER_ROUTE_INVENTORY, +} from "./_internal/route-ownership.js"; import type { RequestRouteInventoryEntry } from "./_internal/request-route-url.js"; import { safeDecode, withBase } from "./_internal/url.js"; import { buildLastUpdatedIndex } from "./_internal/git-last-updated.js"; @@ -153,18 +162,18 @@ const REQUEST_ROUTE_INVENTORY_ENTRYPOINT = new URL( import.meta.url, ); -type PreparedArtifactsModule = typeof import("./_internal/prepared-artifacts.js"); +type AgentEndpointAssetsModule = typeof import("./_internal/agent-endpoint-assets.js"); -function loadPreparedArtifacts(): Promise { +function loadAgentEndpointAssets(): Promise { const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; const specifier = [ "./_internal/", - "prepared-artifacts.", + "agent-endpoint-assets.", extension, ].join(""); return import( new URL(specifier, import.meta.url).href - ) as Promise; + ) as Promise; } export interface SitemapOptions { @@ -383,6 +392,7 @@ export function nimbus( // build materialization knows where to write `.nimbus/routes.json` and // what `base` Astro is using. let projectRootForBuild = ""; + let srcDirForBuild = ""; let astroBaseForBuild = ""; // Captured at config:done / routes:resolved, consumed by the build:done // prerender-invariant reporter. @@ -394,7 +404,9 @@ export function nimbus( let renderingRoutes = new Map(); let requestRenderingConfigured = false; let requestRenderingCollections = new Set(); - let requestRoutePatterns = new Set(); + let managedRoutesForBuild: ManagedRouteDeclaration[] = []; + let userExtensibleRoutesForBuild: UserRouteDeclaration[] = []; + let contentRoutePatternsForBuild = new Set(); let sitemapCustomPages: string[] = []; let sitemapExcludedPaths = new Set(); let sitemapTrailingSlash: "always" | "never" | "ignore" = "ignore"; @@ -428,8 +440,8 @@ export function nimbus( const srcDir = fileURLToPath(astroConfig.srcDir); const projectRoot = fileURLToPath(astroConfig.root); beginPreparedMarkdownSession(astroConfig.root); - const preparedArtifacts = await loadPreparedArtifacts(); - preparedArtifacts.configurePreparedArtifactRoot( + const agentEndpointAssets = await loadAgentEndpointAssets(); + agentEndpointAssets.configureAgentEndpointAssetRoot( astroConfig.root, command === "build" ? "build" : "dev", async () => { @@ -443,7 +455,7 @@ export function nimbus( ), ]), ); - return preparedArtifacts.bakePreparedArtifacts({ + return agentEndpointAssets.bakeAgentEndpointAssets({ root: projectRoot, base: astroConfig.base || "/", site: config.site, @@ -492,7 +504,7 @@ export function nimbus( }); }, () => - preparedArtifacts.bakePreparedHeadings({ + agentEndpointAssets.bakePreparedHeadings({ root: projectRoot, base: astroConfig.base || "/", indexedCollections: indexedCollectionsForBuild, @@ -522,7 +534,7 @@ export function nimbus( ), ) ) { - preparedArtifacts.registerPreparedArtifactDemand(astroConfig.root); + agentEndpointAssets.registerAgentEndpointAssetDemand(astroConfig.root); } const publicDir = astroConfig.publicDir ? fileURLToPath(astroConfig.publicDir) @@ -627,6 +639,7 @@ export function nimbus( // emitted `pages` array as the route truth (single source of truth // — Astro itself tells us which URLs the site serves). projectRootForBuild = projectRoot; + srcDirForBuild = srcDir; astroBaseForBuild = astroConfig.base ?? ""; // Reset here (build cycle's first hook, before `routes:resolved` fills @@ -706,17 +719,28 @@ export function nimbus( renderingRoutes = new Map(); requestRenderingConfigured = false; requestRenderingCollections = new Set(); - requestRoutePatterns = new Set(); + contentRoutePatternsForBuild = new Set(); + managedRoutesForBuild = []; + userExtensibleRoutesForBuild = STARTER_ROUTE_INVENTORY.filter( + (route) => route.allowsContentShadow, + ).map((route) => ({ + pattern: route.pattern, + entrypoint: normalizeSourceRouteEntrypoint( + projectRoot, + srcDir, + route.entrypoint, + )!, + })); + const versions = config.versions + ? { others: config.versions.others ?? [] } + : null; + const candidates = new Set([ + ...indexedCollections, + ...(config.versions?.others ?? []).map( + (version) => `docs-${version}`, + ), + ]); if (config.rendering) { - const versions = config.versions - ? { others: config.versions.others ?? [] } - : null; - const candidates = new Set([ - ...indexedCollections, - ...(config.versions?.others ?? []).map( - (version) => `docs-${version}`, - ), - ]); const unresolvedOverrides = Object.keys( config.rendering.collections ?? {}, ).filter((collection) => !candidates.has(collection)); @@ -732,40 +756,69 @@ export function nimbus( "or overriding a collection Nimbus cannot statically identify.", ); } - const canonicalCollections = [...candidates].filter((collection) => - fs.existsSync( - canonicalCollectionRouteComponent(srcDir, collection, versions), - ), + } + const canonicalCollections = [...candidates].filter((collection) => { + const component = canonicalCollectionRouteComponent( + srcDir, + collection, + versions, ); - const policy = compileRenderingPolicy( - config.rendering, - canonicalCollections, + return ( + (config.rendering !== undefined && + isRequiredCanonicalRouteComponent( + projectRoot, + srcDir, + component, + )) || + fs.existsSync(component) ); - requestRenderingConfigured = Object.values( - policy.collections, - ).includes("request"); - requestRenderingCollections = new Set( - Object.entries(policy.collections) - .filter(([, mode]) => mode === "request") - .map(([collection]) => collection), + }); + const policy = compileRenderingPolicy( + config.rendering, + canonicalCollections, + ); + requestRenderingConfigured = Object.values(policy.collections).includes( + "request", + ); + requestRenderingCollections = new Set( + Object.entries(policy.collections) + .filter(([, mode]) => mode === "request") + .map(([collection]) => collection), + ); + for (const [collection, mode] of Object.entries(policy.collections)) { + const component = canonicalCollectionRouteComponent( + srcDir, + collection, + versions, ); - for (const [collection, mode] of Object.entries(policy.collections)) { - const component = canonicalCollectionRouteComponent( - srcDir, - collection, - versions, - ); + if (config.rendering) { for (const key of routeComponentKeys(projectRoot, component)) { renderingRoutes.set(key, mode); } } - if (building && requestRenderingConfigured) { - injectRoute({ - pattern: REQUEST_ROUTE_INVENTORY_PATTERN, - entrypoint: REQUEST_ROUTE_INVENTORY_ENTRYPOINT, - prerender: true, - }); - } + const mount = collectionMountPrefix(collection, versions); + managedRoutesForBuild.push({ + pattern: mount === "/" ? "/[...slug]" : `${mount}/[...slug]`, + entrypoint: normalizeRouteEntrypoint(projectRoot, component)!, + owner: "canonical", + rendering: mode, + }); + } + if (building) { + injectRoute({ + pattern: REQUEST_ROUTE_INVENTORY_PATTERN, + entrypoint: REQUEST_ROUTE_INVENTORY_ENTRYPOINT, + prerender: true, + }); + managedRoutesForBuild.push({ + pattern: REQUEST_ROUTE_INVENTORY_PATTERN, + entrypoint: normalizeRouteEntrypoint( + projectRoot, + REQUEST_ROUTE_INVENTORY_ENTRYPOINT.href, + )!, + owner: "infrastructure", + rendering: "build", + }); } // Remote refs fold into the citation index but not the manifest (which republishes @@ -848,7 +901,6 @@ export function nimbus( source: `src/content/${entry.relPath}`, kind: "content" as const, })); - const pageOwners: RouteOwner[] = enumerateStaticPageRoutes( path.join(srcDir, "pages"), projectRoot, @@ -1167,7 +1219,34 @@ export function nimbus( coordinates: Object.fromEntries(citationIndex), manifest: coordinatesManifest, })), - preparedArtifacts.preparedHeadingsPlugin(astroConfig.root), + agentEndpointAssets.preparedHeadingsPlugin(astroConfig.root), + agentEndpointAssets.agentEndpointAssetsRuntimePlugin(astroConfig.root), + agentEndpointAssets.agentEndpointAssetLoaderPlugin( + () => adapterNameForBuild, + ), + { + name: "nimbus-docs:agent-endpoint-assets", + enforce: "pre", + applyToEnvironment: (environment) => + environment.name === "client", + async writeBundle(outputOptions) { + if (!outputOptions.dir) return; + const outputRoot = path.resolve(projectRoot, outputOptions.dir); + if ( + outputModeForBuild === "server" && + agentEndpointAssets.isAgentEndpointAssetRequested(projectRoot) + ) { + await agentEndpointAssets.stageAgentEndpointAssets( + projectRoot, + outputRoot, + ); + } else { + await agentEndpointAssets.removeAgentEndpointAssets( + outputRoot, + ); + } + }, + }, virtualApiBuildConfigPlugin(config.api, projectRoot), virtualLastUpdatedPlugin(lastUpdatedByPath), virtualConfigPlugin(config, { @@ -1330,7 +1409,7 @@ export function nimbus( if (!isContentFile(file)) return; const { clearNavCaches } = await import("./index.js"); clearNavCaches(); - (await loadPreparedArtifacts()).invalidatePreparedArtifacts( + (await loadAgentEndpointAssets()).invalidateAgentEndpointAssets( projectRootForBuild, ); server.moduleGraph.invalidateAll(); @@ -1365,7 +1444,7 @@ export function nimbus( ); citationIndex = index; coordinatesManifest = manifest; - (await loadPreparedArtifacts()).invalidatePreparedArtifacts( + (await loadAgentEndpointAssets()).invalidateAgentEndpointAssets( projectRootForBuild, ); server.moduleGraph.invalidateAll(); @@ -1383,43 +1462,32 @@ export function nimbus( "astro:build:start": async () => { const { clearNavCaches } = await import("./index.js"); clearNavCaches(); - const preparedArtifacts = await loadPreparedArtifacts(); + const agentEndpointAssets = await loadAgentEndpointAssets(); if (requestRenderingConfigured) { - preparedArtifacts.registerPreparedArtifactDemand(projectRootForBuild); + agentEndpointAssets.registerAgentEndpointAssetDemand(projectRootForBuild); } - if (preparedArtifacts.isPreparedArtifactRequested(projectRootForBuild)) { - await preparedArtifacts.ensurePreparedArtifacts(projectRootForBuild); + if (agentEndpointAssets.isAgentEndpointAssetRequested(projectRootForBuild)) { + await agentEndpointAssets.ensureAgentEndpointAssets(projectRootForBuild); } }, "astro:routes:resolved": ({ routes }) => { - requestRoutePatterns = - renderingRoutes.size === 0 - ? new Set() - : new Set( - routes - .filter( - (route) => - renderingRoutes.get( - normalizeRouteComponent(route.entrypoint), - ) === "request", - ) - .map((route) => route.pattern), - ); resolvedRoutesForBuild = routes.map((r) => ({ pattern: r.pattern, type: r.type, isPrerendered: r.isPrerendered, origin: r.origin, + entrypoint: normalizeRouteEntrypoint( + projectRootForBuild, + r.entrypoint, + ), })); }, "astro:build:done": async ({ dir, pages, logger }) => { const distDir = fileURLToPath(dir); - const publicPages = requestRenderingConfigured - ? pages.filter( - ({ pathname }) => - !isRequestRouteInventoryPath(pathname, astroBaseForBuild), - ) - : pages; + const publicPages = pages.filter( + ({ pathname }) => + !isRequestRouteInventoryPath(pathname, astroBaseForBuild), + ); const prerenderedRoutes = new Set( publicPages.map(({ pathname }) => canonicalizePathname(pathname)), ); @@ -1430,13 +1498,19 @@ export function nimbus( ), ), ); - const inventory = requestRenderingConfigured + const inventory = building ? readRequestRouteInventory( distDir, astroBaseForBuild, requestRenderingCollections, ) : []; + contentRoutePatternsForBuild = new Set( + inventory.map((entry) => canonicalizePathname(entry.url)), + ); + const prerenderedContentCount = [...contentRoutePatternsForBuild].filter( + (pathname) => prerenderedRoutes.has(pathname), + ).length; const requestRoutes = inventory .filter((entry) => entry.request) .map((entry) => canonicalizePathname(entry.url)) @@ -1459,25 +1533,6 @@ export function nimbus( sitemapCustomPages.push(new URL(sitemapPath, config.site).href); } } - // Materialize the site's route truth from Astro's emitted `pages` - // array — the single source of truth: every URL on this list is a - // page Astro just wrote to disk. No reconstruction, no slug - // mirroring, no Astro-internals coupling. The build/lint - // contract is "after `astro build`, `.nimbus/routes.json` reflects - // exactly what the site serves." Lint that runs without a prior - // build silently skips `internal-link`. - // - // Duplicate-slug detection happens in `astro:config:setup`, not - // here: Astro silently dedupes colliding routes before this hook - // fires, so the collisions are invisible post-build. - materializeRouteTruthFromPages( - projectRootForBuild, - astroBaseForBuild, - publicPages, - requestRoutes, - logger, - ); - // Filled by `astro:routes:resolved`; reset at the next build's // `config:setup`, so a build whose `routes:resolved` never fires trips // the empty-routes guard instead of reusing stale routes. @@ -1489,14 +1544,29 @@ export function nimbus( const footprint = deriveFootprint( readDependencyNames(projectRootForBuild), ); + const activeFeatureRoutes = footprintRoutes(footprint) + .map((route) => ({ + ...route, + entrypoint: + normalizeSourceRouteEntrypoint( + projectRootForBuild, + srcDirForBuild, + route.entrypoint, + ) ?? route.entrypoint, + })) + .filter((route) => + fs.existsSync(path.resolve(projectRootForBuild, route.entrypoint)), + ); const report = analyzeBuild({ outputMode: outputModeForBuild, adapterName: adapterNameForBuild, routes: resolvedRoutes, - prerenderedPageCount: publicPages.length, + prerenderedPageCount: prerenderedContentCount, requestRenderedPageCount: requestRoutes.length, - declaredFeatureRoutes: footprintRoutes(footprint), - declaredRequestRoutes: [...requestRoutePatterns], + managedRoutes: managedRoutesForBuild, + featureRoutes: activeFeatureRoutes, + userExtensibleRoutes: userExtensibleRoutesForBuild, + contentRoutePatterns: [...contentRoutePatternsForBuild], serverFeatures: footprint.map((f) => f.id), }); logger.info(report.summaryLine); @@ -1507,6 +1577,17 @@ export function nimbus( throw new Error(formatInvariantFailure(report.violations)); } + materializeRouteTruthFromPages( + projectRootForBuild, + astroBaseForBuild, + publicPages, + [ + ...requestRoutes, + ...report.onDemandDocRoutes.filter(isConcreteRoutePattern), + ], + logger, + ); + materializeCoordinatesManifest( projectRootForBuild, coordinatesManifest, @@ -1569,8 +1650,8 @@ function materializeLintConfig( /** * Write the site's route truth to `/.nimbus/routes.json` from Astro's - * emitted pages plus the concrete inventory produced for request-rendered - * collections. + * emitted pages, request-rendered collection inventory, and concrete + * on-demand route patterns. * * Best-effort write, same as `materializeLintConfig`. When the file is * missing (e.g. lint ran before any `astro build`), `internal-link` skips @@ -1585,7 +1666,7 @@ function materializeRouteTruthFromPages( projectRoot: string, base: string, pages: readonly { pathname: string }[], - requestRoutes: readonly string[], + onDemandRoutes: readonly string[], logger: { warn: (msg: string) => void; debug?: (msg: string) => void }, ): void { // Normalize and dedupe pathnames into the canonical `/foo` form used by @@ -1597,7 +1678,7 @@ function materializeRouteTruthFromPages( for (const { pathname } of pages) { canonical.add(canonicalizePathname(pathname)); } - for (const pathname of requestRoutes) { + for (const pathname of onDemandRoutes) { canonical.add(canonicalizePathname(pathname)); } @@ -1625,6 +1706,10 @@ function materializeRouteTruthFromPages( } } +function isConcreteRoutePattern(pattern: string): boolean { + return !pattern.includes("["); +} + function isRequestRouteInventoryPath(pathname: string, base: string): boolean { const canonical = canonicalizePathname(pathname); const normalizedBase = canonicalizePathname(base); diff --git a/packages/nimbus-docs/src/publication.ts b/packages/nimbus-docs/src/publication.ts new file mode 100644 index 00000000..7fb21cd6 --- /dev/null +++ b/packages/nimbus-docs/src/publication.ts @@ -0,0 +1,30 @@ +import { + getLlmsPayload, + getLlmsStaticPaths, + getMarkdownPayload, + getMarkdownStaticPaths, +} from "./agent-endpoints.js"; + +/** @deprecated Import `getMarkdownPayload` from `@cloudflare/nimbus-docs/agent-endpoints`. */ +export const getPreparedMarkdownRouteArtifact = getMarkdownPayload; + +/** @deprecated Import `getLlmsPayload` from `@cloudflare/nimbus-docs/agent-endpoints`. */ +export const getPreparedLlmsRouteArtifact = getLlmsPayload; + +/** @deprecated Import `getMarkdownStaticPaths` from `@cloudflare/nimbus-docs/agent-endpoints`; route props use `reference` instead of `artifact`. */ +export async function getPreparedMarkdownRouteStaticPaths( + options: Parameters[0], +) { + return (await getMarkdownStaticPaths(options)).map(({ props, ...path }) => ({ + ...path, + props: { artifact: props.reference }, + })); +} + +/** @deprecated Import `getLlmsStaticPaths` from `@cloudflare/nimbus-docs/agent-endpoints`; route props use `reference` instead of `artifact`. */ +export async function getPreparedLlmsRouteStaticPaths() { + return (await getLlmsStaticPaths()).map(({ props, ...path }) => ({ + ...path, + props: { artifact: props.reference }, + })); +} diff --git a/packages/nimbus-docs/src/runtime.ts b/packages/nimbus-docs/src/runtime.ts index f9b6317e..66bce76a 100644 --- a/packages/nimbus-docs/src/runtime.ts +++ b/packages/nimbus-docs/src/runtime.ts @@ -530,9 +530,9 @@ export async function renderIndexedEntryMarkdown( * - Each entry is a `#`-level block (bodies render at `##` and below). * - The document header cross-references `/llms.txt`. * - * The starter route reads the prepared full-document artifact. A site that wants - * a different policy (per-version, filtered, chunked) should prepare its own - * artifact at build time rather than compose runtime entry renderers, which do + * The starter route reads the prebuilt full-document endpoint payload. A site + * that wants a different policy (per-version, filtered, chunked) should generate + * its own output at build time rather than compose runtime entry renderers, which do * not carry build-only partial or API rendering context. Pass Astro's * `import.meta.env.BASE_URL` as `base` when the site supports sub-path deploys. */ diff --git a/packages/nimbus-docs/src/types.ts b/packages/nimbus-docs/src/types.ts index 88623ac0..153a1c37 100644 --- a/packages/nimbus-docs/src/types.ts +++ b/packages/nimbus-docs/src/types.ts @@ -23,14 +23,17 @@ export interface GeneratedMarkdownPartialResolver { resolve: (attrs: { file: string; product: string | undefined }) => string; } +/** @deprecated Use `MarkdownEndpointSurface` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export type PreparedMarkdownSurface = "markdown" | "source"; +/** @deprecated Use `MarkdownEndpointReference` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export interface PreparedMarkdownReference { collection: string; id: string; surface: PreparedMarkdownSurface; } +/** @deprecated Use `MarkdownEndpointPayload` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export interface PreparedMarkdownArtifact extends PreparedMarkdownReference { digest: string; mediaType: string; @@ -38,10 +41,12 @@ export interface PreparedMarkdownArtifact extends PreparedMarkdownReference { content: string; } +/** @deprecated Use `LlmsEndpointReference` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export type PreparedLlmsReference = | { scope: "site"; surface: "index" | "full" } | { scope: "section"; surface: "index"; section: string }; +/** @deprecated Use `LlmsEndpointPayload` from `@cloudflare/nimbus-docs/agent-endpoints`. */ export type PreparedLlmsArtifact = PreparedLlmsReference & { digest: string; mediaType: string; diff --git a/packages/nimbus-docs/src/types/virtual-modules.d.ts b/packages/nimbus-docs/src/types/virtual-modules.d.ts index 20b39747..29ccb2b6 100644 --- a/packages/nimbus-docs/src/types/virtual-modules.d.ts +++ b/packages/nimbus-docs/src/types/virtual-modules.d.ts @@ -33,6 +33,20 @@ declare module "virtual:nimbus/headings" { export const records: import("../_internal/prepared-headings.js").PreparedHeadingRecord[]; } +declare module "virtual:nimbus/agent-endpoint-assets" { + export const projectRoot: string; + export const base: string; + export const markdownAssets: import("../_internal/agent-endpoint-assets.js").MarkdownEndpointAsset[]; + export const llmsAssets: import("../_internal/agent-endpoint-assets.js").LlmsEndpointAsset[]; +} + +declare module "virtual:nimbus/agent-endpoint-asset-loader" { + export function fetchAgentEndpointAsset( + path: string, + request: Request, + ): Promise | null; +} + declare module "virtual:nimbus/api-build-config" { export const api: import("../types.js").ApiSpec[]; export const root: string; diff --git a/packages/nimbus-docs/test/prepared-artifacts.test.ts b/packages/nimbus-docs/test/agent-endpoint-assets.test.ts similarity index 66% rename from packages/nimbus-docs/test/prepared-artifacts.test.ts rename to packages/nimbus-docs/test/agent-endpoint-assets.test.ts index 9f290457..669b9d8b 100644 --- a/packages/nimbus-docs/test/prepared-artifacts.test.ts +++ b/packages/nimbus-docs/test/agent-endpoint-assets.test.ts @@ -6,6 +6,7 @@ import { readdir, rm, symlink, + writeFile, } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -22,16 +23,20 @@ import { } from "../src/_internal/prepared-markdown-registry.ts"; import { bakePreparedHeadings, - bakePreparedArtifacts, - configurePreparedArtifactRoot, - ensurePreparedArtifacts, - invalidatePreparedArtifacts, - isPreparedArtifactRequested, + bakeAgentEndpointAssets, + configureAgentEndpointAssetRoot, + ensureAgentEndpointAssets, + invalidateAgentEndpointAssets, + isAgentEndpointAssetRequested, + agentEndpointAssetLoaderPlugin, + agentEndpointAssetsRuntimePlugin, preparedHeadingsPlugin, - readPreparedLlmsArtifact, - readPreparedMarkdownArtifact, - registerPreparedArtifactDemand, -} from "../src/_internal/prepared-artifacts.ts"; + readLlmsEndpointPayload, + readMarkdownEndpointPayload, + registerAgentEndpointAssetDemand, + removeAgentEndpointAssets, + stageAgentEndpointAssets, +} from "../src/_internal/agent-endpoint-assets.ts"; const roots: string[] = []; const capability = { generation: 1, base: "/docs" }; @@ -44,7 +49,7 @@ afterEach(async () => { }); async function root(): Promise { - const value = await mkdtemp(path.join(os.tmpdir(), "nimbus-prepared-artifacts-")); + const value = await mkdtemp(path.join(os.tmpdir(), "nimbus-agent-endpoint-assets-")); roots.push(value); beginPreparedMarkdownSession(value); return value; @@ -52,10 +57,10 @@ async function root(): Promise { function configure( projectRoot: string, - options: Parameters[0], + options: Parameters[0], ): void { - configurePreparedArtifactRoot(projectRoot, "build", () => - bakePreparedArtifacts(options), + configureAgentEndpointAssetRoot(projectRoot, "build", () => + bakeAgentEndpointAssets(options), ); } @@ -142,7 +147,7 @@ test("bakes compact headings with a revisioned partial resolver", async () => { }, }; configure(projectRoot, options); - const manifest = await bakePreparedArtifacts(options); + const manifest = await bakeAgentEndpointAssets(options); assert.deepEqual(manifest.headings, [ { collection: "docs", @@ -156,7 +161,7 @@ test("bakes compact headings with a revisioned partial resolver", async () => { ]); assert.match( ( - await readPreparedMarkdownArtifact(projectRoot, { + await readMarkdownEndpointPayload(projectRoot, { collection: "docs", id: "guide", surface: "source", @@ -166,7 +171,7 @@ test("bakes compact headings with a revisioned partial resolver", async () => { ); }); -test("bakes expanded source and transformed Markdown artifacts deterministically", async () => { +test("bakes expanded source and transformed Markdown endpoint assets deterministically", async () => { const projectRoot = await root(); commit(projectRoot, "docs", [ { @@ -215,13 +220,13 @@ test("bakes expanded source and transformed Markdown artifacts deterministically }, }; configure(projectRoot, options); - const first = await bakePreparedArtifacts(options); - const second = await bakePreparedArtifacts(options); + const first = await bakeAgentEndpointAssets(options); + const second = await bakeAgentEndpointAssets(options); assert.deepEqual(second, first); - assert.equal(first.markdownArtifacts.length, 2); - assert.equal(first.llmsArtifacts.length, 2); + assert.equal(first.markdownAssets.length, 2); + assert.equal(first.llmsAssets.length, 2); - const source = await readPreparedMarkdownArtifact(projectRoot, { + const source = await readMarkdownEndpointPayload(projectRoot, { collection: "docs", id: "guide", surface: "source", @@ -231,7 +236,7 @@ test("bakes expanded source and transformed Markdown artifacts deterministically assert.doesNotMatch(source.body, / { +test("bakes site and section llms.txt endpoint assets from public discoverable prose and API pages", async () => { const projectRoot = await root(); commit(projectRoot, "docs", [ { id: "guide/a", body: "Guide A", data: { title: "Guide A" } }, @@ -332,17 +337,17 @@ test("bakes site and section llms.txt artifacts from public discoverable prose a ], }; configure(projectRoot, options); - const manifest = await bakePreparedArtifacts(options); + const manifest = await bakeAgentEndpointAssets(options); assert.deepEqual( - manifest.llmsArtifacts.map((artifact) => - artifact.scope === "site" - ? `${artifact.scope}:${artifact.surface}` - : `${artifact.scope}:${artifact.section}`, + manifest.llmsAssets.map((asset) => + asset.scope === "site" + ? `${asset.scope}:${asset.surface}` + : `${asset.scope}:${asset.section}`, ), ["section:api", "section:blog", "section:guide", "site:full", "site:index"], ); - const index = await readPreparedLlmsArtifact(projectRoot, { + const index = await readLlmsEndpointPayload(projectRoot, { scope: "site", surface: "index", }); @@ -360,7 +365,7 @@ test("bakes site and section llms.txt artifacts from public discoverable prose a ); assert.doesNotMatch(index.body, /v1|Hidden/); - const guide = await readPreparedLlmsArtifact(projectRoot, { + const guide = await readLlmsEndpointPayload(projectRoot, { scope: "section", surface: "index", section: "guide", @@ -369,7 +374,7 @@ test("bakes site and section llms.txt artifacts from public discoverable prose a assert.match(guide.body, /Guide B.*Second guide/); assert.doesNotMatch(guide.body, /Hidden/); - const full = await readPreparedLlmsArtifact(projectRoot, { + const full = await readLlmsEndpointPayload(projectRoot, { scope: "site", surface: "full", }); @@ -381,15 +386,183 @@ test("bakes site and section llms.txt artifacts from public discoverable prose a assert.doesNotMatch(full.body, /# Old|# Hidden|Secret API/); assert.ok( - manifest.markdownArtifacts.some( - (artifact) => artifact.collection === "docs" && artifact.id === "hidden", + manifest.markdownAssets.some( + (asset) => asset.collection === "docs" && asset.id === "hidden", ), ); assert.ok( - manifest.markdownArtifacts.every((artifact) => artifact.collection !== "docs-v1"), + manifest.markdownAssets.every((asset) => asset.collection !== "docs-v1"), ); }); +test("Markdown and llms.txt endpoints expose metadata and stage bodies as assets", async () => { + const projectRoot = await root(); + commit(projectRoot, "docs", [ + { id: "guide", body: "Unique prepared body", data: { title: "Guide" } }, + ]); + const options = { + root: projectRoot, + base: "/docs", + site: "https://example.test", + title: "Test", + indexedCollections: ["docs"], + }; + configure(projectRoot, options); + + const plugin = agentEndpointAssetsRuntimePlugin(projectRoot); + const id = plugin.resolveId("virtual:nimbus/agent-endpoint-assets"); + assert.ok(id); + const source = await plugin.load.call( + { environment: { name: "ssr" } }, + id, + ); + assert.ok(source); + assert.doesNotMatch(source, /Unique prepared body/); + assert.match(source, /assets\//); + assert.equal(isAgentEndpointAssetRequested(projectRoot), true); + + const output = path.join(projectRoot, "dist", "client"); + const stale = path.join( + output, + "_nimbus", + "agent-endpoint-assets", + "assets", + "stale.txt", + ); + await mkdir(path.dirname(stale), { recursive: true }); + await writeFile(stale, "stale"); + const legacyStale = path.join( + output, + "_nimbus", + "prepared-artifacts", + "assets", + "stale.txt", + ); + await mkdir(path.dirname(legacyStale), { recursive: true }); + await writeFile(legacyStale, "legacy stale"); + await stageAgentEndpointAssets(projectRoot, output); + await assert.rejects(readFile(stale, "utf8"), { code: "ENOENT" }); + await assert.rejects(readFile(legacyStale, "utf8"), { code: "ENOENT" }); + const manifest = await ensureAgentEndpointAssets(projectRoot); + for (const asset of [ + ...manifest.markdownAssets, + ...manifest.llmsAssets, + ]) { + assert.equal( + await readFile( + path.join(output, "_nimbus", "agent-endpoint-assets", asset.path), + "utf8", + ), + await readFile( + path.join( + projectRoot, + ".astro", + "nimbus", + "agent-endpoint-assets", + asset.path, + ), + "utf8", + ), + ); + } + await removeAgentEndpointAssets(output); + await assert.rejects( + readdir(path.join(output, "_nimbus", "agent-endpoint-assets")), + { code: "ENOENT" }, + ); +}); + +test("agent-endpoint asset loader uses the Cloudflare assets binding only on Cloudflare", async () => { + const cloudflare = agentEndpointAssetLoaderPlugin(() => "@astrojs/cloudflare"); + const cloudflareId = cloudflare.resolveId( + "virtual:nimbus/agent-endpoint-asset-loader", + ); + assert.ok(cloudflareId); + const cloudflareSource = await cloudflare.load.call( + { environment: { name: "ssr" } }, + cloudflareId, + ); + assert.match(cloudflareSource ?? "", /cloudflare:workers/); + assert.match(cloudflareSource ?? "", /env\.ASSETS/); + const prerenderSource = await cloudflare.load.call( + { environment: { name: "prerender" } }, + cloudflareId, + ); + assert.doesNotMatch(prerenderSource ?? "", /cloudflare:workers|ASSETS/); + + const node = agentEndpointAssetLoaderPlugin(() => "@astrojs/node"); + const nodeId = node.resolveId("virtual:nimbus/agent-endpoint-asset-loader"); + assert.ok(nodeId); + const nodeSource = await node.load(nodeId); + assert.doesNotMatch(nodeSource ?? "", /cloudflare:workers|ASSETS/); +}); + +test("staging rejects manifest paths outside asset roots before cleanup", async () => { + const projectRoot = await root(); + commit(projectRoot, "docs", [ + { id: "guide", body: "Guide", data: { title: "Guide" } }, + ]); + configure(projectRoot, { + root: projectRoot, + base: "/docs", + site: "https://example.test", + title: "Test", + indexedCollections: ["docs"], + }); + const manifest = await ensureAgentEndpointAssets(projectRoot); + const asset = manifest.markdownAssets[0]; + assert.ok(asset); + asset.path = "../outside.md"; + + const output = path.join(projectRoot, "dist", "client"); + const staged = path.join( + output, + "_nimbus", + "agent-endpoint-assets", + "assets", + "retained.md", + ); + const outsideTarget = path.join(output, "_nimbus", "outside.md"); + const outsideSource = path.join( + projectRoot, + ".astro", + "nimbus", + "outside.md", + ); + await mkdir(path.dirname(staged), { recursive: true }); + await writeFile(staged, "retained"); + await writeFile(outsideTarget, "outside retained"); + await writeFile(outsideSource, "poisoned"); + + await assert.rejects( + stageAgentEndpointAssets(projectRoot, output), + /asset path escapes its root/, + ); + assert.equal(await readFile(staged, "utf8"), "retained"); + assert.equal(await readFile(outsideTarget, "utf8"), "outside retained"); +}); + +test("staged agent-endpoint asset cleanup rejects a symlinked output root", async () => { + const projectRoot = await root(); + const realOutput = path.join(projectRoot, "real-output"); + const linkedOutput = path.join(projectRoot, "linked-output"); + const retained = path.join( + realOutput, + "_nimbus", + "agent-endpoint-assets", + "retained.txt", + ); + await mkdir(path.dirname(retained), { recursive: true }); + await writeFile(retained, "retained"); + await symlink(realOutput, linkedOutput, "dir"); + + await assert.rejects( + removeAgentEndpointAssets(linkedOutput), + /contains a symbolic link/, + ); + assert.equal(await readFile(retained, "utf8"), "retained"); +}); + test("waits for API index transactions before caching llms.txt output", async () => { const projectRoot = await root(); commit(projectRoot, "docs", [ @@ -406,8 +579,8 @@ test("waits for API index transactions before caching llms.txt output", async () }; let firstRead = true; let update: Promise | undefined; - configurePreparedArtifactRoot(projectRoot, "dev", () => { - return bakePreparedArtifacts({ + configureAgentEndpointAssetRoot(projectRoot, "dev", () => { + return bakeAgentEndpointAssets({ ...options, loadApiEntries: async () => { const captured = apiEntries; @@ -427,9 +600,9 @@ test("waits for API index transactions before caching llms.txt output", async () }); }); - await ensurePreparedArtifacts(projectRoot); + await ensureAgentEndpointAssets(projectRoot); await update; - const full = await readPreparedLlmsArtifact(projectRoot, { + const full = await readLlmsEndpointPayload(projectRoot, { scope: "site", surface: "full", }); @@ -451,26 +624,26 @@ test("rebakes when invalidated during API input loading", async () => { }; let bakes = 0; let reads = 0; - configurePreparedArtifactRoot(projectRoot, "dev", () => { + configureAgentEndpointAssetRoot(projectRoot, "dev", () => { bakes += 1; - return bakePreparedArtifacts({ + return bakeAgentEndpointAssets({ ...options, loadApiEntries: async () => { reads += 1; - if (reads === 1) invalidatePreparedArtifacts(projectRoot); + if (reads === 1) invalidateAgentEndpointAssets(projectRoot); return [apiPage(reads === 1 ? "Old API" : "New API")]; }, }); }); - const manifest = await ensurePreparedArtifacts(projectRoot); + const manifest = await ensureAgentEndpointAssets(projectRoot); assert.equal(bakes, 2); assert.deepEqual( ( - await readdir(path.join(projectRoot, ".astro/nimbus/prepared-artifacts/artifacts")) + await readdir(path.join(projectRoot, ".astro/nimbus/agent-endpoint-assets/assets")) ).sort(), - [...manifest.markdownArtifacts, ...manifest.llmsArtifacts] - .map((artifact) => path.basename(artifact.path)) + [...manifest.markdownAssets, ...manifest.llmsAssets] + .map((asset) => path.basename(asset.path)) .sort(), ); }); @@ -493,7 +666,7 @@ test("rejects page collisions and unsafe section route parameters", async () => indexedCollections: ["docs"], }; await assert.rejects( - bakePreparedArtifacts(options), + bakeAgentEndpointAssets(options), /guide\/llms\.txt.*collides with the generated llms.txt route/s, ); @@ -502,14 +675,14 @@ test("rejects page collisions and unsafe section route parameters", async () => { id: "guide/index", body: "Index", data: { title: "Index" } }, ]); await assert.rejects( - bakePreparedArtifacts(options), + bakeAgentEndpointAssets(options), /guide\/index.*collides with.*docs:guide.*generated Markdown route/s, ); commit(projectRoot, "docs", [ { id: "../secret", body: "Secret", data: { title: "Secret" } }, ]); - await assert.rejects(bakePreparedArtifacts(options), /section slug is unsafe/); + await assert.rejects(bakeAgentEndpointAssets(options), /section slug is unsafe/); commit(projectRoot, "docs", [ { @@ -518,7 +691,7 @@ test("rejects page collisions and unsafe section route parameters", async () => data: { title: "Secret" }, }, ]); - await assert.rejects(bakePreparedArtifacts(options), /unsafe entry ID/); + await assert.rejects(bakeAgentEndpointAssets(options), /unsafe entry ID/); commit(projectRoot, "docs", [ { id: "%67uide/a", body: "A", data: { title: "A" } }, @@ -530,7 +703,7 @@ test("rejects page collisions and unsafe section route parameters", async () => }, ]); await assert.rejects( - bakePreparedArtifacts(options), + bakeAgentEndpointAssets(options), /guide\/llms\.txt.*collides with the generated llms.txt route/s, ); }); @@ -551,8 +724,8 @@ test("uses locale-independent ordering in llms.txt indexes", async () => { indexedCollections: ["docs"], }; configure(projectRoot, options); - await bakePreparedArtifacts(options); - const index = await readPreparedLlmsArtifact(projectRoot, { + await bakeAgentEndpointAssets(options); + const index = await readLlmsEndpointPayload(projectRoot, { scope: "site", surface: "index", }); @@ -577,8 +750,8 @@ test("preserves protocol-relative social images", async () => { indexedCollections: ["docs"], }; configure(projectRoot, options); - await bakePreparedArtifacts(options); - const markdown = await readPreparedMarkdownArtifact(projectRoot, { + await bakeAgentEndpointAssets(options); + const markdown = await readMarkdownEndpointPayload(projectRoot, { collection: "docs", id: "guide", surface: "markdown", @@ -597,7 +770,7 @@ test("resolves the complete audience before touching partials", async () => { { id: "public", body: "Public", data: { title: "Public" } }, ]); - const manifest = await bakePreparedArtifacts({ + const manifest = await bakeAgentEndpointAssets({ root: projectRoot, base: "/docs", site: "https://example.test", @@ -605,7 +778,7 @@ test("resolves the complete audience before touching partials", async () => { indexedCollections: ["docs"], }); assert.deepEqual( - manifest.markdownArtifacts.map(({ id, surface }) => [id, surface]), + manifest.markdownAssets.map(({ id, surface }) => [id, surface]), [ ["public", "markdown"], ["public", "source"], @@ -633,7 +806,7 @@ test("fails closed for unknown audiences and invalid transitive partials", async indexedCollections: ["docs"], }; await assert.rejects( - bakePreparedArtifacts(options), + bakeAgentEndpointAssets(options), /exclude.*partials:hidden|excluded partial/s, ); @@ -645,7 +818,7 @@ test("fails closed for unknown audiences and invalid transitive partials", async }, ]); await assert.rejects( - bakePreparedArtifacts(options), + bakeAgentEndpointAssets(options), /visibility is unknown.*docs:guide/s, ); }); @@ -653,7 +826,7 @@ test("fails closed for unknown audiences and invalid transitive partials", async test("rejects unprepared collections and stale collection capabilities", async () => { const projectRoot = await root(); await assert.rejects( - bakePreparedArtifacts({ + bakeAgentEndpointAssets({ root: projectRoot, base: "/docs", site: "https://example.test", @@ -676,7 +849,7 @@ test("rejects unprepared collections and stale collection capabilities", async ( [{ id: "guide", body: "Guide", data: { title: "Guide" } }] as never, ); await assert.rejects( - bakePreparedArtifacts({ + bakeAgentEndpointAssets({ root: projectRoot, base: "/docs", site: "https://example.test", @@ -715,13 +888,13 @@ test("prepares headings without requiring every indexed collection to support pr ["docs:guide"], ); - let artifactBakes = 0; - configurePreparedArtifactRoot( + let assetBakes = 0; + configureAgentEndpointAssetRoot( projectRoot, "build", async () => { - artifactBakes += 1; - throw new Error("strict artifact bake should not run"); + assetBakes += 1; + throw new Error("strict asset bake should not run"); }, () => bakePreparedHeadings({ @@ -734,7 +907,7 @@ test("prepares headings without requiring every indexed collection to support pr const plugin = preparedHeadingsPlugin(projectRoot); const resolved = plugin.resolveId("virtual:nimbus/headings")!; const source = await plugin.load(resolved); - assert.equal(artifactBakes, 0); + assert.equal(assetBakes, 0); assert.match(source ?? "", /export const base = "\/docs"/u); assert.match(source ?? "", /"collection":"docs","id":"guide"/u); assert.doesNotMatch(source ?? "", /bodyless|unwrapped/u); @@ -753,7 +926,7 @@ test("prepares headings without requiring every indexed collection to support pr assert.equal(invalidated, headingModule); }); -test("joins concurrent rebakes and rejects symlinked artifact roots", async () => { +test("joins concurrent rebakes and rejects symlinked agent-endpoint asset roots", async () => { const projectRoot = await root(); commit(projectRoot, "docs", [ { id: "guide", body: "Guide", data: { title: "Guide" } }, @@ -766,14 +939,14 @@ test("joins concurrent rebakes and rejects symlinked artifact roots", async () = indexedCollections: ["docs"], }; let calls = 0; - configurePreparedArtifactRoot(projectRoot, "dev", async () => { + configureAgentEndpointAssetRoot(projectRoot, "dev", async () => { calls += 1; await new Promise((resolve) => setTimeout(resolve, 10)); - return bakePreparedArtifacts(options); + return bakeAgentEndpointAssets(options); }); const [first, second] = await Promise.all([ - ensurePreparedArtifacts(projectRoot), - ensurePreparedArtifacts(projectRoot), + ensureAgentEndpointAssets(projectRoot), + ensureAgentEndpointAssets(projectRoot), ]); assert.deepEqual(second, first); assert.equal(calls, 1); @@ -784,9 +957,9 @@ test("joins concurrent rebakes and rejects symlinked artifact roots", async () = ]); const outside = await root(); await mkdir(path.join(escapedRoot, ".astro/nimbus"), { recursive: true }); - await symlink(outside, path.join(escapedRoot, ".astro/nimbus/prepared-artifacts"), "dir"); + await symlink(outside, path.join(escapedRoot, ".astro/nimbus/agent-endpoint-assets"), "dir"); await assert.rejects( - bakePreparedArtifacts({ ...options, root: escapedRoot }), + bakeAgentEndpointAssets({ ...options, root: escapedRoot }), /symbolic link/, ); }); @@ -812,9 +985,9 @@ test("queues a follow-up bake when invalidated during in-flight work", async () const gate = new Promise((resolve) => { release = resolve; }); - configurePreparedArtifactRoot(projectRoot, "dev", async () => { + configureAgentEndpointAssetRoot(projectRoot, "dev", async () => { calls += 1; - const manifest = await bakePreparedArtifacts(options); + const manifest = await bakeAgentEndpointAssets(options); if (calls === 1) { entered?.(); await gate; @@ -822,15 +995,15 @@ test("queues a follow-up bake when invalidated during in-flight work", async () return manifest; }); - const read = ensurePreparedArtifacts(projectRoot); + const read = ensureAgentEndpointAssets(projectRoot); await started; - invalidatePreparedArtifacts(projectRoot); + invalidateAgentEndpointAssets(projectRoot); release?.(); await read; assert.equal(calls, 2); }); -test("removes artifacts made obsolete by edits and deletions", async () => { +test("removes assets made obsolete by edits and deletions", async () => { const projectRoot = await root(); commit(projectRoot, "docs", [ { id: "guide", body: "Old guide", data: { title: "Guide" } }, @@ -844,26 +1017,26 @@ test("removes artifacts made obsolete by edits and deletions", async () => { indexedCollections: ["docs"], }; configure(projectRoot, options); - await bakePreparedArtifacts(options); + await bakeAgentEndpointAssets(options); configure(projectRoot, options); commit(projectRoot, "docs", [ { id: "guide", body: "New guide", data: { title: "Guide" } }, ]); - const manifest = await bakePreparedArtifacts(options); + const manifest = await bakeAgentEndpointAssets(options); const files = await readdir( - path.join(projectRoot, ".astro/nimbus/prepared-artifacts/artifacts"), + path.join(projectRoot, ".astro/nimbus/agent-endpoint-assets/assets"), ); assert.deepEqual( files.sort(), - [...manifest.markdownArtifacts, ...manifest.llmsArtifacts] - .map((artifact) => path.basename(artifact.path)) + [...manifest.markdownAssets, ...manifest.llmsAssets] + .map((asset) => path.basename(asset.path)) .sort(), ); }); -test("scopes artifact demand to the current configuration session", async () => { +test("scopes agent-endpoint asset demand to the current configuration session", async () => { const projectRoot = await root(); const options = { root: projectRoot, @@ -873,9 +1046,9 @@ test("scopes artifact demand to the current configuration session", async () => indexedCollections: ["docs"], }; configure(projectRoot, options); - registerPreparedArtifactDemand(projectRoot); - assert.equal(isPreparedArtifactRequested(projectRoot), true); + registerAgentEndpointAssetDemand(projectRoot); + assert.equal(isAgentEndpointAssetRequested(projectRoot), true); configure(projectRoot, options); - assert.equal(isPreparedArtifactRequested(projectRoot), false); + assert.equal(isAgentEndpointAssetRequested(projectRoot), false); }); diff --git a/packages/nimbus-docs/test/prepared-artifact-lifecycle.test.ts b/packages/nimbus-docs/test/agent-endpoint-lifecycle.test.ts similarity index 75% rename from packages/nimbus-docs/test/prepared-artifact-lifecycle.test.ts rename to packages/nimbus-docs/test/agent-endpoint-lifecycle.test.ts index b47c311d..8b3fb3fc 100644 --- a/packages/nimbus-docs/test/prepared-artifact-lifecycle.test.ts +++ b/packages/nimbus-docs/test/agent-endpoint-lifecycle.test.ts @@ -24,7 +24,7 @@ afterEach(async () => { ); }); -test("bakes prepared artifacts at astro:build:start for prerendered build helpers", async () => { +test("bakes agent-endpoint assets at astro:build:start for prerendered endpoints", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "nimbus-generated-markdown-lifecycle-"), ); @@ -41,8 +41,11 @@ test("bakes prepared artifacts at astro:build:start for prerendered build helper const contentModule = pathToFileURL( path.resolve(import.meta.dirname, "../src/content.ts"), ).href; - const buildModule = pathToFileURL( - path.resolve(import.meta.dirname, "../src/build.ts"), + const endpointsModule = pathToFileURL( + path.resolve(import.meta.dirname, "../src/agent-endpoints.ts"), + ).href; + const publicationModule = pathToFileURL( + path.resolve(import.meta.dirname, "../src/publication.ts"), ).href; await writeFile( path.join(root, "src/content.config.ts"), @@ -77,43 +80,45 @@ export const collections = { ); await writeFile( path.join(root, "src/pages/[...slug]/index.md.ts"), - `import { getPreparedMarkdownArtifact, getPreparedMarkdownStaticPaths } from ${JSON.stringify(buildModule)}; + `import { getMarkdownPayload } from ${JSON.stringify(endpointsModule)}; +import { getPreparedMarkdownRouteStaticPaths } from ${JSON.stringify(publicationModule)}; export const prerender = true; -export const getStaticPaths = () => getPreparedMarkdownStaticPaths({ collection: "docs", surface: "markdown" }); -export async function GET({ props }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { headers: { "content-type": artifact.mediaType } }); +export const getStaticPaths = () => getPreparedMarkdownRouteStaticPaths({ collection: "docs", surface: "markdown" }); +export async function GET({ params, props, request }) { + const payload = await getMarkdownPayload({ collection: "docs", surface: "markdown", slug: params.slug, reference: props.artifact, context: { request } }); + return new Response(payload.body, { headers: { "content-type": payload.mediaType } }); }`, "utf8", ); await writeFile( path.join(root, "src/pages/llms.txt.ts"), - `import { getPreparedLlmsArtifact } from ${JSON.stringify(buildModule)}; + `import { getLlmsPayload } from ${JSON.stringify(endpointsModule)}; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ scope: "site", surface: "index" }); - return new Response(artifact.body, { headers: { "content-type": artifact.mediaType } }); +export async function GET({ request }) { + const payload = await getLlmsPayload({ scope: "site", surface: "index" }, { request }); + return new Response(payload.body, { headers: { "content-type": payload.mediaType } }); }`, "utf8", ); await writeFile( path.join(root, "src/pages/llms-full.txt.ts"), - `import { getPreparedLlmsArtifact } from ${JSON.stringify(buildModule)}; + `import { getLlmsPayload } from ${JSON.stringify(endpointsModule)}; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ scope: "site", surface: "full" }); - return new Response(artifact.body, { headers: { "content-type": artifact.mediaType } }); +export async function GET({ request }) { + const payload = await getLlmsPayload({ scope: "site", surface: "full" }, { request }); + return new Response(payload.body, { headers: { "content-type": payload.mediaType } }); }`, "utf8", ); await writeFile( path.join(root, "src/pages/[section]/llms.txt.ts"), - `import { getPreparedLlmsArtifact, getPreparedLlmsStaticPaths } from ${JSON.stringify(buildModule)}; + `import { getLlmsPayload } from ${JSON.stringify(endpointsModule)}; +import { getPreparedLlmsRouteStaticPaths } from ${JSON.stringify(publicationModule)}; export const prerender = true; -export const getStaticPaths = () => getPreparedLlmsStaticPaths(); -export async function GET({ props }) { - const artifact = await getPreparedLlmsArtifact(props.artifact); - return new Response(artifact.body, { headers: { "content-type": artifact.mediaType } }); +export const getStaticPaths = () => getPreparedLlmsRouteStaticPaths(); +export async function GET({ props, request }) { + const payload = await getLlmsPayload(props.artifact, { request }); + return new Response(payload.body, { headers: { "content-type": payload.mediaType } }); }`, "utf8", ); @@ -180,14 +185,14 @@ export async function GET({ props }) { assert.doesNotMatch(llmsFull, / { + const packageJson = JSON.parse( + readFileSync(new URL("../package.json", import.meta.url), "utf8"), + ) as { exports: Record }; + assert.ok(packageJson.exports["./agent-endpoints"]); + assert.ok(packageJson.exports["./build"]); + assert.ok(packageJson.exports["./publication"]); + assert.deepEqual(Object.keys(agentEndpoints).sort(), [ + "getLlmsPayload", + "getLlmsStaticPaths", + "getMarkdownPayload", + "getMarkdownStaticPaths", + ]); + assert.deepEqual(Object.keys(publication).sort(), [ + "getPreparedLlmsRouteArtifact", + "getPreparedLlmsRouteStaticPaths", + "getPreparedMarkdownRouteArtifact", + "getPreparedMarkdownRouteStaticPaths", + ]); + assert.deepEqual(Object.keys(buildApi).sort(), [ + "getPreparedLlmsArtifact", + "getPreparedLlmsStaticPaths", + "getPreparedMarkdownArtifact", + "getPreparedMarkdownStaticPaths", + ]); + assert.equal( + publication.getPreparedLlmsRouteArtifact, + agentEndpoints.getLlmsPayload, + ); + assert.equal( + publication.getPreparedMarkdownRouteArtifact, + agentEndpoints.getMarkdownPayload, + ); +}); diff --git a/packages/nimbus-docs/test/build-report.test.ts b/packages/nimbus-docs/test/build-report.test.ts index 04004fe1..899b2744 100644 --- a/packages/nimbus-docs/test/build-report.test.ts +++ b/packages/nimbus-docs/test/build-report.test.ts @@ -1,214 +1,486 @@ -import { test } from "node:test"; import assert from "node:assert/strict"; +import { test } from "node:test"; import { analyzeBuild, formatInvariantFailure, + type FeatureRouteDeclaration, + type ManagedRouteDeclaration, type ResolvedRouteLike, + type UserRouteDeclaration, } from "../src/_internal/build-report.js"; +import { STARTER_ROUTE_INVENTORY } from "../src/_internal/route-ownership.js"; -// The real server-build route set captured from Astro 7 (no feature). -const SERVER_ROUTES: ResolvedRouteLike[] = [ - { pattern: "/_server-islands/[name]", type: "page", isPrerendered: false, origin: "internal" }, - { pattern: "/custom-image", type: "endpoint", isPrerendered: false, origin: "internal" }, - { pattern: "/404", type: "page", isPrerendered: false, origin: "internal" }, - { pattern: "/llms.txt", type: "endpoint", isPrerendered: true, origin: "project" }, - { pattern: "/[...slug]/index.md", type: "endpoint", isPrerendered: true, origin: "project" }, - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - { pattern: "/[...slug]", type: "page", isPrerendered: true, origin: "project" }, -]; - -test("server + no feature: infra on-demand routes are explained, 0 violations", () => { - const r = analyzeBuild({ - outputMode: "server", - adapterName: "node", - routes: SERVER_ROUTES, - prerenderedPageCount: 5, - }); - assert.deepEqual(r.violations, []); - assert.deepEqual(r.onDemandDocRoutes, [], "/_ infra routes are not counted as doc on-demand"); - assert.match(r.summaryLine, /output=server/); - assert.match(r.summaryLine, /adapter=node/); - assert.equal(r.fatal, null); - const full = analyzeBuild({ - outputMode: "server", - adapterName: "@astrojs/node", - routes: SERVER_ROUTES, - prerenderedPageCount: 5, - }); - assert.match(full.summaryLine, /adapter=node ·/); - assert.match(r.summaryLine, /on-demand routes=0/); - assert.match(r.summaryLine, /server features=\[\]/); +const canonical = ( + rendering: "build" | "request" = "build", +): ManagedRouteDeclaration => ({ + pattern: "/[...slug]", + entrypoint: "src/pages/[...slug].astro", + owner: "canonical", + rendering, }); -test("Astro Actions are excluded by internal origin", () => { - const r = analyzeBuild({ +const feature: FeatureRouteDeclaration = { + pattern: "/mcp", + entrypoint: "src/pages/mcp.ts", + feature: "hosted-mcp", +}; + +const userExtensible: UserRouteDeclaration = { + pattern: "/", + entrypoint: "src/pages/index.astro", +}; + +function projectRoute( + pattern: string, + entrypoint: string, + isPrerendered: boolean, + type: "page" | "endpoint" = "page", +): ResolvedRouteLike { + return { pattern, entrypoint, type, isPrerendered, origin: "project" }; +} + +function externalRoute( + pattern: string, + entrypoint: string, + isPrerendered: boolean, + type: "page" | "endpoint" = "endpoint", +): ResolvedRouteLike { + return { pattern, entrypoint, type, isPrerendered, origin: "external" }; +} + +function report( + routes: readonly ResolvedRouteLike[], + overrides: Partial[0]> = {}, +) { + return analyzeBuild({ outputMode: "server", adapterName: "node", - routes: [ - ...SERVER_ROUTES, - { pattern: "/_actions/[...path]", type: "endpoint", isPrerendered: false, origin: "internal" }, - ], - prerenderedPageCount: 5, + routes, + prerenderedPageCount: 1, + ...overrides, }); - assert.deepEqual(r.violations, []); - assert.deepEqual(r.onDemandDocRoutes, []); - assert.equal(r.fatal, null); +} + +test("project-owned pages and endpoints may render on request without Nimbus registration", () => { + for (const adapterName of ["cloudflare", "vercel", "netlify", "node"]) { + const result = report( + [ + projectRoute("/foo", "src/pages/foo.astro", false), + projectRoute("/api/ping", "src/pages/api/ping.ts", false, "endpoint"), + projectRoute("/products/[id]", "src/pages/products/[id].astro", false), + ], + { adapterName }, + ); + assert.deepEqual(result.violations, []); + assert.equal(result.fatal, null); + assert.deepEqual(result.customOnDemandRoutes, [ + "/foo", + "/api/ping", + "/products/[id]", + ]); + assert.match(result.summaryLine, /custom on-demand routes=3/); + } }); -test("server build with no resolved routes is a fatal reporter malfunction, not a pass", () => { - const r = analyzeBuild({ - outputMode: "server", - adapterName: "node", - routes: [], - prerenderedPageCount: 0, - }); - assert.deepEqual(r.violations, []); - assert.match(r.fatal ?? "", /CANNOT BE VERIFIED/); +test("a custom prerendered route remains static and is reported separately", () => { + const result = report([ + projectRoute("/static", "src/pages/static.astro", true), + ]); + assert.deepEqual(result.violations, []); + assert.deepEqual(result.customPrerenderedRoutes, ["/static"]); + assert.deepEqual(result.customOnDemandRoutes, []); + assert.match(result.summaryLine, /custom on-demand routes=0/); }); -test("static build with no resolved routes does not trip the fatal guard", () => { - const r = analyzeBuild({ - outputMode: "static", - adapterName: null, - routes: [], - prerenderedPageCount: 0, - }); - assert.equal(r.fatal, null); +test("canonical collection routes must match entrypoint and rendering policy", () => { + const allowed = report( + [projectRoute("/[...slug]", "src/pages/[...slug].astro", false)], + { + managedRoutes: [canonical("request")], + requestRenderedPageCount: 100, + prerenderedPageCount: 3, + }, + ); + assert.deepEqual(allowed.violations, []); + assert.deepEqual(allowed.nimbusRequestRoutes, ["/[...slug]"]); + assert.match(allowed.summaryLine, /docs prerendered=3\/103 \(100 moved\)/); + assert.match(allowed.summaryLine, /nimbus request routes=1/); + + const buildDrift = report( + [projectRoute("/[...slug]", "src/pages/[...slug].astro", false)], + { managedRoutes: [canonical("build")] }, + ); + assert.deepEqual(buildDrift.violations, ["/[...slug]"]); + + const requestDrift = report( + [projectRoute("/[...slug]", "src/pages/[...slug].astro", true)], + { managedRoutes: [canonical("request")] }, + ); + assert.deepEqual(requestDrift.violations, ["/[...slug]"]); }); -test("a project on-demand route is an unexplained violation", () => { - const routes: ResolvedRouteLike[] = [ - { pattern: "/custom-image", type: "endpoint", isPrerendered: false, origin: "internal" }, - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - { pattern: "/[...slug]", type: "page", isPrerendered: false, origin: "project" }, - ]; - const r = analyzeBuild({ - outputMode: "server", - adapterName: "vercel", - routes, - prerenderedPageCount: 1, - }); - assert.deepEqual(r.violations, ["/[...slug]"]); +test("conflicting active declarations for one route are rejected", () => { + assert.throws( + () => + report( + [projectRoute("/[...slug]", "src/pages/[...slug].astro", false)], + { + managedRoutes: [canonical("request")], + featureRoutes: [ + { + pattern: "/[...slug]", + entrypoint: "src/pages/[...slug].astro", + feature: "conflicting-feature", + }, + ], + }, + ), + /matches multiple active declarations/, + ); }); -test("a doc route forced on-demand fails the invariant", () => { - const routes: ResolvedRouteLike[] = [ - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - { pattern: "/llms.txt", type: "endpoint", isPrerendered: false, origin: "project" }, - ]; - const r = analyzeBuild({ - outputMode: "server", - adapterName: "node", - routes, - prerenderedPageCount: 1, - }); - assert.deepEqual(r.violations, ["/llms.txt"]); - assert.match(r.summaryLine, /\(0 moved\)/); +test("scaffolded Markdown and llms.txt endpoints remain user-owned", () => { + for (const route of STARTER_ROUTE_INVENTORY.filter( + (candidate) => candidate.role === "user-owned", + )) { + const result = report([ + projectRoute( + route.pattern, + `src/${route.entrypoint}`, + false, + route.entrypoint.endsWith(".astro") ? "page" : "endpoint", + ), + ]); + assert.deepEqual(result.violations, [], route.entrypoint); + assert.deepEqual(result.customOnDemandRoutes, [route.pattern]); + } }); -test("declared feature routes explain a non-`/_` on-demand route", () => { - const routes: ResolvedRouteLike[] = [ - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - { pattern: "/mcp", type: "endpoint", isPrerendered: false, origin: "external" }, - ]; - const r = analyzeBuild({ - outputMode: "server", - adapterName: "cloudflare", - routes, - prerenderedPageCount: 1, - declaredFeatureRoutes: ["/mcp"], - serverFeatures: ["hosted-mcp"], - }); - assert.deepEqual(r.violations, []); - assert.deepEqual(r.onDemandDocRoutes, ["/mcp"]); - assert.match(r.summaryLine, /on-demand routes=1 \(\/mcp\)/); - assert.match(r.summaryLine, /server features=\[hosted-mcp\]/); +test("feature routes require matching feature, pattern, and entrypoint", () => { + const allowed = report( + [projectRoute("/mcp", "src/pages/mcp.ts", false, "endpoint")], + { featureRoutes: [feature], serverFeatures: ["hosted-mcp"] }, + ); + assert.deepEqual(allowed.violations, []); + assert.deepEqual(allowed.featureRoutes, ["/mcp"]); + assert.deepEqual(allowed.onDemandDocRoutes, ["/mcp"]); + assert.match(allowed.summaryLine, /feature routes=1 \(\/mcp\)/); + assert.match(allowed.summaryLine, /server features=\[hosted-mcp\]/); + + const impersonator = report( + [projectRoute("/mcp", "src/pages/not-mcp.ts", false, "endpoint")], + { featureRoutes: [feature] }, + ); + assert.deepEqual(impersonator.violations, ["/mcp"]); + + const patternDrift = report( + [projectRoute("/other", "src/pages/mcp.ts", false, "endpoint")], + { featureRoutes: [feature] }, + ); + assert.deepEqual(patternDrift.violations, ["/other", "/mcp"]); + + const inactiveProjectRoute = report([ + projectRoute("/mcp", "src/pages/not-mcp.ts", false, "endpoint"), + ]); + assert.deepEqual(inactiveProjectRoute.violations, []); + assert.deepEqual(inactiveProjectRoute.customOnDemandRoutes, ["/mcp"]); + + const inactiveIntegrationRoute = report([ + externalRoute("/mcp", "node_modules/example/mcp.ts", false), + ]); + assert.deepEqual(inactiveIntegrationRoute.violations, []); + assert.deepEqual(inactiveIntegrationRoute.integrationOnDemandRoutes, [ + "/mcp", + ]); }); -test("declared request routes are explained and counted as moved docs", () => { - const r = analyzeBuild({ - outputMode: "server", - adapterName: "cloudflare", - routes: [ +test("every active managed and feature declaration must resolve exactly once", () => { + const missingManaged = report( + [projectRoute("/custom", "src/pages/custom.astro", false)], + { managedRoutes: [canonical("request")] }, + ); + assert.deepEqual(missingManaged.violations, ["/[...slug]"]); + + const missingFeature = report( + [projectRoute("/custom", "src/pages/custom.astro", false)], + { featureRoutes: [feature] }, + ); + assert.deepEqual(missingFeature.violations, ["/mcp"]); + + const duplicateManaged = report( + [ + projectRoute("/[...slug]", "src/pages/[...slug].astro", false), + projectRoute("/[...slug]", "src/pages/[...slug].astro", false), + ], + { managedRoutes: [canonical("request")] }, + ); + assert.deepEqual(duplicateManaged.violations, ["/[...slug]"]); +}); + +test("same-pattern impersonation of a managed route fails closed", () => { + const canonicalImpersonator = report( + [projectRoute("/[...slug]", "src/pages/not-docs.astro", true)], + { managedRoutes: [canonical()] }, + ); + assert.deepEqual(canonicalImpersonator.violations, ["/[...slug]"]); +}); + +test("unrelated integration routes compose and are reported separately", () => { + const result = report([ + externalRoute("/integration/static", "node_modules/example/static.ts", true), + externalRoute( + "/integration/request", + "node_modules/example/request.ts", + false, + ), + ]); + assert.deepEqual(result.violations, []); + assert.deepEqual(result.integrationPrerenderedRoutes, [ + "/integration/static", + ]); + assert.deepEqual(result.integrationOnDemandRoutes, [ + "/integration/request", + ]); + assert.match(result.summaryLine, /integration prerendered routes=1/); + assert.match(result.summaryLine, /integration on-demand routes=1/); +}); + +test("declared Nimbus infrastructure is allowed by exact identity", () => { + const infrastructure: ManagedRouteDeclaration = { + pattern: "/_nimbus/request-route-inventory.json", + entrypoint: + "node_modules/@cloudflare/nimbus-docs/dist/_internal/request-route-inventory.js", + owner: "infrastructure", + rendering: "build", + }; + const result = report( + [externalRoute(infrastructure.pattern, infrastructure.entrypoint, true)], + { managedRoutes: [infrastructure] }, + ); + assert.deepEqual(result.violations, []); + assert.equal(result.fatal, null); + + const renderingDrift = report( + [externalRoute(infrastructure.pattern, infrastructure.entrypoint, false)], + { managedRoutes: [infrastructure] }, + ); + assert.deepEqual(renderingDrift.violations, [infrastructure.pattern]); + + const projectImpersonator = report( + [ + projectRoute( + infrastructure.pattern, + "src/pages/_nimbus/request-route-inventory.json.ts", + true, + "endpoint", + ), + ], + { managedRoutes: [infrastructure] }, + ); + assert.deepEqual(projectImpersonator.violations, [infrastructure.pattern]); +}); + +test("missing entrypoint provenance is a fatal metadata failure", () => { + for (const isPrerendered of [false, true]) { + const result = report([ { - pattern: "/[...slug]", + pattern: "/foo", type: "page", - isPrerendered: false, + isPrerendered, origin: "project", }, + ]); + assert.deepEqual(result.violations, []); + assert.match(result.fatal ?? "", /CANNOT BE VERIFIED/); + assert.match(result.fatal ?? "", /entrypoint metadata/); + } +}); + +test("catch-all overlap is allowed but an exact published-content collision fails", () => { + const free = report( + [ + projectRoute("/[...slug]", "src/pages/[...slug].astro", true), + projectRoute("/foo", "src/pages/foo.astro", false), ], - prerenderedPageCount: 3, - requestRenderedPageCount: 100, - declaredRequestRoutes: ["/[...slug]"], - }); + { + managedRoutes: [canonical("build")], + contentRoutePatterns: ["/bar"], + }, + ); + assert.deepEqual(free.violations, []); - assert.deepEqual(r.violations, []); - assert.deepEqual(r.onDemandDocRoutes, ["/[...slug]"]); - assert.match(r.summaryLine, /docs prerendered=3\/103 \(100 moved\)/); -}); + const dynamicOverlap = report( + [ + projectRoute("/[...slug]", "src/pages/[...slug].astro", true), + projectRoute( + "/[section]/[slug]", + "src/pages/[section]/[slug].astro", + false, + ), + projectRoute("/[...path]", "src/pages/[...path].astro", false), + ], + { managedRoutes: [canonical("build")] }, + ); + assert.deepEqual(dynamicOverlap.violations, []); -test("static build: adapter=none, on-demand routes=0, no server-features field", () => { - const routes: ResolvedRouteLike[] = [ - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - { pattern: "/[...slug]", type: "page", isPrerendered: true, origin: "project" }, - ]; - const r = analyzeBuild({ - outputMode: "static", - adapterName: null, - routes, - prerenderedPageCount: 2, - }); - assert.deepEqual(r.violations, []); - assert.equal( - r.summaryLine, - "nimbus: output=static · adapter=none · docs prerendered=2/2 · on-demand routes=0", + const owned = report( + [ + projectRoute("/[...slug]", "src/pages/[...slug].astro", true), + projectRoute("/foo", "src/pages/foo.astro", false), + ], + { + managedRoutes: [canonical("build")], + contentRoutePatterns: ["/foo"], + }, + ); + assert.deepEqual(owned.violations, ["/foo"]); + + const staticOwned = report( + [projectRoute("/foo", "src/pages/foo.astro", true)], + { contentRoutePatterns: ["/foo"] }, + ); + assert.deepEqual(staticOwned.violations, ["/foo"]); + + const intentionalRoot = report( + [projectRoute("/", "src/pages/index.astro", true)], + { + contentRoutePatterns: ["/"], + userExtensibleRoutes: [userExtensible], + }, ); + assert.deepEqual(intentionalRoot.violations, []); }); -test("redirect/fallback routes are ignored by the invariant", () => { - const routes: ResolvedRouteLike[] = [ - { pattern: "/old", type: "redirect", isPrerendered: false, origin: "project" }, - { pattern: "/fb", type: "fallback", isPrerendered: false, origin: "project" }, - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - ]; - const r = analyzeBuild({ - outputMode: "server", - adapterName: "node", - routes, - prerenderedPageCount: 1, - }); - assert.deepEqual(r.violations, []); +test("Astro internal routes are ignored", () => { + const result = report([ + { + pattern: "/_actions/[...path]", + type: "endpoint", + isPrerendered: false, + origin: "internal", + }, + ]); + assert.deepEqual(result.violations, []); + assert.deepEqual(result.onDemandDocRoutes, []); }); -test("a project route under an Astro-looking path still fails", () => { - const r = analyzeBuild({ - outputMode: "server", - adapterName: "node", - routes: [ - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, - { pattern: "/_actions/custom", type: "endpoint", isPrerendered: false, origin: "project" }, - ], - prerenderedPageCount: 1, - }); - assert.deepEqual(r.violations, ["/_actions/custom"]); +test("unrelated project redirects and fallbacks preserve native Astro behavior", () => { + const result = report([ + projectRoute("/home", "src/pages/home.astro", true), + { + pattern: "/old", + type: "redirect", + isPrerendered: false, + origin: "project", + }, + { + pattern: "/fallback", + type: "fallback", + isPrerendered: false, + origin: "project", + }, + { + pattern: "/integration-redirect", + type: "redirect", + isPrerendered: false, + origin: "external", + }, + { + pattern: "/integration-fallback", + type: "fallback", + isPrerendered: false, + origin: "external", + }, + ]); + assert.deepEqual(result.violations, []); + assert.equal(result.fatal, null); }); -test("server build with only internal routes is a fatal reporter malfunction", () => { - const r = analyzeBuild({ - outputMode: "server", - adapterName: "node", - routes: [ - { pattern: "/custom-image", type: "endpoint", isPrerendered: false, origin: "internal" }, +test("redirects and internal routes cannot impersonate managed patterns", () => { + const infrastructure: ManagedRouteDeclaration = { + pattern: "/_nimbus/request-route-inventory.json", + entrypoint: + "node_modules/@cloudflare/nimbus-docs/dist/_internal/request-route-inventory.js", + owner: "infrastructure", + rendering: "build", + }; + const redirect = report( + [ + projectRoute("/home", "src/pages/home.astro", true), + { + pattern: infrastructure.pattern, + type: "redirect", + isPrerendered: false, + origin: "project", + }, + ], + { managedRoutes: [infrastructure] }, + ); + assert.match(redirect.fatal ?? "", /entrypoint metadata/); + + const internal = report( + [ + projectRoute("/home", "src/pages/home.astro", true), + { + pattern: infrastructure.pattern, + type: "endpoint", + isPrerendered: false, + origin: "internal", + }, + ], + { managedRoutes: [infrastructure] }, + ); + assert.match(internal.fatal ?? "", /entrypoint metadata/); + + const contentRedirect = report( + [ + projectRoute("/home", "src/pages/home.astro", true), + { + pattern: "/guide", + type: "redirect", + isPrerendered: false, + origin: "project", + }, ], - prerenderedPageCount: 0, + { contentRoutePatterns: ["/guide"] }, + ); + assert.deepEqual(contentRedirect.violations, []); +}); + +test("server builds with no reportable routes fail verification", () => { + const empty = report([]); + assert.match(empty.fatal ?? "", /CANNOT BE VERIFIED/); + + const internalOnly = report([ + { + pattern: "/custom-image", + type: "endpoint", + isPrerendered: false, + origin: "internal", + }, + ]); + assert.match(internalOnly.fatal ?? "", /CANNOT BE VERIFIED/); +}); + +test("static builds do not require resolved route metadata", () => { + const result = analyzeBuild({ + outputMode: "static", + adapterName: null, + routes: [], + prerenderedPageCount: 2, }); - assert.match(r.fatal ?? "", /CANNOT BE VERIFIED/); + assert.equal(result.fatal, null); + assert.equal( + result.summaryLine, + "nimbus: output=static · adapter=none · docs prerendered=2/2 · custom prerendered routes=0 · integration prerendered routes=0", + ); }); -test("failure message lists every unexplained route", () => { - const msg = formatInvariantFailure(["/a", "/b"]); - assert.match(msg, /2 unexplained on-demand routes/); - assert.match(msg, /- \/a/); - assert.match(msg, /- \/b/); +test("failure message explains exact ownership requirements", () => { + const message = formatInvariantFailure(["/a", "/b"]); + assert.match(message, /2 route violations/); + assert.match(message, /- \/a/); + assert.match(message, /- \/b/); + assert.match(message, /Custom project and unrelated integration routes/); }); diff --git a/packages/nimbus-docs/test/footprint.test.ts b/packages/nimbus-docs/test/footprint.test.ts index a8f7ecc9..6f98ca60 100644 --- a/packages/nimbus-docs/test/footprint.test.ts +++ b/packages/nimbus-docs/test/footprint.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { deriveFootprint, + footprintRoutes, type FeatureRecipe, } from "../src/_internal/footprint.js"; @@ -18,6 +19,7 @@ const RECIPES: FeatureRecipe[] = [ requires: "server", env: [{ name: "MCP_PROVIDER_TOKEN", kind: "runtime" }], dep: "@cloudflare/nimbus-mcp", + routes: [{ pattern: "/mcp", entrypoint: "src/pages/mcp.ts" }], }, ]; @@ -32,9 +34,21 @@ test("deriveFootprint selects recipes whose dep is present", () => { test("deriveFootprint returns nothing when no recipe dep is present", () => { const deps = new Set(["astro", "@astrojs/mdx"]); - assert.deepEqual(deriveFootprint(deps, RECIPES), []); + const footprint = deriveFootprint(deps, RECIPES); + assert.deepEqual(footprint, []); + assert.deepEqual(footprintRoutes(footprint), []); }); test("deriveFootprint defaults to the (empty) first-party recipe set", () => { assert.deepEqual(deriveFootprint(new Set(["anything"])), []); }); + +test("footprint routes retain feature and entrypoint ownership", () => { + assert.deepEqual(footprintRoutes([RECIPES[1]!, RECIPES[1]!]), [ + { + pattern: "/mcp", + entrypoint: "src/pages/mcp.ts", + feature: "hosted-mcp", + }, + ]); +}); diff --git a/packages/nimbus-docs/test/integration-dist-invariant.test.ts b/packages/nimbus-docs/test/integration-dist-invariant.test.ts index 27671319..64b80f92 100644 --- a/packages/nimbus-docs/test/integration-dist-invariant.test.ts +++ b/packages/nimbus-docs/test/integration-dist-invariant.test.ts @@ -101,6 +101,7 @@ async function driveBuild( const infos: string[] = []; const warnings: string[] = []; + const injectedRoutes: ResolvedRouteLike[] = []; const logger = { info: (m: string) => infos.push(m), warn: (m: string) => warnings.push(m), @@ -138,6 +139,19 @@ async function driveBuild( }, logger, command: "build", + injectRoute: (route: { + pattern: string; + entrypoint: URL; + prerender?: boolean; + }) => { + injectedRoutes.push({ + pattern: route.pattern, + entrypoint: route.entrypoint.href, + type: "endpoint", + isPrerendered: route.prerender === true, + origin: "external", + }); + }, } as never); hooks["astro:config:done"]!({ @@ -149,9 +163,9 @@ async function driveBuild( }, } as never); - if (opts.routes) { - hooks["astro:routes:resolved"]!({ routes: opts.routes } as never); - } + hooks["astro:routes:resolved"]!({ + routes: [...injectedRoutes, ...(opts.routes ?? [])], + } as never); if (opts.seedRedirects !== undefined) { await writeFile( @@ -161,12 +175,23 @@ async function driveBuild( ); } - const runBuild = () => - hooks["astro:build:done"]!({ + const runBuild = async () => { + const inventory = path.join( + distDir, + "_nimbus/request-route-inventory.json", + ); + await mkdir(path.dirname(inventory), { recursive: true }); + await writeFile( + inventory, + JSON.stringify([{ collection: "docs", url: "/" }]), + "utf8", + ); + await hooks["astro:build:done"]!({ dir: dirUrl(distDir), pages: [{ pathname: "/" }], logger, - } as never) as Promise; + } as never); + }; await runBuild(); @@ -251,7 +276,15 @@ test("server output with no adapter is not the static lane → no _redirects", a const { distEntries } = await driveBuild(t, { signal: "cloudflare", output: "server", - routes: [{ pattern: "/", type: "page", isPrerendered: true, origin: "project" }], + routes: [ + { + pattern: "/", + entrypoint: "src/pages/index.astro", + type: "page", + isPrerendered: true, + origin: "project", + }, + ], redirects: { "/old": "/new" }, }); assert.deepEqual(distEntries, BASELINE_DIST); @@ -268,11 +301,92 @@ test("custom Astro infrastructure routes are excluded by origin", async (t) => { isPrerendered: false, origin: "internal", }, - { pattern: "/", type: "page", isPrerendered: true, origin: "project" }, + { + pattern: "/", + entrypoint: "src/pages/index.astro", + type: "page", + isPrerendered: true, + origin: "project", + }, ], }); assert.deepEqual(distEntries, BASELINE_DIST); - assert.ok(infos.some((message) => /on-demand routes=0/.test(message))); + assert.ok(infos.some((message) => /custom on-demand routes=0/.test(message))); +}); + +test("project pages and endpoints reach build completion as custom on-demand routes", async (t) => { + const { infos, projectRoot } = await driveBuild(t, { + output: "server", + adapter: "@astrojs/node", + base: "/docs", + routes: [ + { + pattern: "/foo", + entrypoint: "src/pages/foo.astro", + type: "page", + isPrerendered: false, + origin: "project", + }, + { + pattern: "/api/ping", + entrypoint: "src/pages/api/ping.ts", + type: "endpoint", + isPrerendered: false, + origin: "project", + }, + { + pattern: "/dynamic/[slug]", + entrypoint: "src/pages/dynamic/[slug].ts", + type: "endpoint", + isPrerendered: false, + origin: "project", + }, + ], + }); + assert.ok( + infos.some((message) => + /custom on-demand routes=3 \(\/foo, \/api\/ping, \/dynamic\/\[slug\]\)/.test( + message, + ), + ), + ); + const routeTruth = JSON.parse( + await readFile(path.join(projectRoot, ".nimbus/routes.json"), "utf8"), + ); + assert.equal(routeTruth.base, "/docs"); + assert.deepEqual( + routeTruth.knownRoutes, + ["/", "/api/ping", "/foo"], + ); +}); + +test("unrelated integration routes reach build completion separately", async (t) => { + const { infos, projectRoot } = await driveBuild(t, { + output: "server", + adapter: "@astrojs/node", + routes: [ + { + pattern: "/integration/status", + entrypoint: "node_modules/example-integration/status.ts", + type: "endpoint", + isPrerendered: false, + origin: "external", + }, + ], + }); + assert.ok( + infos.some((message) => + /integration on-demand routes=1 \(\/integration\/status\)/.test( + message, + ), + ), + ); + assert.deepEqual( + JSON.parse( + await readFile(path.join(projectRoot, ".nimbus/routes.json"), "utf8"), + ).knownRoutes, + ["/", "/integration/status"], + ); }); test("a pre-existing dist/_redirects is preserved and the emit is idempotent", async (t) => { diff --git a/packages/nimbus-docs/test/rendering-policy.test.ts b/packages/nimbus-docs/test/rendering-policy.test.ts index f0a60af7..d7835787 100644 --- a/packages/nimbus-docs/test/rendering-policy.test.ts +++ b/packages/nimbus-docs/test/rendering-policy.test.ts @@ -33,6 +33,7 @@ import { preparedMarkdownRootKey, } from "../src/_internal/prepared-markdown-registry.js"; import { + contentInventoryEntryUrl, requestInventoryEntryUrl, requestInventoryVersionStatusKey, } from "../src/_internal/request-route-url.js"; @@ -68,6 +69,16 @@ test("request inventory preserves prose ids and only collapses the API root", () assert.equal(requestInventoryVersionStatusKey("api", true, "v1"), "api@v1"); }); +test("content inventory uses final IDs and each collection's actual render mode", () => { + assert.equal( + contentInventoryEntryUrl("", "1.1.1.1/encryption", false, false), + "/1.1.1.1/encryption", + ); + assert.equal(contentInventoryEntryUrl("", "index", false, false), "/"); + assert.equal(contentInventoryEntryUrl("", "index", false, true), "/index"); + assert.equal(contentInventoryEntryUrl("/api", "index", true, true), "/api"); +}); + test("request inventory reader removes root and base-prefixed candidates", async (t) => { const root = await mkdtemp(path.join(tmpdir(), "nimbus-request-inventory-")); t.after(() => rm(root, { recursive: true, force: true })); @@ -266,7 +277,9 @@ async function setupIntegration( command: "dev" | "build" = "dev", contentConfig = 'export const collections = { docs: {}, blog: {}, "docs-v1": {} };\n', api?: NimbusConfig["api"], - integrationOptions: Partial = {}, + integrationOptions: Partial & { + omitCanonicalDocsRoute?: boolean; + } = {}, base = "", trailingSlash: "always" | "never" | "ignore" = "ignore", ) { @@ -280,7 +293,10 @@ async function setupIntegration( }; await write("src/content.config.ts", contentConfig); await write("src/components.ts", "export const components = {};\n"); - await write("src/pages/[...slug].astro", "---\n---\n"); + const { omitCanonicalDocsRoute = false, ...options } = integrationOptions; + if (!omitCanonicalDocsRoute) { + await write("src/pages/[...slug].astro", "---\n---\n"); + } await write("src/pages/blog/[...slug].astro", "---\n---\n"); await write("src/pages/v1/[...slug].astro", "---\n---\n"); await write("src/pages/api/[...slug].astro", "---\n---\n"); @@ -300,7 +316,7 @@ async function setupIntegration( admonitions: false, sitemap: false, markdown: { processor: {} as never }, - ...integrationOptions, + ...options, }, ); const setup = integration.hooks["astro:config:setup"]; @@ -357,6 +373,48 @@ async function setupIntegration( }; } +function resolvedNimbusRoutes( + injectedRoutes: readonly unknown[], + docsRendering: "build" | "request", +) { + return [ + { + pattern: "/[...slug]", + entrypoint: "src/pages/[...slug].astro", + type: "page", + isPrerendered: docsRendering === "build", + origin: "project", + }, + { + pattern: "/blog/[...slug]", + entrypoint: "src/pages/blog/[...slug].astro", + type: "page", + isPrerendered: true, + origin: "project", + }, + { + pattern: "/v1/[...slug]", + entrypoint: "src/pages/v1/[...slug].astro", + type: "page", + isPrerendered: true, + origin: "project", + }, + ...injectedRoutes.map((route) => { + const injected = route as { pattern: string; entrypoint: string | URL }; + return { + pattern: injected.pattern, + entrypoint: + injected.entrypoint instanceof URL + ? injected.entrypoint.href + : injected.entrypoint, + type: "endpoint", + isPrerendered: true, + origin: "project", + }; + }), + ]; +} + const buildLogger = { info: () => {}, warn: () => {}, @@ -393,15 +451,7 @@ async function generateRequestSitemap( config: { output: "server", adapter: { name: "cloudflare" } }, buildOutput: "server", } as never); - const routes = [ - { - pattern: "/[...slug]", - entrypoint: "src/pages/[...slug].astro", - type: "page", - isPrerendered: false, - origin: "project", - }, - ]; + const routes = resolvedNimbusRoutes(integration.injectedRoutes, "request"); integration.routesResolved({ routes } as never); const sitemapIntegration = integration.configUpdates @@ -671,7 +721,9 @@ test("request inventory is removed before downstream build failures", async (t) config: { output: "server", adapter: { name: "cloudflare" } }, buildOutput: "server", } as never); - integration.routesResolved({ routes: [] } as never); + integration.routesResolved({ + routes: resolvedNimbusRoutes(integration.injectedRoutes, "build"), + } as never); const dist = path.join(integration.root, "dist"); const inventory = path.join(dist, "_nimbus/request-route-inventory.json"); await mkdir(path.dirname(inventory), { recursive: true }); @@ -728,16 +780,25 @@ test("omitted rendering policy leaves existing route decisions untouched", async assert.equal(docs.prerender, false); assert.equal(blog.prerender, true); - assert.equal(integration.injectedRoutes.length, 0); + assert.equal(integration.injectedRoutes.length, 1); integration.configDone({ injectTypes: () => new URL("file:///noop"), config: { output: "static" }, buildOutput: "static", } as never); - integration.routesResolved({ routes: [] } as never); + integration.routesResolved({ + routes: resolvedNimbusRoutes(integration.injectedRoutes, "build"), + } as never); + const dist = path.join(integration.root, "dist"); + await mkdir(path.join(dist, "_nimbus"), { recursive: true }); + await writeFile( + path.join(dist, "_nimbus/request-route-inventory.json"), + "[]", + "utf8", + ); await integration.buildDone({ - dir: pathToFileURL(`${path.join(integration.root, "dist")}${path.sep}`), + dir: pathToFileURL(`${dist}${path.sep}`), pages: [{ pathname: "/_nimbus/request-route-inventory.json" }], logger: { info: () => {}, @@ -752,9 +813,7 @@ test("omitted rendering policy leaves existing route decisions untouched", async const routeTruth = JSON.parse( await readFile(path.join(integration.root, ".nimbus/routes.json"), "utf8"), ); - assert.deepEqual(routeTruth.knownRoutes, [ - "/_nimbus/request-route-inventory.json", - ]); + assert.deepEqual(routeTruth.knownRoutes, []); }); test("opaque version registrations still reach the request inventory", async (t) => { @@ -858,6 +917,40 @@ test("production request rendering requires server output and an adapter", async ); }); +test("required canonical routes retain rendering policy when their file is missing", async (t) => { + const integration = await setupIntegration( + t, + { default: "request" }, + "build", + 'export const collections = { docs: {}, blog: {}, "docs-v1": {} };\n', + undefined, + { omitCanonicalDocsRoute: true }, + ); + const canonical = { + component: "src/pages/[...slug].astro", + prerender: true, + }; + const moved = { + component: "src/pages/docs/[...slug].astro", + prerender: true, + }; + + await integration.routeSetup({ route: canonical } as never); + await integration.routeSetup({ route: moved } as never); + + assert.equal(canonical.prerender, false); + assert.equal(moved.prerender, true); + assert.throws( + () => + integration.configDone({ + injectTypes: () => new URL("file:///noop"), + config: { output: "static", adapter: null }, + buildOutput: "static", + } as never), + /requires Astro `output: "server"` and a compatible adapter/, + ); +}); + test("production API request rendering is accepted with model packaging", async (t) => { const integration = await setupIntegration( t, @@ -908,15 +1001,7 @@ test("configured request routes are explained to the build invariant", async (t) buildOutput: "server", } as never); integration.routesResolved({ - routes: [ - { - pattern: "/[...slug]", - entrypoint: "src/pages/[...slug].astro", - type: "page", - isPrerendered: false, - origin: "project", - }, - ], + routes: resolvedNimbusRoutes(integration.injectedRoutes, "request"), } as never); const preparedRoot = preparedMarkdownRootKey(integration.root); for (const [collection, entries] of [ @@ -977,7 +1062,7 @@ test("configured request routes are explained to the build invariant", async (t) ); assert.equal(route.prerender, false); assert.ok( - infos.some((message) => /docs prerendered=2\/3 \(1 moved\)/.test(message)), + infos.some((message) => /docs prerendered=1\/2 \(1 moved\)/.test(message)), ); assert.deepEqual( JSON.parse( diff --git a/packages/nimbus-docs/test/route-ownership.test.ts b/packages/nimbus-docs/test/route-ownership.test.ts new file mode 100644 index 00000000..eae7115e --- /dev/null +++ b/packages/nimbus-docs/test/route-ownership.test.ts @@ -0,0 +1,97 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { test } from "node:test"; + +import { + isRequiredCanonicalRouteComponent, + normalizeRouteEntrypoint, + normalizeSourceRouteEntrypoint, +} from "../src/_internal/route-ownership.js"; + +test("route entrypoints normalize relative, absolute, and file URL identities", () => { + const root = path.join(path.sep, "workspace", "site"); + const entrypoint = path.join(root, "src", "pages", "foo.astro"); + assert.equal( + normalizeRouteEntrypoint(root, entrypoint), + "src/pages/foo.astro", + ); + assert.equal( + normalizeRouteEntrypoint(root, pathToFileURL(entrypoint).href), + "src/pages/foo.astro", + ); + assert.equal( + normalizeRouteEntrypoint(root, "src\\pages\\foo.astro"), + "src/pages/foo.astro", + ); +}); + +test("source-relative declarations follow a custom Astro srcDir", () => { + const root = path.join(path.sep, "workspace", "site"); + const srcDir = path.join(root, "app"); + assert.equal( + normalizeSourceRouteEntrypoint(root, srcDir, "pages/mcp.ts"), + "app/pages/mcp.ts", + ); + assert.equal( + normalizeSourceRouteEntrypoint(root, srcDir, "src/pages/mcp.ts"), + "app/pages/mcp.ts", + ); +}); + +test("required canonical routes are recognized independently of filesystem state", () => { + const root = path.join(path.sep, "workspace", "site"); + const srcDir = path.join(root, "app"); + assert.equal( + isRequiredCanonicalRouteComponent( + root, + srcDir, + path.join(srcDir, "pages", "[...slug].astro"), + ), + true, + ); + assert.equal( + isRequiredCanonicalRouteComponent( + root, + srcDir, + path.join(srcDir, "pages", "docs", "[...slug].astro"), + ), + false, + ); +}); + +test("Windows entrypoints retain stable project-relative identities", () => { + const root = "C:\\workspace\\site"; + const srcDir = `${root}\\app`; + assert.equal( + normalizeRouteEntrypoint( + root, + `${root}\\src\\pages\\foo.astro?astro&type=script`, + ), + "src/pages/foo.astro", + ); + assert.equal( + normalizeSourceRouteEntrypoint(root, srcDir, "pages/mcp.ts"), + "app/pages/mcp.ts", + ); +}); + +test("unstable route entrypoint metadata is rejected", () => { + const root = path.join(path.sep, "workspace", "site"); + assert.equal(normalizeRouteEntrypoint(root, ""), null); + assert.equal(normalizeRouteEntrypoint(root, null), null); + assert.equal(normalizeRouteEntrypoint(root, 42), null); + assert.equal(normalizeRouteEntrypoint(root, "file://%"), null); + assert.equal(normalizeRouteEntrypoint(root, " virtual:route"), null); + assert.equal(normalizeRouteEntrypoint(root, "virtual:route"), null); + assert.equal(normalizeRouteEntrypoint(root, "https://example.com/route"), null); + assert.equal(normalizeRouteEntrypoint(root, "src/pages/foo.astro\n"), null); +}); + +test("relative entrypoints resolve to stable project-relative identities", () => { + const root = path.join(path.sep, "workspace", "site"); + assert.equal( + normalizeRouteEntrypoint(root, "src/pages/../pages/foo.astro?astro&type=script"), + "src/pages/foo.astro", + ); +}); diff --git a/packages/nimbus-docs/test/transform-citations.test.ts b/packages/nimbus-docs/test/transform-citations.test.ts index 6503209a..665b6502 100644 --- a/packages/nimbus-docs/test/transform-citations.test.ts +++ b/packages/nimbus-docs/test/transform-citations.test.ts @@ -54,7 +54,7 @@ describe("renderEntryAsMarkdown: coordinate citations", () => { test("rejects runtime partial expansion with migration guidance", () => { assert.throws( () => renderEntryAsMarkdown({ body: '' }), - /prepared artifact helpers/, + /getMarkdownPayload/, ); assert.doesNotThrow(() => renderEntryAsMarkdown({ body: '```mdx\n\n```' }), diff --git a/packages/nimbus-docs/tsdown.config.ts b/packages/nimbus-docs/tsdown.config.ts index d380b1bd..4695349e 100644 --- a/packages/nimbus-docs/tsdown.config.ts +++ b/packages/nimbus-docs/tsdown.config.ts @@ -9,6 +9,8 @@ export default defineConfig({ entry: { index: "src/index.ts", runtime: "src/runtime.ts", + "agent-endpoints": "src/agent-endpoints.ts", + publication: "src/publication.ts", build: "src/build.ts", config: "src/config.ts", content: "src/content.ts", @@ -25,8 +27,8 @@ export default defineConfig({ "_internal/request-route-inventory": "src/_internal/request-route-inventory.ts", "_internal/git-last-updated": "src/_internal/git-last-updated.ts", - "_internal/prepared-artifacts": - "src/_internal/prepared-artifacts.ts", + "_internal/agent-endpoint-assets": + "src/_internal/agent-endpoint-assets.ts", "_internal/api-loader": "src/_internal/api-loader.ts", }, format: "esm", diff --git a/packages/nimbus-starter-source/src/pages/[...slug]/index.md.ts b/packages/nimbus-starter-source/src/pages/[...slug]/index.md.ts index 06af0fe0..b56cca5f 100644 --- a/packages/nimbus-starter-source/src/pages/[...slug]/index.md.ts +++ b/packages/nimbus-starter-source/src/pages/[...slug]/index.md.ts @@ -1,21 +1,37 @@ import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; +import { agentEndpointResponse } from "../../utils/agent-endpoint-response"; export const prerender = true; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: "docs", surface: "markdown" }); +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export const getStaticPaths = async () => + getMarkdownStaticPaths({ + collection: "docs", + surface: "markdown", }); + +export async function GET({ params, props, request }: SlugContext) { + return agentEndpointResponse(() => + getMarkdownPayload({ + collection: "docs", + surface: "markdown", + slug: params.slug, + reference: props.reference, + context: { request }, + }), + prerender, + ); } diff --git a/packages/nimbus-starter-source/src/pages/[...slug]/index.mdx.ts b/packages/nimbus-starter-source/src/pages/[...slug]/index.mdx.ts index e94421a0..92a6d266 100644 --- a/packages/nimbus-starter-source/src/pages/[...slug]/index.mdx.ts +++ b/packages/nimbus-starter-source/src/pages/[...slug]/index.mdx.ts @@ -1,21 +1,37 @@ import { - getPreparedMarkdownArtifact, - getPreparedMarkdownStaticPaths, - type PreparedMarkdownReference, -} from "@cloudflare/nimbus-docs/build"; + getMarkdownPayload, + getMarkdownStaticPaths, + type MarkdownEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; +import { agentEndpointResponse } from "../../utils/agent-endpoint-response"; export const prerender = true; interface SlugProps { - artifact: PreparedMarkdownReference; + reference: MarkdownEndpointReference; } -export const getStaticPaths = () => - getPreparedMarkdownStaticPaths({ collection: "docs", surface: "source" }); +interface SlugContext { + params: { slug?: string }; + props: Partial; + request: Request; +} -export async function GET({ props }: { props: SlugProps }) { - const artifact = await getPreparedMarkdownArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, +export const getStaticPaths = async () => + getMarkdownStaticPaths({ + collection: "docs", + surface: "source", }); + +export async function GET({ params, props, request }: SlugContext) { + return agentEndpointResponse(() => + getMarkdownPayload({ + collection: "docs", + surface: "source", + slug: params.slug, + reference: props.reference, + context: { request }, + }), + prerender, + ); } diff --git a/packages/nimbus-starter-source/src/pages/[section]/llms.txt.ts b/packages/nimbus-starter-source/src/pages/[section]/llms.txt.ts index 8fa2cd59..245d86bb 100644 --- a/packages/nimbus-starter-source/src/pages/[section]/llms.txt.ts +++ b/packages/nimbus-starter-source/src/pages/[section]/llms.txt.ts @@ -1,20 +1,40 @@ import { - getPreparedLlmsArtifact, - getPreparedLlmsStaticPaths, - type PreparedLlmsReference, -} from "@cloudflare/nimbus-docs/build"; + getLlmsPayload, + getLlmsStaticPaths, + type LlmsEndpointReference, +} from "@cloudflare/nimbus-docs/agent-endpoints"; +import { agentEndpointResponse } from "../../utils/agent-endpoint-response"; export const prerender = true; interface SectionProps { - artifact: PreparedLlmsReference; + reference: LlmsEndpointReference; } -export const getStaticPaths = () => getPreparedLlmsStaticPaths(); +interface SectionContext { + params: { section?: string }; + props: Partial; + request: Request; +} + +export const getStaticPaths = async () => + getLlmsStaticPaths(); -export async function GET({ props }: { props: SectionProps }) { - const artifact = await getPreparedLlmsArtifact(props.artifact); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, - }); +export async function GET({ params, props, request }: SectionContext) { + const reference = + props.reference ?? + (params.section + ? ({ + scope: "section", + surface: "index", + section: params.section, + } satisfies LlmsEndpointReference) + : null); + if (!reference) return new Response("Not found", { status: 404 }); + return agentEndpointResponse(() => + getLlmsPayload(reference, { + request, + }), + prerender, + ); } diff --git a/packages/nimbus-starter-source/src/pages/llms-full.txt.ts b/packages/nimbus-starter-source/src/pages/llms-full.txt.ts index 3bcee441..24a5fff3 100644 --- a/packages/nimbus-starter-source/src/pages/llms-full.txt.ts +++ b/packages/nimbus-starter-source/src/pages/llms-full.txt.ts @@ -1,13 +1,17 @@ -import { getPreparedLlmsArtifact } from "@cloudflare/nimbus-docs/build"; +import { getLlmsPayload } from "@cloudflare/nimbus-docs/agent-endpoints"; +import { agentEndpointResponse } from "../utils/agent-endpoint-response"; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ - scope: "site", - surface: "full", - }); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, - }); +export async function GET(context: { request: Request }) { + return agentEndpointResponse(() => + getLlmsPayload( + { + scope: "site", + surface: "full", + }, + context, + ), + prerender, + ); } diff --git a/packages/nimbus-starter-source/src/pages/llms.txt.ts b/packages/nimbus-starter-source/src/pages/llms.txt.ts index 142c3a9c..4f69eacd 100644 --- a/packages/nimbus-starter-source/src/pages/llms.txt.ts +++ b/packages/nimbus-starter-source/src/pages/llms.txt.ts @@ -1,13 +1,17 @@ -import { getPreparedLlmsArtifact } from "@cloudflare/nimbus-docs/build"; +import { getLlmsPayload } from "@cloudflare/nimbus-docs/agent-endpoints"; +import { agentEndpointResponse } from "../utils/agent-endpoint-response"; export const prerender = true; -export async function GET() { - const artifact = await getPreparedLlmsArtifact({ - scope: "site", - surface: "index", - }); - return new Response(artifact.body, { - headers: { "Content-Type": artifact.mediaType }, - }); +export async function GET(context: { request: Request }) { + return agentEndpointResponse(() => + getLlmsPayload( + { + scope: "site", + surface: "index", + }, + context, + ), + prerender, + ); } diff --git a/packages/nimbus-starter-source/src/utils/agent-endpoint-response.ts b/packages/nimbus-starter-source/src/utils/agent-endpoint-response.ts new file mode 100644 index 00000000..0068d977 --- /dev/null +++ b/packages/nimbus-starter-source/src/utils/agent-endpoint-response.ts @@ -0,0 +1,21 @@ +interface AgentEndpointPayload { + body: string; + mediaType: string; +} + +export async function agentEndpointResponse( + load: () => Promise, + prerender: boolean, +): Promise { + try { + const payload = await load(); + if (!payload) return new Response("Not found", { status: 404 }); + return new Response(payload.body, { + headers: { "Content-Type": payload.mediaType }, + }); + } catch (error) { + if (prerender) throw error; + console.error(error); + return new Response("Internal Server Error", { status: 500 }); + } +} diff --git a/scripts/templates-check.mjs b/scripts/templates-check.mjs index 4d910cf3..5ce68767 100644 --- a/scripts/templates-check.mjs +++ b/scripts/templates-check.mjs @@ -10,15 +10,18 @@ * scaffold resolves the in-repo code, not whatever is on npm). */ -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { + existsSync, mkdtempSync, + mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; +import { createServer } from "node:net"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { generateTemplates } from "../packages/create-nimbus-docs/scripts/copy-template.mjs"; @@ -69,6 +72,150 @@ function ok(msg) { console.log(`[templates-check] ok — ${msg}`); } +async function availablePort() { + return new Promise((resolvePort, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("could not reserve a runtime verification port")); + return; + } + server.close((error) => (error ? reject(error) : resolvePort(address.port))); + }); + }); +} + +async function verifyRuntime(site, lane) { + const port = await availablePort(); + const origin = `http://127.0.0.1:${port}`; + const command = + lane === "node" + ? { + bin: process.execPath, + args: [join(site, "dist", "server", "entry.mjs")], + env: { HOST: "127.0.0.1", PORT: String(port) }, + } + : { + bin: SCAFFOLD_PM_BIN, + args: [ + ...SCAFFOLD_PM_PREFIX, + "exec", + "wrangler", + "dev", + "--config", + "dist/server/wrangler.json", + "--ip", + "127.0.0.1", + "--port", + String(port), + ], + env: {}, + }; + const child = spawn(command.bin, command.args, { + cwd: site, + env: { ...process.env, ...command.env }, + stdio: "inherit", + }); + const routes = [ + ["/custom-default", "custom-default"], + ["/custom-false", "custom-false"], + ["/api/ping-default", "ping-default"], + ["/api/ping-false", "ping-false"], + ["/404", "Page not found", 404], + ["/robots.txt", "User-agent: *"], + ["/llms.txt", "Renamed route", 200, "Hidden runtime page"], + ["/llms-full.txt", "Renamed route", 200, "Hidden runtime page"], + ["/nimbus-api/coordinates.json", '"version":1'], + ["/owned-by-slug/index.md", "This text lives in"], + ["/owned-by-slug/index.mdx", "This text lives in"], + ["/runtime-section/llms.txt", "Runtime section one", 200, "Hidden runtime page"], + ["/dynamic/free", "dynamic-overlap"], + ]; + try { + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`runtime exited with status ${child.exitCode}`); + } + try { + const response = await fetch(`${origin}${routes[0][0]}`, { + signal: AbortSignal.timeout(1_000), + }); + if (response.ok) break; + } catch {} + await new Promise((resolveWait) => setTimeout(resolveWait, 250)); + } + for (const [route, expected, expectedStatus = 200, unexpected] of routes) { + const response = await fetch(`${origin}${route}`, { + signal: AbortSignal.timeout(5_000), + }); + const body = await response.text(); + if ( + response.status !== expectedStatus || + !body.includes(expected) || + (unexpected && body.includes(unexpected)) + ) { + throw new Error( + `${route} returned ${response.status} without ${JSON.stringify(expected)}: ${JSON.stringify(body.slice(0, 300))}`, + ); + } + } + if (lane === "cloudflare") { + const manifest = JSON.parse( + readFileSync( + join(site, ".astro", "nimbus", "agent-endpoint-assets", "manifest.json"), + "utf8", + ), + ); + const asset = manifest.markdownAssets.find( + (entry) => + entry.collection === "docs" && + entry.id === "owned-by-slug" && + entry.surface === "markdown", + ); + if (!asset) throw new Error("runtime fixture has no known Markdown asset"); + rmSync( + join( + site, + "dist", + "client", + "_nimbus", + "agent-endpoint-assets", + asset.path, + ), + ); + const missingAsset = await fetch(`${origin}/owned-by-slug/index.md`, { + signal: AbortSignal.timeout(5_000), + }); + const missingAssetBody = await missingAsset.text(); + if ( + missingAsset.status !== 500 || + missingAssetBody !== "Internal Server Error" || + !missingAsset.headers.get("content-type")?.startsWith("text/plain") + ) { + throw new Error( + `known missing asset returned ${missingAsset.status}: ${JSON.stringify(missingAssetBody.slice(0, 300))}`, + ); + } + const unknown = await fetch(`${origin}/missing/index.md`, { + signal: AbortSignal.timeout(5_000), + }); + if (unknown.status !== 404) { + throw new Error(`unknown Markdown endpoint returned ${unknown.status}`); + } + } + } finally { + child.kill("SIGTERM"); + await Promise.race([ + new Promise((resolveClose) => child.once("close", resolveClose)), + new Promise((resolveWait) => setTimeout(resolveWait, 5_000)), + ]); + } +} + // 1. Build framework + scaffolder, then generate every variant. console.log( `[templates-check] ${LANE} scaffold install/build via ${SCAFFOLD_PNPM ? `corepack ${SCAFFOLD_PNPM}` : "ambient pnpm"}`, @@ -103,6 +250,100 @@ const scaffoldArgs = [ if (LANE !== "static") scaffoldArgs.push("--adapter", LANE); run("node", scaffoldArgs, { cwd: work }); const site = join(work, "ci-site"); +mkdirSync(join(site, "src", "pages", "api"), { recursive: true }); +writeFileSync( + join(site, "src", "pages", "custom-static.astro"), + "---\nexport const prerender = true;\n---\n

custom-static

\n", +); +if (LANE !== "static") { + writeFileSync( + join(site, "src", "pages", "custom-default.astro"), + "---\n---\n

custom-default

\n", + ); + writeFileSync( + join(site, "src", "pages", "custom-false.astro"), + "---\nexport const prerender = false;\n---\n

custom-false

\n", + ); + writeFileSync( + join(site, "src", "pages", "api", "ping-default.ts"), + 'export function GET() { return new Response("ping-default"); }\n', + ); + writeFileSync( + join(site, "src", "pages", "api", "ping-false.ts"), + 'export const prerender = false;\nexport function GET() { return new Response("ping-false"); }\n', + ); + const notFoundPath = join(site, "src", "pages", "404.astro"); + writeFileSync( + notFoundPath, + readFileSync(notFoundPath, "utf8").replace( + "export const prerender = true;", + "export const prerender = false;", + ), + ); + const robotsPath = join(site, "src", "pages", "robots.txt.ts"); + writeFileSync( + robotsPath, + readFileSync(robotsPath, "utf8").replace( + "export const prerender = true;\n\n", + "", + ), + ); + for (const route of [ + join(site, "src", "pages", "llms.txt.ts"), + join(site, "src", "pages", "llms-full.txt.ts"), + join(site, "src", "pages", "nimbus-api", "coordinates.json.ts"), + join(site, "src", "pages", "og.png.ts"), + join(site, "src", "pages", "og", "[...slug].ts"), + join(site, "src", "pages", "[...slug]", "index.md.ts"), + join(site, "src", "pages", "[...slug]", "index.mdx.ts"), + join(site, "src", "pages", "[section]", "llms.txt.ts"), + ]) { + writeFileSync( + route, + readFileSync(route, "utf8").replace( + "export const prerender = true;", + "export const prerender = false;", + ), + ); + } + mkdirSync(join(site, "src", "pages", "dynamic"), { recursive: true }); + writeFileSync( + join(site, "src", "pages", "dynamic", "[slug].astro"), + "---\n---\n

dynamic-overlap

\n", + ); +} +const contentConfigPath = join(site, "src", "content.config.ts"); +const contentConfig = readFileSync(contentConfigPath, "utf8"); +const schemaFields = "schemaFields: {"; +if (!contentConfig.includes(schemaFields)) { + fail("starter content config has no schemaFields fixture seam"); +} +writeFileSync( + contentConfigPath, + contentConfig.replace( + schemaFields, + `${schemaFields}\n slug: z.string().optional(),`, + ), +); +writeFileSync( + join(site, "src", "content", "docs", "route-source.mdx"), + '---\ntitle: Renamed route\nslug: owned-by-slug\n---\n\nFinal Astro IDs own routes.\n\n\n', +); +mkdirSync(join(site, "src", "content", "docs", "runtime-section"), { + recursive: true, +}); +writeFileSync( + join(site, "src", "content", "docs", "runtime-section", "one.mdx"), + "---\ntitle: Runtime section one\n---\n\nOne.\n", +); +writeFileSync( + join(site, "src", "content", "docs", "runtime-section", "two.mdx"), + "---\ntitle: Runtime section two\n---\n\nTwo.\n", +); +writeFileSync( + join(site, "src", "content", "docs", "runtime-section", "hidden.mdx"), + "---\ntitle: Hidden runtime page\nnoindex: true\nslug: runtime-hidden\n---\n\nHidden.\n", +); const nimbusJson = JSON.parse(readFileSync(join(site, "nimbus.json"), "utf8")); if (LANE === "static") { if (nimbusJson.serverOutput !== undefined) { @@ -141,6 +382,76 @@ run(SCAFFOLD_PM_BIN, [...SCAFFOLD_PM_PREFIX, "install", "--no-frozen-lockfile"], run(SCAFFOLD_PM_BIN, [...SCAFFOLD_PM_PREFIX, "typecheck"], { cwd: site }); run(SCAFFOLD_PM_BIN, [...SCAFFOLD_PM_PREFIX, "build"], { cwd: site }); +const staticRouteCandidates = [ + join(site, "dist", "custom-static", "index.html"), + join(site, "dist", "client", "custom-static", "index.html"), +]; +if (!staticRouteCandidates.some(existsSync)) { + fail(`${LANE} scaffold did not emit the explicit prerender=true route`); +} +if (LANE === "node" || LANE === "cloudflare") { + const routeTruth = JSON.parse( + readFileSync(join(site, ".nimbus", "routes.json"), "utf8"), + ); + for (const route of ["/custom-default", "/custom-false", "/api/ping-false"]) { + if (!routeTruth.knownRoutes.includes(route)) { + fail(`${LANE} route truth omits custom on-demand route ${route}`); + } + } + if (routeTruth.knownRoutes.includes("/dynamic/[slug]")) { + fail(`${LANE} route truth includes a non-concrete dynamic route pattern`); + } + if (LANE === "node") { + rmSync(join(site, ".astro", "nimbus", "agent-endpoint-assets"), { + recursive: true, + force: true, + }); + } + try { + await verifyRuntime(site, LANE); + } catch (error) { + fail(`${LANE} runtime verification failed: ${error.message}`); + } + ok(`${LANE} serves custom, scaffolded, and dynamic request routes`); +} +if (LANE === "cloudflare") { + rmSync(join(site, "src", "pages", "[...slug].astro")); + const missingCanonical = spawnSync( + SCAFFOLD_PM_BIN, + [...SCAFFOLD_PM_PREFIX, "build"], + { cwd: site, encoding: "utf8" }, + ); + const output = `${missingCanonical.stdout ?? ""}\n${missingCanonical.stderr ?? ""}`; + if ( + missingCanonical.status === 0 || + !/route ownership invariant FAILED/.test(output) || + !output.includes("/[...slug]") + ) { + fail("missing canonical request route did not fail ownership validation"); + } + ok("missing canonical request route fails ownership validation"); +} +if (LANE === "node") { + writeFileSync( + join(site, "src", "pages", "owned-by-slug.astro"), + "---\nexport const prerender = true;\n---\n

collision

\n", + ); + const collision = spawnSync( + SCAFFOLD_PM_BIN, + [...SCAFFOLD_PM_PREFIX, "build"], + { cwd: site, encoding: "utf8" }, + ); + const output = `${collision.stdout ?? ""}\n${collision.stderr ?? ""}`; + if ( + collision.status === 0 || + !/route ownership invariant FAILED/.test(output) || + !output.includes("/owned-by-slug") + ) { + fail("final Astro content IDs did not block a custom static route collision"); + } + ok("final Astro content IDs block custom route collisions"); +} + const installed = JSON.parse( readFileSync(join(site, "node_modules", NIMBUS_NAME, "package.json"), "utf8"), ); diff --git a/scripts/worker-size-budget.json b/scripts/worker-size-budget.json index 96d43ef1..94089538 100644 --- a/scripts/worker-size-budget.json +++ b/scripts/worker-size-budget.json @@ -6,7 +6,7 @@ "maxBytes": 6500000, "maxGzipBytes": 1500000 }, - "preparedSource": { + "agentEndpointSource": { "baselineBytes": 1024, "baselineGzipBytes": 768, "maxBytes": 4096, diff --git a/scripts/workers-feasibility-check.mjs b/scripts/workers-feasibility-check.mjs index 253bea36..a07a6883 100644 --- a/scripts/workers-feasibility-check.mjs +++ b/scripts/workers-feasibility-check.mjs @@ -548,13 +548,18 @@ function assertSizeBudgets(site) { `Worker output is ${workerGzipBytes} gzip bytes; budget is ${SIZE_BUDGET.worker.maxGzipBytes}`, ); - const preparedArtifactRoot = join(site, ".astro", "nimbus", "prepared-artifacts"); + const agentEndpointAssetRoot = join( + site, + ".astro", + "nimbus", + "agent-endpoint-assets", + ); const manifest = JSON.parse( - readFileSync(join(preparedArtifactRoot, "manifest.json"), "utf8"), + readFileSync(join(agentEndpointAssetRoot, "manifest.json"), "utf8"), ); - const sourceBodies = manifest.markdownArtifacts - .filter((artifact) => artifact.surface === "source") - .map((artifact) => readFileSync(join(preparedArtifactRoot, artifact.path))); + const sourceBodies = manifest.markdownAssets + .filter((asset) => asset.surface === "source") + .map((asset) => readFileSync(join(agentEndpointAssetRoot, asset.path))); const sourceBytes = sourceBodies.reduce( (total, body) => total + body.length, 0, @@ -564,12 +569,12 @@ function assertSizeBudgets(site) { 0, ); assert( - sourceBytes <= SIZE_BUDGET.preparedSource.maxBytes, - `prepared source is ${sourceBytes} bytes; budget is ${SIZE_BUDGET.preparedSource.maxBytes}`, + sourceBytes <= SIZE_BUDGET.agentEndpointSource.maxBytes, + `agent-endpoint source is ${sourceBytes} bytes; budget is ${SIZE_BUDGET.agentEndpointSource.maxBytes}`, ); assert( - sourceGzipBytes <= SIZE_BUDGET.preparedSource.maxGzipBytes, - `prepared source is ${sourceGzipBytes} gzip bytes; budget is ${SIZE_BUDGET.preparedSource.maxGzipBytes}`, + sourceGzipBytes <= SIZE_BUDGET.agentEndpointSource.maxGzipBytes, + `agent-endpoint source is ${sourceGzipBytes} gzip bytes; budget is ${SIZE_BUDGET.agentEndpointSource.maxGzipBytes}`, ); } @@ -647,9 +652,9 @@ const WORKER_TEXT_DENYLIST = [ /(?:@cloudflare\/nimbus-docs\/build|nimbus-docs[\\/](?:(?:src|dist)[\\/])?build\.(?:[cm]?[jt]s)|(?:from\s*|import\s*\(?|require\s*\()\s*["'](?:\.\.?[\\/])+build\.js["']|(?:^|[\\/])build-markdown(?:-[^\\/"']+)?\.js)/m, }, { - category: "prepared artifact", + category: "agent-endpoint asset", pattern: - /\.astro[\\/]nimbus[\\/]prepared-artifacts|nimbus\/prepared-artifacts\/manifest\.json/, + /\.astro[\\/]nimbus[\\/]agent-endpoint-assets|nimbus\/agent-endpoint-assets\/manifest\.json/, }, { category: "native binding", @@ -708,7 +713,7 @@ function assertWorkerPurityScanner() { ["/server/chunks/build-markdown-CX42.js", "build helper"], ["/node_modules/@cloudflare/nimbus-docs/src/build.ts", "build helper"], ["/node_modules/@cloudflare/nimbus-docs/dist/build.js", "build helper"], - [".astro/nimbus/prepared-artifacts/manifest.json", "prepared artifact"], + [".astro/nimbus/agent-endpoint-assets/manifest.json", "agent-endpoint asset"], ['require("binding.node")', "native binding"], ["/server/binding.node", "native binding"], ['WebAssembly.instantiate(atob("AGFzbAAAA"))', "embedded wasm"], @@ -872,24 +877,24 @@ for (const output of ["dist", ".astro", join("node_modules", ".vite")]) { } build(site, { docs: "build", api: "build" }); const firstWorkerBuild = directorySnapshot(join(site, "dist", "server")); -const firstPreparedArtifactBuild = directorySnapshot( - join(site, ".astro", "nimbus", "prepared-artifacts"), +const firstAgentEndpointAssetBuild = directorySnapshot( + join(site, ".astro", "nimbus", "agent-endpoint-assets"), ); for (const output of ["dist", ".astro", join("node_modules", ".vite")]) { rmSync(join(site, output), { recursive: true, force: true }); } build(site, { docs: "build", api: "build" }); const secondWorkerBuild = directorySnapshot(join(site, "dist", "server")); -const secondPreparedArtifactBuild = directorySnapshot( - join(site, ".astro", "nimbus", "prepared-artifacts"), +const secondAgentEndpointAssetBuild = directorySnapshot( + join(site, ".astro", "nimbus", "agent-endpoint-assets"), ); assert( JSON.stringify(secondWorkerBuild) === JSON.stringify(firstWorkerBuild), `two clean Worker builds differed: ${snapshotDifference(firstWorkerBuild, secondWorkerBuild).join(", ")}`, ); assert( - JSON.stringify(secondPreparedArtifactBuild) === JSON.stringify(firstPreparedArtifactBuild), - `two clean prepared-artifact builds differed: ${snapshotDifference(firstPreparedArtifactBuild, secondPreparedArtifactBuild).join(", ")}`, + JSON.stringify(secondAgentEndpointAssetBuild) === JSON.stringify(firstAgentEndpointAssetBuild), + `two clean agent-endpoint asset builds differed: ${snapshotDifference(firstAgentEndpointAssetBuild, secondAgentEndpointAssetBuild).join(", ")}`, ); const staticPages = captureStaticPages(site); const proseStatic = prosePages(staticPages);