diff --git a/layer/modules/markdown-rewrite.ts b/layer/modules/markdown-rewrite.ts index 45839aaf7..d762b69ce 100644 --- a/layer/modules/markdown-rewrite.ts +++ b/layer/modules/markdown-rewrite.ts @@ -1,144 +1,104 @@ -import { defineNuxtModule, logger } from '@nuxt/kit' +import { addServerHandler, createResolver, defineNuxtModule } from '@nuxt/kit' +import { readFile, rm, writeFile } from 'node:fs/promises' import { resolve } from 'node:path' -import { readFile, writeFile } from 'node:fs/promises' - -const log = logger.withTag('docus') +import { createCloudflareModuleWorkerRoutes, createMarkdownRoutes, createVercelNegotiationRoutes, getPrerenderedHtmlPaths, type VercelRoute } from './runtime/server/utils/markdown-negotiation' type I18nLocale = string | { code: string } type DocusI18nOptions = { locales?: I18nLocale[] } +type DocusRuntimeConfig = { + docus?: { + markdownNegotiation?: { + locales?: string[] + routes?: Record + } + } +} +type DocusCloudflareConfig = { + cloudflare?: { + wrangler?: { + assets?: { + run_worker_first?: boolean | string[] + } + } + } +} export default defineNuxtModule({ meta: { name: 'markdown-rewrite', }, setup(_options, nuxt) { + const { resolve: resolveLayer } = createResolver(import.meta.url) + const i18nOptions = (nuxt.options as typeof nuxt.options & { i18n?: DocusI18nOptions }).i18n + const runtimeConfig = nuxt.options.runtimeConfig as DocusRuntimeConfig + runtimeConfig.docus ||= {} + runtimeConfig.docus.markdownNegotiation = { + locales: (i18nOptions?.locales || []).map(locale => typeof locale === 'string' ? locale : locale.code), + routes: {}, + } + + addServerHandler({ + handler: resolveLayer('./runtime/server/middleware/markdown-negotiation'), + middleware: true, + }) + nuxt.hooks.hook('nitro:init', (nitro) => { - if (nitro.options.dev || !nitro.options.preset.includes('vercel')) { + if (nitro.options.dev) { return } - nitro.hooks.hook('compiled', async () => { - const vcJSON = resolve(nitro.options.output.dir, 'config.json') - const vcConfig = JSON.parse(await readFile(vcJSON, 'utf8')) - - // Check if llms.txt exists before setting up any routes - let llmsTxt - const llmsTxtPath = resolve(nitro.options.output.publicDir, 'llms.txt') - try { - llmsTxt = await readFile(llmsTxtPath, 'utf-8') - } - catch { - log.warn('llms.txt not found, skipping markdown redirect routes') - return - } - - // Always redirect / to /llms.txt and ensure plain text content type - const markdownHeaders = { - 'content-type': 'text/markdown; charset=utf-8', + nitro.hooks.hook('prerender:done', async () => { + const llmsText = await readFile(resolve(nitro.options.output.publicDir, 'llms.txt'), 'utf8') + .catch(() => '') + + const nitroRuntimeConfig = nitro.options.runtimeConfig as DocusRuntimeConfig + nitroRuntimeConfig.docus ||= {} + const routes = createMarkdownRoutes( + llmsText, + runtimeConfig.docus?.markdownNegotiation?.locales, + ) + nitroRuntimeConfig.docus.markdownNegotiation = { + locales: runtimeConfig.docus?.markdownNegotiation?.locales, + routes, } - const routes = [ - { - src: '^/$', - dest: '/llms.txt', - headers: markdownHeaders, - has: [{ type: 'header', key: 'accept', value: '(.*)text/markdown(.*)' }], - }, - { - src: '^/$', - dest: '/llms.txt', - headers: markdownHeaders, - has: [{ type: 'header', key: 'user-agent', value: 'curl/.*' }], - }, - ] - - // Check if i18n is enabled - const i18nOptions = (nuxt.options as typeof nuxt.options & { i18n?: DocusI18nOptions }).i18n - const isI18nEnabled = !!i18nOptions?.locales - let localeCodes: string[] = [] - - if (isI18nEnabled) { - // Get locale codes - const locales = i18nOptions?.locales || [] - localeCodes = locales.map((locale: I18nLocale) => { - return typeof locale === 'string' ? locale : locale.code - }) - - // Create a regex pattern for all locales (e.g., "en|fr|es") - const localePattern = localeCodes.join('|') - - // Add routes for each locale homepage: /{locale} → /llms.txt - routes.push( - { - src: `^/(${localePattern})$`, - dest: '/llms.txt', - headers: markdownHeaders, - has: [{ type: 'header', key: 'accept', value: '(.*)text/markdown(.*)' }], - }, - { - src: `^/(${localePattern})$`, - dest: '/llms.txt', - headers: markdownHeaders, - has: [{ type: 'header', key: 'user-agent', value: 'curl/.*' }], - }, + if (nitro.options.preset.includes('cloudflare-module')) { + const options = nitro.options as DocusCloudflareConfig + options.cloudflare ||= {} + options.cloudflare.wrangler ||= {} + options.cloudflare.wrangler.assets ||= {} + options.cloudflare.wrangler.assets.run_worker_first = createCloudflareModuleWorkerRoutes( + routes, + options.cloudflare.wrangler.assets.run_worker_first, ) } - - // Parse llms.txt to get all documentation pages - const urlRegex = /\[([^\]]+)\]\((https?:\/\/[^)]+)\)/g - const matches = [...llmsTxt.matchAll(urlRegex)] - - for (const match of matches) { - const url = match[2] - if (!url) continue - - try { - // Extract path from URL - const urlObj = new URL(url) - const rawPath = urlObj.pathname - - // Skip root path (already handled) - if (rawPath === '/') continue - - // Only process raw markdown URLs from llms.txt - if (!rawPath.startsWith('/raw/')) continue - - // Convert /raw/en/getting-started/installation.md to /en/getting-started/installation - const pagePath = rawPath.replace('/raw', '').replace(/\.md$/, '') - - // Skip locale homepages (e.g., /en, /fr) - they already redirect to /llms.txt - if (isI18nEnabled) { - const isLocaleHomepage = localeCodes.some(code => pagePath === `/${code}`) - if (isLocaleHomepage) continue - } - - // Add redirect routes: page URL → raw markdown URL - const docsRoutes = [ - { - src: `^${pagePath}$`, - dest: rawPath, - headers: markdownHeaders, - has: [{ type: 'header', key: 'accept', value: '(.*)text/markdown(.*)' }], - }, - { - src: `^${pagePath}$`, - dest: rawPath, - headers: markdownHeaders, - has: [{ type: 'header', key: 'user-agent', value: 'curl/.*' }], - }, - ] - routes.push(...docsRoutes) - } - catch { - // Skip invalid URLs - } + else if (!nitro.options.preset.includes('vercel')) { + await Promise.all(getPrerenderedHtmlPaths(routes).map(path => ( + rm(resolve(nitro.options.output.publicDir, path), { force: true }) + ))) } + }) - vcConfig.routes.unshift(...routes) + if (nitro.options.preset.includes('vercel')) { + nitro.hooks.hook('compiled', async () => { + const configPath = resolve(nitro.options.output.dir, 'config.json') + const vercelConfig = JSON.parse(await readFile(configPath, 'utf8')) as { + routes?: VercelRoute[] + } + const config = nitro.options.runtimeConfig as DocusRuntimeConfig + const routes = vercelConfig.routes || [] + const fallbackDestination = routes.find(route => route.src === '/(.*)' && route.dest)?.dest + const negotiationRoutes = createVercelNegotiationRoutes( + config.docus?.markdownNegotiation?.routes, + fallbackDestination, + ) + const filesystemIndex = routes.findIndex(route => route.handle === 'filesystem') + routes.splice(filesystemIndex < 0 ? 0 : filesystemIndex, 0, ...negotiationRoutes) + vercelConfig.routes = routes - await writeFile(vcJSON, JSON.stringify(vcConfig, null, 2), 'utf8') - log.info(`Successfully wrote ${routes.length} routes to ${vcJSON} (serve markdown content to AI agents)`) - }) + await writeFile(configPath, JSON.stringify(vercelConfig, null, 2), 'utf8') + }) + } }) }, }) diff --git a/layer/modules/runtime/server/middleware/markdown-negotiation.ts b/layer/modules/runtime/server/middleware/markdown-negotiation.ts new file mode 100644 index 000000000..09053519a --- /dev/null +++ b/layer/modules/runtime/server/middleware/markdown-negotiation.ts @@ -0,0 +1,49 @@ +import { appendResponseHeader, defineEventHandler, getRequestHeader, getRequestURL, setResponseHeader, setResponseStatus } from 'h3' +import { getMarkdownPath, negotiateContentType, withMarkdownHeaders } from '../utils/markdown-negotiation' + +type DocusRuntimeConfig = { + docus?: { + markdownNegotiation?: { + routes?: Record + } + } +} + +export default defineEventHandler(async (event) => { + if (event.method !== 'GET' && event.method !== 'HEAD') return + + const runtimeConfig = useRuntimeConfig(event) as ReturnType & DocusRuntimeConfig + const markdownPath = getMarkdownPath( + getRequestURL(event).pathname, + runtimeConfig.docus?.markdownNegotiation?.routes, + ) + if (!markdownPath) return + + const contentType = negotiateContentType( + getRequestHeader(event, 'accept'), + getRequestHeader(event, 'user-agent'), + ) + + if (!contentType) { + appendResponseHeader(event, 'vary', 'Accept') + setResponseStatus(event, 406, 'Not Acceptable') + setResponseHeader(event, 'content-type', 'text/plain; charset=utf-8') + return 'Not Acceptable\n\nAvailable: text/html, text/markdown\n' + } + + if (contentType === 'text/html') { + appendResponseHeader(event, 'vary', 'Accept') + appendResponseHeader(event, 'link', `<${markdownPath}>; rel="alternate"; type="text/markdown"`) + return + } + + try { + const response = await event.fetch(markdownPath, { method: 'GET', headers: { accept: '*/*' } }) + if (response.ok) return withMarkdownHeaders(response) + } + catch { + // Fall through to the original route. + } + + appendResponseHeader(event, 'vary', 'Accept') +}) diff --git a/layer/modules/runtime/server/utils/markdown-negotiation.ts b/layer/modules/runtime/server/utils/markdown-negotiation.ts new file mode 100644 index 000000000..68e5b9a9a --- /dev/null +++ b/layer/modules/runtime/server/utils/markdown-negotiation.ts @@ -0,0 +1,134 @@ +import Negotiator from 'negotiator' + +export type VercelRoute = { + handle?: string + src?: string + dest?: string + [key: string]: unknown +} + +export function negotiateContentType(accept?: string, userAgent?: string): 'text/html' | 'text/markdown' | undefined { + const mediaTypes = accept?.split(',').map(range => range.split(';')[0]?.trim().toLowerCase()) || [] + const hasExplicitDocumentType = mediaTypes.some(type => type === 'text/html' || type === 'text/markdown') + if (/^curl\//i.test(userAgent || '') && (!accept || (mediaTypes.includes('*/*') && !hasExplicitDocumentType))) { + return 'text/markdown' + } + + return new Negotiator({ headers: { accept } }).mediaType(['text/html', 'text/markdown']) as 'text/html' | 'text/markdown' | undefined +} + +export function createMarkdownRoutes( + llmsText = '', + locales: string[] = [], +): Record { + const routes: Record = {} + if (!llmsText) return routes + + routes['/'] = '/llms.txt' + for (const locale of locales) routes[`/${locale}`] = '/llms.txt' + + for (const [, link] of llmsText.matchAll(/\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g)) { + try { + const rawPath = new URL(link!, 'https://docus.local').pathname + if (!rawPath.startsWith('/raw/') || !rawPath.endsWith('.md')) continue + const pagePath = rawPath.slice(4, -3).replace(/\/index$/, '') || '/' + routes[pagePath] = rawPath + } + catch { + // Ignore malformed links in generated or user-authored llms.txt content. + } + } + + return routes +} + +export function getMarkdownPath(path: string, routes: Record = {}): string | undefined { + return routes[path === '/' ? path : path.replace(/\/+$/, '')] +} + +export function getPrerenderedHtmlPaths(routes: Record = {}): string[] { + return Object.keys(routes).flatMap((route) => { + const path = route === '/' ? 'index' : route.slice(1) + return [`${path}.html`, `${path}/index.html`] + }) +} + +export function createCloudflareModuleWorkerRoutes( + markdownRoutes: Record = {}, + current: boolean | string[] | undefined = [], +): boolean | string[] { + if (current === true) { + return true + } + + const workerRoutes = new Set(Array.isArray(current) ? current : []) + for (const path of Object.keys(markdownRoutes)) { + if (path === '/') { + workerRoutes.add(path) + continue + } + + const topLevelPath = `/${path.split('/')[1]}` + workerRoutes.add(topLevelPath) + workerRoutes.add(`${topLevelPath}/*`) + } + + return [...workerRoutes] +} + +export function createVercelNegotiationRoutes( + markdownRoutes: Record = {}, + destination = '/__fallback', +): VercelRoute[] { + return Object.keys(markdownRoutes).flatMap((path) => { + const src = `^${path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}${path === '/' ? '' : '/?'}$` + + return [ + { + src, + dest: destination, + has: [{ + type: 'header', + key: 'accept', + value: '.*[tT][eE][xX][tT]/[mM][aA][rR][kK][dD][oO][wW][nN].*', + }], + }, + { + src, + dest: destination, + has: [{ + type: 'header', + key: 'user-agent', + value: '[cC][uU][rR][lL]/.*', + }], + }, + { + src, + headers: { + link: `<${markdownRoutes[path]}>; rel="alternate"; type="text/markdown"`, + vary: 'Accept', + }, + continue: true, + }, + ] + }) +} + +export function withMarkdownHeaders(response: Response): Response { + const headers = new Headers(response.headers) + const vary = headers.get('vary') + headers.set('content-type', 'text/markdown; charset=utf-8') + + if (!vary) { + headers.set('vary', 'Accept') + } + else if (vary !== '*' && !vary.split(',').some(value => value.trim().toLowerCase() === 'accept')) { + headers.set('vary', `${vary}, Accept`) + } + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }) +} diff --git a/layer/package.json b/layer/package.json index 194ee84ea..22bfeaf85 100644 --- a/layer/package.json +++ b/layer/package.json @@ -44,12 +44,14 @@ "@shikijs/stream": "^4.3.1", "@shikijs/themes": "^4.3.1", "@takumi-rs/core": "^2.2.0", + "@types/negotiator": "^0.6.5", "@vueuse/core": "^14.3.0", "ai": "^7.0.29", "defu": "^6.1.7", "exsolve": "^1.1.0", "git-url-parse": "^16.1.0", "motion-v": "^2.3.0", + "negotiator": "1.0.0", "nuxt-llms": "^0.2.0", "nuxt-og-image": "~6.7.2", "pathe": "^2.0.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 72217ef7a..8c783a126 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,6 +187,9 @@ importers: '@takumi-rs/core': specifier: ^2.2.0 version: 2.2.0(csstype@3.2.3) + '@types/negotiator': + specifier: ^0.6.5 + version: 0.6.5 '@vueuse/core': specifier: ^14.3.0 version: 14.3.0(vue@3.5.39(typescript@6.0.3)) @@ -208,6 +211,9 @@ importers: motion-v: specifier: ^2.3.0 version: 2.3.0(@vueuse/core@14.3.0(vue@3.5.39(typescript@6.0.3)))(vue@3.5.39(typescript@6.0.3)) + negotiator: + specifier: 1.0.0 + version: 1.0.0 nuxt: specifier: 4.x version: 4.4.8(@babel/plugin-syntax-jsx@7.29.7(@babel/core@7.29.7))(@babel/plugin-syntax-typescript@7.29.7(@babel/core@7.29.7))(@parcel/watcher@2.5.6)(@types/node@26.1.1)(@vue/compiler-sfc@3.5.39)(better-sqlite3@12.11.1)(db0@0.3.4(better-sqlite3@12.11.1))(esbuild@0.28.1)(eslint@10.7.0(jiti@2.7.0))(ioredis@5.11.1)(lightningcss@1.32.0)(magicast@0.5.3)(meow@13.2.0)(optionator@0.9.4)(rollup-plugin-visualizer@7.0.1(rollup@4.62.2))(rollup@4.62.2)(srvx@0.11.22)(terser@5.49.0)(tsx@4.23.1)(typescript@6.0.3)(vite@7.3.6(@types/node@26.1.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.49.0)(tsx@4.23.1)(yaml@2.9.0))(vue-tsc@3.3.7(typescript@6.0.3))(yaml@2.9.0) @@ -3540,6 +3546,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/negotiator@0.6.5': + resolution: {integrity: sha512-MPOlB48mfWhoUlynY0ga7CFsXIPcH6vGPkjzXMn2p+4PH1QUyn2KPtw0hrLLmO6SaX4zse3X6h2x/083vveAlA==} + '@types/node@26.1.1': resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} @@ -11631,6 +11640,8 @@ snapshots: '@types/ms@2.1.0': {} + '@types/negotiator@0.6.5': {} + '@types/node@26.1.1': dependencies: undici-types: 8.3.0 diff --git a/test/markdown-negotiation.test.mts b/test/markdown-negotiation.test.mts new file mode 100644 index 000000000..492e7f942 --- /dev/null +++ b/test/markdown-negotiation.test.mts @@ -0,0 +1,128 @@ +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import { describe, it } from 'node:test' +import { + createCloudflareModuleWorkerRoutes, + createMarkdownRoutes, + createVercelNegotiationRoutes, + getMarkdownPath, + getPrerenderedHtmlPaths, + negotiateContentType, + withMarkdownHeaders, +} from '../layer/modules/runtime/server/utils/markdown-negotiation.ts' + +describe('Markdown negotiation', () => { + it('publishes negotiator with its declarations', async () => { + const packageJson = JSON.parse(await readFile(new URL('../layer/package.json', import.meta.url), 'utf8')) + + assert.equal(typeof packageJson.dependencies.negotiator, 'string') + assert.equal(typeof packageJson.dependencies['@types/negotiator'], 'string') + }) + + it('honors Accept quality and curl fallback', () => { + assert.equal(negotiateContentType('text/html, text/markdown;q=0.5'), 'text/html') + assert.equal(negotiateContentType('text/markdown, text/html;q=0.5'), 'text/markdown') + assert.equal(negotiateContentType('application/pdf'), undefined) + assert.equal(negotiateContentType('text/markdown;q=0'), undefined) + assert.equal(negotiateContentType('text/markdown;q=invalid'), undefined) + assert.equal(negotiateContentType('text/markdown;q=0, text/markdown;q=0.2'), 'text/markdown') + assert.equal(negotiateContentType('*/*', 'curl/8.7.1'), 'text/markdown') + assert.equal(negotiateContentType('*/*,image/webp', 'curl/8.7.1'), 'text/markdown') + assert.equal(negotiateContentType('text/html', 'curl/8.7.1'), 'text/html') + }) + + it('maps pages listed in llms.txt and accepts canonical trailing slashes', () => { + const routes = createMarkdownRoutes(` +- [Guide](https://docs.example.com/raw/docs/guide.md) +- [Blog](/raw/blog/agents.md) +- [Home](/raw/index.md) +- [French home](/raw/fr/index.md) +- [External](https://example.com/guide.md) +`, ['fr']) + + assert.deepEqual(routes, { + '/': '/raw/index.md', + '/fr': '/raw/fr/index.md', + '/docs/guide': '/raw/docs/guide.md', + '/blog/agents': '/raw/blog/agents.md', + }) + assert.equal(getMarkdownPath('/docs/guide/', routes), '/raw/docs/guide.md') + assert.equal(getMarkdownPath('/blog/agents/', routes), '/raw/blog/agents.md') + assert.equal(getMarkdownPath('/missing', routes), undefined) + assert.deepEqual(createMarkdownRoutes('', ['fr']), {}) + }) + + it('finds prerendered HTML files that would bypass negotiation', () => { + assert.deepEqual(getPrerenderedHtmlPaths({ + '/': '/raw/index.md', + '/docs/guide': '/raw/docs/guide.md', + }), [ + 'index.html', + 'index/index.html', + 'docs/guide.html', + 'docs/guide/index.html', + ]) + }) + + it('routes negotiated page groups through Cloudflare Workers Assets', () => { + const routes = createCloudflareModuleWorkerRoutes({ + '/': '/llms.txt', + '/about': '/raw/about.md', + '/docs/guide': '/raw/docs/guide.md', + '/blog/agents': '/raw/blog/agents.md', + }, ['/api/*']) + + assert.deepEqual(routes, [ + '/api/*', + '/', + '/about', + '/about/*', + '/docs', + '/docs/*', + '/blog', + '/blog/*', + ]) + assert.deepEqual(createCloudflareModuleWorkerRoutes({ '/docs/guide': '/raw/docs/guide.md' }, false), ['/docs', '/docs/*']) + assert.equal(createCloudflareModuleWorkerRoutes({}, true), true) + }) + + it('routes negotiated pages and trailing slashes through Vercel', () => { + const routes = createVercelNegotiationRoutes({ + '/': '/llms.txt', + '/guide/v1.0/(intro)': '/raw/guide/v1.0/(intro).md', + }, '/__fallback') + + assert.equal(routes.length, 6) + assert.equal(routes[0]?.src, '^/$') + assert.equal(routes[3]?.src, '^/guide/v1\\.0/\\(intro\\)/?$') + assert.deepEqual(routes[5], { + src: '^/guide/v1\\.0/\\(intro\\)/?$', + headers: { + link: '; rel="alternate"; type="text/markdown"', + vary: 'Accept', + }, + continue: true, + }) + }) + + it('returns Markdown without discarding existing response metadata', async () => { + const response = withMarkdownHeaders(new Response('# Guide', { + status: 206, + headers: { vary: 'Accept-Encoding' }, + })) + + assert.equal(response.status, 206) + assert.equal(response.headers.get('content-type'), 'text/markdown; charset=utf-8') + assert.equal(response.headers.get('vary'), 'Accept-Encoding, Accept') + assert.equal(await response.text(), '# Guide') + }) + + it('preserves existing Accept and wildcard Vary headers', () => { + assert.equal(withMarkdownHeaders(new Response('', { + headers: { vary: 'accept, Origin' }, + })).headers.get('vary'), 'accept, Origin') + assert.equal(withMarkdownHeaders(new Response('', { + headers: { vary: '*' }, + })).headers.get('vary'), '*') + }) +})