feat: refactor llms with Astro routing, MDX stripping, .md chat URLs (#493)

Author: nijoe1

packages/docs/docs-plugin.ts

@@ -1,3 +1,4 @@
+import { createRequire } from 'node:module'
 import path from 'node:path'
 import url from 'node:url'
 import type { StarlightPlugin } from '@astrojs/starlight/types'
@@ -105,13 +106,25 @@ export function docsPlugin(options: DocsPluginOptions = {}): StarlightPlugin {
             urlPath: getUrlPath(directory, astroConfig.base),
           }
 
+          const projectRoot = url.fileURLToPath(astroConfig.root)
+          const _require = createRequire(path.join(projectRoot, 'package.json'))
+          const resolvedPlugins = (options.typeDocOptions?.plugin ?? []).map(
+            (p) => {
+              try {
+                return _require.resolve(p.toString())
+              } catch {
+                logger.warn(
+                  `Could not resolve TypeDoc plugin "${p}", using as-is`
+                )
+                return p
+              }
+            }
+          )
+
           const app = await Application.bootstrapWithPlugins({
             ..._defaultTypeDocOptions,
             ...options.typeDocOptions,
-            plugin: [
-              ...(options.typeDocOptions?.plugin ?? []),
-              'typedoc-plugin-markdown',
-            ],
+            plugin: [...resolvedPlugins, 'typedoc-plugin-markdown'],
             outputs: [{ name: 'markdown', path: output.path }],
             readme: 'none',
             packageOptions: {

packages/docs/llms-plugin/PageTitle.astro

@@ -10,13 +10,13 @@ interface OptionsProps {
 }
 
 const currentPath = Astro.url.pathname.replace(/\/$/, "");
-const currentUrl = Astro.url.href;
+const mdUrl = `${Astro.site!}${currentPath.replace(/^\//, "")}.md`;
 const { actions, prompt: userPrompt } = config;
 const { pageActions } = (Astro.locals as any).starlightRoute.entry.data;
 
 const prompt = userPrompt?.includes("{url}")
-  ? userPrompt?.replace("{url}", currentUrl)
-  : `${userPrompt} ${currentUrl}`;
+  ? userPrompt?.replace("{url}", mdUrl)
+  : `${userPrompt} ${mdUrl}`;
 const encodedPrompt = encodeURIComponent(prompt as string);
 
 const defaultOptions: OptionsProps[] = [

packages/docs/llms-plugin/env.d.ts

@@ -1,5 +1,16 @@
 /// <reference types="node_modules/astro/types.d.ts" />
 
+declare module 'astro:config/client' {
+  export const site: string
+}
+
+declare module 'astro:content' {
+  export function getCollection(
+    collection: 'docs',
+    filter?: (doc: { data: { draft?: boolean } }) => boolean
+  ): Promise<import('./strip').DocEntry[]>
+}
+
 declare module 'virtual:llms-plugin/config' {
   const config: import('./index').PageActionsConfig
   export default config

packages/docs/llms-plugin/index.ts

@@ -1,7 +1,5 @@
 import type { StarlightPlugin } from '@astrojs/starlight/types'
 import { AstroError } from 'astro/errors'
-import { normalizePath } from 'vite'
-import { viteStaticCopy } from 'vite-plugin-static-copy'
 
 interface Actions {
   chatgpt?: boolean
@@ -23,6 +21,7 @@ export interface PageActionsConfig {
   actions?: Actions
   title?: string
   description?: string
+  maxDepth?: number
 }
 
 /**
@@ -37,7 +36,7 @@ export interface PageActionsConfig {
  * @param {string} [userConfig.prompt] - The prompt template for AI chat services. Use `{url}` as placeholder for the markdown URL.
  * @param {string} [userConfig.baseUrl] - The base URL of your site, required for generating the `llms.txt` file.
  * @param {Actions} [userConfig.actions] - Configure which built-in actions to display and define custom actions.
- *
+ * @param {number} [userConfig.maxDepth] - The maximum depth of the documentation tree to include in the `llms.txt` file. Defaults to 4.
  *
  * @example
  * ```javascript
@@ -52,6 +51,7 @@ export interface PageActionsConfig {
  *         llmsPlugin({
  *           prompt: "Read {url} and explain its main points briefly.",
  *           baseUrl: "https://mydocs.example.com",
+ *           maxDepth: 3,
  *           actions: {
  *            chatgpt: false,
  *            v0: true,
@@ -125,128 +125,45 @@ export function llmsPlugin(userConfig?: PageActionsConfig): StarlightPlugin {
         }
 
         addIntegration({
-          name: 'llms-plugin-integration',
+          name: 'llms',
           hooks: {
-            'astro:config:setup': ({ updateConfig }) => {
+            'astro:config:setup'({ updateConfig, injectRoute }) {
               updateConfig({
                 vite: {
                   plugins: [
                     {
                       name: 'llms-plugin-config',
-                      resolveId(id) {
+                      resolveId(id: string) {
                         if (id === 'virtual:llms-plugin/config') {
                           return `\0${id}`
                         }
+                        return undefined
                       },
-                      load(id) {
+                      load(id: string) {
                         if (id === '\0virtual:llms-plugin/config') {
                           return `export default ${JSON.stringify(config)}`
                         }
+                        return undefined
                       },
                     },
-                    viteStaticCopy({
-                      targets: [
-                        {
-                          src: 'src/content/docs/**/*.{md,mdx}',
-                          dest: '',
-                          transform: (content: string) => {
-                            const frontmatterRegex =
-                              /^---\s*\n([\s\S]*?)\n---\s*\n([\s\S]*)$/
-                            const match = content.match(frontmatterRegex)
-
-                            let title = ''
-                            let markdownContent = content
-
-                            if (
-                              match &&
-                              match[1] !== undefined &&
-                              match[2] !== undefined
-                            ) {
-                              const frontmatter = match[1]
-                              markdownContent = match[2]
-
-                              const titleMatch = frontmatter.match(
-                                /title:\s*["']?([^"'\n]+)["']?/
-                              )
-                              if (titleMatch && titleMatch[1] !== undefined) {
-                                title = titleMatch[1].trim()
-                              }
-                            }
-
-                            const contentWithoutImports = markdownContent
-                            // .split('\n')
-                            // .filter(
-                            //   (line) => !line.trim().startsWith('import ')
-                            // )
-                            // .join('\n')
-
-                            let newContent = ''
-
-                            if (title) {
-                              newContent = `# ${title}\n\n`
-                            }
-
-                            newContent += contentWithoutImports.trim()
-
-                            return newContent
-                          },
-                          rename: (
-                            fileName: string,
-                            fileExtension: string,
-                            fullPath: string
-                          ) => {
-                            const fullPathNormalized = normalizePath(fullPath)
-                            const relativePath = (
-                              fullPathNormalized.split(
-                                'src/content/docs/'
-                              )[1] as string
-                            ).replace(new RegExp(`\\.${fileExtension}$`), '')
-                            const pathSegments = relativePath.split('/')
-
-                            if (fileName === 'index') {
-                              if (pathSegments.length === 1) {
-                                return 'index.md'
-                              } else {
-                                const directories = pathSegments
-                                  .slice(0, -2)
-                                  .join('/')
-                                const folderName =
-                                  pathSegments[pathSegments.length - 2]
-
-                                return directories
-                                  ? `${directories}/${folderName}.md`
-                                  : `${folderName}.md`
-                              }
-                            }
-
-                            const directories = pathSegments
-                              .slice(0, -1)
-                              .join('/')
-                            const finalPath = directories
-                              ? `${directories}/${fileName}.md`
-                              : `${fileName}.md`
-
-                            return finalPath.replace('@', '').toLowerCase()
-                          },
-                        },
-                      ],
-                    }),
                   ],
                 },
               })
-            },
-          },
-        })
 
-        addIntegration({
-          name: 'llms',
-          hooks: {
-            'astro:config:setup'({ injectRoute }) {
-              const entrypoint = new URL('llms.txt.ts', import.meta.url)
               injectRoute({
-                entrypoint,
+                entrypoint: '@hugomrdias/docs/llms.txt',
                 pattern: '/llms.txt',
               })
+
+              injectRoute({
+                entrypoint: '@hugomrdias/docs/llms-full.txt',
+                pattern: '/llms-full.txt',
+              })
+
+              injectRoute({
+                entrypoint: '@hugomrdias/docs/markdown',
+                pattern: '/[...slug].md',
+              })
             },
           },
         })

packages/docs/llms-plugin/llms-full.txt.ts

@@ -0,0 +1,8 @@
+import { site } from 'astro:config/client'
+import config from 'virtual:llms-plugin/config'
+import type { APIRoute } from 'astro'
+import { generateLlmsIndex } from './utils'
+
+export const GET: APIRoute = () => {
+  return generateLlmsIndex(site, config, Number.POSITIVE_INFINITY)
+}

packages/docs/llms-plugin/llms.txt.ts

@@ -1,83 +1,8 @@
-// @ts-expect-error - TODO: fix this
 import { site } from 'astro:config/client'
-// @ts-expect-error - TODO: fix this
-import { getCollection, type InferEntrySchema } from 'astro:content'
 import config from 'virtual:llms-plugin/config'
 import type { APIRoute } from 'astro'
+import { generateLlmsIndex } from './utils'
 
-/**
- * Route that generates a single plaintext Markdown document from the full website content.
- */
-export const GET: APIRoute = async () => {
-  type SectionNode = {
-    docs: { id: string; data: InferEntrySchema<'docs'> }[]
-    children: Map<string, SectionNode>
-  }
-
-  function toTitleCase(str: string): string {
-    return str
-      .split('-')
-      .map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
-      .join(' ')
-  }
-
-  const root: SectionNode = { docs: [], children: new Map() }
-  // @ts-expect-error - TODO: fix this
-  const docs = await getCollection('docs', (doc) => !doc.data.draft)
-  let body = `# ${config.title}\n\n${config.description}\n\n`
-
-  for (const doc of docs) {
-    const segments = doc.id.split('/')
-    let current = root
-    for (let i = 0; i < segments.length - 1; i++) {
-      const segment = segments[i]
-      if (!current.children.has(segment)) {
-        current.children.set(segment, { docs: [], children: new Map() })
-      }
-      const next = current.children.get(segment)
-      if (next) {
-        current = next
-      }
-    }
-    current.docs.push({ id: doc.id, data: doc.data })
-  }
-
-  function renderSection(
-    node: SectionNode,
-    level: number,
-    path: string[]
-  ): string {
-    let output = ''
-    const headerLevel = '#'.repeat(level + 1)
-    if (node.docs.length > 0 && path.length === 0) {
-      for (const doc of node.docs) {
-        const description = doc.data.description
-          ? `: ${doc.data.description}`
-          : ''
-        output += `- [${doc.data.title}](${site}/${doc.id}.md)${description}\n`
-      }
-      output += '\n'
-    }
-
-    for (const [segment, childNode] of node.children.entries()) {
-      const currentPath = [...path, segment]
-      output += `${headerLevel} ${toTitleCase(segment)}\n\n`
-      if (childNode.docs.length > 0) {
-        for (const doc of childNode.docs) {
-          const description = doc.data.description
-            ? `: ${doc.data.description}`
-            : ''
-          output += `- [${doc.data.title}](${site}/${doc.id}.md)${description}\n`
-        }
-        output += '\n'
-      }
-      output += renderSection(childNode, level + 1, currentPath)
-    }
-
-    return output
-  }
-
-  body += renderSection(root, 1, [])
-
-  return new Response(body)
+export const GET: APIRoute = () => {
+  return generateLlmsIndex(site, config, config.maxDepth ?? 4)
 }

packages/docs/llms-plugin/markdown.ts

@@ -0,0 +1,26 @@
+import { getCollection } from 'astro:content'
+import type { APIRoute, GetStaticPaths } from 'astro'
+import { type DocEntry, getMarkdownPath, stripMdxSyntax } from './strip'
+
+export const getStaticPaths: GetStaticPaths = async () => {
+  const docs = await getCollection(
+    'docs',
+    (doc: { data: { draft?: boolean } }) => !doc.data.draft
+  )
+
+  return docs.map((doc) => ({
+    params: { slug: getMarkdownPath(doc.id) },
+    props: { doc },
+  }))
+}
+
+export const GET: APIRoute = ({ props }) => {
+  const { doc } = props as { doc: DocEntry }
+  const body = `# ${doc.data.title}\n\n${stripMdxSyntax(doc.body).trim()}`
+
+  return new Response(body, {
+    headers: {
+      'content-type': 'text/markdown; charset=utf-8',
+    },
+  })
+}