refactor: use full URLs throughout indexing and search system (#4)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: scalvert

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Setup mise
         uses: jdx/mise-action@v2
         with:
-          install_args: "node@${{ matrix.node }}"
+          install_args: 'node@${{ matrix.node }}'
 
       - name: Install dependencies
         run: npm ci

scripts/test-mcp-server.mjs

@@ -91,23 +91,26 @@ Access tokens are used to authenticate API requests.`,
   },
 ];
 
-// Convert to Record for lookups
+const BASE_URL = 'https://docs.example.com';
+
+// Convert to Record for lookups - keyed by full URL
 const sampleDocs = {};
 for (const doc of sampleDocsArray) {
-  sampleDocs[doc.route] = doc;
+  const fullUrl = `${BASE_URL}${doc.route}`;
+  sampleDocs[fullUrl] = doc;
 }
 
 async function main() {
-  // Build search index
-  const searchIndex = buildSearchIndex(sampleDocsArray);
+  // Build search index with baseUrl so IDs are full URLs
+  const searchIndex = buildSearchIndex(sampleDocsArray, BASE_URL);
   const searchIndexData = await exportSearchIndex(searchIndex);
 
   const mcpServer = new McpDocsServer({
     name: 'test-docs',
     version: '1.0.0',
     docs: sampleDocs,
     searchIndexData,
-    baseUrl: 'https://docs.example.com',
+    baseUrl: BASE_URL,
   });
 
   await mcpServer.initialize();

src/index.ts

@@ -14,8 +14,7 @@ export type {
   McpServerDataConfig,
   DocsIndex,
   DocsSearchParams,
-  DocsGetPageParams,
-  DocsGetSectionParams,
+  DocsFetchParams,
   ExtractedContent,
 } from './types/index.js';
 

src/mcp/server.ts

@@ -103,9 +103,7 @@ export class McpDocsServer {
         try {
           const results = await this.searchProvider.search(query, { limit });
           return {
-            content: [
-              { type: 'text' as const, text: formatSearchResults(results, this.config.baseUrl) },
-            ],
+            content: [{ type: 'text' as const, text: formatSearchResults(results) }],
           };
         } catch (error) {
           console.error('[MCP] Search error:', error);
@@ -123,13 +121,15 @@ export class McpDocsServer {
         description:
           'Fetch the complete content of a documentation page. Use this when you need the full content of a specific page.',
         inputSchema: {
-          route: z
+          url: z
             .string()
-            .min(1)
-            .describe('The page route path (e.g., "/docs/getting-started" or "/api/reference")'),
+            .url()
+            .describe(
+              'The full URL of the page to fetch (e.g., "https://docs.example.com/docs/getting-started")'
+            ),
         },
       },
-      async ({ route }) => {
+      async ({ url }) => {
         await this.initialize();
 
         if (!this.searchProvider || !this.searchProvider.isReady()) {
@@ -140,9 +140,9 @@ export class McpDocsServer {
         }
 
         try {
-          const doc = await this.getDocument(route);
+          const doc = await this.getDocument(url);
           return {
-            content: [{ type: 'text' as const, text: formatPageContent(doc, this.config.baseUrl) }],
+            content: [{ type: 'text' as const, text: formatPageContent(doc) }],
           };
         } catch (error) {
           console.error('[MCP] Get page error:', error);
@@ -156,33 +156,15 @@ export class McpDocsServer {
   }
 
   /**
-   * Get a document by route using the search provider
+   * Get a document by URL using the search provider
    */
-  private async getDocument(route: string): Promise<ProcessedDoc | null> {
+  private async getDocument(url: string): Promise<ProcessedDoc | null> {
     if (!this.searchProvider) {
       return null;
     }
 
-    // Use the provider's getDocument if available
     if (this.searchProvider.getDocument) {
-      return this.searchProvider.getDocument(route);
-    }
-
-    // For FlexSearchProvider, we can access docs directly
-    if (this.searchProvider instanceof FlexSearchProvider) {
-      const docs = this.searchProvider.getDocs();
-      if (!docs) return null;
-
-      // Try exact match first
-      if (docs[route]) {
-        return docs[route];
-      }
-
-      // Try with/without leading slash
-      const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
-      const withoutSlash = route.startsWith('/') ? route.slice(1) : route;
-
-      return docs[normalizedRoute] ?? docs[withoutSlash] ?? null;
+      return this.searchProvider.getDocument(url);
     }
 
     return null;

src/mcp/tools/docs-fetch.ts

@@ -1,4 +1,4 @@
-import type { ProcessedDoc, DocsGetPageParams } from '../../types/index.js';
+import type { ProcessedDoc } from '../../types/index.js';
 
 /**
  * Tool definition for docs_fetch
@@ -10,59 +10,23 @@ export const docsFetchTool = {
   inputSchema: {
     type: 'object' as const,
     properties: {
-      route: {
+      url: {
         type: 'string',
-        description: 'The route path of the page (e.g., /docs/getting-started)',
+        format: 'uri',
+        description:
+          'The full URL of the page to fetch (e.g., "https://docs.example.com/docs/getting-started")',
       },
     },
-    required: ['route'],
+    required: ['url'],
   },
 };
 
-/**
- * Execute the docs_get_page tool
- */
-export function executeDocsGetPage(
-  params: DocsGetPageParams,
-  docs: Record<string, ProcessedDoc>
-): ProcessedDoc | null {
-  const { route } = params;
-
-  // Validate parameters
-  if (!route || typeof route !== 'string') {
-    throw new Error('Route parameter is required and must be a string');
-  }
-
-  // Normalize route (ensure leading slash, remove trailing slash)
-  let normalizedRoute = route.trim();
-  if (!normalizedRoute.startsWith('/')) {
-    normalizedRoute = '/' + normalizedRoute;
-  }
-  if (normalizedRoute.length > 1 && normalizedRoute.endsWith('/')) {
-    normalizedRoute = normalizedRoute.slice(0, -1);
-  }
-
-  // Look up the document
-  const doc = docs[normalizedRoute];
-
-  if (!doc) {
-    // Try without leading slash
-    const altRoute = normalizedRoute.slice(1);
-    if (docs[altRoute]) {
-      return docs[altRoute] ?? null;
-    }
-    return null;
-  }
-
-  return doc;
-}
-
 /**
  * Format page content for MCP response
  */
-export function formatPageContent(doc: ProcessedDoc | null, baseUrl?: string): string {
+export function formatPageContent(doc: ProcessedDoc | null): string {
   if (!doc) {
-    return 'Page not found. Please check the route path and try again.';
+    return 'Page not found. Please check the URL and try again.';
   }
 
   const lines: string[] = [];
@@ -77,12 +41,6 @@ export function formatPageContent(doc: ProcessedDoc | null, baseUrl?: string): s
     lines.push('');
   }
 
-  // Include full URL if baseUrl is provided
-  if (baseUrl) {
-    const fullUrl = `${baseUrl.replace(/\/$/, '')}${doc.route}`;
-    lines.push(`**URL:** ${fullUrl}`);
-  }
-
   lines.push(`**Route:** ${doc.route}`);
   lines.push('');
 

src/mcp/tools/docs-search.ts

@@ -53,7 +53,7 @@ export function executeDocsSearch(
 /**
  * Format search results for MCP response
  */
-export function formatSearchResults(results: SearchResult[], baseUrl?: string): string {
+export function formatSearchResults(results: SearchResult[]): string {
   if (results.length === 0) {
     return 'No matching documents found.';
   }
@@ -65,14 +65,7 @@ export function formatSearchResults(results: SearchResult[], baseUrl?: string):
     if (!result) continue;
 
     lines.push(`${i + 1}. **${result.title}**`);
-
-    // Include full URL if baseUrl is provided
-    if (baseUrl) {
-      const fullUrl = `${baseUrl.replace(/\/$/, '')}${result.route}`;
-      lines.push(`   URL: ${fullUrl}`);
-    }
-
-    lines.push(`   Route: ${result.route}`);
+    lines.push(`   URL: ${result.url}`);
 
     if (result.matchingHeadings && result.matchingHeadings.length > 0) {
       lines.push(`   Matching sections: ${result.matchingHeadings.join(', ')}`);
@@ -82,5 +75,7 @@ export function formatSearchResults(results: SearchResult[], baseUrl?: string):
     lines.push('');
   }
 
+  lines.push('Use docs_fetch with the URL to retrieve the full page content.');
+
   return lines.join('\n');
 }

src/providers/indexers/flexsearch-indexer.ts

@@ -6,12 +6,13 @@ import { buildSearchIndex, exportSearchIndex } from '../../search/flexsearch-ind
  * Built-in FlexSearch content indexer.
  *
  * This indexer builds a local FlexSearch index and produces:
- * - docs.json: All processed documents keyed by route
+ * - docs.json: All processed documents keyed by full URL
  * - search-index.json: Exported FlexSearch index for runtime search
  */
 export class FlexSearchIndexer implements ContentIndexer {
   readonly name = 'flexsearch';
 
+  private baseUrl = '';
   private docsIndex: Record<string, ProcessedDoc> = {};
   private exportedIndex: unknown = null;
   private docCount = 0;
@@ -24,8 +25,9 @@ export class FlexSearchIndexer implements ContentIndexer {
     return true;
   }
 
-  async initialize(_context: ProviderContext): Promise<void> {
+  async initialize(context: ProviderContext): Promise<void> {
     // Reset state for fresh indexing
+    this.baseUrl = context.baseUrl.replace(/\/$/, '');
     this.docsIndex = {};
     this.exportedIndex = null;
     this.docCount = 0;
@@ -34,14 +36,15 @@ export class FlexSearchIndexer implements ContentIndexer {
   async indexDocuments(docs: ProcessedDoc[]): Promise<void> {
     this.docCount = docs.length;
 
-    // Build docs index (keyed by route)
+    // Build docs index (keyed by full URL)
     for (const doc of docs) {
-      this.docsIndex[doc.route] = doc;
+      const fullUrl = `${this.baseUrl}${doc.route}`;
+      this.docsIndex[fullUrl] = doc;
     }
 
-    // Build and export FlexSearch index
+    // Build and export FlexSearch index (use full URLs as document IDs)
     console.log('[FlexSearch] Building search index...');
-    const searchIndex = buildSearchIndex(docs);
+    const searchIndex = buildSearchIndex(docs, this.baseUrl);
     this.exportedIndex = await exportSearchIndex(searchIndex);
     console.log(`[FlexSearch] Indexed ${this.docCount} documents`);
   }

src/providers/search/flexsearch-provider.ts

@@ -75,21 +75,13 @@ export class FlexSearchProvider implements SearchProvider {
     return searchIndex(this.searchIndex, this.docs, query, { limit });
   }
 
-  async getDocument(route: string): Promise<ProcessedDoc | null> {
+  async getDocument(url: string): Promise<ProcessedDoc | null> {
     if (!this.docs) {
       throw new Error('[FlexSearch] Provider not initialized');
     }
 
-    // Try exact match first
-    if (this.docs[route]) {
-      return this.docs[route];
-    }
-
-    // Try with/without leading slash
-    const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
-    const withoutSlash = route.startsWith('/') ? route.slice(1) : route;
-
-    return this.docs[normalizedRoute] ?? this.docs[withoutSlash] ?? null;
+    // Direct lookup by URL
+    return this.docs[url] ?? null;
   }
 
   async healthCheck(): Promise<{ healthy: boolean; message?: string }> {

src/search/flexsearch-indexer.ts

@@ -96,10 +96,21 @@ export function createSearchIndex(): FlexSearchDocument {
 
 /**
  * Add a document to the search index
+ *
+ * @param index - The FlexSearch index
+ * @param doc - The document to add
+ * @param baseUrl - Optional base URL to construct full URL as document ID
  */
-export function addDocumentToIndex(index: FlexSearchDocument, doc: ProcessedDoc): void {
+export function addDocumentToIndex(
+  index: FlexSearchDocument,
+  doc: ProcessedDoc,
+  baseUrl?: string
+): void {
+  // Use full URL as ID if baseUrl is provided, otherwise use route
+  const id = baseUrl ? `${baseUrl.replace(/\/$/, '')}${doc.route}` : doc.route;
+
   const indexable: IndexableDocument = {
-    id: doc.route,
+    id,
     title: doc.title,
     content: doc.markdown,
     headings: doc.headings.map((h) => h.text).join(' '),
@@ -111,12 +122,15 @@ export function addDocumentToIndex(index: FlexSearchDocument, doc: ProcessedDoc)
 
 /**
  * Build the search index from processed documents
+ *
+ * @param docs - Documents to index
+ * @param baseUrl - Optional base URL to construct full URLs as document IDs
  */
-export function buildSearchIndex(docs: ProcessedDoc[]): FlexSearchDocument {
+export function buildSearchIndex(docs: ProcessedDoc[], baseUrl?: string): FlexSearchDocument {
   const index = createSearchIndex();
 
   for (const doc of docs) {
-    addDocumentToIndex(index, doc);
+    addDocumentToIndex(index, doc, baseUrl);
   }
 
   return index;
@@ -180,6 +194,7 @@ export function searchIndex(
     if (!doc) continue;
 
     results.push({
+      url: docId, // docId is the full URL when indexed with baseUrl
       route: doc.route,
       title: doc.title,
       score,