refactor: use full URLs throughout the indexing and search system

All documents are now indexed and looked up by full URL rather than
route paths. This provides a consistent interface regardless of the
search provider:

- FlexSearchIndexer keys docs by full URL (e.g., https://example.com/docs/intro)
- FlexSearch index uses full URLs as document IDs
- docs_fetch tool accepts only URL parameter (not route)
- Search results return routes but lookups use URLs

This simplifies integration with external search providers like Glean
that work with URLs natively.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: scalvert

scripts/test-mcp-server.mjs

@@ -91,23 +91,26 @@ Access tokens are used to authenticate API requests.`,
   },
 ];
 
-// Convert to Record for lookups
+const BASE_URL = 'https://docs.example.com';
+
+// Convert to Record for lookups - keyed by full URL
 const sampleDocs = {};
 for (const doc of sampleDocsArray) {
-  sampleDocs[doc.route] = doc;
+  const fullUrl = `${BASE_URL}${doc.route}`;
+  sampleDocs[fullUrl] = doc;
 }
 
 async function main() {
-  // Build search index
-  const searchIndex = buildSearchIndex(sampleDocsArray);
+  // Build search index with baseUrl so IDs are full URLs
+  const searchIndex = buildSearchIndex(sampleDocsArray, BASE_URL);
   const searchIndexData = await exportSearchIndex(searchIndex);
 
   const mcpServer = new McpDocsServer({
     name: 'test-docs',
     version: '1.0.0',
     docs: sampleDocs,
     searchIndexData,
-    baseUrl: 'https://docs.example.com',
+    baseUrl: BASE_URL,
   });
 
   await mcpServer.initialize();

src/index.ts

@@ -15,7 +15,6 @@ export type {
   DocsIndex,
   DocsSearchParams,
   DocsFetchParams,
-  DocsGetPageParams,
   ExtractedContent,
 } from './types/index.js';
 

src/mcp/server.ts

@@ -123,15 +123,15 @@ export class McpDocsServer {
         description:
           'Fetch the complete content of a documentation page. Use this when you need the full content of a specific page.',
         inputSchema: {
-          page: z
+          url: z
             .string()
-            .min(1)
+            .url()
             .describe(
-              'The page to fetch - either a route path (e.g., "/docs/getting-started") or a full URL (e.g., "https://docs.example.com/docs/getting-started")'
+              'The full URL of the page to fetch (e.g., "https://docs.example.com/docs/getting-started")'
             ),
         },
       },
-      async ({ page }) => {
+      async ({ url }) => {
         await this.initialize();
 
         if (!this.searchProvider || !this.searchProvider.isReady()) {
@@ -142,9 +142,9 @@ export class McpDocsServer {
         }
 
         try {
-          const doc = await this.getDocument(page);
+          const doc = await this.getDocument(url);
           return {
-            content: [{ type: 'text' as const, text: formatPageContent(doc, this.config.baseUrl) }],
+            content: [{ type: 'text' as const, text: formatPageContent(doc) }],
           };
         } catch (error) {
           console.error('[MCP] Get page error:', error);
@@ -158,16 +158,15 @@ export class McpDocsServer {
   }
 
   /**
-   * Get a document by page identifier (route or URL) using the search provider
+   * Get a document by URL using the search provider
    */
-  private async getDocument(page: string): Promise<ProcessedDoc | null> {
+  private async getDocument(url: string): Promise<ProcessedDoc | null> {
     if (!this.searchProvider) {
       return null;
     }
 
-    // Use the provider's getDocument if available (handles both URLs and routes)
     if (this.searchProvider.getDocument) {
-      return this.searchProvider.getDocument(page);
+      return this.searchProvider.getDocument(url);
     }
 
     return null;

src/mcp/tools/docs-fetch.ts

@@ -1,4 +1,4 @@
-import type { ProcessedDoc, DocsGetPageParams } from '../../types/index.js';
+import type { ProcessedDoc } from '../../types/index.js';
 
 /**
  * Tool definition for docs_fetch
@@ -10,60 +10,23 @@ export const docsFetchTool = {
   inputSchema: {
     type: 'object' as const,
     properties: {
-      page: {
+      url: {
         type: 'string',
+        format: 'uri',
         description:
-          'The page to fetch - either a route path (e.g., "/docs/getting-started") or a full URL (e.g., "https://docs.example.com/docs/getting-started")',
+          'The full URL of the page to fetch (e.g., "https://docs.example.com/docs/getting-started")',
       },
     },
-    required: ['page'],
+    required: ['url'],
   },
 };
 
-/**
- * Execute the docs_get_page tool
- */
-export function executeDocsGetPage(
-  params: DocsGetPageParams,
-  docs: Record<string, ProcessedDoc>
-): ProcessedDoc | null {
-  const { route } = params;
-
-  // Validate parameters
-  if (!route || typeof route !== 'string') {
-    throw new Error('Route parameter is required and must be a string');
-  }
-
-  // Normalize route (ensure leading slash, remove trailing slash)
-  let normalizedRoute = route.trim();
-  if (!normalizedRoute.startsWith('/')) {
-    normalizedRoute = '/' + normalizedRoute;
-  }
-  if (normalizedRoute.length > 1 && normalizedRoute.endsWith('/')) {
-    normalizedRoute = normalizedRoute.slice(0, -1);
-  }
-
-  // Look up the document
-  const doc = docs[normalizedRoute];
-
-  if (!doc) {
-    // Try without leading slash
-    const altRoute = normalizedRoute.slice(1);
-    if (docs[altRoute]) {
-      return docs[altRoute] ?? null;
-    }
-    return null;
-  }
-
-  return doc;
-}
-
 /**
  * Format page content for MCP response
  */
-export function formatPageContent(doc: ProcessedDoc | null, baseUrl?: string): string {
+export function formatPageContent(doc: ProcessedDoc | null): string {
   if (!doc) {
-    return 'Page not found. Please check the route path and try again.';
+    return 'Page not found. Please check the URL and try again.';
   }
 
   const lines: string[] = [];
@@ -78,12 +41,6 @@ export function formatPageContent(doc: ProcessedDoc | null, baseUrl?: string): s
     lines.push('');
   }
 
-  // Include full URL if baseUrl is provided
-  if (baseUrl) {
-    const fullUrl = `${baseUrl.replace(/\/$/, '')}${doc.route}`;
-    lines.push(`**URL:** ${fullUrl}`);
-  }
-
   lines.push(`**Route:** ${doc.route}`);
   lines.push('');
 

src/providers/indexers/flexsearch-indexer.ts

@@ -6,12 +6,13 @@ import { buildSearchIndex, exportSearchIndex } from '../../search/flexsearch-ind
  * Built-in FlexSearch content indexer.
  *
  * This indexer builds a local FlexSearch index and produces:
- * - docs.json: All processed documents keyed by route
+ * - docs.json: All processed documents keyed by full URL
  * - search-index.json: Exported FlexSearch index for runtime search
  */
 export class FlexSearchIndexer implements ContentIndexer {
   readonly name = 'flexsearch';
 
+  private baseUrl = '';
   private docsIndex: Record<string, ProcessedDoc> = {};
   private exportedIndex: unknown = null;
   private docCount = 0;
@@ -24,8 +25,9 @@ export class FlexSearchIndexer implements ContentIndexer {
     return true;
   }
 
-  async initialize(_context: ProviderContext): Promise<void> {
+  async initialize(context: ProviderContext): Promise<void> {
     // Reset state for fresh indexing
+    this.baseUrl = context.baseUrl.replace(/\/$/, '');
     this.docsIndex = {};
     this.exportedIndex = null;
     this.docCount = 0;
@@ -34,14 +36,15 @@ export class FlexSearchIndexer implements ContentIndexer {
   async indexDocuments(docs: ProcessedDoc[]): Promise<void> {
     this.docCount = docs.length;
 
-    // Build docs index (keyed by route)
+    // Build docs index (keyed by full URL)
     for (const doc of docs) {
-      this.docsIndex[doc.route] = doc;
+      const fullUrl = `${this.baseUrl}${doc.route}`;
+      this.docsIndex[fullUrl] = doc;
     }
 
-    // Build and export FlexSearch index
+    // Build and export FlexSearch index (use full URLs as document IDs)
     console.log('[FlexSearch] Building search index...');
-    const searchIndex = buildSearchIndex(docs);
+    const searchIndex = buildSearchIndex(docs, this.baseUrl);
     this.exportedIndex = await exportSearchIndex(searchIndex);
     console.log(`[FlexSearch] Indexed ${this.docCount} documents`);
   }

src/providers/search/flexsearch-provider.ts

@@ -75,32 +75,13 @@ export class FlexSearchProvider implements SearchProvider {
     return searchIndex(this.searchIndex, this.docs, query, { limit });
   }
 
-  async getDocument(page: string): Promise<ProcessedDoc | null> {
+  async getDocument(url: string): Promise<ProcessedDoc | null> {
     if (!this.docs) {
       throw new Error('[FlexSearch] Provider not initialized');
     }
 
-    // If it's a URL, extract the pathname as the route
-    let route = page;
-    if (page.startsWith('http://') || page.startsWith('https://')) {
-      try {
-        const url = new URL(page);
-        route = url.pathname;
-      } catch {
-        // Invalid URL, treat as route
-      }
-    }
-
-    // Try exact match first
-    if (this.docs[route]) {
-      return this.docs[route] ?? null;
-    }
-
-    // Try with/without leading slash
-    const normalizedRoute = route.startsWith('/') ? route : `/${route}`;
-    const withoutSlash = route.startsWith('/') ? route.slice(1) : route;
-
-    return this.docs[normalizedRoute] ?? this.docs[withoutSlash] ?? null;
+    // Direct lookup by URL
+    return this.docs[url] ?? null;
   }
 
   async healthCheck(): Promise<{ healthy: boolean; message?: string }> {

src/search/flexsearch-indexer.ts

@@ -96,10 +96,21 @@ export function createSearchIndex(): FlexSearchDocument {
 
 /**
  * Add a document to the search index
+ *
+ * @param index - The FlexSearch index
+ * @param doc - The document to add
+ * @param baseUrl - Optional base URL to construct full URL as document ID
  */
-export function addDocumentToIndex(index: FlexSearchDocument, doc: ProcessedDoc): void {
+export function addDocumentToIndex(
+  index: FlexSearchDocument,
+  doc: ProcessedDoc,
+  baseUrl?: string
+): void {
+  // Use full URL as ID if baseUrl is provided, otherwise use route
+  const id = baseUrl ? `${baseUrl.replace(/\/$/, '')}${doc.route}` : doc.route;
+
   const indexable: IndexableDocument = {
-    id: doc.route,
+    id,
     title: doc.title,
     content: doc.markdown,
     headings: doc.headings.map((h) => h.text).join(' '),
@@ -111,12 +122,15 @@ export function addDocumentToIndex(index: FlexSearchDocument, doc: ProcessedDoc)
 
 /**
  * Build the search index from processed documents
+ *
+ * @param docs - Documents to index
+ * @param baseUrl - Optional base URL to construct full URLs as document IDs
  */
-export function buildSearchIndex(docs: ProcessedDoc[]): FlexSearchDocument {
+export function buildSearchIndex(docs: ProcessedDoc[], baseUrl?: string): FlexSearchDocument {
   const index = createSearchIndex();
 
   for (const doc of docs) {
-    addDocumentToIndex(index, doc);
+    addDocumentToIndex(index, doc, baseUrl);
   }
 
   return index;

src/types/index.ts

@@ -206,16 +206,8 @@ export interface DocsSearchParams {
  * Input parameters for docs_fetch tool
  */
 export interface DocsFetchParams {
-  /** Page to fetch - either a route path or full URL */
-  page: string;
-}
-
-/**
- * @deprecated Use DocsFetchParams instead
- */
-export interface DocsGetPageParams {
-  /** Route path of the page */
-  route: string;
+  /** Full URL of the page to fetch */
+  url: string;
 }
 
 /**

tests/playwright/mcp.spec.ts

@@ -63,29 +63,26 @@ test.describe('docs_search Tool', () => {
 });
 
 test.describe('docs_fetch Tool', () => {
-  test('retrieves full page content by route', async ({ mcp }) => {
-    const result = await mcp.callTool('docs_fetch', { page: '/docs/intro' });
-    expect(result).not.toBeToolError();
-    expect(result).toContainToolText('Introduction');
-    expect(result).toContainToolText('Welcome to the documentation');
-  });
-
   test('retrieves full page content by URL', async ({ mcp }) => {
     const result = await mcp.callTool('docs_fetch', {
-      page: 'https://docs.example.com/docs/intro',
+      url: 'https://docs.example.com/docs/intro',
     });
     expect(result).not.toBeToolError();
     expect(result).toContainToolText('Introduction');
     expect(result).toContainToolText('Welcome to the documentation');
   });
 
   test('returns page with markdown content', async ({ mcp }) => {
-    const result = await mcp.callTool('docs_fetch', { page: '/docs/installation' });
+    const result = await mcp.callTool('docs_fetch', {
+      url: 'https://docs.example.com/docs/installation',
+    });
     expect(result).toContainToolText('npm install my-platform');
   });
 
   test('returns error for non-existent page', async ({ mcp }) => {
-    const result = await mcp.callTool('docs_fetch', { page: '/docs/nonexistent' });
+    const result = await mcp.callTool('docs_fetch', {
+      url: 'https://docs.example.com/docs/nonexistent',
+    });
     expect(result).toContainToolText('Page not found');
   });
 });