fix(context): recover semantic injection when scoped Chroma metadata misses matches (closes #2979)

src/services/sqlite/SessionStore.ts

@@ -1491,7 +1491,7 @@ export class SessionStore {
     const { orderBy = 'date_desc', limit, project, type, concepts, files } = options;
     const preserveIdOrder = orderBy === 'relevance';
     const orderClause = preserveIdOrder ? '' : `ORDER BY created_at_epoch ${orderBy === 'date_asc' ? 'ASC' : 'DESC'}`;
-    const limitClause = limit ? `LIMIT ${limit}` : '';
+    const limitClause = limit && !preserveIdOrder ? `LIMIT ${limit}` : '';
 
     const placeholders = ids.map(() => '?').join(',');
     const params: any[] = [...ids];
@@ -1549,7 +1549,9 @@ export class SessionStore {
     if (!preserveIdOrder) return rows;
 
     const rowMap = new Map(rows.map(r => [r.id, r]));
-    return ids.map(id => rowMap.get(id)).filter((r): r is ObservationSearchResult => !!r);
+    const orderedRows = ids.map(id => rowMap.get(id)).filter((r): r is ObservationSearchResult => !!r);
+    // Relevance order comes from the caller, so apply any limit after reordering.
+    return limit ? orderedRows.slice(0, limit) : orderedRows;
   }
 
   getSummaryForSession(memorySessionId: string): {

src/services/sync/ChromaSync.ts

@@ -866,11 +866,11 @@ export class ChromaSync {
       if (isConnectionError) {
         this.collectionCreated = false;
         logger.error('CHROMA_SYNC', 'Connection lost during query',
-          { project: this.project, query }, error as Error);
+          { project: this.project, queryLength: query.length }, error as Error);
         throw new Error(`Chroma query failed - connection lost: ${errorMessage}`);
       }
 
-      logger.error('CHROMA_SYNC', 'Query failed', { project: this.project, query }, error as Error);
+      logger.error('CHROMA_SYNC', 'Query failed', { project: this.project, queryLength: query.length }, error as Error);
       throw error;
     }
 

src/services/worker/SearchManager.ts

@@ -32,6 +32,10 @@ export interface SearchTelemetryEnvelope {
   fallback_reason?: 'none' | 'chroma_connection' | 'chroma_error' | 'chroma_not_initialized';
 }
 
+interface SearchExecutionOptions {
+  semanticHydrationLimit?: number;
+}
+
 export class SearchManager {
   private orchestrator: SearchOrchestrator;
 
@@ -299,9 +303,20 @@ export class SearchManager {
     return normalized;
   }
 
-  async search(args: any, telemetryOut?: SearchTelemetryEnvelope): Promise<any> {
+  async search(args: any, telemetryOut?: SearchTelemetryEnvelope, executionOptions: SearchExecutionOptions = {}): Promise<any> {
     const normalized = this.normalizeParams(args);
-    const { query, type, obs_type, concepts, files, format, ...options } = normalized;
+    const { query, type, obs_type, concepts, files, format, semanticLimit: _ignoredPublicSemanticLimit, ...options } = normalized;
+    const requestedLimit = Math.max(
+      Number.parseInt(String(options.limit ?? SEARCH_CONSTANTS.DEFAULT_LIMIT), 10) || SEARCH_CONSTANTS.DEFAULT_LIMIT,
+      1
+    );
+    const semanticCandidateLimit = Math.min(
+      Math.max(requestedLimit, executionOptions.semanticHydrationLimit ?? SEARCH_CONSTANTS.CHROMA_BATCH_SIZE),
+      SEARCH_CONSTANTS.CHROMA_BATCH_SIZE
+    );
+    const semanticHydrationLimit = executionOptions.semanticHydrationLimit === undefined
+      ? requestedLimit
+      : semanticCandidateLimit;
     let observations: ObservationSearchResult[] = [];
     let sessions: SessionSummarySearchResult[] = [];
     let prompts: UserPromptSearchResult[] = [];
@@ -352,7 +367,7 @@ export class SearchManager {
       }
 
       try {
-        const chromaResults = await this.queryChroma(query, 100, whereFilter);
+        const chromaResults = await this.queryChroma(query, semanticCandidateLimit, whereFilter);
         chromaSucceeded = true; 
         logger.debug('SEARCH', 'ChromaDB returned semantic matches', { matchCount: chromaResults.ids.length });
 
@@ -402,7 +417,7 @@ export class SearchManager {
           }
 
           if (obsIds.length > 0) {
-            const obsOptions = { ...options, type: obs_type, concepts, files };
+            const obsOptions = { ...options, type: obs_type, concepts, files, limit: semanticHydrationLimit };
             observations = this.sessionStore.getObservationsByIds(obsIds, obsOptions);
           }
           if (sessionIds.length > 0) {
@@ -484,8 +499,11 @@ export class SearchManager {
     }
 
     if (format === 'json') {
+      const jsonObservations = executionOptions.semanticHydrationLimit
+        ? observations
+        : observations.slice(0, requestedLimit);
       return {
-        observations,
+        observations: jsonObservations,
         sessions,
         prompts,
         totalResults,

src/services/worker/http/routes/SearchRoutes.ts

@@ -5,6 +5,7 @@ import path from 'path';
 import { z } from 'zod';
 import { SearchManager } from '../../SearchManager.js';
 import type { SearchTelemetryEnvelope } from '../../SearchManager.js';
+import { SEARCH_CONSTANTS } from '../../search/types.js';
 import { BaseRouteHandler } from '../BaseRouteHandler.js';
 import { validateBody } from '../middleware/validateBody.js';
 import { logger } from '../../../../utils/logger.js';
@@ -463,6 +464,27 @@ export class SearchRoutes extends BaseRouteHandler {
     const query = (req.body?.q || req.query.q) as string;
     const project = (req.body?.project || req.query.project) as string;
     const limit = Math.min(Math.max(parseInt(String(req.body?.limit || req.query.limit || '5'), 10) || 5, 1), 20);
+    const semanticWindowLimit = SEARCH_CONSTANTS.CHROMA_BATCH_SIZE;
+    const scopedSearchArgs = project
+      ? {
+          query,
+          type: 'observations',
+          project,
+          limit: String(limit),
+          format: 'json',
+          orderBy: 'relevance',
+        }
+      : {
+          query,
+          type: 'observations',
+          limit: String(limit),
+          format: 'json',
+          orderBy: 'relevance',
+        };
+    const observationKey = (obs: any): string => String(
+      obs?.id
+      ?? `${obs?.title || ''}:${obs?.created_at || ''}:${obs?.project || ''}:${obs?.merged_into_project || ''}`
+    );
 
     if (!query || query.length < 20) {
       res.json({ context: '', count: 0 });
@@ -471,31 +493,95 @@ export class SearchRoutes extends BaseRouteHandler {
 
     let result: any;
     try {
-      result = await this.searchManager.search({
-        query, type: 'observations', project, limit: String(limit), format: 'json'
-      });
+      const scopedTelemetry: SearchTelemetryEnvelope = {};
+      result = await this.searchManager.search(
+        scopedSearchArgs,
+        scopedTelemetry,
+        { semanticHydrationLimit: semanticWindowLimit }
+      );
+      const scopedObservations = result?.observations || [];
+      if (project) {
+        try {
+          const fallbackTelemetry: SearchTelemetryEnvelope = {};
+          // Always run: the scoped result cannot self-report missing adopted rows,
+          // and CHROMA_BATCH_SIZE keeps the recovery window bounded.
+          const fallbackResult = await this.searchManager.search({
+            query,
+            type: 'observations',
+            limit: String(limit),
+            format: 'json',
+            orderBy: 'relevance'
+          }, fallbackTelemetry, { semanticHydrationLimit: semanticWindowLimit });
+          const fallbackUsedKeywordSearch =
+            fallbackTelemetry.search_strategy === 'fts'
+            || fallbackTelemetry.search_strategy === 'filter_only';
+          if (fallbackUsedKeywordSearch) {
+            result = {
+              ...(result || {}),
+              observations: scopedObservations,
+            };
+          } else {
+            const scopedObservationsByKey = new Map(
+              scopedObservations.map((obs: any) => [observationKey(obs), obs])
+            );
+            const seenObservationKeys = new Set<string>();
+            result = {
+              ...(result || {}),
+              observations: [
+                ...(fallbackResult?.observations || [])
+                  .filter((obs: any) => obs.project === project || obs.merged_into_project === project)
+                  .filter((obs: any) => {
+                    const key = observationKey(obs);
+                    if (seenObservationKeys.has(key)) return false;
+                    seenObservationKeys.add(key);
+                    return true;
+                  })
+                  .map((obs: any) => scopedObservationsByKey.get(observationKey(obs)) || obs),
+                ...scopedObservations.filter((obs: any) => {
+                  const key = observationKey(obs);
+                  if (seenObservationKeys.has(key)) return false;
+                  seenObservationKeys.add(key);
+                  return true;
+                }),
+              ],
+            };
+          }
+        } catch (fallbackError) {
+          if (!scopedObservations.length) throw fallbackError;
+          const normalizedFallbackError = fallbackError instanceof Error
+            ? fallbackError
+            : new Error(String(fallbackError));
+          logger.warn(
+            'HTTP',
+            'Semantic context fallback failed, keeping scoped results',
+            { queryLength: query.length, project },
+            normalizedFallbackError
+          );
+        }
+      }
     } catch (error) {
       const normalizedError = error instanceof Error ? error : new Error(String(error));
-      logger.error('HTTP', 'Semantic context query failed', { query, project }, normalizedError);
+      logger.error('HTTP', 'Semantic context query failed', { queryLength: query.length, project }, normalizedError);
       res.json({ context: '', count: 0 });
       return;
     }
 
     const observations = result?.observations || [];
-    if (!observations.length) {
+    const renderedObservations = observations.slice(0, limit);
+    if (!renderedObservations.length) {
       res.json({ context: '', count: 0 });
       return;
     }
 
     const lines: string[] = ['## Relevant Past Work (semantic match)\n'];
-    for (const obs of observations.slice(0, limit)) {
+    for (const obs of renderedObservations) {
       const date = obs.created_at?.slice(0, 10) || '';
       lines.push(`### ${obs.title || 'Observation'} (${date})`);
       if (obs.narrative) lines.push(obs.narrative);
       lines.push('');
     }
 
-    res.json({ context: lines.join('\n'), count: observations.length });
+    res.json({ context: lines.join('\n'), count: renderedObservations.length });
   });
 
   private handleOnboardingExplainer = this.wrapHandler((_req: Request, res: Response): void => {

tests/services/sqlite/get-observations-by-ids-relevance.test.ts

@@ -46,6 +46,40 @@ describe('SessionStore.*ByIds — orderBy: "relevance" preserves caller ID order
     expect(results.map(r => r.id)).toEqual(callerOrder);
   });
 
+  it('applies a relevance limit after restoring the caller-provided ID order', () => {
+    const sdkId = store.createSDKSession('content-relevance-limit', 'p', 'prompt');
+    store.updateMemorySessionId(sdkId, 'session-relevance-limit');
+
+    const baseTs = 1_700_000_000_000;
+    const inserted: number[] = [];
+    for (let i = 0; i < 5; i++) {
+      const result = store.storeObservations(
+        'session-relevance-limit',
+        'p',
+        [{
+          type: 'test',
+          title: `obs-limit-${i}`,
+          subtitle: null,
+          facts: [`fact ${i}`],
+          narrative: null,
+          concepts: [],
+          files_read: [],
+          files_modified: [],
+        }],
+        null,
+        i,
+        0,
+        baseTs + i * 1000,
+      );
+      inserted.push(result.observationIds[0]);
+    }
+
+    const callerOrder = [...inserted].reverse();
+    const results = store.getObservationsByIds(callerOrder, { orderBy: 'relevance', limit: 2 });
+
+    expect(results.map(r => r.id)).toEqual(callerOrder.slice(0, 2));
+  });
+
   it('getObservationsByIds still respects date_desc when orderBy defaults', () => {
     const sdkId = store.createSDKSession('content-date', 'p', 'prompt');
     store.updateMemorySessionId(sdkId, 'session-date');