Merge remote-tracking branch 'origin/feat/221-rag-sources' into feat/rag-hybrid

# Conflicts:
#	__tests__/retrieval.test.ts
#	utils/hybridRetrieval.ts

Author: kfaracik

__tests__/llmStore.test.ts

@@ -761,6 +761,12 @@ describe('runBenchmark', () => {
     await loadModel();
     useLLMStore.setState({ model: baseModel });
 
+    // The RN jest preset aliases performance.now to Date.now (1 ms resolution),
+    // so on a fast machine startTime and the first token can share a millisecond
+    // and the measured delta collapses to 0. Advance a virtual clock instead.
+    let now = 0;
+    jest.spyOn(performance, 'now').mockImplementation(() => (now += 10));
+
     mockInstance.generate.mockImplementation(async () => {
       await flushFrame();
       capturedTokenCallback!('tok');

constants/citations.ts

@@ -8,12 +8,9 @@ export const CITATION_DOCUMENT_NAME_TOKEN_PATTERN = /[^a-z0-9ąćęłńóśźż]
 export const THINK_OPEN = '<think>';
 export const THINK_CLOSE = '</think>';
 
-// English negation cues. Terms inside a negated clause say what a source does NOT
-// support, so they must not count as overlap evidence for citing it.
 export const NEGATION_CUE_EN =
   /\b(no|not|n['’]t|never|none|neither|nor|without|lacks?|lacking)\b/i;
 
-// Clause boundaries, so "covers X but does not mention Y" keeps X and drops only Y.
 export const CLAUSE_SPLIT_PATTERN =
   /[,;]|\b(?:but|however|although|though|whereas|while)\b/i;
 

hooks/useAttachment.ts

@@ -74,9 +74,6 @@ export const useAttachment = () => {
   const vectorStoreRef = useRef(vectorStore);
   vectorStoreRef.current = vectorStore;
 
-  // A document is embedded as a source the moment it is attached, but it is only
-  // tied to a chat on send. Sweep whenever one is abandoned instead, or it stays
-  // in the store forever. cleanupOrphanedSources only removes unreferenced rows.
   const sweepAbandonedSources = useCallback(() => {
     const store = vectorStoreRef.current;
     if (store) useSourceStore.getState().cleanupOrphanedSources(store);

utils/embeddingModelMigration.ts

@@ -52,8 +52,6 @@ export const migrateEmbeddingModelIfNeeded = async (
     const persistedDim = await readPersistedVectorDim(vectorStore);
     const incompatible =
       persistedDim !== null && persistedDim !== currentModelDim;
-    // Leave the key unset when the wipe was partial so the next launch retries;
-    // deleting again is idempotent.
     if (incompatible && !(await clearImportedSources(vectorStore, db))) {
       return true;
     }

utils/messageSources.ts

@@ -123,7 +123,6 @@ export const visibleAnswer = (answer: string): string => {
   while (open !== -1) {
     parts.push(answer.slice(cursor, open));
     const close = answer.indexOf(THINK_CLOSE, open + THINK_OPEN.length);
-    // Unterminated: the model is still reasoning, so nothing after it is visible.
     if (close === -1) return `${parts.join(' ')} `;
     cursor = close + THINK_CLOSE.length;
     open = answer.indexOf(THINK_OPEN, cursor);
@@ -133,9 +132,6 @@ export const visibleAnswer = (answer: string): string => {
   return parts.join(' ');
 };
 
-// Keep only the clauses the reply actually asserts; a negated clause names a topic
-// the source does not cover, and scoring it as overlap cites the source for the
-// opposite of what it says. English-only for now.
 const affirmativeAnswer = (visibleReply: string): string =>
   (visibleReply.match(CITATION_SENTENCE_PATTERN) ?? [visibleReply])
     .flatMap((sentence) => sentence.split(CLAUSE_SPLIT_PATTERN))