feat: optional embedding-based dedup on reflection memory writes

Adds two new AppSettings — reflectionSemanticDedupEnabled (boolean,
default false) and reflectionSemanticDedupThreshold (number, default
0.88) — that layer cosine-similarity dedup on top of the existing
text-equality cross-run dedup in writeReflectionMemories.

Motivation: the existing cross-run dedup catches reflections only when
the LLM produces literally-identical normalized text. In practice the
classifier rederives the same insight in different words across runs
("Always verify before acting" vs "Confirm state first"); text-equality
misses these, and 95% of memory rows end up as routine reflection/*.
Embedding-based dedup catches the near-duplicates.

How it works:
  - In writeReflectionMemories, after the existing text-equality cross-run
    dedup loop, if reflectionSemanticDedupEnabled is set: pull recent
    reflection memories' embeddings (new memoryDb.recentReflectionEmbeddings
    method), embed each candidate note with the configured embedder, and
    skip notes whose cosine similarity to any recent reflection exceeds
    the threshold.
  - Falls back gracefully when embeddings aren't configured (getEmbedding
    returns null), when recent memories have no stored embeddings, or on
    any error — never blocks the write.
  - writeReflectionMemories becomes async (sole caller already in an async
    fn, so the change is contained).

Why opt-in: each candidate note costs one embedder call (~10-20 ms local
nomic-embed-text, more for remote providers). For deployments with the
local Ollama setup running already, the cost is negligible; for OpenAI
embedding users it adds tokens. Default false leaves behavior unchanged.

Files:
  - src/types/app-settings.ts: 2 new optional fields
  - src/lib/server/memory/memory-db.ts: recentReflectionEmbeddings method
  - src/lib/server/autonomy/supervisor-reflection.ts: semantic dedup pass
    in writeReflectionMemories (now async; call site awaits)

Author: siglimumuni

src/lib/server/autonomy/supervisor-reflection.ts

@@ -744,7 +744,7 @@ function inferFollowUpAt(note: string, createdAt: number): number {
   return createdAt + 7 * 24 * 3600_000
 }
 
-function writeReflectionMemories(params: {
+async function writeReflectionMemories(params: {
   reflectionId: string
   runId: string
   sessionId: string
@@ -761,7 +761,7 @@ function writeReflectionMemories(params: {
   profile: string[]
   boundaries: string[]
   openLoops: string[]
-}): string[] {
+}): Promise<string[]> {
   const memoryDb = getMemoryDb()
   const memoryIds: string[] = []
   const incidentIds = params.incidents.map((incident) => incident.id)
@@ -809,11 +809,53 @@ function writeReflectionMemories(params: {
     // dedup only rather than blocking the reflection write.
   }
 
+  // Semantic dedup (opt-in): on top of the text-equality cross-run dedup
+  // above, compare each candidate note's embedding against recent reflection
+  // memories' embeddings. Catches near-duplicates the LLM re-derives in
+  // different words ("Always verify before acting" / "Confirm state first").
+  // Falls back gracefully when embeddings aren't configured.
+  let appSettings: AppSettings | null = null
+  try { appSettings = loadSettings() } catch { appSettings = null }
+  const semanticDedupEnabled = Boolean(appSettings?.reflectionSemanticDedupEnabled)
+  const semanticDedupThreshold = typeof appSettings?.reflectionSemanticDedupThreshold === 'number'
+    ? appSettings.reflectionSemanticDedupThreshold
+    : 0.88
+  const semanticSkip = new Set<string>()
+  if (semanticDedupEnabled && params.agentId) {
+    try {
+      const recentEmb = memoryDb.recentReflectionEmbeddings(params.agentId, crossRunDedupCutoff, 500)
+        .filter((r) => Array.isArray(r.embedding) && r.embedding.length > 0) as Array<{ id: string; content: string; embedding: number[] }>
+      if (recentEmb.length > 0) {
+        const { getEmbedding, cosineSimilarity } = await import('@/lib/server/embeddings')
+        for (const group of groups) {
+          for (const note of group.notes) {
+            const trimmed = (note || '').trim()
+            if (!trimmed) continue
+            const norm = normalizeNote(trimmed)
+            if (!norm || seenNormalized.has(norm) || semanticSkip.has(norm)) continue
+            const emb = await getEmbedding(trimmed)
+            if (!emb) continue
+            for (const r of recentEmb) {
+              if (cosineSimilarity(emb, r.embedding) >= semanticDedupThreshold) {
+                semanticSkip.add(norm)
+                break
+              }
+            }
+          }
+        }
+      }
+    } catch {
+      // Best-effort: any failure (embedder offline, DB blip) falls through to
+      // the existing text-equality dedup. Never block the write.
+    }
+  }
+
   for (const group of groups) {
     for (const note of group.notes) {
       const norm = normalizeNote(note)
       if (!norm) continue
       if (seenNormalized.has(norm)) continue
+      if (semanticSkip.has(norm)) continue
       seenNormalized.add(norm)
       const metadata: Record<string, unknown> = {
         origin: 'autonomy-reflection',
@@ -1085,10 +1127,6 @@ export async function observeAutonomyRunOutcome(
   if (parsed.skip) return { incidents, reflection: null }
 
   const reflectionId = genId()
-  // Quality gate: if reflectionMinQuality is set above 0, only write memories
-  // when the reflection's qualityScore meets or exceeds it. Null/undefined
-  // qualityScore is admitted (we don't want a model that omits the score to
-  // silently lose all memory writes).
   const minQuality = typeof settings.reflectionMinQuality === 'number' ? settings.reflectionMinQuality : 0
   const qualityScore = parsed.qualityScore
   const qualityGateOpen = minQuality <= 0
@@ -1100,7 +1138,7 @@ export async function observeAutonomyRunOutcome(
     )
   }
   const autoMemoryIds = settings.reflectionAutoWriteMemory && qualityGateOpen
-    ? writeReflectionMemories({
+    ? await writeReflectionMemories({
         reflectionId,
         runId: input.runId,
         sessionId: input.sessionId,

src/lib/server/memory/memory-db.ts

@@ -1270,6 +1270,30 @@ function initDb() {
       return (stmts.listByAgent.all(agentId, safeLimit) as any[]).map(rowToEntry)
     },
 
+    /** Return recent reflection/* memories with their embeddings deserialized
+     *  for semantic dedup. Memories without an embedding (older rows, or
+     *  embedding still being computed in background) are included with a
+     *  null embedding so callers can fall back to text dedup. */
+    recentReflectionEmbeddings(
+      agentId: string,
+      sinceMs: number,
+      limit = 200,
+    ): Array<{ id: string; content: string; embedding: number[] | null }> {
+      const safeLimit = Math.max(1, Math.min(500, Math.trunc(limit)))
+      const rows = db.prepare(
+        `SELECT id, content, embedding FROM memories
+         WHERE (agentId = ? OR sharedWith LIKE ?)
+           AND category LIKE 'reflection/%'
+           AND updatedAt >= ?
+         ORDER BY updatedAt DESC LIMIT ?`,
+      ).all(agentId, `%"${agentId}"%`, sinceMs, safeLimit) as Array<{ id: string; content: string; embedding: Buffer | null }>
+      return rows.map((row) => ({
+        id: row.id,
+        content: row.content || '',
+        embedding: row.embedding ? deserializeEmbedding(row.embedding) : null,
+      }))
+    },
+
     getFrequentlyAccessedByAgent(agentId: string, minAccessCount = 3, sinceDays = 7): MemoryEntry[] {
       const cutoff = Date.now() - sinceDays * 86_400_000
       const rows = stmts.frequentlyAccessedByAgent.all(agentId, minAccessCount, cutoff) as Record<string, unknown>[]

src/types/app-settings.ts

@@ -111,15 +111,12 @@ export interface AppSettings {
   autonomyResumeApprovalsEnabled?: boolean
   reflectionEnabled?: boolean
   reflectionAutoWriteMemory?: boolean
-  /** Minimum reflection quality score (0-1, as emitted by the reflection
-   *  classifier in `parsed.qualityScore`) required to auto-write memories.
-   *  Defaults to 0 (no gating, behavior unchanged). When raised, low-quality
-   *  reflections still produce a RunReflection record but skip the per-kind
-   *  memory writes — reducing the bloat where 95%+ of memory rows are
-   *  routine reflection/* entries that never get retrieved.
-   *  Reflections with a null qualityScore are admitted regardless (so an
-   *  upstream parse failure can't silently lose all memory writes). */
+  /** Minimum reflection quality score (0-1) required to auto-write memories. */
   reflectionMinQuality?: number
+  /** Enable embedding-based dedup for reflection memory writes. */
+  reflectionSemanticDedupEnabled?: boolean
+  /** Cosine threshold above which a reflection note is considered duplicate. */
+  reflectionSemanticDedupThreshold?: number
   memoryReferenceDepth?: number
   maxMemoriesPerLookup?: number
   maxLinkedMemoriesExpanded?: number