fix: enforce read-only vector opens, batch hybrid FTS leg, MCP scope param

Author: wesm

docs/semantic-search-internals.md

@@ -289,8 +289,12 @@ Generation activation always happens under the single writer. Search opens
   are unchanged on every backend.
 - **`scope` governs unit visibility and supersedes `include_children`.**
   `scope=top|all|subordinate` (default `all`) filters each leg's hits before
-  the RRF merge and before the limit, so a scoped search still fills up to the
-  limit from the over-fetched candidates. In semantic/hybrid modes the
+  the RRF merge and before the limit. The hybrid FTS leg fetches additional
+  rank-ordered batches until it holds the fusion depth `k` of surviving
+  entries (capped at `maxHybridFTSBatches`), so scope discards and same-unit
+  collapse do not starve it; the semantic (KNN) leg cannot page, so scoped or
+  collapse-heavy searches can still under-fill past those caps even when more
+  matches exist deeper in the ranking. In semantic/hybrid modes the
   sidebar-child session exclusion is lifted (`semanticSessionScopeSubquery`) —
   both hybrid legs must see the same universe for fusion to be sound — and an
   explicit `include_children` is accepted but superseded. Subagent/fork-typed

docs/semantic-search.md

@@ -342,8 +342,10 @@ MCP tool error, carrying the same remediation text.
   fixed minimum if that's larger), then drop hits whose session fails
   `--project`/`--agent`/`--date*`/etc., then truncate to the requested limit.
   At small corpus sizes or with a narrow filter, this can return fewer than
-  `--limit` results even though more exist. This is a known v1 tradeoff, not a
-  bug.
+  `--limit` results even though more exist. A narrow `--scope` (and, in
+  hybrid, matches concentrated in one long run) can likewise return fewer than
+  `--limit` even when more matches exist deeper in the ranking. This is a
+  known v1 tradeoff, not a bug.
 - **Legacy no-`source_uuid` rows re-embed on ordinal shifts.** Each embedded
   document is keyed by its first message's stable per-message UUID when the
   parser recorded one, or by `(session_id, ordinal)` when it didn't.

internal/db/search_content.go

@@ -1033,6 +1033,14 @@ func (db *DB) hybridVectorLeg(
 	return leg, nil
 }
 
+// maxHybridFTSBatches caps how many k-row FTS batches hybridFTSLeg fetches.
+// It bounds the worst-case work when discard dominates — many rows collapsing
+// into one unit, or a narrow f.Scope dropping most rows — while letting the
+// leg keep paging past discarded rows instead of under-filling after the
+// first batch. The residual is documented: a leg needing survivors deeper
+// than maxHybridFTSBatches x k rows can still under-fill.
+const maxHybridFTSBatches = 4
+
 // hybridFTSLeg runs a rank-ordered FTS query over the embedded universe
 // (role user/assistant, is_system = 0, system-prefix excluded -- the same
 // predicate ScanEmbeddableUnits uses), scoped in SQL to sessions passing
@@ -1045,9 +1053,38 @@ func (db *DB) hybridVectorLeg(
 // hits in one unit collapse to the best-ranked one. A hit with no containing
 // unit keeps a message-granularity key, is never subordinate-penalized, and
 // so survives fusion on its own.
+//
+// Rows are fetched in rank-ordered batches of k with OFFSET continuation:
+// collapse and scope filtering can discard most of a batch, so the leg keeps
+// fetching until it holds k entries, the stream is exhausted, or
+// maxHybridFTSBatches is hit. The display seen-check dedups across batches;
+// earlier batches rank better, so the best-ranked hit per unit always wins.
 func (db *DB) hybridFTSLeg(
 	ctx context.Context, f ContentSearchFilter, searcher VectorSearcher, k int,
 ) (hybridLeg, error) {
+	leg := hybridLeg{display: make(map[string]hybridDisplay, k)}
+	for batch := range maxHybridFTSBatches {
+		hits, err := db.fetchHybridFTSBatch(ctx, f, k, batch*k)
+		if err != nil {
+			return hybridLeg{}, err
+		}
+		if err := appendHybridFTSHits(ctx, searcher, f.Scope, hits, &leg); err != nil {
+			return hybridLeg{}, err
+		}
+		if len(hits) < k || len(leg.ranked) >= k {
+			break
+		}
+	}
+	return leg, nil
+}
+
+// fetchHybridFTSBatch fetches one rank-ordered batch of at most k FTS message
+// rows for hybridFTSLeg, starting at offset. The ORDER BY carries m.id as a
+// deterministic tiebreak so OFFSET continuation is stable across batches when
+// ranks tie.
+func (db *DB) fetchHybridFTSBatch(
+	ctx context.Context, f ContentSearchFilter, k, offset int,
+) ([]hybridDisplay, error) {
 	scope, scopeArgs := semanticSessionScopeSubquery(f)
 	query := fmt.Sprintf(`
 		SELECT m.session_id, m.ordinal,
@@ -1056,45 +1093,57 @@ func (db *DB) hybridFTSLeg(
 		WHERE messages_fts MATCH ? AND m.role IN ('user','assistant')
 		  AND m.is_system = 0 AND %s
 		  AND m.%s
-		ORDER BY f.rank LIMIT ?`,
+		ORDER BY f.rank, m.id LIMIT ? OFFSET ?`,
 		SystemPrefixSQL("m.content", "m.role"), scope)
 
 	args := []any{PrepareFTSQuery(f.Pattern)}
 	args = append(args, scopeArgs...)
-	args = append(args, k)
+	args = append(args, k, offset)
 
 	rows, err := db.getReader().QueryContext(ctx, query, args...)
 	if err != nil {
-		return hybridLeg{}, classifyFTSError(fmt.Errorf("hybrid search fts leg: %w", err))
+		return nil, classifyFTSError(fmt.Errorf("hybrid search fts leg: %w", err))
 	}
 	defer rows.Close()
 
 	var hits []hybridDisplay
 	for rows.Next() {
 		var hit hybridDisplay
 		if err := rows.Scan(&hit.sessionID, &hit.ordinal, &hit.snippet); err != nil {
-			return hybridLeg{}, fmt.Errorf("scan hybrid fts hit: %w", err)
+			return nil, fmt.Errorf("scan hybrid fts hit: %w", err)
 		}
 		hits = append(hits, hit)
 	}
 	if err := rows.Err(); err != nil {
-		return hybridLeg{}, err
+		return nil, err
 	}
+	return hits, nil
+}
 
+// appendHybridFTSHits resolves one batch of FTS message hits to their
+// containing units and accumulates the survivors into leg: hits outside
+// scope are dropped, and a unit already seen (within or across batches)
+// keeps its earlier, better-ranked entry.
+func appendHybridFTSHits(
+	ctx context.Context, searcher VectorSearcher, scope string,
+	hits []hybridDisplay, leg *hybridLeg,
+) error {
+	if len(hits) == 0 {
+		return nil
+	}
 	refs := make([]MessageRef, len(hits))
 	for i, hit := range hits {
 		refs[i] = MessageRef{SessionID: hit.sessionID, Ordinal: hit.ordinal}
 	}
 	units, err := searcher.ResolveMessageUnits(ctx, refs)
 	if err != nil {
-		return hybridLeg{}, fmt.Errorf("resolving fts hits to units: %w", err)
+		return fmt.Errorf("resolving fts hits to units: %w", err)
 	}
 	if len(units) != len(refs) {
-		return hybridLeg{}, fmt.Errorf(
+		return fmt.Errorf(
 			"resolving fts hits to units: got %d units for %d refs", len(units), len(refs))
 	}
 
-	leg := hybridLeg{display: make(map[string]hybridDisplay, len(hits))}
 	for i, hit := range hits {
 		key := messageFusionKey(hit.sessionID, hit.ordinal)
 		hit.ordinalStart, hit.ordinalEnd = hit.ordinal, hit.ordinal
@@ -1104,7 +1153,7 @@ func (db *DB) hybridFTSLeg(
 			hit.ordinalEnd = units[i].OrdinalEnd
 			hit.subordinate = units[i].Subordinate
 		}
-		if scopeExcludes(f.Scope, hit.subordinate) {
+		if scopeExcludes(scope, hit.subordinate) {
 			continue
 		}
 		if _, seen := leg.display[key]; seen {
@@ -1113,7 +1162,7 @@ func (db *DB) hybridFTSLeg(
 		leg.ranked = append(leg.ranked, unitRanked{Key: key, Subordinate: hit.subordinate})
 		leg.display[key] = hit
 	}
-	return leg, nil
+	return nil
 }
 
 // enrichHybridMatches looks up session/message metadata for the fused units
@@ -1325,7 +1374,8 @@ func (db *DB) enrichSemanticHits(
 // approximate snippet text semantic/hybrid modes locate within a message's
 // full content: the vector index's trailing unicode ellipsis
 // (internal/vector's truncateRunes) and FTS5 snippet()'s literal "..." marker
-// (used at both ends), configured as hybridFTSLeg's 5th snippet() argument.
+// (used at both ends), configured as fetchHybridFTSBatch's 5th snippet()
+// argument.
 var snippetTruncationMarkers = []string{"...", "…"}
 
 // approxSnippetSpan locates approx (a searcher-provided chunk/snippet or

internal/db/search_content_hybrid_test.go

@@ -458,6 +458,79 @@ func TestSearchContentHybridMatchCarriesUnitRangeAndLineage(t *testing.T) {
 	assert.True(t, m.Sidechain, "anchor message is_sidechain")
 }
 
+// TestSearchContentHybridFTSLegCollapseRefillsFromDeeperRanks pins the
+// batched FTS leg against unit collapse: when more than k (the
+// semanticOverfetchMin=200 fusion depth) rank-ordered FTS rows all fall
+// inside ONE run-unit, they collapse to a single leg entry, and a match in a
+// different unit ranked below all of them must still be fetched and returned
+// rather than being cut off by the first batch's window.
+func TestSearchContentHybridFTSLegCollapseRefillsFromDeeperRanks(t *testing.T) {
+	d := testDB(t)
+	if !d.HasFTS() {
+		t.Skip("fts5 not available")
+	}
+	// One run-unit spanning 205 assistant messages, each repeating the term
+	// so bm25 ranks every one of them above the single-occurrence session.
+	const runLen = 205
+	runMsgs := make([][2]string, runLen)
+	for i := range runMsgs {
+		runMsgs[i] = [2]string{"assistant", "zebra zebra zebra zebra"}
+	}
+	seedSearchSession(t, d, "bigrun", "proj", runMsgs)
+	seedSearchSession(t, d, "other", "proj", [][2]string{
+		{"user", "zebra appears once in a much longer unrelated sentence"},
+	})
+	// The vector leg is empty; the resolver knows the whole run as one unit.
+	d.SetVectorSearcher(&fakeVectorSearcher{units: []UnitRef{
+		{DocKey: "r:bigrun:0", SessionID: "bigrun",
+			OrdinalStart: 0, OrdinalEnd: runLen - 1},
+	}})
+
+	page, err := d.SearchContent(context.Background(), ContentSearchFilter{
+		Pattern: "zebra", Mode: "hybrid", Limit: 10,
+	})
+	require.NoError(t, err, "SearchContent hybrid")
+	require.Len(t, page.Matches, 2,
+		"the lower-ranked unit past the collapsed run must be fetched")
+	ids := []string{page.Matches[0].SessionID, page.Matches[1].SessionID}
+	assert.ElementsMatch(t, []string{"bigrun", "other"}, ids)
+}
+
+// TestSearchContentHybridFTSLegScopeExcludedRowsRefill pins the batched FTS
+// leg against scope discard: with scope=subordinate, when the first k FTS
+// rows are all top-level (and so all dropped), a subordinate match ranked
+// below them must still be fetched and returned.
+func TestSearchContentHybridFTSLegScopeExcludedRowsRefill(t *testing.T) {
+	d := testDB(t)
+	if !d.HasFTS() {
+		t.Skip("fts5 not available")
+	}
+	// 205 top-level sessions would be slow; one top-level session with 205
+	// matching messages fills the first batch the same way, since each
+	// message is its own unit-less (never-subordinate) row.
+	const topLen = 205
+	topMsgs := make([][2]string, topLen)
+	for i := range topMsgs {
+		topMsgs[i] = [2]string{"user", "zebra zebra zebra zebra"}
+	}
+	seedSearchSession(t, d, "toplots", "proj", topMsgs)
+	seedSubagentSession(t, d, "sub", "toplots", "proj", [][2]string{
+		{"user", "zebra appears once in a much longer subagent sentence"},
+	})
+	d.SetVectorSearcher(&fakeVectorSearcher{units: []UnitRef{
+		{DocKey: "u:sub:0", SessionID: "sub",
+			OrdinalStart: 0, OrdinalEnd: 0, Subordinate: true},
+	}})
+
+	page, err := d.SearchContent(context.Background(), ContentSearchFilter{
+		Pattern: "zebra", Mode: "hybrid", Scope: "subordinate", Limit: 10,
+	})
+	require.NoError(t, err, "SearchContent hybrid")
+	require.Len(t, page.Matches, 1,
+		"the subordinate match past the excluded top-level rows must be fetched")
+	assert.Equal(t, "sub", page.Matches[0].SessionID)
+}
+
 // TestSearchContentHybridVectorOnlyMatchCarriesUnitRange pins the
 // vector-leg display path: a unit only the semantic leg found keeps its
 // chunk anchor and still exposes the unit range and subordinate flag.

internal/mcp/tools.go

@@ -475,6 +475,7 @@ func (t *toolset) getMessagesAround(
 type searchContentIn struct {
 	Pattern       string `json:"pattern" jsonschema:"Exact substring or regex to find across message text and tool inputs/results."`
 	Mode          string `json:"mode,omitempty" jsonschema:"substring (default), regex, semantic, or hybrid."`
+	Scope         string `json:"scope,omitempty" jsonschema:"Semantic/hybrid result scope: top, all, or subordinate (default all). Only valid with mode semantic or hybrid."`
 	Project       string `json:"project,omitempty" jsonschema:"Restrict to one project."`
 	Agent         string `json:"agent,omitempty" jsonschema:"Restrict to one agent."`
 	DateFrom      string `json:"date_from,omitempty" jsonschema:"Only sessions on or after this date (YYYY-MM-DD)."`
@@ -533,9 +534,17 @@ type searchContentOut struct {
 func (t *toolset) searchContent(
 	ctx context.Context, _ *mcp.CallToolRequest, in searchContentIn,
 ) (*mcp.CallToolResult, searchContentOut, error) {
+	// The db layer silently ignores Scope outside semantic/hybrid, so reject
+	// it here with the same message the HTTP transport uses
+	// (internal/server/huma_routes_search.go).
+	if in.Scope != "" && in.Mode != "semantic" && in.Mode != "hybrid" {
+		return nil, searchContentOut{}, fmt.Errorf(
+			"scope is only supported for semantic and hybrid search modes")
+	}
 	res, err := t.svc.SearchContent(ctx, service.ContentSearchRequest{
 		Pattern:  in.Pattern,
 		Mode:     in.Mode,
+		Scope:    in.Scope,
 		Project:  in.Project,
 		Agent:    in.Agent,
 		DateFrom: in.DateFrom,

internal/mcp/tools_test.go

@@ -848,6 +848,48 @@ func TestSearchContent_SemanticUnavailableMapsToRemediationError(t *testing.T) {
 	assert.Equal(t, "semantic", fake.lastReq.Mode)
 }
 
+// search_content must reject scope outside semantic/hybrid with the same
+// message the HTTP transport uses (the db layer silently ignores Scope for
+// lexical modes, so the guard lives in the transport), and must not reach
+// the service at all on rejection.
+func TestSearchContent_ScopeRejectedOnLexicalModes(t *testing.T) {
+	fake := &fakeContentSearchService{result: &service.ContentSearchResult{}}
+	ts := &toolset{svc: fake, now: func() time.Time { return fixedNow }}
+
+	for _, mode := range []string{"", "substring", "regex", "fts"} {
+		t.Run("mode="+mode, func(t *testing.T) {
+			_, _, err := ts.searchContent(context.Background(), nil, searchContentIn{
+				Pattern: "needle", Mode: mode, Scope: "top", IncludeActive: true,
+			})
+			require.Error(t, err)
+			assert.EqualError(t, err,
+				"scope is only supported for semantic and hybrid search modes")
+		})
+	}
+	assert.Empty(t, fake.lastReq.Pattern,
+		"a rejected request must not reach the service")
+}
+
+// search_content must pass Scope through to the service untouched for
+// semantic and hybrid modes; the db layer owns scope-value validation from
+// there.
+func TestSearchContent_ScopeForwardedForSemanticModes(t *testing.T) {
+	for _, mode := range []string{"semantic", "hybrid"} {
+		t.Run(mode, func(t *testing.T) {
+			fake := &fakeContentSearchService{result: &service.ContentSearchResult{}}
+			ts := &toolset{svc: fake, now: func() time.Time { return fixedNow }}
+
+			_, _, err := ts.searchContent(context.Background(), nil, searchContentIn{
+				Pattern: "retries", Mode: mode, Scope: "subordinate", IncludeActive: true,
+			})
+			require.NoError(t, err)
+			assert.Equal(t, mode, fake.lastReq.Mode)
+			assert.Equal(t, "subordinate", fake.lastReq.Scope,
+				"scope must reach the service untouched")
+		})
+	}
+}
+
 // search_content's Context parameter must reach the service, and each
 // match's ContextBefore/ContextAfter (full service-level db.Message) must
 // map to the MCP layer's truncated contextMessage shape, along with Score.

internal/vector/index.go

@@ -134,6 +134,11 @@ type GenerationInfo struct {
 // concurrently rewritten by another agentsview process, and carries its own
 // busy timeout so a reader waits out a concurrent writer's lock instead of
 // failing immediately with SQLITE_BUSY.
+//
+// Both branches emit a file: URI. mattn/go-sqlite3 forwards the `_`-prefixed
+// pragma params either way, but it only honors mode=ro when the DSN carries
+// the file: scheme — a bare path silently opens read-write, so the ro
+// contract depends on the prefix.
 func vectorDSN(path string, readOnly bool) string {
 	params := url.Values{}
 	if readOnly {
@@ -144,7 +149,7 @@ func vectorDSN(path string, readOnly bool) string {
 		params.Set("_busy_timeout", "5000")
 		params.Set("_synchronous", "NORMAL")
 	}
-	return path + "?" + params.Encode()
+	return "file:" + path + "?" + params.Encode()
 }
 
 // ChunkOverlap derives the SplitOptions.Overlap rune count from