diff --git a/AGENTS.md b/AGENTS.md index 2dd2d16f2..5db98b2b7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,6 +120,8 @@ CLI (agentsview) -> Config -> DB (SQLite/FTS5) - `internal/parser/` - Per-agent session file parsers and content extraction. - `internal/server/` - HTTP handlers, SSE, middleware, search, and export. - `internal/sync/` - Sync engine, file watcher, discovery, and hashing. +- `internal/vector/` - Semantic search: embeddings encoder, `vectors.db` + mirror/index, build orchestration, and semantic/hybrid search. - `internal/timeutil/` - Time parsing utilities. - `internal/web/` - Embedded frontend copied from `frontend/dist/` at build time. @@ -128,30 +130,33 @@ CLI (agentsview) -> Config -> DB (SQLite/FTS5) ## Key Files -| Path | Purpose | -| -------------------------------- | --------------------------------------------- | -| `cmd/agentsview/main.go` | CLI entry point, server startup, file watcher | -| `cmd/agentsview/pg.go` | `pg` command group: push, status, serve | -| `internal/server/server.go` | HTTP router and handler setup | -| `internal/server/sessions.go` | Session list/detail API handlers | -| `internal/server/search.go` | Full-text search API | -| `internal/server/events.go` | SSE event streaming | -| `internal/db/db.go` | Database open, migrations, schema | -| `internal/db/sessions.go` | Session CRUD queries | -| `internal/db/search.go` | FTS5 search queries | -| `internal/sync/engine.go` | Sync orchestration | -| `internal/parser/types.go` | Agent registry with one `AgentDef` per agent | -| `internal/parser/*.go` | Per-agent session parsers | -| `internal/postgres/connect.go` | Connection setup, SSL checks, DSN helpers | -| `internal/postgres/schema.go` | PG DDL and schema management | -| `internal/postgres/push.go` | Push logic and fingerprinting | -| `internal/postgres/sync.go` | Push sync lifecycle | -| `internal/postgres/store.go` | PostgreSQL read-only store | -| `internal/postgres/sessions.go` | PG session queries on the read side | -| `internal/postgres/messages.go` | PG message queries and ILIKE search | -| `internal/postgres/analytics.go` | PG analytics queries | -| `internal/postgres/time.go` | Timestamp conversion helpers | -| `internal/config/config.go` | Config loading and flag registration | +| Path | Purpose | +| -------------------------------- | --------------------------------------------------------- | +| `cmd/agentsview/main.go` | CLI entry point, server startup, file watcher | +| `cmd/agentsview/pg.go` | `pg` command group: push, status, serve | +| `cmd/agentsview/embeddings.go` | `embeddings` command group: build, list, activate, retire | +| `internal/server/server.go` | HTTP router and handler setup | +| `internal/server/sessions.go` | Session list/detail API handlers | +| `internal/server/search.go` | Full-text search API | +| `internal/server/events.go` | SSE event streaming | +| `internal/db/db.go` | Database open, migrations, schema | +| `internal/db/sessions.go` | Session CRUD queries | +| `internal/db/search.go` | FTS5 search queries | +| `internal/vector/index.go` | `vectors.db` schema, generations, staleness gate | +| `internal/vector/search.go` | Semantic + hybrid search, RRF merge | +| `internal/sync/engine.go` | Sync orchestration | +| `internal/parser/types.go` | Agent registry with one `AgentDef` per agent | +| `internal/parser/*.go` | Per-agent session parsers | +| `internal/postgres/connect.go` | Connection setup, SSL checks, DSN helpers | +| `internal/postgres/schema.go` | PG DDL and schema management | +| `internal/postgres/push.go` | Push logic and fingerprinting | +| `internal/postgres/sync.go` | Push sync lifecycle | +| `internal/postgres/store.go` | PostgreSQL read-only store | +| `internal/postgres/sessions.go` | PG session queries on the read side | +| `internal/postgres/messages.go` | PG message queries and ILIKE search | +| `internal/postgres/analytics.go` | PG analytics queries | +| `internal/postgres/time.go` | Timestamp conversion helpers | +| `internal/config/config.go` | Config loading and flag registration | ## Development diff --git a/README.md b/README.md index 022719899..1df78dc97 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,11 @@ agentsview stats --include-git-outcomes | ![Search](https://agentsview.io/assets/generated/screenshots/search-results.png) | ![Heatmap](https://agentsview.io/assets/generated/screenshots/heatmap.png) | - **Full-text search** across all message content (FTS5) +- **Semantic search** (opt-in) -- index session content with any + OpenAI-compatible embeddings endpoint and search by meaning with + `agentsview session search --semantic` or `--hybrid`; every content-search + match cites the conversation unit it came from + ([docs](https://agentsview.io/semantic-search/)) - **Token usage and cost dashboard** -- per-session and per-model cost breakdowns, daily spend charts, all in the web UI - **Analytics dashboard** -- activity heatmaps, tool usage, velocity metrics, diff --git a/cmd/agentsview/cli.go b/cmd/agentsview/cli.go index b4de5931b..7884e999b 100644 --- a/cmd/agentsview/cli.go +++ b/cmd/agentsview/cli.go @@ -115,12 +115,14 @@ func newRootCommand() *cobra.Command { root.AddCommand(newActivityCommand()) root.AddCommand(newPGCommand()) root.AddCommand(newDuckDBCommand()) + root.AddCommand(newEmbeddingsCommand()) root.AddCommand(newSessionCommand()) root.AddCommand(newMCPCommand()) root.AddCommand(newStatsCommand()) root.AddCommand(newParseDiffCommand()) root.AddCommand(newClassifierCommand()) root.AddCommand(newSecretsCommand()) + root.AddCommand(newSkillsCommand()) root.AddCommand(newDoctorCommand()) root.AddCommand(newVersionCommand()) root.AddCommand(newOpenAPICommand()) diff --git a/cmd/agentsview/doctor.go b/cmd/agentsview/doctor.go index e4de4bdb0..b5c1835ad 100644 --- a/cmd/agentsview/doctor.go +++ b/cmd/agentsview/doctor.go @@ -202,12 +202,17 @@ func inspectDoctorDB(path string) doctorDBInspection { return insp } +// doctorReadOnlyDSN builds a read-only sqlite3 DSN. The file: scheme is +// required for mattn/go-sqlite3 to honor mode=ro (a bare path silently opens +// read-write), and the path is percent-encoded so `%`, `?`, or `#` in a real +// path cannot be misparsed as URI syntax. func doctorReadOnlyDSN(path string) string { params := url.Values{} params.Set("mode", "ro") params.Set("_busy_timeout", "5000") params.Set("_foreign_keys", "ON") - return path + "?" + params.Encode() + escaped := (&url.URL{Path: path}).EscapedPath() + return "file:" + escaped + "?" + params.Encode() } func listDoctorResyncTempFiles(dbPath string) []string { diff --git a/cmd/agentsview/embed_scheduler.go b/cmd/agentsview/embed_scheduler.go new file mode 100644 index 000000000..68903d8c2 --- /dev/null +++ b/cmd/agentsview/embed_scheduler.go @@ -0,0 +1,486 @@ +// ABOUTME: after-sync embedding scheduler and the daemon's vector subsystem +// ABOUTME: wiring — index open, encoder/Manager construction, searcher adapter. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "time" + + kitvec "go.kenn.io/kit/vector" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/server" + "go.kenn.io/agentsview/internal/sync" + "go.kenn.io/agentsview/internal/vector" +) + +// vectorsWriteLockRetryInterval and vectorsWriteLockRetryTimeout bound how +// long setupVectorServing waits for vectors.write.lock before giving up and +// disabling vector serving for this daemon run. Package vars so tests can +// shrink them. +var ( + vectorsWriteLockRetryInterval = 200 * time.Millisecond + vectorsWriteLockRetryTimeout = 5 * time.Second +) + +// acquireVectorsWriteLockWithRetry tries to acquire vectors.write.lock, +// retrying briefly if another process (typically a long-running direct +// `embeddings build`) currently holds it. It returns ok=false, err=nil — not +// an error — once the retry window elapses with the lock still held, so +// setupVectorServing can degrade (disable vector serving for this run) +// rather than fail daemon startup: a long direct build must never block the +// daemon from booting. +func acquireVectorsWriteLockWithRetry( + ctx context.Context, dataDir string, +) (*writeOwnerLock, bool, error) { + deadline := time.Now().Add(vectorsWriteLockRetryTimeout) + for { + lock, err := tryAcquireNamedLock(dataDir, vectorsWriteLockFile) + if err == nil { + return lock, true, nil + } + var held writeOwnerLockHeldError + if !errors.As(err, &held) { + return nil, false, err + } + if !time.Now().Before(deadline) { + return nil, false, nil + } + select { + case <-ctx.Done(): + return nil, false, nil + case <-time.After(vectorsWriteLockRetryInterval): + } + } +} + +// embedDebounceInterval is the fixed quiet period the after-sync scheduler +// waits, after the last sync-completion signal, before running a build. +const embedDebounceInterval = 30 * time.Second + +// embedManager is the subset of *vector.Manager the scheduler needs, +// letting tests substitute a fake that records TryBuild calls instead of +// driving a real build. +type embedManager interface { + TryBuild(ctx context.Context, req vector.BuildRequest) (bool, error) +} + +// embedScheduler debounces sync-completion signals into background +// embedding builds: a burst of Notify calls collapses into one TryBuild +// after debounce has elapsed with no further signal, and a backstop ticker +// periodically forces a full mirror reconciliation regardless of sync +// activity. +type embedScheduler struct { + mgr embedManager + debounce time.Duration + backstop time.Duration + // includeAutomated is the configured [vector].include_automated scope, + // carried into every scheduler-driven BuildRequest so scheduled builds + // stay config-authoritative rather than drifting from a CLI-only + // override (see EmbeddingsBuildOptions.IncludeAutomatedSet). + includeAutomated bool + + dirty chan struct{} + stop chan struct{} + done chan struct{} +} + +// newEmbedScheduler builds a scheduler over mgr. backstop <= 0 disables the +// periodic backstop ticker entirely, leaving only the after-sync debounce +// path. includeAutomated is the configured [vector].include_automated scope +// applied to every build this scheduler triggers. +func newEmbedScheduler(mgr embedManager, debounce, backstop time.Duration, includeAutomated bool) *embedScheduler { + return &embedScheduler{ + mgr: mgr, + debounce: debounce, + backstop: backstop, + includeAutomated: includeAutomated, + dirty: make(chan struct{}, 1), + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +// Notify signals that new data may need embedding. It never blocks: dirty +// has capacity 1, so a burst of calls while Run is busy (or not yet +// started) coalesces into a single pending signal. +func (s *embedScheduler) Notify() { + select { + case s.dirty <- struct{}{}: + default: + } +} + +// Stop signals Run to exit and blocks until it has, so a caller that +// closes the underlying Index right after Stop can never race a build +// still in flight. +func (s *embedScheduler) Stop() { + close(s.stop) + <-s.done +} + +// Run is the scheduler's goroutine body: it debounces Notify signals into +// TryBuild calls and, independently, fires a Backstop TryBuild on every +// backstop tick. It returns when ctx is done or Stop is called. +func (s *embedScheduler) Run(ctx context.Context) { + defer close(s.done) + + debounceTimer := time.NewTimer(s.debounce) + stopTimer(debounceTimer) + defer debounceTimer.Stop() + + var backstopC <-chan time.Time + if s.backstop > 0 { + ticker := time.NewTicker(s.backstop) + defer ticker.Stop() + backstopC = ticker.C + } + + // pendingBackstop remembers a backstop tick that collided with a build + // already running elsewhere (a long manual `embeddings build`, or the + // HTTP API) and so was dropped: without it, that reconciliation pass + // would be silently deferred until the next backstop tick (24h by + // default) instead of running as soon as any build slot frees up. It + // is read and written only from this single goroutine, so it needs no + // synchronization of its own. + var pendingBackstop bool + + for { + select { + case <-ctx.Done(): + return + case <-s.stop: + return + case <-s.dirty: + resetTimer(debounceTimer, s.debounce) + case <-debounceTimer.C: + req := vector.BuildRequest{Backstop: pendingBackstop, IncludeAutomated: s.includeAutomated} + started, err := s.mgr.TryBuild(ctx, req) + if err != nil { + log.Printf("embed scheduler: build failed: %v", err) + } + if !started { + // A build was already running elsewhere; re-arm rather + // than drop the pass entirely. pendingBackstop, if set, + // stays set so the retry still carries it. + resetTimer(debounceTimer, s.debounce) + continue + } + // Only clear a carried backstop once the build both started and + // succeeded: a started-but-failed build (started=true, err!=nil) + // never actually ran the full reconciliation it carried, so + // clearing pendingBackstop here would silently defer that + // reconciliation to the next backstop tick (24h by default) + // instead of retrying on the very next debounced build. + if err == nil { + pendingBackstop = false + } + case <-backstopC: + started, err := s.mgr.TryBuild(ctx, + vector.BuildRequest{Backstop: true, IncludeAutomated: s.includeAutomated}) + if err != nil { + log.Printf("embed scheduler: backstop build failed: %v", err) + } + // Same started-but-failed rule as the debounced path above: + // a build that started but errored never completed its + // reconciliation, so the pass must still be retried rather + // than deferred to the next backstop tick. + pendingBackstop = !started || err != nil + } + } +} + +// stopTimer stops t, draining an already-fired-but-unread channel value so +// a following Reset starts from a clean state. +func stopTimer(t *time.Timer) { + if !t.Stop() { + select { + case <-t.C: + default: + } + } +} + +// resetTimer stops and drains t before rearming it for d, the safe +// stop-then-reset sequence for a timer whose channel may already hold an +// unread tick. +func resetTimer(t *time.Timer, d time.Duration) { + stopTimer(t) + t.Reset(d) +} + +// teeEmitter fans a sync completion out to the production SSE emitter and, +// when after-sync embedding is enabled, the embed scheduler. The scheduler +// side never blocks (embedScheduler.Notify is non-blocking), so wrapping +// the emitter this way cannot slow down the sync pipeline. +type teeEmitter struct { + primary sync.Emitter + scheduler *embedScheduler + runAfterSync bool +} + +func (t teeEmitter) Emit(scope string) { + t.primary.Emit(scope) + if t.runAfterSync { + t.scheduler.Notify() + } +} + +// searcherAdapter implements db.VectorSearcher over a vector.Index, +// translating its error taxonomy into db.ErrSemanticUnavailable-wrapped +// errors and enforcing the config-drift staleness gate before every query. +type searcherAdapter struct { + ix *vector.Index + enc kitvec.EncodeFunc + fingerprint string +} + +// newSearcherAdapter builds a searcherAdapter for gen's configured +// embedding identity. +func newSearcherAdapter(ix *vector.Index, enc kitvec.EncodeFunc, gen kitvec.Generation) searcherAdapter { + return searcherAdapter{ix: ix, enc: enc, fingerprint: gen.Fingerprint()} +} + +// SemanticSearch implements db.VectorSearcher. A stale active generation +// (the configured model/dimension no longer matches what was last built) +// is a hard error checked before querying at all, rather than silently +// searching the mismatched old generation. +func (a searcherAdapter) SemanticSearch( + ctx context.Context, query string, limit int, +) ([]db.VectorHit, error) { + stale, err := a.ix.StaleActive(ctx, a.fingerprint) + if err != nil { + // StaleActive shares Search's error taxonomy (notably + // vector.ErrMirrorVersionMismatch from a version-mismatched + // read-only vectors.db), so it is translated the same way; + // errors outside the taxonomy pass through with this context. + return nil, translateSearchError( + fmt.Errorf("checking embedding index staleness: %w", err)) + } + if stale { + return nil, fmt.Errorf( + "%w: index is stale (embedding config changed): run "+ + "'agentsview embeddings build --full-rebuild'", + db.ErrSemanticUnavailable) + } + + hits, err := a.ix.Search(ctx, a.enc, query, limit) + if err != nil { + return nil, translateSearchError(err) + } + + out := make([]db.VectorHit, len(hits)) + for i, h := range hits { + out[i] = db.VectorHit{ + SessionID: h.SessionID, + Ordinal: h.Ordinal, + OrdinalStart: h.OrdinalStart, + OrdinalEnd: h.OrdinalEnd, + Subordinate: h.Subordinate, + Score: h.Score, + Snippet: h.Snippet, + } + } + return out, nil +} + +// ResolveMessageUnits implements db.VectorSearcher by delegating to the +// index's resolver, translating its error taxonomy (notably +// vector.ErrMirrorVersionMismatch from a version-mismatched read-only +// vectors.db) the same way SemanticSearch does. It needs no staleness gate +// of its own: the hybrid path always calls SemanticSearch — which enforces +// the gate — before resolving FTS hits. +func (a searcherAdapter) ResolveMessageUnits( + ctx context.Context, refs []db.MessageRef, +) ([]db.UnitRef, error) { + units, err := a.ix.ResolveMessageUnits(ctx, refs) + if err != nil { + return nil, translateSearchError(err) + } + return units, nil +} + +// translateSearchError maps vector.Index.Search's error taxonomy to +// server-facing sentinels. ErrNoActiveGeneration and BuildingError both +// mean nothing is queryable yet, so they map to db.ErrSemanticUnavailable +// (ErrNoActiveGeneration needs no extra cause text: db.ErrSemanticUnavailable's +// own message already is the "run the build" remediation). +// ErrMirrorVersionMismatch (a read-only vectors.db written by an +// incompatible mirror schema version) also maps to +// db.ErrSemanticUnavailable, carrying the sentinel's rebuild-required +// message as the cause. A QueryEncodeError means the index itself is ready +// but this particular query-time embed call failed (the embeddings endpoint +// is down, slow, or erroring); that maps to the distinct +// db.ErrSemanticTransient so a caller can tell "not configured" apart from +// "configured, but this request failed and can be retried". +func translateSearchError(err error) error { + var buildingErr *vector.BuildingError + var queryEncErr *vector.QueryEncodeError + switch { + case errors.As(err, &buildingErr): + return fmt.Errorf("%w: index is building: %d%% complete", + db.ErrSemanticUnavailable, buildingErr.Percent) + case errors.Is(err, vector.ErrMirrorVersionMismatch): + return fmt.Errorf("%w: %v", db.ErrSemanticUnavailable, err) + case errors.Is(err, vector.ErrNoActiveGeneration): + return db.ErrSemanticUnavailable + case errors.As(err, &queryEncErr): + // Double-wrap so callers can still match the underlying cause — + // notably context.Canceled/DeadlineExceeded from a dead client — + // alongside the transient sentinel. + return fmt.Errorf("%w: %w", db.ErrSemanticTransient, queryEncErr.Err) + default: + return err + } +} + +// vectorServing bundles what runServe needs to wire the vector subsystem +// into the daemon. All fields are zero when [vector] is disabled, so +// callers can treat it uniformly without a separate enabled check. +type vectorServing struct { + ServerOpts []server.Option + Scheduler *embedScheduler + Close func() error +} + +// setupVectorServing acquires vectors.write.lock, opens vectors.db +// read-write, builds the embeddings encoder and Manager, wires database's +// semantic searcher, and constructs the after-sync scheduler. database is +// passed directly as the Manager's UnitSource since *db.DB already +// implements vector.UnitSource. +// +// The write lock is held for the daemon's lifetime (released by the +// returned Close) so a concurrent direct `embeddings build` cannot race the +// daemon's own builds over vectors.db — both writers park evicted rows at +// sentinel ordinals, and a race between them can trip unique-index +// conflicts or silently discard embeddings. If the lock is already held +// (typically by a long-running direct build), setupVectorServing retries +// briefly and, failing that, disables vector serving for this run — logging +// a warning — rather than blocking or failing daemon startup. +func setupVectorServing( + ctx context.Context, cfg config.Config, database *db.DB, +) (vectorServing, error) { + if !cfg.Vector.Enabled { + return vectorServing{}, nil + } + + lock, ok, err := acquireVectorsWriteLockWithRetry(ctx, cfg.DataDir) + if err != nil { + return vectorServing{}, fmt.Errorf("acquiring vectors write lock: %w", err) + } + if !ok { + log.Printf( + "serve: vectors.write.lock held by another process after %s; "+ + "disabling vector serving for this run", + vectorsWriteLockRetryTimeout, + ) + return vectorServing{}, nil + } + + ix, err := vector.Open( + ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, cfg.Vector.Embeddings.MaxInputChars, + ) + if err != nil { + _ = lock.Close() + return vectorServing{}, fmt.Errorf("opening vectors.db: %w", err) + } + + encoders, err := vectorEncoderSet(cfg.Vector.Embeddings) + if err != nil { + ix.Close() + _ = lock.Close() + return vectorServing{}, err + } + // Search-time query encoding always uses the default server; builds may + // pick any named entry via BuildRequest.Using. + queryEnc := encoders.ByName[encoders.Default].Encode + + backstop, err := time.ParseDuration(cfg.Vector.Embed.BackstopInterval) + if err != nil { + ix.Close() + _ = lock.Close() + return vectorServing{}, fmt.Errorf( + "parsing [vector.embed] backstop_interval %q: %w", + cfg.Vector.Embed.BackstopInterval, err) + } + + gen := vectorGeneration(cfg.Vector.Embeddings) + mgr := vector.NewManager(ix, database, encoders, gen) + database.SetVectorSearcher(newSearcherAdapter(ix, queryEnc, gen)) + scheduler := newEmbedScheduler(mgr, embedDebounceInterval, backstop, cfg.Vector.IncludeAutomated) + + return vectorServing{ + ServerOpts: []server.Option{server.WithEmbeddingsManager(mgr)}, + Scheduler: scheduler, + Close: func() error { + ixErr := ix.Close() + lockErr := lock.Close() + if ixErr != nil { + return ixErr + } + return lockErr + }, + }, nil +} + +// installDirectVectorSearcher wires a read-only vectors.db into d's +// semantic searcher for direct (non-daemon) CLI reads, e.g. `session +// search --semantic` with no daemon running. It is a no-op — leaving d +// without a VectorSearcher, so callers see db.ErrSemanticUnavailable +// naturally — when [vector] is disabled, vectors.db does not exist yet, or +// vectors.db cannot be opened at all (e.g. corrupt or truncated). +// +// A vectors.db written by an incompatible mirror schema version is NOT one +// of those no-op cases: the read-only open succeeds with the mismatch +// recorded on the Index, so the searcher is wired and every semantic query +// surfaces the rebuild-required error (vector.ErrMirrorVersionMismatch, +// mapped by translateSearchError onto db.ErrSemanticUnavailable with the +// remediation attached) instead of semantic search silently reading as +// "not enabled". +// +// Vector wiring failures never fail direct service construction: every +// direct read command (e.g. `session list`) opens vectors.db eagerly +// through this path, so a bad vectors.db must not break unrelated reads +// against an otherwise-healthy sessions.db archive. Failures are logged as +// a warning and degrade to semantic search returning +// db.ErrSemanticUnavailable, matching the disabled/missing-file cases. +// +// The returned close func is nil whenever no searcher was wired (the +// no-op cases above, and the degraded-on-error case); otherwise the +// caller must call it when done with d to release the read-only index +// handle. +func installDirectVectorSearcher(cfg config.Config, d *db.DB) func() error { + if !cfg.Vector.Enabled { + return nil + } + path := cfg.Vector.ResolvedDBPath(cfg.DataDir) + if _, err := os.Stat(path); err != nil { + return nil + } + + ix, err := vector.Open(context.Background(), path, true, cfg.Vector.Embeddings.MaxInputChars) + if err != nil { + log.Printf( + "warning: opening vectors.db for semantic search: %v; "+ + "continuing without semantic search", err, + ) + return nil + } + // Query encoding uses the default server. + enc, err := newVectorEncoder(cfg.Vector.Embeddings, "") + if err != nil { + ix.Close() + log.Printf( + "warning: building embeddings encoder for semantic search: %v; "+ + "continuing without semantic search", err, + ) + return nil + } + d.SetVectorSearcher(newSearcherAdapter(ix, enc, vectorGeneration(cfg.Vector.Embeddings))) + return ix.Close +} diff --git a/cmd/agentsview/embed_scheduler_test.go b/cmd/agentsview/embed_scheduler_test.go new file mode 100644 index 000000000..d1809aa13 --- /dev/null +++ b/cmd/agentsview/embed_scheduler_test.go @@ -0,0 +1,844 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/server" + "go.kenn.io/agentsview/internal/vector" +) + +// --- fake embedManager --- + +// fakeEmbedManager records every TryBuild call and returns scripted +// (started, err) results in order, repeating the last scripted result once +// the script is exhausted (default: started=true, err=nil). +type fakeEmbedManager struct { + mu sync.Mutex + calls []vector.BuildRequest + results []fakeTryBuildResult +} + +type fakeTryBuildResult struct { + started bool + err error +} + +func (f *fakeEmbedManager) TryBuild( + _ context.Context, req vector.BuildRequest, +) (bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, req) + idx := len(f.calls) - 1 + if idx < len(f.results) { + r := f.results[idx] + return r.started, r.err + } + if len(f.results) > 0 { + r := f.results[len(f.results)-1] + return r.started, r.err + } + return true, nil +} + +func (f *fakeEmbedManager) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +func (f *fakeEmbedManager) callsSnapshot() []vector.BuildRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]vector.BuildRequest(nil), f.calls...) +} + +// waitForSchedulerCondition polls cond until it is true or 2s pass, failing +// the test with msg otherwise. Avoids fixed sleeps that would either flake +// under load or slow the suite down needlessly. +func waitForSchedulerCondition(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + require.Fail(t, "timed out waiting for condition", msg) +} + +func TestEmbedSchedulerBurstOfNotifyProducesExactlyOneBuild(t *testing.T) { + fake := &fakeEmbedManager{} + s := newEmbedScheduler(fake, 20*time.Millisecond, 0, false) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + for range 10 { + s.Notify() + time.Sleep(2 * time.Millisecond) + } + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "expected a build after the burst quieted") + // Give any spurious extra build a chance to show up before asserting + // there is exactly one. + time.Sleep(60 * time.Millisecond) + assert.Equal(t, 1, fake.callCount(), "a burst of Notify must collapse to one build") + assert.Equal(t, []vector.BuildRequest{{}}, fake.callsSnapshot()) +} + +// TestEmbedSchedulerIncludeAutomatedThreadsIntoBuildRequests asserts the +// scheduler's configured include-automated scope rides along on every +// BuildRequest it issues -- both the debounced after-sync path and the +// backstop ticker -- so scheduled builds stay config-authoritative rather +// than silently reverting to includeAutomated=false. +func TestEmbedSchedulerIncludeAutomatedThreadsIntoBuildRequests(t *testing.T) { + fake := &fakeEmbedManager{} + s := newEmbedScheduler(fake, 5*time.Millisecond, 20*time.Millisecond, true) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + s.Notify() + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "expected a debounced build") + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 2 }, + "expected a backstop build") + + for _, req := range fake.callsSnapshot() { + assert.True(t, req.IncludeAutomated, + "every scheduler-issued BuildRequest must carry the configured scope") + } +} + +func TestEmbedSchedulerNotifyDuringRunningBuildRearmsForFollowUpPass(t *testing.T) { + fake := &fakeEmbedManager{ + results: []fakeTryBuildResult{ + {started: false, err: nil}, // a build is already running elsewhere + {started: true, err: nil}, // the follow-up pass actually runs + }, + } + s := newEmbedScheduler(fake, 15*time.Millisecond, 0, false) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + s.Notify() + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 2 }, + "expected the scheduler to re-arm and retry after a dropped build") + calls := fake.callsSnapshot() + require.Len(t, calls, 2) + assert.Equal(t, vector.BuildRequest{}, calls[0]) + assert.Equal(t, vector.BuildRequest{}, calls[1]) +} + +func TestEmbedSchedulerBackstopTickIssuesBackstopBuild(t *testing.T) { + fake := &fakeEmbedManager{} + // A very long debounce so only the backstop ticker can fire a build. + s := newEmbedScheduler(fake, time.Hour, 20*time.Millisecond, false) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "expected a backstop build") + calls := fake.callsSnapshot() + require.NotEmpty(t, calls) + assert.True(t, calls[0].Backstop, "backstop tick must set BuildRequest.Backstop") +} + +// TestEmbedSchedulerDroppedBackstopRetriesOnNextDebouncedBuild is the fix-4 +// regression test: a backstop tick that collides with a build already +// running elsewhere must not be silently dropped for a full backstop +// interval (24h in production). The scheduler must remember it and fold +// Backstop: true into the next debounced build request instead. +func TestEmbedSchedulerDroppedBackstopRetriesOnNextDebouncedBuild(t *testing.T) { + fake := &fakeEmbedManager{ + results: []fakeTryBuildResult{ + {started: false, err: nil}, // the backstop tick collides with a build elsewhere + {started: true, err: nil}, // the following debounced build recovers it + }, + } + // A long backstop interval relative to the debounce interval and the + // test's own buffers keeps a second, unrelated backstop tick from + // firing mid-test and making the call count non-deterministic. + s := newEmbedScheduler(fake, 10*time.Millisecond, 500*time.Millisecond, false) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "expected the backstop tick to fire and be dropped") + + s.Notify() + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 2 }, + "expected the debounced build to recover the dropped backstop") + time.Sleep(50 * time.Millisecond) + + calls := fake.callsSnapshot() + require.Len(t, calls, 2, "the dropped backstop must be retried exactly once, not repeatedly") + assert.True(t, calls[0].Backstop, "the original (dropped) backstop tick request") + assert.True(t, calls[1].Backstop, + "the debounced build must carry the pending backstop forward instead of dropping it") + + // Once the recovered build actually started, the pending flag must + // clear: a further, unrelated debounced build must not keep carrying + // Backstop: true forever. + s.Notify() + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 3 }, + "expected a further debounced build after the recovered one") + time.Sleep(50 * time.Millisecond) + + calls = fake.callsSnapshot() + require.Len(t, calls, 3) + assert.False(t, calls[2].Backstop, + "the recovered backstop must not leak into a later unrelated debounced build") +} + +// TestEmbedSchedulerBackstopTickStartedButFailedKeepsPendingBackstop is the +// fix-5 regression test: a backstop tick whose TryBuild call actually started +// but then returned an error must not clear pendingBackstop -- the +// reconciliation it carried never completed, so it must be retried on the +// very next debounced build rather than silently deferred to the next +// backstop interval (24h in production). +func TestEmbedSchedulerBackstopTickStartedButFailedKeepsPendingBackstop(t *testing.T) { + buildErr := errors.New("embeddings endpoint unreachable") + fake := &fakeEmbedManager{ + results: []fakeTryBuildResult{ + {started: true, err: buildErr}, // the backstop tick starts but fails + {started: true, err: nil}, // the following debounced build recovers it + }, + } + s := newEmbedScheduler(fake, 10*time.Millisecond, 500*time.Millisecond, false) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "expected the backstop tick to fire and fail") + + s.Notify() + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 2 }, + "expected the debounced build to retry the failed backstop") + time.Sleep(50 * time.Millisecond) + + calls := fake.callsSnapshot() + require.Len(t, calls, 2, "the failed backstop must be retried exactly once, not repeatedly") + assert.True(t, calls[0].Backstop, "the original (started-but-failed) backstop tick request") + assert.True(t, calls[1].Backstop, + "the debounced build must carry the failed backstop forward instead of dropping it") + + // Once the recovered build actually succeeded, the pending flag must + // clear: a further, unrelated debounced build must not keep carrying + // Backstop: true forever. + s.Notify() + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 3 }, + "expected a further debounced build after the recovered one") + time.Sleep(50 * time.Millisecond) + + calls = fake.callsSnapshot() + require.Len(t, calls, 3) + assert.False(t, calls[2].Backstop, + "the recovered backstop must not leak into a later unrelated debounced build") +} + +// TestEmbedSchedulerDebouncedBuildStartedButFailedKeepsPendingBackstop is the +// fix-5 regression test for the debounced-build path: once a dropped +// backstop tick is being carried by pendingBackstop, a debounced build that +// starts but then fails must not clear it either -- the same +// started-but-failed rule applies on both paths that can clear the flag. +func TestEmbedSchedulerDebouncedBuildStartedButFailedKeepsPendingBackstop(t *testing.T) { + buildErr := errors.New("embeddings endpoint unreachable") + fake := &fakeEmbedManager{ + results: []fakeTryBuildResult{ + {started: false, err: nil}, // the backstop tick collides with a build elsewhere + {started: true, err: buildErr}, // the recovering debounced build starts but fails + {started: true, err: nil}, // a further debounced build finally succeeds + }, + } + s := newEmbedScheduler(fake, 10*time.Millisecond, 500*time.Millisecond, false) + + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "expected the backstop tick to fire and be dropped") + + s.Notify() + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 2 }, + "expected the debounced build to attempt recovering the dropped backstop") + + s.Notify() + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 3 }, + "expected a further debounced build to retry after the recovering build failed") + time.Sleep(50 * time.Millisecond) + + calls := fake.callsSnapshot() + require.Len(t, calls, 3) + assert.True(t, calls[0].Backstop, "the original (dropped) backstop tick request") + assert.True(t, calls[1].Backstop, "the recovering (started-but-failed) debounced build") + assert.True(t, calls[2].Backstop, + "a started-but-failed build must not clear pendingBackstop: the retry must still carry it") +} + +func TestEmbedSchedulerStopTerminatesRun(t *testing.T) { + fake := &fakeEmbedManager{} + s := newEmbedScheduler(fake, time.Hour, 0, false) + + go s.Run(context.Background()) + + // Stop blocks until Run has actually exited, so its returning at all + // (within a generous timeout) is the proof Run terminated. + stopped := make(chan struct{}) + go func() { + s.Stop() + close(stopped) + }() + + select { + case <-stopped: + case <-time.After(2 * time.Second): + require.Fail(t, "Stop did not return; Run likely never terminated") + } +} + +func TestEmbedSchedulerNotifyNeverBlocksWithoutAReader(t *testing.T) { + fake := &fakeEmbedManager{} + s := newEmbedScheduler(fake, time.Hour, 0, false) + + done := make(chan struct{}) + go func() { + for range 100 { + s.Notify() + } + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + require.Fail(t, "Notify blocked with no Run consuming dirty") + } +} + +// --- teeEmitter --- + +type recordingEmitter struct { + mu sync.Mutex + scopes []string +} + +func (e *recordingEmitter) Emit(scope string) { + e.mu.Lock() + defer e.mu.Unlock() + e.scopes = append(e.scopes, scope) +} + +func (e *recordingEmitter) count() int { + e.mu.Lock() + defer e.mu.Unlock() + return len(e.scopes) +} + +func TestTeeEmitterAlwaysCallsPrimaryAndGatesSchedulerOnRunAfterSync(t *testing.T) { + primary := &recordingEmitter{} + fake := &fakeEmbedManager{} + s := newEmbedScheduler(fake, 10*time.Millisecond, 0, false) + ctx := t.Context() + go s.Run(ctx) + defer s.Stop() + + disabled := teeEmitter{primary: primary, scheduler: s, runAfterSync: false} + disabled.Emit("sessions") + assert.Equal(t, 1, primary.count()) + time.Sleep(30 * time.Millisecond) + assert.Equal(t, 0, fake.callCount(), "runAfterSync=false must not notify the scheduler") + + enabled := teeEmitter{primary: primary, scheduler: s, runAfterSync: true} + enabled.Emit("sessions") + assert.Equal(t, 2, primary.count()) + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "runAfterSync=true must notify the scheduler") +} + +// TestRunRemoteHostSyncLoop_EmitsThroughTeeNotifiesScheduler is a +// regression test for a bug where runServe wired startPeriodicSync's +// scheduled remote-host sync path to the bare SSE broadcaster instead of +// the wrapped teeEmitter, so remote-synced sessions notified SSE clients +// but never reached the embed scheduler until an unrelated local sync or +// the 24h backstop ran. It drives runRemoteHostSyncLoop — the function +// startPeriodicSync spawns per configured remote host — with a teeEmitter +// and a syncFn reporting synced sessions, and asserts the scheduler +// actually receives a build trigger through that path. +func TestRunRemoteHostSyncLoop_EmitsThroughTeeNotifiesScheduler(t *testing.T) { + primary := &recordingEmitter{} + fake := &fakeEmbedManager{} + s := newEmbedScheduler(fake, 5*time.Millisecond, 0, false) + schedCtx := t.Context() + go s.Run(schedCtx) + defer s.Stop() + + tee := teeEmitter{primary: primary, scheduler: s, runAfterSync: true} + + syncFn := func() (int, error) { + return 1, nil // one session synced on this remote host + } + + loopCtx := t.Context() + done := make(chan struct{}) + go runRemoteHostSyncLoop( + loopCtx, "remote-host", 5*time.Millisecond, syncFn, tee, nil, done, + ) + defer close(done) + + waitForSchedulerCondition(t, func() bool { return fake.callCount() >= 1 }, + "a scheduled remote sync reaching the tee emitter must notify the embed scheduler") + assert.NotZero(t, primary.count(), + "the tee must still forward remote sync completions to the SSE broadcaster") +} + +// --- searcherAdapter error taxonomy --- + +func TestTranslateSearchErrorMapsVectorErrorsToSemanticUnavailable(t *testing.T) { + t.Run("no active generation", func(t *testing.T) { + err := translateSearchError(vector.ErrNoActiveGeneration) + assert.ErrorIs(t, err, db.ErrSemanticUnavailable) + }) + t.Run("building", func(t *testing.T) { + err := translateSearchError(&vector.BuildingError{Percent: 62}) + assert.ErrorIs(t, err, db.ErrSemanticUnavailable) + assert.Contains(t, err.Error(), "index is building: 62% complete") + }) + t.Run("other error passes through", func(t *testing.T) { + boom := errors.New("boom") + assert.Same(t, boom, translateSearchError(boom)) + }) + t.Run("query encode failure maps to semantic transient, not unavailable", func(t *testing.T) { + queryErr := &vector.QueryEncodeError{Err: errors.New("dial tcp: connection refused")} + got := translateSearchError(queryErr) + assert.ErrorIs(t, got, db.ErrSemanticTransient) + assert.False(t, errors.Is(got, db.ErrSemanticUnavailable), + "a query-time endpoint failure must not read as semantic search being disabled") + assert.Contains(t, got.Error(), "connection refused") + }) + t.Run("query encode failure preserves the underlying cause chain", func(t *testing.T) { + queryErr := &vector.QueryEncodeError{ + Err: fmt.Errorf("encoding query: %w", context.Canceled), + } + got := translateSearchError(queryErr) + assert.ErrorIs(t, got, db.ErrSemanticTransient) + assert.ErrorIs(t, got, context.Canceled, + "context errors must stay matchable so cancellation handling still fires") + }) + t.Run("mirror version mismatch maps to semantic unavailable with rebuild message", func(t *testing.T) { + got := translateSearchError( + fmt.Errorf("checking embedding index staleness: %w", vector.ErrMirrorVersionMismatch)) + assert.ErrorIs(t, got, db.ErrSemanticUnavailable) + assert.Contains(t, got.Error(), "embeddings build", + "the rebuild remediation must survive translation") + }) +} + +// TestSearcherAdapterVersionMismatchedIndexReturnsSemanticUnavailable is the +// adapter-level regression test for the mirror version gate: a +// searcherAdapter over a read-only vectors.db written by a different mirror +// schema version must return an error matching db.ErrSemanticUnavailable and +// mentioning the rebuild remediation — not a raw SQL error or a wrong +// staleness verdict from StaleActive querying an incompatible mirror. +func TestSearcherAdapterVersionMismatchedIndexReturnsSemanticUnavailable(t *testing.T) { + dataDir := t.TempDir() + cfg := vectorTestConfig(dataDir) + path := cfg.Vector.ResolvedDBPath(dataDir) + + // Create a current vectors.db, then restamp it as written by the + // previous mirror schema version, simulating a file left behind by an + // older agentsview build. + seed, err := vector.Open(context.Background(), path, false, cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + require.NoError(t, seed.Close()) + raw, err := sql.Open("sqlite3", path) + require.NoError(t, err) + _, err = raw.Exec(`UPDATE vector_meta SET value = '2' WHERE key = 'mirror_schema_version'`) + require.NoError(t, err) + require.NoError(t, raw.Close()) + + ix, err := vector.Open(context.Background(), path, true, cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err, "read-only Open must succeed against a mismatched vectors.db") + defer ix.Close() + + enc, err := newVectorEncoder(cfg.Vector.Embeddings, "") + require.NoError(t, err) + adapter := newSearcherAdapter(ix, enc, vectorGeneration(cfg.Vector.Embeddings)) + + _, err = adapter.SemanticSearch(context.Background(), "any query", 5) + require.Error(t, err) + assert.ErrorIs(t, err, db.ErrSemanticUnavailable, + "a version-mismatched index must surface the semantic-unavailable taxonomy") + assert.Contains(t, err.Error(), "embeddings build", + "the error must tell the user to rebuild the index") +} + +// --- integration: real serve/server construction path --- + +// vectorTestConfig returns a config.Config with a small-but-valid [vector] +// section (accepted by config.Validate) pointed at an embeddings endpoint +// that is never actually called in these tests — setupVectorServing only +// constructs the encoder, it does not exercise it. +func vectorTestConfig(dataDir string) config.Config { + return config.Config{ + DataDir: dataDir, + DBPath: filepath.Join(dataDir, "sessions.db"), + Vector: config.VectorConfig{ + Enabled: true, + Embeddings: config.VectorEmbeddingsConfig{ + Model: "test-model", + Dimension: 3, + MaxInputChars: 1000, + Servers: map[string]config.VectorEmbeddingsServerConfig{ + "local": { + Endpoint: "http://127.0.0.1:1/v1", + BatchSize: 10, + Concurrency: 1, + Timeout: "5s", + MaxRetries: 1, + }, + }, + }, + Embed: config.VectorEmbedConfig{ + BackstopInterval: "24h", + }, + }, + } +} + +// listenLoopback opens a loopback listener on an OS-assigned port, +// returning it alongside the port so a caller can set cfg.Port to it +// before constructing the server: the host-check middleware validates +// the Host header against cfg.Host:cfg.Port exactly, so the request's +// destination port must be known up front rather than assigned by +// httptest.NewServer after the fact. +func listenLoopback(t *testing.T) (net.Listener, int) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + return ln, ln.Addr().(*net.TCPAddr).Port +} + +// startTestServer serves handler on ln (already bound to the port cfg.Port +// was set to) and returns an *httptest.Server whose URL and lifecycle +// helpers work as usual. +func startTestServer(t *testing.T, ln net.Listener, handler http.Handler) *httptest.Server { + t.Helper() + ts := httptest.NewUnstartedServer(handler) + require.NoError(t, ts.Listener.Close()) + ts.Listener = ln + ts.Start() + t.Cleanup(ts.Close) + return ts +} + +// TestServeConstructionRegistersEmbeddingsRoutesWhenVectorEnabled drives the +// real setupVectorServing + server.New construction path (not a fake mux) +// with a vector-enabled config and asserts the embeddings status endpoint +// responds, then a sibling test rebuilds with vector disabled and asserts +// it does not. +func TestServeConstructionRegistersEmbeddingsRoutesWhenVectorEnabled(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + + ln, port := listenLoopback(t) + cfg := vectorTestConfig(dataDir) + cfg.Host, cfg.Port = "127.0.0.1", port + vs, err := setupVectorServing(context.Background(), cfg, database) + require.NoError(t, err) + require.NotNil(t, vs.Scheduler) + require.NotNil(t, vs.Close) + defer func() { require.NoError(t, vs.Close()) }() + + srv := server.New(cfg, database, nil, vs.ServerOpts...) + ts := startTestServer(t, ln, srv.Handler()) + + resp, err := http.Get(ts.URL + "/api/v1/embeddings/status") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "application/json", mediaType(t, resp.Header.Get("Content-Type"))) +} + +// TestEmbeddingsDaemonClientBuildSucceedsThroughRealMiddleware drives a POST +// build request from embeddingsDaemonClient (the same client `embeddings +// build` uses to talk to a running daemon) through the server's real +// middleware chain (srv.Handler(), not a bare mux), proving the CSRF guard +// in internal/server/server.go's corsMiddleware does not reject it. Every +// other embeddings-daemon-dispatch test in embeddings_test.go serves off an +// httptest.NewServeMux() directly, bypassing that middleware entirely, which +// is exactly how the missing Origin header regression went unnoticed. +func TestEmbeddingsDaemonClientBuildSucceedsThroughRealMiddleware(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + + ln, port := listenLoopback(t) + cfg := vectorTestConfig(dataDir) + cfg.Host, cfg.Port = "127.0.0.1", port + vs, err := setupVectorServing(context.Background(), cfg, database) + require.NoError(t, err) + defer func() { require.NoError(t, vs.Close()) }() + + srv := server.New(cfg, database, nil, vs.ServerOpts...) + ts := startTestServer(t, ln, srv.Handler()) + + client := embeddingsDaemonClient{baseURL: ts.URL} + err = client.startBuild(context.Background(), vector.BuildRequest{}) + require.NoError(t, err, + "a POST build must succeed once the client sets Origin to satisfy the CSRF guard") +} + +// TestSetupVectorServingDisablesWhenWriteLockHeld asserts a held +// vectors.write.lock (simulating a concurrent direct `embeddings build`) +// makes setupVectorServing degrade to a fully-disabled vectorServing after a +// short retry window, rather than blocking or failing daemon startup, and +// that it logs a clear warning explaining why. +func TestSetupVectorServingDisablesWhenWriteLockHeld(t *testing.T) { + origInterval, origTimeout := vectorsWriteLockRetryInterval, vectorsWriteLockRetryTimeout + vectorsWriteLockRetryInterval = time.Millisecond + vectorsWriteLockRetryTimeout = 10 * time.Millisecond + t.Cleanup(func() { + vectorsWriteLockRetryInterval = origInterval + vectorsWriteLockRetryTimeout = origTimeout + }) + + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + cfg := vectorTestConfig(dataDir) + + held, err := tryAcquireNamedLock(dataDir, vectorsWriteLockFile) + require.NoError(t, err) + defer func() { require.NoError(t, held.Close()) }() + + logBuf := captureLogOutput(t) + + vs, err := setupVectorServing(context.Background(), cfg, database) + require.NoError(t, err, "a held lock must degrade, not fail, daemon startup") + assert.Nil(t, vs.Scheduler) + assert.Nil(t, vs.Close) + assert.Empty(t, vs.ServerOpts) + assert.Contains(t, logBuf.String(), "vectors.write.lock") + assert.Contains(t, logBuf.String(), "disabling vector serving") +} + +// TestSetupVectorServingAcquiresAndReleasesWriteLock asserts a free +// vectors.write.lock is acquired for the daemon's lifetime and released by +// Close, so a second setupVectorServing call after Close succeeds rather +// than finding the lock still held. +func TestSetupVectorServingAcquiresAndReleasesWriteLock(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + cfg := vectorTestConfig(dataDir) + + vs, err := setupVectorServing(context.Background(), cfg, database) + require.NoError(t, err) + require.NotNil(t, vs.Close) + + // While vs holds the lock, a competing direct acquire must fail. + _, lockErr := tryAcquireNamedLock(dataDir, vectorsWriteLockFile) + require.Error(t, lockErr, "setupVectorServing must hold vectors.write.lock while running") + + require.NoError(t, vs.Close()) + + // Once released, a fresh acquire — standing in for a second + // setupVectorServing call — must succeed. + held, err := tryAcquireNamedLock(dataDir, vectorsWriteLockFile) + require.NoError(t, err, "Close must release vectors.write.lock") + require.NoError(t, held.Close()) +} + +func TestServeConstructionKeepsEmbeddingsRoutesUnavailableWhenVectorDisabled(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + + ln, port := listenLoopback(t) + cfg := config.Config{DataDir: dataDir, DBPath: filepath.Join(dataDir, "sessions.db"), + Host: "127.0.0.1", Port: port} + vs, err := setupVectorServing(context.Background(), cfg, database) + require.NoError(t, err) + assert.Nil(t, vs.Scheduler) + assert.Nil(t, vs.Close) + assert.Empty(t, vs.ServerOpts) + + srv := server.New(cfg, database, nil, vs.ServerOpts...) + ts := startTestServer(t, ln, srv.Handler()) + + resp, err := http.Get(ts.URL + "/api/v1/embeddings/status") + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusNotImplemented, resp.StatusCode, + "the embeddings API should be documented and registered, but unavailable when vector is disabled") + assert.Equal(t, "application/json", mediaType(t, resp.Header.Get("Content-Type"))) +} + +// mediaType extracts the bare MIME type from a Content-Type header value, +// dropping any "; charset=..." parameters, so tests can compare it exactly. +func mediaType(t *testing.T, contentType string) string { + t.Helper() + mt, _, err := mime.ParseMediaType(contentType) + require.NoError(t, err) + return mt +} + +// --- installDirectVectorSearcher (direct, non-daemon CLI path) --- + +func TestInstallDirectVectorSearcherNoOpWhenVectorDisabled(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + + cfg := config.Config{DataDir: dataDir} + closeFn := installDirectVectorSearcher(cfg, database) + assert.Nil(t, closeFn) + assert.False(t, database.HasSemantic()) +} + +func TestInstallDirectVectorSearcherNoOpWhenVectorsDBMissing(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + + cfg := vectorTestConfig(dataDir) + closeFn := installDirectVectorSearcher(cfg, database) + assert.Nil(t, closeFn) + assert.False(t, database.HasSemantic(), + "no searcher wired means callers see db.ErrSemanticUnavailable naturally") +} + +func TestInstallDirectVectorSearcherWiresSearcherWhenVectorsDBExists(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + cfg := vectorTestConfig(dataDir) + + // Create vectors.db up front, as a prior `embeddings build` would. + seed, err := vector.Open(context.Background(), cfg.Vector.ResolvedDBPath(dataDir), false, 1000) + require.NoError(t, err) + require.NoError(t, seed.Close()) + + closeFn := installDirectVectorSearcher(cfg, database) + require.NotNil(t, closeFn) + defer func() { assert.NoError(t, closeFn()) }() + + assert.True(t, database.HasSemantic(), + "an existing vectors.db must wire a read-only searcher for direct CLI reads") +} + +// TestInstallDirectVectorSearcherDegradesOnCorruptVectorsDB is a regression +// test for a bug where a corrupt or incompatible vectors.db broke direct +// service construction entirely, taking down unrelated commands like +// `session list` that never touch the vector index. A garbage vectors.db +// file must degrade to "no searcher installed" rather than propagate an +// error, leaving non-semantic reads unaffected and semantic search falling +// back to the standard unavailable error. +func TestInstallDirectVectorSearcherDegradesOnCorruptVectorsDB(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + cfg := vectorTestConfig(dataDir) + + // Not a SQLite file at all — simulates corruption or an incompatible + // build rather than a partially-written one. + vectorsPath := cfg.Vector.ResolvedDBPath(dataDir) + require.NoError(t, os.MkdirAll(filepath.Dir(vectorsPath), 0o755)) + require.NoError(t, os.WriteFile(vectorsPath, []byte("not a sqlite database"), 0o644)) + + closeFn := installDirectVectorSearcher(cfg, database) + assert.Nil(t, closeFn, + "a corrupt vectors.db must not return a handle to close") + assert.False(t, database.HasSemantic(), + "a corrupt vectors.db must degrade to no searcher, not fail construction") + + _, err := database.SearchContent(context.Background(), db.ContentSearchFilter{ + Pattern: "query", + Mode: "semantic", + Limit: 5, + }) + assert.ErrorIs(t, err, db.ErrSemanticUnavailable, + "semantic search must return the standard unavailable error once degraded") +} + +// TestInstallDirectVectorSearcherVersionMismatchServesRebuildRequired pins +// the direct-install path's handling of a version-mismatched vectors.db: +// the read-only open succeeds, so the searcher must be WIRED (not silently +// dropped by the corrupt-file degradation branch) and semantic search over +// HTTP must answer with the 501 rebuild-required taxonomy whose body names +// the incompatible-version remediation — not the generic "not enabled" +// message and not an empty page. +func TestInstallDirectVectorSearcherVersionMismatchServesRebuildRequired(t *testing.T) { + dataDir := t.TempDir() + database := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + cfg := vectorTestConfig(dataDir) + path := cfg.Vector.ResolvedDBPath(dataDir) + + // Create a current vectors.db, then restamp it as written by the + // previous mirror schema version. + seed, err := vector.Open(context.Background(), path, false, cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + require.NoError(t, seed.Close()) + raw, err := sql.Open("sqlite3", path) + require.NoError(t, err) + _, err = raw.Exec(`UPDATE vector_meta SET value = '2' WHERE key = 'mirror_schema_version'`) + require.NoError(t, err) + require.NoError(t, raw.Close()) + + closeFn := installDirectVectorSearcher(cfg, database) + require.NotNil(t, closeFn, + "a version-mismatched vectors.db must wire a searcher, not silently unwire semantic search") + defer func() { assert.NoError(t, closeFn()) }() + require.True(t, database.HasSemantic()) + + ln, port := listenLoopback(t) + cfg.Host, cfg.Port = "127.0.0.1", port + srv := server.New(cfg, database, nil) + ts := startTestServer(t, ln, srv.Handler()) + + req, err := http.NewRequest(http.MethodGet, + ts.URL+"/api/v1/search/content?pattern=anything&mode=semantic", nil) + require.NoError(t, err) + req.Header.Set("X-AgentsView-Search-Intent", "semantic") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, http.StatusNotImplemented, resp.StatusCode, + "body: %s", body) + assert.Contains(t, string(body), "incompatible version", + "the error body must carry the rebuild-required remediation") + assert.Contains(t, string(body), "embeddings build", + "the error body must tell the user how to rebuild") +} diff --git a/cmd/agentsview/embeddings.go b/cmd/agentsview/embeddings.go new file mode 100644 index 000000000..4bffbd64f --- /dev/null +++ b/cmd/agentsview/embeddings.go @@ -0,0 +1,781 @@ +// ABOUTME: `embeddings` command group — build, list, activate, and retire +// ABOUTME: semantic-search embedding generations, via the daemon or directly. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + kitvec "go.kenn.io/kit/vector" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/vector" +) + +// embeddingsPollInterval bounds how often the daemon build path polls +// /api/v1/embeddings/status. A package var so tests can shrink it. +var embeddingsPollInterval = 2 * time.Second + +// directBuildProgressInterval bounds how often the direct build path polls +// the in-process Manager's Status for a progress line. A package var so +// tests can shrink it. +var directBuildProgressInterval = 2 * time.Second + +// fingerprintDisplayLen is how many leading characters of a generation +// fingerprint `embeddings list` prints. +const fingerprintDisplayLen = 12 + +// embeddingsDaemonHTTPClient bounds each individual request the embeddings +// daemon client makes (build/status/list/activate/retire) so a wedged +// daemon cannot hang the CLI forever, matching the timeout other daemon +// HTTP clients in this codebase use (internal/service/http.go's +// httpBackend.client). This is a per-request timeout, not a deadline on the +// overall command: buildViaDaemon's poll loop issues one status call every +// embeddingsPollInterval, so a build that legitimately runs for longer than +// this timeout keeps polling fine — only an individual unresponsive call is +// cut off. +var embeddingsDaemonHTTPClient = &http.Client{Timeout: 30 * time.Second} + +func newEmbeddingsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "embeddings", + Short: "Manage the semantic search embedding index", + GroupID: groupData, + SilenceUsage: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return cmd.Help() + }, + } + cmd.AddCommand(newEmbeddingsBuildCommand()) + cmd.AddCommand(newEmbeddingsListCommand()) + cmd.AddCommand(newEmbeddingsActivateCommand()) + cmd.AddCommand(newEmbeddingsRetireCommand()) + return cmd +} + +// EmbeddingsBuildOptions holds the parsed `embeddings build` flags. +type EmbeddingsBuildOptions struct { + FullRebuild bool + Backstop bool + Yes bool + // IncludeAutomated is the --include-automated flag's parsed value; only + // meaningful when IncludeAutomatedSet is true (see that field). + IncludeAutomated bool + // IncludeAutomatedSet reports whether --include-automated was + // explicitly passed (cmd.Flags().Changed), overriding + // [vector].include_automated to IncludeAutomated's parsed value (true or + // false) for this one build. + IncludeAutomatedSet bool + // Using names the [vector.embeddings.servers.] entry this build + // encodes against; empty uses default_server. + Using string +} + +func newEmbeddingsBuildCommand() *cobra.Command { + var opts EmbeddingsBuildOptions + cmd := &cobra.Command{ + Use: "build", + Short: "Build or refresh the embedding index", + SilenceUsage: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + opts.IncludeAutomatedSet = cmd.Flags().Changed("include-automated") + return runEmbeddingsBuild( + cmd.Context(), cmd.OutOrStdout(), cmd.InOrStdin(), opts, + ) + }, + } + cmd.Flags().BoolVar(&opts.FullRebuild, "full-rebuild", false, + "Re-embed every document, even ones already embedded under the active generation") + cmd.Flags().BoolVar(&opts.Backstop, "backstop", false, + "Force a full mirror reconciliation scan without forcing a re-embed") + cmd.Flags().BoolVar(&opts.Yes, "yes", false, + "Skip the full-rebuild confirmation prompt") + cmd.Flags().StringVar(&opts.Using, "using", "", + "Named embeddings server from [vector.embeddings.servers] to run this "+ + "build against (default: the config's default_server)") + cmd.Flags().BoolVar(&opts.IncludeAutomated, "include-automated", false, + "Override [vector].include_automated for this build only: bare "+ + "--include-automated embeds automated (non-interactive) sessions "+ + "too, and --include-automated=false force-excludes them even if "+ + "the config default is true. Prefer setting the config key for "+ + "scheduled builds: mixing this flag with a different config "+ + "default flips the index's scope on every other build, forcing "+ + "a full mirror reconciliation each time.") + return cmd +} + +func newEmbeddingsListCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "List embedding generations", + SilenceUsage: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runEmbeddingsList( + cmd.Context(), cmd.OutOrStdout(), outputFormat(cmd) == "json", + ) + }, + } + registerFormatFlags(cmd.Flags()) + return cmd +} + +func newEmbeddingsActivateCommand() *cobra.Command { + var force bool + cmd := &cobra.Command{ + Use: "activate ", + Short: "Activate an embedding generation", + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := parseGenerationID(args[0]) + if err != nil { + return err + } + return runEmbeddingsGenerationAction( + cmd.Context(), cmd.OutOrStdout(), id, force, false, + ) + }, + } + cmd.Flags().BoolVar(&force, "force", false, + "Activate even if the generation has incomplete coverage") + return cmd +} + +func newEmbeddingsRetireCommand() *cobra.Command { + var force bool + cmd := &cobra.Command{ + Use: "retire ", + Short: "Retire an embedding generation", + SilenceUsage: true, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + id, err := parseGenerationID(args[0]) + if err != nil { + return err + } + return runEmbeddingsGenerationAction( + cmd.Context(), cmd.OutOrStdout(), id, force, true, + ) + }, + } + cmd.Flags().BoolVar(&force, "force", false, + "Retire even if the generation is currently active") + return cmd +} + +func parseGenerationID(raw string) (int64, error) { + id, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid generation id %q: %w", raw, err) + } + return id, nil +} + +// requireVectorEnabled rejects every embeddings subcommand up front when +// [vector] is not configured, with the exact message the brief specifies. +func requireVectorEnabled(cfg config.Config) error { + if !cfg.Vector.Enabled { + return errors.New( + "vector search is not enabled: set [vector] enabled = true in config.toml", + ) + } + return nil +} + +// vectorGeneration maps the configured embeddings model to the kit +// Generation identity Build/Fill fingerprint against. Defined once here so +// the daemon serve wiring (Task 16) can reuse the exact same mapping. +// doc_unit_scheme and chunk_overlap_chars are part of the fingerprint so +// that changing the run-grouped document scheme or the chunk overlap +// formula (vector.ChunkOverlap) cuts a new generation rather than silently +// reusing embeddings built under the old scheme. +func vectorGeneration(c config.VectorEmbeddingsConfig) kitvec.Generation { + params := map[string]string{ + "max_input_chars": strconv.Itoa(c.MaxInputChars), + "doc_unit_scheme": "run_v1", + "chunk_overlap_chars": strconv.Itoa(vector.ChunkOverlap(c.MaxInputChars)), + } + // input_suffix joins the fingerprint only when set: an empty suffix must + // hash identically to configs written before the key existed, so adding + // the field does not orphan every existing generation. + if c.InputSuffix != "" { + params["input_suffix"] = c.InputSuffix + } + return kitvec.Generation{ + Model: c.Model, + Dimensions: c.Dimension, + Params: params, + } +} + +// newVectorEncoder builds the OpenAI-compatible embeddings encoder for one +// named server ("" means the default), combining the global model identity +// with that server's transport settings. +func newVectorEncoder(c config.VectorEmbeddingsConfig, serverName string) (kitvec.EncodeFunc, error) { + name, server, err := c.Server(serverName) + if err != nil { + return nil, err + } + timeout, err := time.ParseDuration(server.Timeout) + if err != nil { + return nil, fmt.Errorf( + "parsing [vector.embeddings.servers.%s] timeout %q: %w", name, server.Timeout, err) + } + return vector.NewEncoder(vector.EncoderConfig{ + Endpoint: server.Endpoint, + APIKey: server.APIKey(), + Model: c.Model, + Dimension: c.Dimension, + Timeout: timeout, + MaxRetries: server.MaxRetries, + InputSuffix: c.InputSuffix, + }), nil +} + +// vectorEncoderSet builds one encoder per configured embeddings server, so +// a Manager can run any build against the server the request names. +func vectorEncoderSet(c config.VectorEmbeddingsConfig) (vector.EncoderSet, error) { + set := vector.EncoderSet{ + Default: c.ResolvedDefaultServer(), + ByName: make(map[string]vector.ManagedEncoder, len(c.Servers)), + } + for name, server := range c.Servers { + enc, err := newVectorEncoder(c, name) + if err != nil { + return vector.EncoderSet{}, err + } + set.ByName[name] = vector.ManagedEncoder{ + Encode: enc, + Settings: vector.EncodeSettings{ + BatchSize: server.BatchSize, + Concurrency: server.Concurrency, + }, + } + } + return set, nil +} + +// runEmbeddingsBuild loads config, gates on [vector] enabled, confirms a +// requested full rebuild (unless --yes), and then dispatches to the daemon +// or direct build path depending on whether a writable local daemon owns +// the archive. +func runEmbeddingsBuild( + ctx context.Context, out io.Writer, in io.Reader, opts EmbeddingsBuildOptions, +) error { + cfg, err := config.LoadMinimal() + if err != nil { + return fmt.Errorf("loading config: %w", err) + } + if err := requireVectorEnabled(cfg); err != nil { + return err + } + + includeAutomated := cfg.Vector.IncludeAutomated + if opts.IncludeAutomatedSet { + includeAutomated = opts.IncludeAutomated + } + + if opts.FullRebuild && !opts.Yes { + proceed, err := confirmFullRebuild(ctx, in, out, cfg, includeAutomated) + if err != nil { + return err + } + if !proceed { + fmt.Fprintln(out, "Aborted.") + return nil + } + } + + // Resolve --using against this config up front so a mistyped name fails + // with the full server list instead of a daemon-side error. + if _, _, err := cfg.Vector.Embeddings.Server(opts.Using); err != nil { + return err + } + + req := vector.BuildRequest{ + FullRebuild: opts.FullRebuild, + Backstop: opts.Backstop, + IncludeAutomated: includeAutomated, + Using: opts.Using, + } + if IsLocalDaemonActive(cfg.DataDir, cfg.AuthToken) { + return runEmbeddingsBuildDaemon(ctx, out, cfg, req) + } + return runEmbeddingsBuildDirect(ctx, out, cfg, req) +} + +// confirmFullRebuild prints and reads the "This re-embeds all N documents." +// confirmation prompt, where N is the current count of embeddable unit +// documents (user messages and assistant runs) in the archive under +// includeAutomated's scope — the exact set a full rebuild re-embeds. +func confirmFullRebuild( + ctx context.Context, in io.Reader, out io.Writer, cfg config.Config, includeAutomated bool, +) (bool, error) { + n, err := countEmbeddableUnits(ctx, cfg, includeAutomated) + if err != nil { + return false, fmt.Errorf("counting documents: %w", err) + } + msg := fmt.Sprintf("This re-embeds all %d documents. Continue?", n) + return confirm(in, out, msg), nil +} + +// countEmbeddableUnits opens the archive database read-only and counts +// every unit document eligible for embedding under includeAutomated's +// scope, for the full-rebuild confirmation prompt. +func countEmbeddableUnits(ctx context.Context, cfg config.Config, includeAutomated bool) (int, error) { + archiveDB, err := openReadOnlyDB(cfg) + if err != nil { + return 0, err + } + defer archiveDB.Close() + + var n int + if _, err := archiveDB.ScanEmbeddableUnits( + ctx, "", includeAutomated, func(db.EmbeddableUnit) error { + n++ + return nil + }, + ); err != nil { + return 0, err + } + return n, nil +} + +// runEmbeddingsBuildDirect acquires the vectors write lock, opens the +// archive read-only (so it never competes with a daemon for the SQLite +// write lock) and vectors.db read-write, and runs one build synchronously. +func runEmbeddingsBuildDirect( + ctx context.Context, out io.Writer, cfg config.Config, req vector.BuildRequest, +) error { + lock, err := tryAcquireNamedLock(cfg.DataDir, vectorsWriteLockFile) + if err != nil { + return err + } + defer lock.Close() + + archiveDB, err := openReadOnlyDB(cfg) + if err != nil { + return fmt.Errorf("opening archive database: %w", err) + } + defer archiveDB.Close() + + ix, err := vector.Open( + ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, cfg.Vector.Embeddings.MaxInputChars, + ) + if err != nil { + return fmt.Errorf("opening vectors.db: %w", err) + } + defer ix.Close() + + encoders, err := vectorEncoderSet(cfg.Vector.Embeddings) + if err != nil { + return err + } + + m := vector.NewManager(ix, archiveDB, encoders, vectorGeneration(cfg.Vector.Embeddings)) + return runDirectBuild(ctx, out, m, req) +} + +// runDirectBuild runs m.TryBuild synchronously on a background goroutine +// while the caller's goroutine polls m.Status at directBuildProgressInterval +// to print progress lines, then prints the final summary once the build +// completes. +func runDirectBuild( + ctx context.Context, out io.Writer, m *vector.Manager, req vector.BuildRequest, +) error { + type outcome struct { + started bool + err error + } + resultCh := make(chan outcome, 1) + go func() { + started, err := m.TryBuild(ctx, req) + resultCh <- outcome{started: started, err: err} + }() + + ticker := time.NewTicker(directBuildProgressInterval) + defer ticker.Stop() + for { + select { + case res := <-resultCh: + if !res.started { + return errors.New("a build is already running") + } + if res.err != nil { + return res.err + } + if status := m.Status(); status.LastResult != nil { + printBuildSummary(out, *status.LastResult) + } + return nil + case <-ticker.C: + if status := m.Status(); status.Running { + printBuildProgress(out, status.Done, status.Total) + } + } + } +} + +// runEmbeddingsBuildDaemon resolves the local daemon and runs the build +// through its HTTP API. +func runEmbeddingsBuildDaemon( + ctx context.Context, out io.Writer, cfg config.Config, req vector.BuildRequest, +) error { + client, err := resolveEmbeddingsDaemonClient(cfg) + if err != nil { + return err + } + return buildViaDaemon(ctx, out, client, req) +} + +// buildViaDaemon starts a build via POST /build, printing a status line +// instead of failing when one is already running (409), then polls +// /status until the build stops running. +func buildViaDaemon( + ctx context.Context, out io.Writer, client embeddingsDaemonClient, req vector.BuildRequest, +) error { + if err := client.startBuild(ctx, req); err != nil { + var apiErr *daemonAPIError + if errors.As(err, &apiErr) && apiErr.status == http.StatusConflict { + fmt.Fprintln(out, "a build is already running (daemon)") + } else { + return err + } + } + return pollDaemonBuildStatus(ctx, out, client) +} + +// pollDaemonBuildStatus polls the daemon's build status at +// embeddingsPollInterval, printing a progress line on every poll while the +// build is running, until it reports Running == false. +func pollDaemonBuildStatus( + ctx context.Context, out io.Writer, client embeddingsDaemonClient, +) error { + for { + status, err := client.status(ctx) + if err != nil { + return err + } + if !status.Running { + return finalizeBuildStatus(out, status) + } + printBuildProgress(out, status.Done, status.Total) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(embeddingsPollInterval): + } + } +} + +// finalizeBuildStatus reports a stopped build's outcome: a non-empty +// LastError becomes the returned (non-zero-exit) error, otherwise the +// LastResult prints the same final summary the direct path prints. +func finalizeBuildStatus(out io.Writer, status vector.BuildStatus) error { + if status.LastError != "" { + return errors.New(status.LastError) + } + if status.LastResult != nil { + printBuildSummary(out, *status.LastResult) + } + return nil +} + +// printBuildProgress writes one progress line for either build path. +func printBuildProgress(w io.Writer, done, total int64) { + pct := 0.0 + if total > 0 { + pct = float64(done) / float64(total) * 100 + } + fmt.Fprintf(w, "progress: %d/%d chunks (%.1f%%)\n", done, total, pct) +} + +// printBuildSummary writes the final build summary line (and, when the +// build auto-activated its generation, the activation line) for either +// build path. +func printBuildSummary(w io.Writer, result vector.BuildResult) { + fmt.Fprintf(w, "Embedded %d documents (%d chunks), skipped %d, stale %d\n", + result.Fill.Documents, result.Fill.Chunks, result.Fill.Skipped, result.Fill.Stale) + if result.Activated { + fmt.Fprintln(w, "Generation activated.") + } +} + +// runEmbeddingsList loads config, gates on [vector] enabled, lists every +// generation via the daemon or directly, and renders it as a table or JSON. +func runEmbeddingsList(ctx context.Context, out io.Writer, jsonOutput bool) error { + cfg, err := config.LoadMinimal() + if err != nil { + return fmt.Errorf("loading config: %w", err) + } + if err := requireVectorEnabled(cfg); err != nil { + return err + } + + var gens []vector.GenerationInfo + if IsLocalDaemonActive(cfg.DataDir, cfg.AuthToken) { + client, err := resolveEmbeddingsDaemonClient(cfg) + if err != nil { + return err + } + gens, err = client.generations(ctx) + if err != nil { + return err + } + } else { + gens, err = directListGenerations(ctx, cfg) + if err != nil { + return err + } + } + + if jsonOutput { + if gens == nil { + gens = []vector.GenerationInfo{} + } + return json.NewEncoder(out).Encode(struct { + Generations []vector.GenerationInfo `json:"generations"` + }{gens}) + } + printGenerationsTable(out, gens) + return nil +} + +// directListGenerations opens vectors.db read-only and lists its +// generations, or returns an empty list without error when the index has +// never been built. +func directListGenerations(ctx context.Context, cfg config.Config) ([]vector.GenerationInfo, error) { + path := cfg.Vector.ResolvedDBPath(cfg.DataDir) + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + ix, err := vector.Open(ctx, path, true, cfg.Vector.Embeddings.MaxInputChars) + if err != nil { + return nil, fmt.Errorf("opening vectors.db: %w", err) + } + defer ix.Close() + return ix.Generations(ctx) +} + +// printGenerationsTable renders gens as the `embeddings list` tabwriter +// table, truncating each fingerprint to fingerprintDisplayLen characters. +func printGenerationsTable(out io.Writer, gens []vector.GenerationInfo) { + tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tSTATE\tMODEL\tDIM\tEMBEDDED\tMISSING\tFINGERPRINT") + for _, g := range gens { + fmt.Fprintf(tw, "%d\t%s\t%s\t%d\t%d\t%d\t%s\n", + g.ID, g.State, g.Model, g.Dimension, g.Embedded, g.Missing, + truncateFingerprint(g.Fingerprint)) + } + _ = tw.Flush() +} + +func truncateFingerprint(fp string) string { + if len(fp) <= fingerprintDisplayLen { + return fp + } + return fp[:fingerprintDisplayLen] +} + +// runEmbeddingsGenerationAction implements both `activate ` and +// `retire `: it loads config, gates on [vector] enabled, and dispatches +// the requested action to the daemon or directly. A 409/refusal error's +// message is returned verbatim (no extra wrapping) so it displays exactly +// as the manager phrased it. +func runEmbeddingsGenerationAction( + ctx context.Context, out io.Writer, id int64, force, retire bool, +) error { + cfg, err := config.LoadMinimal() + if err != nil { + return fmt.Errorf("loading config: %w", err) + } + if err := requireVectorEnabled(cfg); err != nil { + return err + } + + if IsLocalDaemonActive(cfg.DataDir, cfg.AuthToken) { + client, err := resolveEmbeddingsDaemonClient(cfg) + if err != nil { + return err + } + if retire { + err = client.retire(ctx, id, force) + } else { + err = client.activate(ctx, id, force) + } + if err != nil { + return err + } + } else if err := directGenerationAction(ctx, cfg, id, force, retire); err != nil { + return err + } + + verb := "activated" + if retire { + verb = "retired" + } + fmt.Fprintf(out, "Generation %d %s.\n", id, verb) + return nil +} + +// directGenerationAction acquires the vectors write lock and applies the +// activate/retire state change directly against vectors.db. +func directGenerationAction( + ctx context.Context, cfg config.Config, id int64, force, retire bool, +) error { + lock, err := tryAcquireNamedLock(cfg.DataDir, vectorsWriteLockFile) + if err != nil { + return err + } + defer lock.Close() + + ix, err := vector.Open( + ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, cfg.Vector.Embeddings.MaxInputChars, + ) + if err != nil { + return fmt.Errorf("opening vectors.db: %w", err) + } + defer ix.Close() + + // Activate/retire never encode, so the manager needs no encoder set. + m := vector.NewManager(ix, nil, vector.EncoderSet{}, vectorGeneration(cfg.Vector.Embeddings)) + if retire { + return m.Retire(ctx, id, force) + } + return m.Activate(ctx, id, force) +} + +// resolveEmbeddingsDaemonClient finds the active local daemon and builds an +// HTTP client for its embeddings API. Callers only reach it after +// IsLocalDaemonActive reported true. +func resolveEmbeddingsDaemonClient(cfg config.Config) (embeddingsDaemonClient, error) { + rt := FindDaemonRuntime(cfg.DataDir, cfg.AuthToken) + if rt == nil { + return embeddingsDaemonClient{}, errors.New("no reachable local agentsview daemon found") + } + return embeddingsDaemonClient{baseURL: urlFromDaemonRuntime(rt), token: cfg.AuthToken}, nil +} + +// embeddingsDaemonClient is a small HTTP client for the Task 14 embeddings +// build lifecycle endpoints. +type embeddingsDaemonClient struct { + baseURL string + token string +} + +// daemonAPIError carries an embeddings API error response's HTTP status +// alongside its message, so callers can distinguish 409 (conflict/refusal) +// from other failures. +type daemonAPIError struct { + status int + message string +} + +func (e *daemonAPIError) Error() string { return e.message } + +func (c embeddingsDaemonClient) startBuild(ctx context.Context, req vector.BuildRequest) error { + return c.do(ctx, http.MethodPost, "/api/v1/embeddings/build", req, nil) +} + +func (c embeddingsDaemonClient) status(ctx context.Context) (vector.BuildStatus, error) { + var st vector.BuildStatus + err := c.do(ctx, http.MethodGet, "/api/v1/embeddings/status", nil, &st) + return st, err +} + +func (c embeddingsDaemonClient) generations(ctx context.Context) ([]vector.GenerationInfo, error) { + var body struct { + Generations []vector.GenerationInfo `json:"generations"` + } + err := c.do(ctx, http.MethodGet, "/api/v1/embeddings/generations", nil, &body) + return body.Generations, err +} + +func (c embeddingsDaemonClient) activate(ctx context.Context, id int64, force bool) error { + path := fmt.Sprintf("/api/v1/embeddings/generations/%d/activate", id) + return c.do(ctx, http.MethodPost, path, map[string]bool{"force": force}, nil) +} + +func (c embeddingsDaemonClient) retire(ctx context.Context, id int64, force bool) error { + path := fmt.Sprintf("/api/v1/embeddings/generations/%d/retire", id) + return c.do(ctx, http.MethodPost, path, map[string]bool{"force": force}, nil) +} + +// do performs one HTTP call against the daemon's embeddings API, +// marshaling reqBody (when non-nil) as the request body and decoding the +// response into out (when non-nil). A non-2xx response becomes a +// *daemonAPIError carrying the status and the server's "error" message. +func (c embeddingsDaemonClient) do( + ctx context.Context, method, path string, reqBody, out any, +) error { + var bodyReader io.Reader + if reqBody != nil { + data, err := json.Marshal(reqBody) + if err != nil { + return err + } + bodyReader = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext( + ctx, method, strings.TrimSuffix(c.baseURL, "/")+path, bodyReader, + ) + if err != nil { + return err + } + if reqBody != nil { + req.Header.Set("Content-Type", "application/json") + } + // The daemon's CSRF guard rejects mutating requests whose Origin is not + // in the allowlist. Setting Origin to the daemon's own baseURL satisfies + // that check for the CLI, which has no real browser origin. + req.Header.Set("Origin", c.baseURL) + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := embeddingsDaemonHTTPClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return &daemonAPIError{status: resp.StatusCode, message: daemonErrorMessage(resp.StatusCode, body)} + } + if out == nil { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} + +// daemonErrorMessage extracts the {"error": "..."} message huma's error +// responses carry, falling back to a generic "HTTP : " when +// the body isn't in that shape. +func daemonErrorMessage(status int, body []byte) string { + var apiErr struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &apiErr) == nil && apiErr.Error != "" { + return apiErr.Error + } + return fmt.Sprintf("HTTP %d: %s", status, strings.TrimSpace(string(body))) +} diff --git a/cmd/agentsview/embeddings_test.go b/cmd/agentsview/embeddings_test.go new file mode 100644 index 000000000..12a5ff6e4 --- /dev/null +++ b/cmd/agentsview/embeddings_test.go @@ -0,0 +1,911 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/kit/daemon" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" + + "go.kenn.io/agentsview/internal/config" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/vector" +) + +// writeEmbeddingsTestConfig writes a config.toml under dataDir with [vector] +// enabled and pointed at endpoint, using small-but-valid operational values +// so config.Validate accepts it. +func writeEmbeddingsTestConfig(t *testing.T, dataDir, endpoint string) { + t.Helper() + writeTestConfig(t, dataDir, fmt.Sprintf(` +[vector] +enabled = true + +[vector.embeddings] +model = "test-model" +dimension = 3 +max_input_chars = 1000 + +[vector.embeddings.servers.local] +endpoint = %q +batch_size = 10 +timeout = "5s" +max_retries = 1 +`, endpoint)) +} + +// newEmbeddingsStubServer returns an httptest server that answers the +// OpenAI-compatible /embeddings endpoint with dimension-length vectors for +// every input, mirroring the shape internal/vector/encoder_test.go's stub +// uses. +func newEmbeddingsStubServer(t *testing.T, dimension int) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Model string `json:"model"` + Input []string `json:"input"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + + data := make([]map[string]any, len(req.Input)) + for i := range req.Input { + vec := make([]float32, dimension) + for j := range vec { + vec[j] = float32(i + 1) + } + data[i] = map[string]any{"index": i, "embedding": vec} + } + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"data": data})) + })) +} + +// seedEmbeddableArchive creates sessions.db at cfg's default path with one +// session carrying one user and one assistant message, both eligible for +// embedding. +func seedEmbeddableArchive(t *testing.T, dataDir string) { + t.Helper() + d := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + // Three messages but only TWO embeddable units: the user message plus + // the contiguous assistant run [1,2]. Tests asserting document counts + // against this seed are therefore discriminating between unit-grouped + // and per-message counting. + dbtest.SeedSessionWithMessages(t, d, "sess-1", "proj", []db.Message{ + dbtest.UserMsg("sess-1", 0, "hello there"), + dbtest.AsstMsg("sess-1", 1, "hi back"), + dbtest.AsstMsg("sess-1", 2, "and a follow-up thought"), + }) +} + +// seedEmbeddableArchiveWithAutomated seeds the same normal session +// seedEmbeddableArchive does, plus one automated session with one +// embeddable message, for the include-automated scope tests. +func seedEmbeddableArchiveWithAutomated(t *testing.T, dataDir string) { + t.Helper() + d := dbtest.OpenTestDBAt(t, filepath.Join(dataDir, "sessions.db")) + dbtest.SeedSessionWithMessages(t, d, "sess-1", "proj", []db.Message{ + dbtest.UserMsg("sess-1", 0, "hello there"), + dbtest.AsstMsg("sess-1", 1, "hi back"), + dbtest.AsstMsg("sess-1", 2, "and a follow-up thought"), + }) + dbtest.SeedSessionWithMessages(t, d, "sess-auto", "proj", []db.Message{ + dbtest.UserMsg("sess-auto", 0, "roborev output"), + }, func(s *db.Session) { s.IsAutomated = true }) +} + +// TestVectorGenerationParams asserts vectorGeneration's Params map carries +// exactly the three run_v1 fingerprint keys the plan requires, with +// chunk_overlap_chars derived from vector.ChunkOverlap so a future change to +// that formula cannot silently drift from the fingerprint. An empty +// input_suffix must be absent from the map (not present as ""), so configs +// written before the key existed keep their fingerprints; a non-empty suffix +// joins the fingerprint and cuts a new generation. +func TestVectorGenerationParams(t *testing.T) { + c := config.VectorEmbeddingsConfig{ + Model: "test-model", + Dimension: 3, + MaxInputChars: 4000, + } + + gen := vectorGeneration(c) + + assert.Equal(t, "test-model", gen.Model) + assert.Equal(t, 3, gen.Dimensions) + assert.Equal(t, map[string]string{ + "max_input_chars": "4000", + "doc_unit_scheme": "run_v1", + "chunk_overlap_chars": strconv.Itoa(vector.ChunkOverlap(4000)), + }, gen.Params) + + c.InputSuffix = "<|endoftext|>" + gen = vectorGeneration(c) + assert.Equal(t, map[string]string{ + "max_input_chars": "4000", + "doc_unit_scheme": "run_v1", + "chunk_overlap_chars": strconv.Itoa(vector.ChunkOverlap(4000)), + "input_suffix": "<|endoftext|>", + }, gen.Params) +} + +// TestEmbeddingsDisabledReturnsError asserts every subcommand refuses with +// the exact "vector search is not enabled" message when [vector] is off +// (the default), before attempting any daemon detection or I/O. +func TestEmbeddingsDisabledReturnsError(t *testing.T) { + tests := []struct { + name string + newCmd func() *cobra.Command + args []string + }{ + {"build", newEmbeddingsBuildCommand, nil}, + {"list", newEmbeddingsListCommand, nil}, + {"activate", newEmbeddingsActivateCommand, []string{"1"}}, + {"retire", newEmbeddingsRetireCommand, []string{"1"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + testDataDir(t) + + cmd := tt.newCmd() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(tt.args) + + err := cmd.Execute() + require.Error(t, err) + assert.Equal(t, + "vector search is not enabled: set [vector] enabled = true in config.toml", + err.Error()) + }) + } +} + +// TestEmbeddingsListRendersTable seeds vectors.db directly with one +// generation (bypassing a full build) and asserts `embeddings list` +// renders it as a table with the documented columns and a +// fingerprint truncated to 12 characters. +func TestEmbeddingsListRendersTable(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + cfg, err := config.LoadMinimal() + require.NoError(t, err) + + ctx := context.Background() + ix, err := vector.Open(ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, + cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + fp, err := ix.EnsureGeneration(ctx, vectorGeneration(cfg.Vector.Embeddings), sqlitevec.StateActive) + require.NoError(t, err) + require.NoError(t, ix.Close()) + + cmd := newEmbeddingsListCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + require.Len(t, lines, 2, "expected a header line and one generation row, got: %q", out.String()) + assert.Contains(t, lines[0], "ID") + assert.Contains(t, lines[0], "STATE") + assert.Contains(t, lines[0], "MODEL") + assert.Contains(t, lines[0], "DIM") + assert.Contains(t, lines[0], "EMBEDDED") + assert.Contains(t, lines[0], "MISSING") + assert.Contains(t, lines[0], "FINGERPRINT") + + assert.Contains(t, lines[1], "active") + assert.Contains(t, lines[1], "test-model") + assert.Contains(t, lines[1], truncateFingerprint(fp)) + assert.NotContains(t, lines[1], fp, "fingerprint column must be truncated to 12 chars") +} + +// TestEmbeddingsListJSONFormat asserts `embeddings list --format json` +// wraps the generation list in the same {"generations": [...]} shape the +// Task 14 HTTP endpoint uses. +func TestEmbeddingsListJSONFormat(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + cfg, err := config.LoadMinimal() + require.NoError(t, err) + ctx := context.Background() + ix, err := vector.Open(ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, + cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + _, err = ix.EnsureGeneration(ctx, vectorGeneration(cfg.Vector.Embeddings), sqlitevec.StateActive) + require.NoError(t, err) + require.NoError(t, ix.Close()) + + cmd := newEmbeddingsListCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"--format", "json"}) + require.NoError(t, cmd.Execute()) + + var body struct { + Generations []vector.GenerationInfo `json:"generations"` + } + require.NoError(t, json.Unmarshal(out.Bytes(), &body)) + require.Len(t, body.Generations, 1) + assert.Equal(t, "active", body.Generations[0].State) +} + +// TestEmbeddingsListEmptyIndexReturnsNoRows asserts `embeddings list` +// against a data dir where vectors.db was never built prints only the +// header, without erroring. +func TestEmbeddingsListEmptyIndexReturnsNoRows(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + cmd := newEmbeddingsListCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + require.Len(t, lines, 1, "expected only the header line, got: %q", out.String()) +} + +// TestEmbeddingsBuildDirectEndToEnd seeds a temp archive with one user +// message and a two-message assistant run (two embeddable units), points +// [vector.embeddings] at an httptest OpenAI-compatible stub, and asserts the +// direct build path embeds both units and prints the exact final summary and +// activation line. +func TestEmbeddingsBuildDirectEndToEnd(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchive(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + assert.Contains(t, out.String(), "Embedded 2 documents (2 chunks), skipped 0, stale 0") + assert.Contains(t, out.String(), "Generation activated.") +} + +// TestEmbeddingsBuildDirectPrintsProgress shrinks the direct path's +// progress ticker and slows the embeddings stub down so the build is still +// running when the ticker fires, asserting at least one progress line in +// the documented format is printed before the final summary. +func TestEmbeddingsBuildDirectPrintsProgress(t *testing.T) { + orig := directBuildProgressInterval + directBuildProgressInterval = 5 * time.Millisecond + t.Cleanup(func() { directBuildProgressInterval = orig }) + + dataDir := testDataDir(t) + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Input []string `json:"input"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + time.Sleep(150 * time.Millisecond) + + data := make([]map[string]any, len(req.Input)) + for i := range req.Input { + data[i] = map[string]any{"index": i, "embedding": []float32{1, 2, 3}} + } + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"data": data})) + })) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchive(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + assert.Regexp(t, `progress: \d+/\d+ chunks \(\d+\.\d%\)`, out.String(), + "a running build must print at least one progress line") + assert.Contains(t, out.String(), "Embedded 2 documents (2 chunks), skipped 0, stale 0") +} + +// TestEmbeddingsBuildDirectConcurrentFlockFails asserts a second direct +// build invocation, run while another process (simulated by acquiring the +// lock in the test) holds vectors.write.lock, fails immediately with the +// write-lock-held error rather than racing the first build. +func TestEmbeddingsBuildDirectConcurrentFlockFails(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchive(t, dataDir) + + held, err := tryAcquireNamedLock(dataDir, vectorsWriteLockFile) + require.NoError(t, err) + defer held.Close() + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + + err = cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "held by another process") + assert.Contains(t, err.Error(), vectorsWriteLockFile) +} + +// TestEmbeddingsBuildFullRebuildPromptsAndAborts asserts --full-rebuild +// without --yes prints the exact confirmation prompt with the true +// embeddable-unit count (the seeded user message and the assistant run are +// one document each), and a "no" answer aborts without building. +func TestEmbeddingsBuildFullRebuildPromptsAndAborts(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchive(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetIn(strings.NewReader("n\n")) + cmd.SetArgs([]string{"--full-rebuild"}) + require.NoError(t, cmd.Execute()) + + assert.Contains(t, out.String(), "This re-embeds all 2 documents. Continue?") + assert.Contains(t, out.String(), "Aborted.") + assert.NotContains(t, out.String(), "Embedded") +} + +// TestEmbeddingsBuildFullRebuildYesSkipsPrompt asserts --yes skips the +// confirmation prompt entirely and proceeds straight to the build. +func TestEmbeddingsBuildFullRebuildYesSkipsPrompt(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchive(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"--full-rebuild", "--yes"}) + require.NoError(t, cmd.Execute()) + + assert.NotContains(t, out.String(), "Continue?") + assert.Contains(t, out.String(), "Embedded 2 documents (2 chunks), skipped 0, stale 0") +} + +// TestEmbeddingsBuildDirectExcludesAutomatedByDefault asserts the direct +// build path's default scope (no config include_automated, no +// --include-automated flag) never embeds an automated session's messages. +func TestEmbeddingsBuildDirectExcludesAutomatedByDefault(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchiveWithAutomated(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + assert.Contains(t, out.String(), "Embedded 2 documents (2 chunks), skipped 0, stale 0", + "the automated session's message must not be embedded by default") +} + +// TestEmbeddingsBuildDirectIncludeAutomatedFlagOverridesConfig asserts +// --include-automated forces the automated session's message into the build +// even though [vector].include_automated defaults to false. +func TestEmbeddingsBuildDirectIncludeAutomatedFlagOverridesConfig(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchiveWithAutomated(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"--include-automated"}) + require.NoError(t, cmd.Execute()) + + assert.Contains(t, out.String(), "Embedded 3 documents (3 chunks), skipped 0, stale 0", + "--include-automated must embed the automated session's message too") +} + +// TestEmbeddingsBuildDirectIncludeAutomatedConfigDefault asserts +// [vector].include_automated = true embeds automated sessions without +// needing the --include-automated flag on every build. +func TestEmbeddingsBuildDirectIncludeAutomatedConfigDefault(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeTestConfig(t, dataDir, fmt.Sprintf(` +[vector] +enabled = true +include_automated = true + +[vector.embeddings] +model = "test-model" +dimension = 3 +max_input_chars = 1000 + +[vector.embeddings.servers.local] +endpoint = %q +batch_size = 10 +timeout = "5s" +max_retries = 1 +`, stub.URL+"/v1")) + seedEmbeddableArchiveWithAutomated(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + assert.Contains(t, out.String(), "Embedded 3 documents (3 chunks), skipped 0, stale 0") +} + +// TestEmbeddingsBuildDirectIncludeAutomatedFlagFalseOverridesConfigTrue +// asserts --include-automated=false narrows the scope back down for this one +// build even when [vector].include_automated defaults to true: the parsed +// flag value must win, not just "the flag was passed forces true". +func TestEmbeddingsBuildDirectIncludeAutomatedFlagFalseOverridesConfigTrue(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeTestConfig(t, dataDir, fmt.Sprintf(` +[vector] +enabled = true +include_automated = true + +[vector.embeddings] +model = "test-model" +dimension = 3 +max_input_chars = 1000 + +[vector.embeddings.servers.local] +endpoint = %q +batch_size = 10 +timeout = "5s" +max_retries = 1 +`, stub.URL+"/v1")) + seedEmbeddableArchiveWithAutomated(t, dataDir) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"--include-automated=false"}) + require.NoError(t, cmd.Execute()) + + assert.Contains(t, out.String(), "Embedded 2 documents (2 chunks), skipped 0, stale 0", + "--include-automated=false must exclude the automated session's message "+ + "even though the config default is true") +} + +// TestEmbeddingsBuildIncludeAutomatedFlagThreadsToDaemonRequest drives the +// daemon build path and asserts --include-automated forces +// BuildRequest.IncludeAutomated to true in the request body, overriding the +// (default false) config value for this one build. +func TestEmbeddingsBuildIncludeAutomatedFlagThreadsToDaemonRequest(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + var gotIncludeAutomated atomic.Bool + startEmbeddingsTestDaemon(t, dataDir, map[string]http.HandlerFunc{ + "POST /api/v1/embeddings/build": func(w http.ResponseWriter, r *http.Request) { + var req vector.BuildRequest + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + gotIncludeAutomated.Store(req.IncludeAutomated) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]bool{"started": true}) + }, + "GET /api/v1/embeddings/status": func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(vector.BuildStatus{Running: false}) + }, + }) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"--include-automated"}) + require.NoError(t, cmd.Execute()) + + assert.True(t, gotIncludeAutomated.Load(), "--include-automated must pass through to the daemon") +} + +// TestEmbeddingsBuildDaemonRequestDefaultsToConfigScope asserts that without +// --include-automated, the daemon request body still carries the resolved +// config value explicitly (true here), rather than silently falling back to +// the zero value the daemon's own (possibly different) config would use. +func TestEmbeddingsBuildDaemonRequestDefaultsToConfigScope(t *testing.T) { + dataDir := testDataDir(t) + writeTestConfig(t, dataDir, ` +[vector] +enabled = true +include_automated = true + +[vector.embeddings] +model = "test-model" +dimension = 3 +max_input_chars = 1000 + +[vector.embeddings.servers.local] +endpoint = "http://127.0.0.1:1" +batch_size = 10 +timeout = "5s" +max_retries = 1 +`) + + var gotIncludeAutomated atomic.Bool + startEmbeddingsTestDaemon(t, dataDir, map[string]http.HandlerFunc{ + "POST /api/v1/embeddings/build": func(w http.ResponseWriter, r *http.Request) { + var req vector.BuildRequest + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + gotIncludeAutomated.Store(req.IncludeAutomated) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]bool{"started": true}) + }, + "GET /api/v1/embeddings/status": func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(vector.BuildStatus{Running: false}) + }, + }) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + assert.True(t, gotIncludeAutomated.Load(), + "the CLI must resolve its local config's include_automated=true and send it explicitly") +} + +// TestEmbeddingsRetireDirectRoundTrip builds an active generation directly, +// asserts retiring it without --force is refused with the manager's exact +// message, and that --force performs the retirement and prints the success +// line. +func TestEmbeddingsRetireDirectRoundTrip(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + cfg, err := config.LoadMinimal() + require.NoError(t, err) + ctx := context.Background() + ix, err := vector.Open(ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, + cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + _, err = ix.EnsureGeneration(ctx, vectorGeneration(cfg.Vector.Embeddings), sqlitevec.StateActive) + require.NoError(t, err) + require.NoError(t, ix.Close()) + + retireCmd := newEmbeddingsRetireCommand() + var out bytes.Buffer + retireCmd.SetOut(&out) + retireCmd.SetArgs([]string{"1"}) + err = retireCmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "is active") + assert.Contains(t, err.Error(), "use --force") + + forceCmd := newEmbeddingsRetireCommand() + var forceOut bytes.Buffer + forceCmd.SetOut(&forceOut) + forceCmd.SetArgs([]string{"1", "--force"}) + require.NoError(t, forceCmd.Execute()) + assert.Equal(t, "Generation 1 retired.\n", forceOut.String()) +} + +// TestEmbeddingsActivateDirectRefusalThenForce builds generation 1 +// end-to-end (active, 2 docs embedded), registers a second never-filled +// building generation with Missing = 2, and asserts the direct path +// surfaces Manager.Activate's refusal wording verbatim without --force, +// then succeeds with --force. +func TestEmbeddingsActivateDirectRefusalThenForce(t *testing.T) { + dataDir := testDataDir(t) + stub := newEmbeddingsStubServer(t, 3) + defer stub.Close() + writeEmbeddingsTestConfig(t, dataDir, stub.URL+"/v1") + seedEmbeddableArchive(t, dataDir) + + buildCmd := newEmbeddingsBuildCommand() + var buildOut bytes.Buffer + buildCmd.SetOut(&buildOut) + buildCmd.SetArgs(nil) + require.NoError(t, buildCmd.Execute()) + + cfg, err := config.LoadMinimal() + require.NoError(t, err) + ctx := context.Background() + ix, err := vector.Open(ctx, cfg.Vector.ResolvedDBPath(cfg.DataDir), false, + cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + otherGen := kitvec.Generation{Model: "other-model", Dimensions: 3} + _, err = ix.EnsureGeneration(ctx, otherGen, sqlitevec.StateBuilding) + require.NoError(t, err) + require.NoError(t, ix.Close()) + + activateCmd := newEmbeddingsActivateCommand() + var out bytes.Buffer + activateCmd.SetOut(&out) + activateCmd.SetArgs([]string{"2"}) + err = activateCmd.Execute() + require.Error(t, err) + assert.Equal(t, + "generation 2 still has 2 documents needing embedding; use --force", + err.Error(), "the manager's refusal message must surface verbatim") + + forceCmd := newEmbeddingsActivateCommand() + var forceOut bytes.Buffer + forceCmd.SetOut(&forceOut) + forceCmd.SetArgs([]string{"2", "--force"}) + require.NoError(t, forceCmd.Execute()) + assert.Equal(t, "Generation 2 activated.\n", forceOut.String()) +} + +// TestEmbeddingsActivateInvalidIDReturnsError asserts a non-numeric +// argument is rejected before any config or I/O work happens. +func TestEmbeddingsActivateInvalidIDReturnsError(t *testing.T) { + testDataDir(t) + cmd := newEmbeddingsActivateCommand() + cmd.SetArgs([]string{"not-a-number"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid generation id") +} + +// startEmbeddingsTestDaemon starts an httptest server that answers the kit +// daemon ping (so FindDaemonRuntime's probe succeeds) plus the given +// embeddings endpoint handlers, writes a live writable runtime record for +// it into dataDir (so IsLocalDaemonActive reports true), and registers +// cleanup. Handlers are keyed by "METHOD /path". +func startEmbeddingsTestDaemon( + t *testing.T, dataDir string, handlers map[string]http.HandlerFunc, +) { + t.Helper() + mux := http.NewServeMux() + mux.Handle("GET /api/ping", daemon.NewPingHandler(daemon.PingHandlerOptions{ + Service: daemonService, + Version: "test", + })) + for pattern, h := range handlers { + mux.HandleFunc(pattern, h) + } + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + + endpoint := serverEndpoint(t, srv) + writeDaemonRuntimeForTest(t, dataDir, endpoint.Host, endpoint.Port, "test", false) +} + +// TestEmbeddingsListDispatchesToDaemon drives the real `embeddings list` +// command with a live writable daemon runtime record present, asserting +// the command routes through the daemon's /generations endpoint (never +// touching vectors.db, which does not exist) and renders its response. +func TestEmbeddingsListDispatchesToDaemon(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + var listCalled atomic.Bool + startEmbeddingsTestDaemon(t, dataDir, map[string]http.HandlerFunc{ + "GET /api/v1/embeddings/generations": func(w http.ResponseWriter, r *http.Request) { + listCalled.Store(true) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "generations": []vector.GenerationInfo{{ + ID: 1, State: "active", Model: "daemon-model", Dimension: 3, + Fingerprint: "abcdef0123456789", Embedded: 7, + }}, + }) + }, + }) + + cmd := newEmbeddingsListCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs(nil) + require.NoError(t, cmd.Execute()) + + assert.True(t, listCalled.Load(), "list must route through the daemon endpoint") + assert.Contains(t, out.String(), "daemon-model") + assert.Contains(t, out.String(), "abcdef012345") + assert.NoFileExists(t, filepath.Join(dataDir, "vectors.db"), + "daemon-dispatched list must not open vectors.db directly") +} + +// TestEmbeddingsBuildDispatchesToDaemon drives the real `embeddings build` +// command with a live writable daemon runtime record present: the daemon +// accepts the build (202) and immediately reports a completed run, so the +// poll terminates after one status call and prints the final summary. +func TestEmbeddingsBuildDispatchesToDaemon(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + var buildCalled, statusCalled atomic.Bool + startEmbeddingsTestDaemon(t, dataDir, map[string]http.HandlerFunc{ + "POST /api/v1/embeddings/build": func(w http.ResponseWriter, r *http.Request) { + buildCalled.Store(true) + var req vector.BuildRequest + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + assert.True(t, req.Backstop, "--backstop must pass through to the daemon") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]bool{"started": true}) + }, + "GET /api/v1/embeddings/status": func(w http.ResponseWriter, r *http.Request) { + statusCalled.Store(true) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(vector.BuildStatus{ + Running: false, + LastResult: &vector.BuildResult{ + Activated: true, + Fill: kitvec.FillStats{Documents: 5, Chunks: 6}, + }, + }) + }, + }) + + cmd := newEmbeddingsBuildCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"--backstop"}) + require.NoError(t, cmd.Execute()) + + assert.True(t, buildCalled.Load(), "build must route through the daemon endpoint") + assert.True(t, statusCalled.Load(), "build must poll the daemon status endpoint") + assert.Contains(t, out.String(), "Embedded 5 documents (6 chunks), skipped 0, stale 0") + assert.Contains(t, out.String(), "Generation activated.") + assert.NoFileExists(t, filepath.Join(dataDir, "vectors.db"), + "daemon-dispatched build must not open vectors.db directly") +} + +// TestEmbeddingsActivateDispatchesToDaemon drives the real `embeddings +// activate` command with a live writable daemon runtime record present, +// asserting the id and --force flag pass through to the daemon endpoint. +func TestEmbeddingsActivateDispatchesToDaemon(t *testing.T) { + dataDir := testDataDir(t) + writeEmbeddingsTestConfig(t, dataDir, "http://127.0.0.1:1") + + var gotForce atomic.Bool + startEmbeddingsTestDaemon(t, dataDir, map[string]http.HandlerFunc{ + "POST /api/v1/embeddings/generations/3/activate": func(w http.ResponseWriter, r *http.Request) { + var body struct { + Force bool `json:"force"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + gotForce.Store(body.Force) + w.WriteHeader(http.StatusNoContent) + }, + }) + + cmd := newEmbeddingsActivateCommand() + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetArgs([]string{"3", "--force"}) + require.NoError(t, cmd.Execute()) + + assert.True(t, gotForce.Load(), "--force must pass through to the daemon") + assert.Equal(t, "Generation 3 activated.\n", out.String()) +} + +// TestBuildViaDaemonConflictThenPolls drives buildViaDaemon (the daemon +// build path's core logic) against a fake HTTP server that refuses the +// first build attempt with 409 and reports Running until the second status +// poll, asserting the CLI prints the "already running" notice and then the +// same final summary format the direct path uses. +func TestBuildViaDaemonConflictThenPolls(t *testing.T) { + orig := embeddingsPollInterval + embeddingsPollInterval = time.Millisecond + t.Cleanup(func() { embeddingsPollInterval = orig }) + + var statusCalls atomic.Int32 + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/embeddings/build", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]string{ + "error": "an embeddings build is already running", + }) + }) + mux.HandleFunc("/api/v1/embeddings/status", func(w http.ResponseWriter, r *http.Request) { + n := statusCalls.Add(1) + status := vector.BuildStatus{Running: n < 2} + if n >= 2 { + status.LastResult = &vector.BuildResult{ + Activated: true, + Fill: kitvec.FillStats{Documents: 3, Chunks: 3}, + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(status) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + client := embeddingsDaemonClient{baseURL: srv.URL} + var out bytes.Buffer + err := buildViaDaemon(context.Background(), &out, client, vector.BuildRequest{}) + require.NoError(t, err) + + assert.Contains(t, out.String(), "a build is already running (daemon)") + assert.Contains(t, out.String(), "Embedded 3 documents (3 chunks), skipped 0, stale 0") + assert.Contains(t, out.String(), "Generation activated.") + assert.GreaterOrEqual(t, statusCalls.Load(), int32(2)) +} + +// TestBuildViaDaemonLastErrorReturnsNonZero asserts a stopped build with a +// non-empty LastError becomes the returned error, so the CLI exits +// non-zero, matching the direct path's error propagation. +func TestBuildViaDaemonLastErrorReturnsNonZero(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("/api/v1/embeddings/build", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"started": true}) + }) + mux.HandleFunc("/api/v1/embeddings/status", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(vector.BuildStatus{ + Running: false, + LastError: "encoder rejected input", + }) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + client := embeddingsDaemonClient{baseURL: srv.URL} + var out bytes.Buffer + err := buildViaDaemon(context.Background(), &out, client, vector.BuildRequest{}) + require.Error(t, err) + assert.Equal(t, "encoder rejected input", err.Error()) +} + +// TestDirectListGenerationsVersionMismatchSurfacesRebuildRequired pins the +// `embeddings list` direct path's version gate: against a vectors.db written +// by an older mirror schema version it must surface the rebuild-required +// error instead of listing stale-shape generation data. +func TestDirectListGenerationsVersionMismatchSurfacesRebuildRequired(t *testing.T) { + dataDir := t.TempDir() + cfg := vectorTestConfig(dataDir) + path := cfg.Vector.ResolvedDBPath(dataDir) + + seed, err := vector.Open(context.Background(), path, false, cfg.Vector.Embeddings.MaxInputChars) + require.NoError(t, err) + require.NoError(t, seed.Close()) + raw, err := sql.Open("sqlite3", path) + require.NoError(t, err) + _, err = raw.Exec(`UPDATE vector_meta SET value = '2' WHERE key = 'mirror_schema_version'`) + require.NoError(t, err) + require.NoError(t, raw.Close()) + + _, err = directListGenerations(context.Background(), cfg) + require.Error(t, err) + assert.ErrorIs(t, err, vector.ErrMirrorVersionMismatch, + "a stale-shape vectors.db must not be listed as if it were current") + assert.Contains(t, err.Error(), "embeddings build", + "the error must carry the rebuild remediation") +} diff --git a/cmd/agentsview/main.go b/cmd/agentsview/main.go index 8037eb4ed..d56ec5503 100644 --- a/cmd/agentsview/main.go +++ b/cmd/agentsview/main.go @@ -182,13 +182,34 @@ func runServe(cfg config.Config, opts serveOptions) { broadcaster := server.NewBroadcaster(cfg.EventsCoalesceInterval) + vectorServe, err := setupVectorServing(ctx, cfg, database) + if err != nil { + fatal("setting up vector index: %v", err) + } + if vectorServe.Close != nil { + defer func() { + if cerr := vectorServe.Close(); cerr != nil { + log.Printf("close vectors.db: %v", cerr) + } + }() + } + + var emitter sync.Emitter = broadcaster + if vectorServe.Scheduler != nil { + emitter = teeEmitter{ + primary: broadcaster, + scheduler: vectorServe.Scheduler, + runAfterSync: cfg.Vector.Embed.RunAfterSyncEnabled(), + } + } + var engine *sync.Engine if !cfg.NoSync { engine = sync.NewEngine(database, sync.EngineConfig{ AgentDirs: cfg.AgentDirs, Machine: "local", BlockedResultCategories: cfg.ResultContentBlockedCategories, - Emitter: broadcaster, + Emitter: emitter, }) if database.NeedsResync() { @@ -228,7 +249,7 @@ func runServe(cfg config.Config, opts serveOptions) { log.Printf("warning: remote_hosts config invalid, skipping periodic remote sync: %v", err) validRemotes = false } - go startPeriodicSync(ctx, cfg, engine, database, idleTracker, validRemotes, broadcaster) + go startPeriodicSync(ctx, cfg, engine, database, idleTracker, validRemotes, emitter) } // Seed model_pricing so a fresh database (first run, or a @@ -251,7 +272,7 @@ func runServe(cfg config.Config, opts serveOptions) { } cfg = preparedCfg - srv := server.New(cfg, database, engine, + srvOpts := []server.Option{ server.WithVersion(server.VersionInfo{ Version: version, Commit: commit, @@ -262,7 +283,9 @@ func runServe(cfg config.Config, opts serveOptions) { server.WithBroadcaster(broadcaster), server.WithIdleTracker(idleTracker), server.WithPprof(opts.Pprof), - ) + } + srvOpts = append(srvOpts, vectorServe.ServerOpts...) + srv := server.New(cfg, database, engine, srvOpts...) startupProgress.SetPhase("starting HTTP server") rt, err := startServerWithOptionalCaddy(ctx, cfg, srv, rtOpts) @@ -315,6 +338,14 @@ func runServe(cfg config.Config, opts serveOptions) { startTelemetryPings(ctx, telemetryReporter) + if vectorServe.Scheduler != nil { + go vectorServe.Scheduler.Run(ctx) + // Registered after the vectors.db Close defer above, so LIFO + // unwind order runs Stop (which waits for any in-flight + // TryBuild to return) before vectors.db is closed. + defer vectorServe.Scheduler.Stop() + } + if engine != nil { // Registered before stopWatcher so LIFO defer order stops // the watcher first, then Close flushes any pending diff --git a/cmd/agentsview/session.go b/cmd/agentsview/session.go index 4321e1bcc..fe551b6bd 100644 --- a/cmd/agentsview/session.go +++ b/cmd/agentsview/session.go @@ -7,11 +7,13 @@ import ( "fmt" "os" "strings" + "time" "github.com/spf13/cobra" "github.com/spf13/pflag" "go.kenn.io/agentsview/internal/config" "go.kenn.io/agentsview/internal/service" + "go.kenn.io/agentsview/internal/timeutil" ) func newSessionCommand() *cobra.Command { @@ -91,6 +93,27 @@ func resolveService( return newService(cfg, tr) } +// resolveSinceFlag validates the --since/--active-since pair shared by +// `session list` and `session search`: setting both is an error, since they +// describe the same active-window filter two different ways. When --since is +// set, it resolves against the current time via timeutil.ParseSince and +// returns the RFC3339 string to use as ActiveSince; otherwise activeSince +// passes through unchanged. +func resolveSinceFlag(since, activeSince string) (string, error) { + if since == "" { + return activeSince, nil + } + if activeSince != "" { + return "", errors.New( + "--since and --active-since are mutually exclusive") + } + t, err := timeutil.ParseSince(time.Now(), since) + if err != nil { + return "", err + } + return t.UTC().Format(time.RFC3339), nil +} + // resolveWritableService constructs a write-capable SessionService: // HTTP when a writable daemon is reachable, otherwise a direct // backend wired with a real sync.Engine. It refuses read-only daemons diff --git a/cmd/agentsview/session_list.go b/cmd/agentsview/session_list.go index a2df60432..93a5cf69d 100644 --- a/cmd/agentsview/session_list.go +++ b/cmd/agentsview/session_list.go @@ -20,6 +20,7 @@ func newSessionListCommand() *cobra.Command { var ( project, excludeProject, machine, agent string date, dateFrom, dateTo, activeSince string + since string minMessages, maxMessages int minUserMessages int includeOneShot bool @@ -39,6 +40,12 @@ func newSessionListCommand() *cobra.Command { Args: cobra.NoArgs, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { + resolvedActiveSince, err := resolveSinceFlag(since, activeSince) + if err != nil { + return err + } + activeSince = resolvedActiveSince + svc, cleanup, err := resolveService(cmd) if err != nil { return err @@ -73,10 +80,12 @@ func newSessionListCommand() *cobra.Command { // quick relaunch: push a now-15m active_since window to the // service so the limit is applied after the filter, and let the // default recent sort keep newest-first ordering. An explicit - // --active-since takes precedence so callers can widen or narrow - // the window. + // --active-since or --since takes precedence so callers can + // widen or narrow the window. now := time.Now() - if (resume || active) && !cmd.Flags().Changed("active-since") { + if (resume || active) && + !cmd.Flags().Changed("active-since") && + !cmd.Flags().Changed("since") { f.ActiveSince = now.Add(-resumeActiveWindow). UTC().Format(time.RFC3339) } @@ -133,6 +142,8 @@ func newSessionListCommand() *cobra.Command { "Filter sessions started on or before YYYY-MM-DD") flags.StringVar(&activeSince, "active-since", "", "Filter sessions active since RFC3339 timestamp") + flags.StringVar(&since, "since", "", + "Only sessions active since a relative duration (12h, 14d, 2w, 3m = 3 months, 1y) or YYYY-MM-DD") flags.IntVar(&minMessages, "min-messages", 0, "Minimum total message count") flags.IntVar(&maxMessages, "max-messages", 0, diff --git a/cmd/agentsview/session_list_test.go b/cmd/agentsview/session_list_test.go new file mode 100644 index 000000000..5b2662e99 --- /dev/null +++ b/cmd/agentsview/session_list_test.go @@ -0,0 +1,119 @@ +// ABOUTME: `session list --since` relative time filter tests -- flag +// ABOUTME: validation, actual filtering behavior, and --resume interaction. +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSessionListSinceMutuallyExclusiveWithActiveSince verifies the error is +// returned before any service/DB access is attempted (no data dir is set up +// here): --since resolution runs ahead of resolveService in the command. +func TestSessionListSinceMutuallyExclusiveWithActiveSince(t *testing.T) { + _, err := executeCommand(newRootCommand(), + "session", "list", "--since", "14d", + "--active-since", "2024-01-01T00:00:00Z") + require.Error(t, err) + assert.Contains(t, err.Error(), + "--since and --active-since are mutually exclusive") +} + +func TestSessionListSinceRejectsInvalidFormat(t *testing.T) { + _, err := executeCommand(newRootCommand(), + "session", "list", "--since", "3x") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid --since") +} + +// TestSessionListSinceFiltersByActivity is the end-to-end regression test: +// --since must narrow results the same way --active-since already does, +// by resolving to an RFC3339 boundary and threading it through unchanged. +func TestSessionListSinceFiltersByActivity(t *testing.T) { + dataDir := newAgentDataDir(t) + seedSessionsWithOpts(t, dataDir, + activitySeed("fresh", 2*time.Hour), + activitySeed("stale", 20*24*time.Hour), + ) + + out, err := executeCommand(newRootCommand(), + "session", "list", "--since", "7d", "--format", "json") + require.NoError(t, err) + + assert.Equal(t, []string{"fresh"}, sessionListIDs(t, out)) +} + +// TestSessionListSinceAcceptsAbsoluteDate covers the YYYY-MM-DD form of +// --since alongside the relative-duration form already covered above. +func TestSessionListSinceAcceptsAbsoluteDate(t *testing.T) { + dataDir := newAgentDataDir(t) + seedSessionsWithOpts(t, dataDir, + activitySeed("fresh", 2*time.Hour), + activitySeed("stale", 20*24*time.Hour), + ) + + yesterday := time.Now().Add(-24 * time.Hour).Format("2006-01-02") + out, err := executeCommand(newRootCommand(), + "session", "list", "--since", yesterday, "--format", "json") + require.NoError(t, err) + + assert.Equal(t, []string{"fresh"}, sessionListIDs(t, out)) +} + +// TestSessionListResumeRespectsExplicitSince is the CRITICAL interaction +// regression test: --resume/--active push a default 15-minute active_since +// window unless an explicit --active-since was given. That guard must also +// recognize an explicit --since, or a session outside the 15-minute default +// but inside the requested --since window would be silently dropped. +func TestSessionListResumeRespectsExplicitSince(t *testing.T) { + dataDir := newAgentDataDir(t) + seedSessionsWithOpts(t, dataDir, + activitySeed("within-since", 2*time.Hour), + activitySeed("outside-since", 20*24*time.Hour), + ) + + // 2 hours ago is well outside --resume's default 15-minute window, so + // this would incorrectly return no sessions if --since were overridden. + out, err := executeCommand(newRootCommand(), + "session", "list", "--resume", "--since", "1d", "--format", "json") + require.NoError(t, err) + + assert.Equal(t, []string{"within-since"}, sessionListIDs(t, out)) +} + +// TestResolveSinceFlag_ResolvesRelativeWindow is a fast unit test of the +// shared flag-resolution helper: since ParseSince resolves against +// time.Now() with no clock-injection seam in this command, the returned +// RFC3339 boundary is asserted to fall within a tolerance window around +// now minus the requested duration rather than an exact instant. +func TestResolveSinceFlag_ResolvesRelativeWindow(t *testing.T) { + before := time.Now().Add(-14 * 24 * time.Hour) + got, err := resolveSinceFlag("14d", "") + after := time.Now().Add(-14 * 24 * time.Hour) + require.NoError(t, err) + + parsed, err := time.Parse(time.RFC3339, got) + require.NoError(t, err) + assert.False(t, parsed.Before(before.Add(-time.Second)), + "resolved active_since %v earlier than expected window start %v", + parsed, before) + assert.False(t, parsed.After(after.Add(time.Second)), + "resolved active_since %v later than expected window end %v", + parsed, after) +} + +func TestResolveSinceFlag_PassesThroughActiveSinceWhenSinceUnset(t *testing.T) { + got, err := resolveSinceFlag("", "2024-01-01T00:00:00Z") + require.NoError(t, err) + assert.Equal(t, "2024-01-01T00:00:00Z", got) +} + +func TestResolveSinceFlag_RejectsBothSet(t *testing.T) { + _, err := resolveSinceFlag("14d", "2024-01-01T00:00:00Z") + require.Error(t, err) + assert.Contains(t, err.Error(), + "--since and --active-since are mutually exclusive") +} diff --git a/cmd/agentsview/session_messages.go b/cmd/agentsview/session_messages.go index e1ead6540..e0f7befc3 100644 --- a/cmd/agentsview/session_messages.go +++ b/cmd/agentsview/session_messages.go @@ -6,44 +6,30 @@ import ( "encoding/json" "fmt" "io" + "strings" "github.com/spf13/cobra" "go.kenn.io/agentsview/internal/service" ) func newSessionMessagesCommand() *cobra.Command { - var ( - from int - limit int - direction string - ) cmd := &cobra.Command{ Use: "messages ", Short: "Show a window of messages from a session", Args: cobra.ExactArgs(1), SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - if direction != "asc" && direction != "desc" { - return fmt.Errorf( - "invalid --direction %q: must be asc or desc", direction, - ) + filter, err := messagesFilterFromFlags(cmd) + if err != nil { + return err } + svc, cleanup, err := resolveService(cmd) if err != nil { return err } defer cleanup() - filter := service.MessageFilter{ - Limit: limit, - Direction: direction, - } - // Preserve presence: an explicit --from 0 means "start - // at ordinal 0", not "use the default tail/head". - if cmd.Flags().Changed("from") { - filter.From = &from - } - id, err := resolveServiceSessionID(cmd.Context(), svc, args[0]) if err != nil { return err @@ -59,16 +45,126 @@ func newSessionMessagesCommand() *cobra.Command { }, } flags := cmd.Flags() - flags.IntVar(&from, "from", 0, + flags.Int("from", 0, "Starting ordinal (inclusive). Omit for the newest page in "+ "--direction desc; explicit 0 starts at ordinal 0.") - flags.IntVar(&limit, "limit", 0, + flags.Int("limit", 0, "Maximum messages to return (0 = server default)") - flags.StringVar(&direction, "direction", "asc", + flags.String("direction", "asc", "Sort direction: asc or desc") + flags.Int("around", 0, + "Center a window on this ordinal (use with --before/--after)") + flags.Int("before", 5, "Messages before --around (default 5)") + flags.Int("after", 5, "Messages after --around (default 5)") + flags.String("role", "", + "Comma-separated roles to include, e.g. user,assistant") return cmd } +// messagesFilterFromFlags builds a service.MessageFilter from the `session +// messages` command's flags. It enforces the CLI-level check that +// --before/--after require --around, and handles the critical +// around-vs-direction/from gotcha: --direction defaults to "asc" and --from +// defaults to 0, so both flags always carry a non-empty value even when the +// user never passed them. Forwarding those defaults on the around path +// would trip the service's around-vs-direction/from mutual-exclusion check +// on every plain `--around N` call, so Direction/From are only forwarded +// when the user actually set the flag; an explicit --from/--direction +// alongside --around still reaches the service, whose error surfaces +// unchanged. +func messagesFilterFromFlags(cmd *cobra.Command) (service.MessageFilter, error) { + flags := cmd.Flags() + + direction, err := flags.GetString("direction") + if err != nil { + return service.MessageFilter{}, err + } + if direction != "asc" && direction != "desc" { + return service.MessageFilter{}, fmt.Errorf( + "invalid --direction %q: must be asc or desc", direction, + ) + } + aroundSet := flags.Changed("around") + if !aroundSet && (flags.Changed("before") || flags.Changed("after")) { + return service.MessageFilter{}, fmt.Errorf( + "--before/--after require --around", + ) + } + + limit, err := flags.GetInt("limit") + if err != nil { + return service.MessageFilter{}, err + } + filter := service.MessageFilter{Limit: limit} + + if aroundSet { + around, err := flags.GetInt("around") + if err != nil { + return service.MessageFilter{}, err + } + filter.Around = &around + if flags.Changed("before") { + before, err := flags.GetInt("before") + if err != nil { + return service.MessageFilter{}, err + } + filter.Before = &before + } + if flags.Changed("after") { + after, err := flags.GetInt("after") + if err != nil { + return service.MessageFilter{}, err + } + filter.After = &after + } + if flags.Changed("direction") { + filter.Direction = direction + } + if flags.Changed("from") { + from, err := flags.GetInt("from") + if err != nil { + return service.MessageFilter{}, err + } + filter.From = &from + } + } else { + filter.Direction = direction + // Preserve presence: an explicit --from 0 means "start at + // ordinal 0", not "use the default tail/head". + if flags.Changed("from") { + from, err := flags.GetInt("from") + if err != nil { + return service.MessageFilter{}, err + } + filter.From = &from + } + } + + if role, err := flags.GetString("role"); err != nil { + return service.MessageFilter{}, err + } else if role != "" { + filter.Roles = splitTrimmedNonEmpty(role) + } + + return filter, nil +} + +// splitTrimmedNonEmpty splits s on commas, trims surrounding whitespace from +// each part, and drops empty parts. This matches `session search --in`'s +// convention (see resolveContentSearchMode's caller in session_search.go) so +// a trailing or doubled comma (e.g. "user,") narrows the filter by one +// intended value instead of silently adding a spurious "" role that matches +// nothing. +func splitTrimmedNonEmpty(s string) []string { + var out []string + for part := range strings.SplitSeq(s, ",") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out +} + // printMessagesHuman prints each message as a header block followed // by its content. Timestamp is trimmed to YYYY-MM-DDTHH:MM:SS. // Session-derived fields are sanitized so escape sequences embedded diff --git a/cmd/agentsview/session_messages_test.go b/cmd/agentsview/session_messages_test.go new file mode 100644 index 000000000..b218c0eab --- /dev/null +++ b/cmd/agentsview/session_messages_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/service" +) + +// parseMessagesFlags builds a `session messages` command, parses args +// against it, and returns the resulting filter/error from +// messagesFilterFromFlags without running the command's RunE (which would +// require a live service). +func parseMessagesFlags(t *testing.T, args []string) (service.MessageFilter, error) { + t.Helper() + cmd := newSessionMessagesCommand() + require.NoError(t, cmd.ParseFlags(args)) + return messagesFilterFromFlags(cmd) +} + +func TestSessionMessagesFlags_InvalidDirection(t *testing.T) { + _, err := parseMessagesFlags(t, []string{"--direction", "backwards"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid --direction") +} + +func TestSessionMessagesFlags_BeforeAfterRequireAround(t *testing.T) { + _, err := parseMessagesFlags(t, []string{"--before", "2"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--around") + + _, err = parseMessagesFlags(t, []string{"--after", "2"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--around") +} + +// TestSessionMessagesFlags_AroundOnlyOmitsDirectionAndFrom is the CRITICAL +// flag-default-gotcha regression test: --direction defaults to "asc" and +// --from defaults to 0, so a naive implementation would always forward +// them, tripping the service's around-vs-direction/from mutual-exclusion +// check on every plain `--around N` call. With only --around set, the +// built filter must leave Direction empty and From nil. +func TestSessionMessagesFlags_AroundOnlyOmitsDirectionAndFrom(t *testing.T) { + filter, err := parseMessagesFlags(t, []string{"--around", "5"}) + require.NoError(t, err) + require.NotNil(t, filter.Around) + assert.Equal(t, 5, *filter.Around) + assert.Empty(t, filter.Direction, + "Direction must stay empty when --direction was never set, "+ + "even though the flag default is asc") + assert.Nil(t, filter.From, + "From must stay nil when --from was never set") + assert.Nil(t, filter.Before, "Before must stay nil when --before was never set") + assert.Nil(t, filter.After, "After must stay nil when --after was never set") +} + +// TestSessionMessagesFlags_AroundWithExplicitFromForwardsIt verifies that +// an explicit --from alongside --around is still forwarded to the +// service (whose mutual-exclusion error then surfaces), rather than +// silently dropped. +func TestSessionMessagesFlags_AroundWithExplicitFromForwardsIt(t *testing.T) { + filter, err := parseMessagesFlags(t, []string{"--around", "5", "--from", "1"}) + require.NoError(t, err) + require.NotNil(t, filter.From) + assert.Equal(t, 1, *filter.From) +} + +// TestSessionMessagesFlags_AroundWithExplicitDirectionForwardsIt mirrors +// the From case for --direction. +func TestSessionMessagesFlags_AroundWithExplicitDirectionForwardsIt(t *testing.T) { + filter, err := parseMessagesFlags(t, []string{"--around", "5", "--direction", "desc"}) + require.NoError(t, err) + assert.Equal(t, "desc", filter.Direction) +} + +func TestSessionMessagesFlags_AroundWithBeforeAfter(t *testing.T) { + filter, err := parseMessagesFlags(t, []string{ + "--around", "5", "--before", "2", "--after", "3", + }) + require.NoError(t, err) + require.NotNil(t, filter.Before) + require.NotNil(t, filter.After) + assert.Equal(t, 2, *filter.Before) + assert.Equal(t, 3, *filter.After) +} + +func TestSessionMessagesFlags_RoleSplitsOnComma(t *testing.T) { + filter, err := parseMessagesFlags(t, []string{"--role", "user,assistant"}) + require.NoError(t, err) + assert.Equal(t, []string{"user", "assistant"}, filter.Roles) +} + +// TestSessionMessagesFlags_RoleTrimsSpacesAndDropsEmpty covers a trailing +// comma or stray whitespace (e.g. "user, " or "user,") which must not +// narrow the filter with a spurious "" role that matches nothing. +func TestSessionMessagesFlags_RoleTrimsSpacesAndDropsEmpty(t *testing.T) { + filter, err := parseMessagesFlags(t, []string{"--role", "user, assistant, "}) + require.NoError(t, err) + assert.Equal(t, []string{"user", "assistant"}, filter.Roles) +} + +// TestSessionMessagesAroundNoOtherFlagsSucceeds is the brief-mandated +// end-to-end check: `session messages --around 5` with no other flags +// must actually succeed when run as a full CLI command (through +// resolveService's normal local-SQLite discovery), not just build a filter +// that looks right in isolation. This is the regression test for the +// CRITICAL flag-default gotcha: the command previously always forwarded +// Direction (flag default "asc"), which would have tripped the +// around-vs-direction validation on every default `--around` call. +func TestSessionMessagesAroundNoOtherFlagsSucceeds(t *testing.T) { + dataDir := newAgentDataDir(t) + seedSession(t, dataDir, "s-around", "proj") + seedMessages(t, dataDir, "s-around", 12) // ordinals 1..12 + + out, err := executeCommand(newRootCommand(), + "session", "messages", "s-around", "--around", "5", "--format", "json") + require.NoError(t, err, "--around 5 with no other flags must succeed") + + got := decodeCLIJSON[cliMessageList](t, out) + // Ordinals start at 1 (seedMessages convention): only 4 messages exist + // below ordinal 5, so before-window is capped at 4 even though the + // default asks for 5; after-window gets the full 5 (6..10). + assert.Equal(t, 10, got.Count, + "before is capped at 4 available messages; after takes the full 5") + assert.Equal(t, float64(1), got.Messages[0]["ordinal"]) + assert.Equal(t, float64(10), got.Messages[len(got.Messages)-1]["ordinal"]) +} diff --git a/cmd/agentsview/session_search.go b/cmd/agentsview/session_search.go index 54fff1f27..a4293f5df 100644 --- a/cmd/agentsview/session_search.go +++ b/cmd/agentsview/session_search.go @@ -9,20 +9,21 @@ import ( "strings" "github.com/spf13/cobra" + "go.kenn.io/agentsview/internal/db" "go.kenn.io/agentsview/internal/service" ) func newSessionSearchCommand() *cobra.Command { var ( - useRegex, useFTS bool - in string - excludeSystem, reveal bool - project, excludeProject, agent string - machine, date, dateFrom, dateTo string - activeSince string - includeChildren, includeAutomated bool - includeOneShot bool - limit, cursor int + useRegex, useFTS, useSemantic, useHybrid bool + in, scope string + excludeSystem, reveal bool + project, excludeProject, agent string + machine, date, dateFrom, dateTo string + activeSince, since string + includeChildren, includeAutomated bool + includeOneShot bool + limit, cursor, contextN int ) cmd := &cobra.Command{ Use: "search ", @@ -30,29 +31,23 @@ func newSessionSearchCommand() *cobra.Command { Args: cobra.ExactArgs(1), SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - if useRegex && useFTS { - return fmt.Errorf("--regex and --fts are mutually exclusive") - } var sources []string for s := range strings.SplitSeq(in, ",") { if s = strings.TrimSpace(s); s != "" { sources = append(sources, s) } } - if useFTS { - for _, s := range sources { - if s != "messages" { - return fmt.Errorf( - "--fts searches messages only; drop --in or --fts") - } - } + mode, err := resolveContentSearchMode( + useRegex, useFTS, useSemantic, useHybrid, sources) + if err != nil { + return err + } + if err := validateScopeFlag(scope, useSemantic, useHybrid); err != nil { + return err } - mode := "substring" - switch { - case useRegex: - mode = "regex" - case useFTS: - mode = "fts" + activeSince, err = resolveSinceFlag(since, activeSince) + if err != nil { + return err } svc, cleanup, err := resolveService(cmd) if err != nil { @@ -77,8 +72,10 @@ func newSessionSearchCommand() *cobra.Command { IncludeChildren: includeChildren, IncludeAutomated: includeAutomated, IncludeOneShot: includeOneShot, + Scope: scope, Limit: limit, Cursor: cursor, + Context: contextN, }) if err != nil { return err @@ -97,6 +94,10 @@ func newSessionSearchCommand() *cobra.Command { flags := cmd.Flags() flags.BoolVar(&useRegex, "regex", false, "Treat pattern as an RE2 regex") flags.BoolVar(&useFTS, "fts", false, "Fast tokenized FTS over messages only") + flags.BoolVar(&useSemantic, "semantic", false, + "Semantic (vector) search over user/assistant messages") + flags.BoolVar(&useHybrid, "hybrid", false, + "Hybrid semantic + full-text search (reciprocal rank fusion)") flags.StringVar(&in, "in", "", "Comma-separated sources: messages,tool_input,tool_result (default all)") flags.BoolVar(&excludeSystem, "exclude-system", false, @@ -110,15 +111,90 @@ func newSessionSearchCommand() *cobra.Command { flags.StringVar(&dateFrom, "date-from", "", "Sessions on or after YYYY-MM-DD") flags.StringVar(&dateTo, "date-to", "", "Sessions on or before YYYY-MM-DD") flags.StringVar(&activeSince, "active-since", "", "Active since RFC3339 timestamp") + flags.StringVar(&since, "since", "", + "Only sessions active since a relative duration (12h, 14d, 2w, 3m = 3 months, 1y) or YYYY-MM-DD") + flags.StringVar(&scope, "scope", "", + "Semantic/hybrid result scope: top, all, or subordinate (default all)") flags.BoolVar(&includeChildren, "include-children", false, "Include subagent sessions") flags.BoolVar(&includeAutomated, "include-automated", false, "Include automated sessions") flags.BoolVar(&includeOneShot, "include-one-shot", false, "Include one-shot sessions") flags.IntVar(&limit, "limit", 0, "Max results (default 50, max 500)") flags.IntVar(&cursor, "cursor", 0, "Pagination cursor from a previous response") + flags.IntVar(&contextN, "context", 0, + "Include N messages of context before and after each match (max 10)") return cmd } +// validateScopeFlag gates --scope at the CLI boundary: it is only +// meaningful for --semantic/--hybrid and must name a known scope. +func validateScopeFlag(scope string, useSemantic, useHybrid bool) error { + if scope == "" { + return nil + } + if !useSemantic && !useHybrid { + return fmt.Errorf("--scope requires --semantic or --hybrid") + } + switch scope { + case "top", "all", "subordinate": + return nil + } + return fmt.Errorf("--scope must be top, all, or subordinate (got %q)", scope) +} + +// resolveContentSearchMode picks the search mode from the mutually exclusive +// --regex/--fts/--semantic/--hybrid flags and, for the modes that only search +// message content ("fts", "semantic", "hybrid"), rejects an explicit --in +// naming other sources. +func resolveContentSearchMode( + useRegex, useFTS, useSemantic, useHybrid bool, sources []string, +) (string, error) { + modes := 0 + for _, b := range []bool{useRegex, useFTS, useSemantic, useHybrid} { + if b { + modes++ + } + } + if modes > 1 { + return "", fmt.Errorf( + "--regex, --fts, --semantic and --hybrid are mutually exclusive") + } + mode := "substring" + switch { + case useRegex: + mode = "regex" + case useFTS: + mode = "fts" + case useSemantic: + mode = "semantic" + case useHybrid: + mode = "hybrid" + } + if useFTS { + for _, s := range sources { + if s != "messages" { + return "", fmt.Errorf( + "--fts searches messages only; drop --in or --fts") + } + } + } + if useSemantic || useHybrid { + for _, s := range sources { + if s != "messages" { + return "", fmt.Errorf( + "--%s searches messages only; drop --in", mode) + } + } + } + return mode, nil +} + // printContentMatchesHuman writes one line per match, terminal-sanitized. +// Scored matches (semantic/hybrid modes) show "score=0.83" after the +// ordinal; unscored matches (substring/regex/fts) omit it. A match spanning +// a multi-message unit renders "#- @" instead of the +// plain "#", and a subordinate unit gains a "sub" marker. When +// --context requested inline context, ContextBefore/ContextAfter print as +// indented "role: content" lines around the match line. func printContentMatchesHuman(w io.Writer, res *service.ContentSearchResult) error { if len(res.Matches) == 0 { fmt.Fprintln(w, "(no matches)") @@ -129,14 +205,56 @@ func printContentMatchesHuman(w io.Writer, res *service.ContentSearchResult) err if m.ToolName != "" { loc = m.Location + ":" + m.ToolName } - fmt.Fprintf(w, "%s #%d %s %s\n", - sanitizeTerminal(m.SessionID), m.Ordinal, + for _, cm := range m.ContextBefore { + printContentContextLine(w, cm) + } + fmt.Fprintf(w, "%s %s", sanitizeTerminal(m.SessionID), formatMatchOrdinal(m)) + if m.Subordinate { + fmt.Fprint(w, " sub") + } + if m.Score != nil { + fmt.Fprintf(w, " score=%.2f", *m.Score) + } + fmt.Fprintf(w, " %s %s\n", sanitizeTerminal(m.Project), sanitizeTerminal(loc)) fmt.Fprintf(w, " %s\n", sanitizeTerminal(strings.ReplaceAll(m.Snippet, "\n", " "))) + for _, cm := range m.ContextAfter { + printContentContextLine(w, cm) + } } if res.NextCursor != 0 { fmt.Fprintf(w, "\nMore results: --cursor %d\n", res.NextCursor) } return nil } + +// formatMatchOrdinal renders a match's position. A match whose unit is a +// single message keeps the plain "#" form; a match whose +// conversation unit spans multiple messages — possible in every mode now +// that lexical rows carry derived unit ranges — renders the range with the +// anchor marked, e.g. "#12-40 @19". +func formatMatchOrdinal(m db.ContentMatch) string { + if m.OrdinalRange[1] > m.OrdinalRange[0] { + return fmt.Sprintf("#%d-%d @%d", m.OrdinalRange[0], m.OrdinalRange[1], m.Ordinal) + } + return fmt.Sprintf("#%d", m.Ordinal) +} + +// contentContextLineMaxChars caps a printed context line's length so a long +// stored message cannot blow out the human-format search output. +const contentContextLineMaxChars = 200 + +// printContentContextLine writes one --context line: two-space indent, +// "role: " prefix, terminal-sanitized and truncated to +// contentContextLineMaxChars runes (with an ellipsis marker when cut). +func printContentContextLine(w io.Writer, m db.Message) { + content := strings.ReplaceAll(m.Content, "\n", " ") + if truncated, cut := truncateRunes(content, contentContextLineMaxChars); cut { + content = truncated + "…" + } else { + content = truncated + } + fmt.Fprintf(w, " %s: %s\n", + sanitizeTerminal(m.Role), sanitizeTerminal(content)) +} diff --git a/cmd/agentsview/session_search_test.go b/cmd/agentsview/session_search_test.go index caee7d7cf..ec5ea9167 100644 --- a/cmd/agentsview/session_search_test.go +++ b/cmd/agentsview/session_search_test.go @@ -1,10 +1,17 @@ package main import ( + "bytes" + "encoding/json" + "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/service" ) func TestSessionSearchFlagValidation(t *testing.T) { @@ -15,6 +22,68 @@ func TestSessionSearchFlagValidation(t *testing.T) { assert.Contains(t, err.Error(), "mutually exclusive") } +// TestSessionSearchSinceMutuallyExclusiveWithActiveSince verifies the error +// is returned before any service/DB access is attempted (no data dir is set +// up here), matching --regex/--fts's fail-fast validation style. +func TestSessionSearchSinceMutuallyExclusiveWithActiveSince(t *testing.T) { + cmd := newSessionSearchCommand() + cmd.SetArgs([]string{ + "needle", "--since", "14d", "--active-since", "2024-01-01T00:00:00Z", + }) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), + "--since and --active-since are mutually exclusive") +} + +func TestSessionSearchSinceRejectsInvalidFormat(t *testing.T) { + cmd := newSessionSearchCommand() + cmd.SetArgs([]string{"needle", "--since", "3x"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid --since") +} + +// seedSearchMessage inserts a single message for sessionID carrying content, +// so `session search ` has something to match. +func seedSearchMessage(t *testing.T, dataDir, sessionID, content string) { + t.Helper() + d, err := db.Open(filepath.Join(dataDir, "sessions.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = d.Close() }) + require.NoError(t, d.InsertMessages([]db.Message{{ + SessionID: sessionID, + Ordinal: 1, + Role: "user", + Content: content, + ContentLength: len(content), + Timestamp: "2026-04-01T00:00:00Z", + }})) +} + +// TestSessionSearchSinceFiltersByActivity is the end-to-end regression test +// for the CRITICAL requirement that --since actually narrows results by +// resolving to the same active_since window --active-since already +// threads through to the search filter: a session active within the +// window survives, one outside it does not. +func TestSessionSearchSinceFiltersByActivity(t *testing.T) { + dataDir := newAgentDataDir(t) + seedSessionsWithOpts(t, dataDir, + activitySeed("fresh", 2*time.Hour), + activitySeed("stale", 20*24*time.Hour), + ) + seedSearchMessage(t, dataDir, "fresh", "needle in fresh session") + seedSearchMessage(t, dataDir, "stale", "needle in stale session") + + out, err := executeCommand(newRootCommand(), + "session", "search", "needle", "--since", "7d", "--format", "json") + require.NoError(t, err) + + got := decodeCLIJSON[service.ContentSearchResult](t, out) + require.Len(t, got.Matches, 1) + assert.Equal(t, "fresh", got.Matches[0].SessionID) +} + func TestSessionSearchFTSWithToolSource(t *testing.T) { cmd := newSessionSearchCommand() cmd.SetArgs([]string{"needle", "--fts", "--in", "tool_result"}) @@ -22,3 +91,270 @@ func TestSessionSearchFTSWithToolSource(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "messages only") } + +func TestResolveContentSearchModeMapping(t *testing.T) { + tests := []struct { + name string + useRegex, useFTS, useSemantic, useHybrid bool + wantMode string + }{ + {name: "default substring", wantMode: "substring"}, + {name: "regex", useRegex: true, wantMode: "regex"}, + {name: "fts", useFTS: true, wantMode: "fts"}, + {name: "semantic", useSemantic: true, wantMode: "semantic"}, + {name: "hybrid", useHybrid: true, wantMode: "hybrid"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, err := resolveContentSearchMode( + tt.useRegex, tt.useFTS, tt.useSemantic, tt.useHybrid, nil) + require.NoError(t, err) + assert.Equal(t, tt.wantMode, mode) + }) + } +} + +func TestResolveContentSearchModeMutualExclusion(t *testing.T) { + tests := []struct { + name string + useRegex, useFTS, useSemantic, useHybrid bool + }{ + {name: "regex and fts", useRegex: true, useFTS: true}, + {name: "semantic and hybrid", useSemantic: true, useHybrid: true}, + {name: "regex and semantic", useRegex: true, useSemantic: true}, + {name: "fts and hybrid", useFTS: true, useHybrid: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := resolveContentSearchMode( + tt.useRegex, tt.useFTS, tt.useSemantic, tt.useHybrid, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "mutually exclusive") + }) + } +} + +func TestSessionSearchSemanticWithToolSource(t *testing.T) { + cmd := newSessionSearchCommand() + cmd.SetArgs([]string{"needle", "--semantic", "--in", "tool_input"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "messages only") +} + +func TestSessionSearchHybridWithToolSource(t *testing.T) { + cmd := newSessionSearchCommand() + cmd.SetArgs([]string{"needle", "--hybrid", "--in", "tool_result"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "messages only") +} + +// TestSessionSearchScopeRequiresSemanticOrHybrid verifies --scope fails +// fast (before any service/DB access) when set without --semantic/--hybrid. +func TestSessionSearchScopeRequiresSemanticOrHybrid(t *testing.T) { + for _, args := range [][]string{ + {"needle", "--scope", "top"}, + {"needle", "--fts", "--scope", "all"}, + {"needle", "--regex", "--scope", "subordinate"}, + } { + cmd := newSessionSearchCommand() + cmd.SetArgs(args) + err := cmd.Execute() + require.Error(t, err, "args %v", args) + assert.Contains(t, err.Error(), "--semantic or --hybrid", "args %v", args) + } +} + +// TestSessionSearchScopeRejectsInvalidValue verifies the value gate fires +// at the CLI boundary rather than deep in the store. +func TestSessionSearchScopeRejectsInvalidValue(t *testing.T) { + cmd := newSessionSearchCommand() + cmd.SetArgs([]string{"needle", "--semantic", "--scope", "bogus"}) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "top, all, or subordinate") +} + +func TestValidateScopeFlag(t *testing.T) { + tests := []struct { + name string + scope string + useSemantic, useHybrid bool + wantErr string + }{ + {name: "empty scope always valid"}, + {name: "top with semantic", scope: "top", useSemantic: true}, + {name: "all with hybrid", scope: "all", useHybrid: true}, + {name: "subordinate with semantic", scope: "subordinate", useSemantic: true}, + {name: "scope without mode flag", scope: "top", + wantErr: "--semantic or --hybrid"}, + {name: "invalid value", scope: "bogus", useSemantic: true, + wantErr: "top, all, or subordinate"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateScopeFlag(tt.scope, tt.useSemantic, tt.useHybrid) + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestPrintContentMatchesHumanShowsScoreForScoredMatches(t *testing.T) { + score := 0.834 + res := &service.ContentSearchResult{ + Matches: []db.ContentMatch{ + { + SessionID: "sess1", + Project: "proj", + Location: "message", + Ordinal: 3, + Snippet: "hello world", + Score: &score, + }, + { + SessionID: "sess2", + Project: "proj", + Location: "message", + Ordinal: 1, + Snippet: "no score here", + }, + }, + } + var buf bytes.Buffer + require.NoError(t, printContentMatchesHuman(&buf, res)) + out := buf.String() + assert.Contains(t, out, "score=0.83") + lines := bytes.Split(buf.Bytes(), []byte("\n")) + require.NotEmpty(t, lines) + assert.NotContains(t, string(lines[2]), "score=", + "unscored match should not print a score") +} + +func TestPrintContentMatchesHumanShowsContext(t *testing.T) { + res := &service.ContentSearchResult{ + Matches: []db.ContentMatch{ + { + SessionID: "sess1", Project: "proj", Location: "message", + Ordinal: 5, Snippet: "the match line", + ContextBefore: []db.Message{ + {Role: "user", Content: "earlier question"}, + }, + ContextAfter: []db.Message{ + {Role: "assistant", Content: "later reply"}, + }, + }, + }, + } + var buf bytes.Buffer + require.NoError(t, printContentMatchesHuman(&buf, res)) + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 4) + assert.Equal(t, " user: earlier question", lines[0]) + assert.Contains(t, lines[1], "sess1") + assert.Contains(t, lines[2], "the match line") + assert.Equal(t, " assistant: later reply", lines[3]) +} + +func TestPrintContentMatchesHumanTruncatesContextLine(t *testing.T) { + longContent := strings.Repeat("a", 250) + res := &service.ContentSearchResult{ + Matches: []db.ContentMatch{ + { + SessionID: "sess1", Ordinal: 1, Snippet: "match", + ContextBefore: []db.Message{{Role: "user", Content: longContent}}, + }, + }, + } + var buf bytes.Buffer + require.NoError(t, printContentMatchesHuman(&buf, res)) + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.NotEmpty(t, lines) + require.True(t, strings.HasPrefix(lines[0], " user: ")) + body := strings.TrimPrefix(lines[0], " user: ") + assert.LessOrEqual(t, len([]rune(body)), 201) + assert.True(t, strings.HasSuffix(body, "…")) +} + +func TestContentMatchJSONRoundTripsContext(t *testing.T) { + res := service.ContentSearchResult{ + Matches: []db.ContentMatch{ + { + SessionID: "sess1", Ordinal: 5, + ContextBefore: []db.Message{{Role: "user", Ordinal: 3, Content: "before"}}, + ContextAfter: []db.Message{{Role: "assistant", Ordinal: 7, Content: "after"}}, + }, + {SessionID: "sess2", Ordinal: 1}, + }, + } + data, err := json.Marshal(res) + require.NoError(t, err) + assert.Contains(t, string(data), `"context_before"`) + assert.Contains(t, string(data), `"context_after"`) + + var decoded service.ContentSearchResult + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Len(t, decoded.Matches, 2) + require.Len(t, decoded.Matches[0].ContextBefore, 1) + assert.Equal(t, "before", decoded.Matches[0].ContextBefore[0].Content) + assert.Empty(t, decoded.Matches[1].ContextBefore) +} + +func TestContentMatchJSONRoundTripsScore(t *testing.T) { + score := 0.5 + res := service.ContentSearchResult{ + Matches: []db.ContentMatch{ + {SessionID: "sess1", Ordinal: 1, Score: &score}, + {SessionID: "sess2", Ordinal: 2}, + }, + } + data, err := json.Marshal(res) + require.NoError(t, err) + assert.Contains(t, string(data), `"score":0.5`) + + var decoded service.ContentSearchResult + require.NoError(t, json.Unmarshal(data, &decoded)) + require.Len(t, decoded.Matches, 2) + require.NotNil(t, decoded.Matches[0].Score) + assert.InDelta(t, score, *decoded.Matches[0].Score, 0.0001) + assert.Nil(t, decoded.Matches[1].Score) +} + +// TestPrintContentMatchesHumanRendersUnitRangeAndSubMarker pins the human +// rendering for run-grouped semantic/hybrid hits: a multi-message unit +// renders "#- @", a subordinate hit gains a "sub" +// marker, and a single-ordinal hit keeps today's plain "#" form. +func TestPrintContentMatchesHumanRendersUnitRangeAndSubMarker(t *testing.T) { + score := 0.91 + res := &service.ContentSearchResult{ + Matches: []db.ContentMatch{ + { + SessionID: "sess1", Project: "proj", Location: "message", + Ordinal: 19, OrdinalRange: [2]int{12, 40}, + Subordinate: true, Score: &score, Snippet: "ranged hit", + }, + { + SessionID: "sess2", Project: "proj", Location: "message", + Ordinal: 5, OrdinalRange: [2]int{5, 5}, + Snippet: "single-message unit", + }, + }, + } + var buf bytes.Buffer + require.NoError(t, printContentMatchesHuman(&buf, res)) + lines := strings.Split(strings.TrimRight(buf.String(), "\n"), "\n") + require.Len(t, lines, 4) + + assert.Contains(t, lines[0], "#12-40 @19", "range with anchor marker") + assert.Contains(t, lines[0], " sub", "subordinate marker") + assert.Contains(t, lines[0], "score=0.91") + + assert.Contains(t, lines[2], "#5", "single-ordinal hit keeps the plain form") + assert.NotContains(t, lines[2], "@", "no anchor marker for single-ordinal hits") + assert.NotContains(t, lines[2], " sub", "no subordinate marker for top-level hits") +} diff --git a/cmd/agentsview/skills.go b/cmd/agentsview/skills.go new file mode 100644 index 000000000..369c2949a --- /dev/null +++ b/cmd/agentsview/skills.go @@ -0,0 +1,280 @@ +// ABOUTME: `skills` command group: install and list the AgentsView skill +// ABOUTME: files that teach coding-agent harnesses to search session history. +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" + gitrepo "go.kenn.io/kit/git/repo" + + "go.kenn.io/agentsview/internal/skills" +) + +// skillFileName is the file every harness's skill directory installs, as +// documented on skills.TargetDir. +const skillFileName = "SKILL.md" + +func newSkillsCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "skills", + Short: "Install and list AgentsView skills for coding-agent harnesses", + GroupID: groupMeta, + SilenceUsage: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + } + cmd.AddCommand(newSkillsInstallCommand()) + cmd.AddCommand(newSkillsListCommand()) + return cmd +} + +func newSkillsInstallCommand() *cobra.Command { + var ( + harnessNames []string + project bool + force bool + ) + cmd := &cobra.Command{ + Use: "install", + Short: "Install AgentsView skill files for coding-agent harnesses", + Args: cobra.NoArgs, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + harnesses, err := resolveSkillHarnesses(harnessNames) + if err != nil { + return err + } + base, err := skillsBaseDir(cmd.Context(), project) + if err != nil { + return err + } + return runSkillsInstall(cmd.OutOrStdout(), harnesses, base, force) + }, + } + flags := cmd.Flags() + flags.StringArrayVar(&harnessNames, "harness", nil, + "Harness to install for (claude or agents); repeatable, default both") + flags.BoolVar(&project, "project", false, + "Install into the project (git root of the current directory) "+ + "instead of the user home directory") + flags.BoolVar(&force, "force", false, + "Overwrite files that were modified or were not generated by agentsview") + return cmd +} + +// runSkillsInstall renders and writes each harness's skill file under base, +// printing one line per target. It processes every target before returning +// so a refusal on one harness never blocks another, then reports a non-nil +// error if any target was refused. +func runSkillsInstall(out io.Writer, harnesses []skills.Harness, base string, force bool) error { + var refused []string + for _, h := range harnesses { + rendered, err := skills.Render(h, version) + if err != nil { + return err + } + dir := skills.TargetDir(h, base) + path := filepath.Join(dir, skillFileName) + + existing, err := readSkillFile(path) + if err != nil { + return err + } + state := skills.Classify(existing, rendered) + + if !force && (state == skills.StateModified || state == skills.StateForeign) { + fmt.Fprintf(out, "%s was modified (or not generated); use --force to overwrite\n", path) + refused = append(refused, path) + continue + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("skills: create %s: %w", dir, err) + } + if err := os.WriteFile(path, []byte(rendered.Content), 0o644); err != nil { + return fmt.Errorf("skills: write %s: %w", path, err) + } + + switch state { + case skills.StateMissing: + fmt.Fprintf(out, "installed %s\n", path) + case skills.StateCurrent: + fmt.Fprintf(out, "up to date %s\n", path) + default: // StateStale, or StateModified/StateForeign forced + fmt.Fprintf(out, "updated %s\n", path) + } + } + + if len(refused) > 0 { + return fmt.Errorf( + "skills install: %d target(s) refused; rerun with --force to overwrite", + len(refused), + ) + } + return nil +} + +func newSkillsListCommand() *cobra.Command { + var project bool + cmd := &cobra.Command{ + Use: "list", + Short: "List AgentsView skill files and their install state", + Args: cobra.NoArgs, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + base, err := skillsBaseDir(cmd.Context(), project) + if err != nil { + return err + } + rows, err := listSkillRows(base, project) + if err != nil { + return err + } + if outputFormat(cmd) == "json" { + return json.NewEncoder(cmd.OutOrStdout()).Encode(rows) + } + return printSkillListHuman(cmd.OutOrStdout(), rows) + }, + } + flags := cmd.Flags() + flags.BoolVar(&project, "project", false, + "List project-level (git root of the current directory) installs "+ + "instead of the user home directory") + registerFormatFlags(flags) + return cmd +} + +// skillListRow is one row of `skills list` output, in both human and JSON form. +type skillListRow struct { + Harness string `json:"harness"` + Level string `json:"level"` + State string `json:"state"` + Path string `json:"path"` +} + +// listSkillRows classifies every harness's skill file under base against a +// fresh render. +func listSkillRows(base string, project bool) ([]skillListRow, error) { + level := "user" + if project { + level = "project" + } + + harnesses := skills.AllHarnesses() + rows := make([]skillListRow, 0, len(harnesses)) + for _, h := range harnesses { + rendered, err := skills.Render(h, version) + if err != nil { + return nil, err + } + dir := skills.TargetDir(h, base) + path := filepath.Join(dir, skillFileName) + + existing, err := readSkillFile(path) + if err != nil { + return nil, err + } + state := skills.Classify(existing, rendered) + + rows = append(rows, skillListRow{ + Harness: string(h), + Level: level, + State: skillStateString(state), + Path: path, + }) + } + return rows, nil +} + +func printSkillListHuman(w io.Writer, rows []skillListRow) error { + fmt.Fprintf(w, "%-8s %-8s %-8s %s\n", "HARNESS", "LEVEL", "STATE", "PATH") + for _, r := range rows { + fmt.Fprintf(w, "%-8s %-8s %-8s %s\n", r.Harness, r.Level, r.State, r.Path) + } + return nil +} + +func skillStateString(s skills.InstalledState) string { + switch s { + case skills.StateMissing: + return "missing" + case skills.StateCurrent: + return "current" + case skills.StateStale: + return "stale" + case skills.StateModified: + return "modified" + case skills.StateForeign: + return "foreign" + default: + return "unknown" + } +} + +// readSkillFile reads path, returning nil (not an empty slice) when the file +// does not exist so the result matches skills.Classify's "missing" contract. +func readSkillFile(path string) ([]byte, error) { + content, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("skills: read %s: %w", path, err) + } + return content, nil +} + +// resolveSkillHarnesses maps --harness flag values to skills.Harness, +// defaulting to every harness when none were given. +func resolveSkillHarnesses(names []string) ([]skills.Harness, error) { + if len(names) == 0 { + return skills.AllHarnesses(), nil + } + out := make([]skills.Harness, 0, len(names)) + for _, name := range names { + switch name { + case string(skills.HarnessClaude): + out = append(out, skills.HarnessClaude) + case string(skills.HarnessAgents): + out = append(out, skills.HarnessAgents) + default: + return nil, fmt.Errorf("skills: unknown --harness %q (want claude or agents)", name) + } + } + return out, nil +} + +// skillsBaseDir resolves the install base: the user home directory by +// default, or the git root of the current directory (falling back to the +// current directory itself outside a repo) when project is true. +func skillsBaseDir(ctx context.Context, project bool) (string, error) { + if !project { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("skills: resolve home directory: %w", err) + } + return home, nil + } + + cwd, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("skills: resolve working directory: %w", err) + } + + opCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + root, err := gitrepo.Root(opCtx, cwd) + if err != nil || root == "" { + return cwd, nil + } + return root, nil +} diff --git a/cmd/agentsview/skills_test.go b/cmd/agentsview/skills_test.go new file mode 100644 index 000000000..8f4502b7c --- /dev/null +++ b/cmd/agentsview/skills_test.go @@ -0,0 +1,388 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/skills" +) + +// skillHeaderFormat mirrors the private header format in internal/skills so +// tests here can synthesize stale/modified fixtures without reaching into +// unexported package internals. +const skillHeaderFormat = "# generated-by: agentsview %s hash:%s — do not edit; " + + "re-run `agentsview skills install`" + +// sha256Hex returns the hex sha256 digest of body, matching internal/skills' +// own bodyHash so synthesized headers classify the way production ones do. +func sha256Hex(body string) string { + sum := sha256.Sum256([]byte(body)) + return hex.EncodeToString(sum[:]) +} + +// claudeSkillPath returns the SKILL.md path the CLI installs for the Claude +// harness under home. +func claudeSkillPath(home string) string { + return filepath.Join(skills.TargetDir(skills.HarnessClaude, home), skillFileName) +} + +// agentsSkillPath returns the SKILL.md path the CLI installs for the Agents +// harness under home. +func agentsSkillPath(home string) string { + return filepath.Join(skills.TargetDir(skills.HarnessAgents, home), skillFileName) +} + +func freshClaudeSkill(t *testing.T) skills.Rendered { + t.Helper() + rendered, err := skills.Render(skills.HarnessClaude, version) + require.NoError(t, err) + return rendered +} + +// writeSkillFile writes content at path, creating parent directories. +func writeSkillFile(t *testing.T, path, content string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) +} + +// setTestHome points the process home at dir for both Unix (HOME) and +// Windows (USERPROFILE), since os.UserHomeDir reads a different variable +// per platform. +func setTestHome(t *testing.T, dir string) { + t.Helper() + t.Setenv("HOME", dir) + t.Setenv("USERPROFILE", dir) +} + +// staleClaudeContent returns a well-formed generated-by file whose recorded +// hash matches an older body, so Classify reports StateStale. +func staleClaudeContent() string { + oldBody := "---\nname: agentsview-finding-history\n---\n" + + "an earlier revision of the skill body, no longer current\n" + header := fmt.Sprintf(skillHeaderFormat, "0.0.1", sha256Hex(oldBody)) + return "---\n" + header + "\n" + strings.TrimPrefix(oldBody, "---\n") +} + +// modifiedClaudeContent returns a fresh render whose body was hand-edited +// after the header hash was recorded, so Classify reports StateModified. +func modifiedClaudeContent(t *testing.T) string { + t.Helper() + return freshClaudeSkill(t).Content + "\nan uninvited local edit\n" +} + +const foreignClaudeContent = "# Just a hand-written file\n\nNo generated-by header here.\n" + +func TestSkillsInstall_StatesAndForce(t *testing.T) { + const refusalMsg = "was modified (or not generated); use --force to overwrite" + + tests := []struct { + name string + seed func(t *testing.T, path string) // nil means the file is missing + wantMsgNoForce string + wantMsgForced string // message once --force overrides a refusal; "" when force changes nothing + }{ + { + name: "missing", + seed: nil, + wantMsgNoForce: "installed", + }, + { + name: "current", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, freshClaudeSkill(t).Content) + }, + wantMsgNoForce: "up to date", + }, + { + name: "stale", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, staleClaudeContent()) + }, + wantMsgNoForce: "updated", + }, + { + name: "modified", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, modifiedClaudeContent(t)) + }, + wantMsgNoForce: refusalMsg, + wantMsgForced: "updated", + }, + { + name: "foreign", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, foreignClaudeContent) + }, + wantMsgNoForce: refusalMsg, + wantMsgForced: "updated", + }, + } + + for _, tt := range tests { + for _, force := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/force=%v", tt.name, force), func(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + path := claudeSkillPath(home) + + var seedContent string + if tt.seed != nil { + tt.seed(t, path) + seedContent = readFileString(t, path) + } + + args := []string{"skills", "install", "--harness", "claude"} + if force { + args = append(args, "--force") + } + out, err := executeCommand(newRootCommand(), args...) + + assert.Contains(t, out, path) + + refused := tt.wantMsgForced != "" && !force + if refused { + assert.Contains(t, out, tt.wantMsgNoForce) + require.Error(t, err, "expected a refusal error") + assert.Equal(t, seedContent, readFileString(t, path), + "refused install must not touch the file") + return + } + + wantMsg := tt.wantMsgNoForce + if force && tt.wantMsgForced != "" { + wantMsg = tt.wantMsgForced + } + assert.Contains(t, out, wantMsg) + + require.NoError(t, err, "output: %s", out) + assert.Equal(t, freshClaudeSkill(t).Content, readFileString(t, path)) + }) + } + } +} + +// readFileString reads path, failing the test on error. +func readFileString(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + require.NoError(t, err, "read %s", path) + return string(b) +} + +func TestSkillsInstall_DefaultHarnessesInstallBoth(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + out, err := executeCommand(newRootCommand(), "skills", "install") + require.NoError(t, err, "output: %s", out) + + assert.Contains(t, out, claudeSkillPath(home)) + assert.Contains(t, out, agentsSkillPath(home)) + assert.FileExists(t, claudeSkillPath(home)) + assert.FileExists(t, agentsSkillPath(home)) +} + +func TestSkillsInstall_UnknownHarnessErrors(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + _, err := executeCommand(newRootCommand(), "skills", "install", "--harness", "bogus") + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown --harness") +} + +func TestSkillsInstall_RefusalStillInstallsOtherTargets(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + writeSkillFile(t, claudeSkillPath(home), foreignClaudeContent) + + out, err := executeCommand(newRootCommand(), "skills", "install") + require.Error(t, err, "one refused target must still fail the command") + + assert.Contains(t, out, "was modified (or not generated); use --force to overwrite") + assert.Contains(t, out, "installed "+agentsSkillPath(home)) + assert.FileExists(t, agentsSkillPath(home)) + assert.Equal(t, foreignClaudeContent, readFileString(t, claudeSkillPath(home)), + "the refused claude target must be untouched") +} + +func TestSkillsInstall_FilePermissions(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + _, err := executeCommand(newRootCommand(), "skills", "install", "--harness", "claude") + require.NoError(t, err) + + info, err := os.Stat(claudeSkillPath(home)) + require.NoError(t, err) + // The process umask may strip group/other bits from the 0644 requested by + // os.WriteFile, so only assert the file is a regular, non-executable file + // readable/writable by its owner rather than the exact resulting mode. + assert.True(t, info.Mode().IsRegular()) + assert.Zero(t, info.Mode().Perm()&0o111, "installed skill file must not be executable") + assert.Equal(t, os.FileMode(0o600), info.Mode().Perm()&0o600, + "owner must be able to read and write the installed skill file") +} + +func TestSkillsList_ReportsEachState(t *testing.T) { + tests := []struct { + name string + seed func(t *testing.T, path string) + want string + }{ + {name: "missing", seed: nil, want: "missing"}, + { + name: "current", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, freshClaudeSkill(t).Content) + }, + want: "current", + }, + { + name: "stale", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, staleClaudeContent()) + }, + want: "stale", + }, + { + name: "modified", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, modifiedClaudeContent(t)) + }, + want: "modified", + }, + { + name: "foreign", + seed: func(t *testing.T, path string) { + writeSkillFile(t, path, foreignClaudeContent) + }, + want: "foreign", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + path := claudeSkillPath(home) + if tt.seed != nil { + tt.seed(t, path) + } + + out, err := executeCommand(newRootCommand(), "skills", "list", "--format", "json") + require.NoError(t, err, "output: %s", out) + + var rows []skillListRow + require.NoError(t, json.Unmarshal([]byte(out), &rows), "output: %s", out) + + var claudeRow *skillListRow + for i := range rows { + if rows[i].Harness == string(skills.HarnessClaude) { + claudeRow = &rows[i] + } + } + require.NotNil(t, claudeRow, "no claude row in %+v", rows) + assert.Equal(t, tt.want, claudeRow.State) + assert.Equal(t, "user", claudeRow.Level) + assert.Equal(t, path, claudeRow.Path) + }) + } +} + +func TestSkillsList_HumanTableHasHeaderAndColumns(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + out, err := executeCommand(newRootCommand(), "skills", "list") + require.NoError(t, err, "output: %s", out) + + assert.Contains(t, out, "HARNESS") + assert.Contains(t, out, "LEVEL") + assert.Contains(t, out, "STATE") + assert.Contains(t, out, "PATH") + assert.Contains(t, out, "claude") + assert.Contains(t, out, "agents") + assert.Contains(t, out, "missing") + assert.Contains(t, out, claudeSkillPath(home)) +} + +// initTestGitRepo runs `git init` in a fresh temp directory. No commit is +// required: gitrepo.Root only needs a `.git` directory to resolve a root. +func initTestGitRepo(t *testing.T) string { + t.Helper() + repo := t.TempDir() + cmd := exec.Command("git", "init", "-q", "-b", "main") + cmd.Dir = repo + out, err := cmd.CombinedOutput() + require.NoError(t, err, "git init: %s", out) + resolved, err := filepath.EvalSymlinks(repo) + require.NoError(t, err) + return resolved +} + +func TestSkillsInstall_ProjectFlagUsesGitRoot(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + repo := initTestGitRepo(t) + nested := filepath.Join(repo, "a", "b") + require.NoError(t, os.MkdirAll(nested, 0o755)) + t.Chdir(nested) + + out, err := executeCommand(newRootCommand(), "skills", "install", "--harness", "claude", "--project") + require.NoError(t, err, "output: %s", out) + + wantPath := claudeSkillPath(repo) + assert.Contains(t, out, wantPath) + assert.FileExists(t, wantPath) + // Must not have installed under the user home directory instead. + assert.NoFileExists(t, claudeSkillPath(home)) +} + +func TestSkillsList_ProjectFlagReportsProjectLevel(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + repo := initTestGitRepo(t) + t.Chdir(repo) + + out, err := executeCommand(newRootCommand(), "skills", "list", "--project", "--format", "json") + require.NoError(t, err, "output: %s", out) + + var rows []skillListRow + require.NoError(t, json.Unmarshal([]byte(out), &rows), "output: %s", out) + require.NotEmpty(t, rows) + for _, r := range rows { + assert.Equal(t, "project", r.Level) + assert.True(t, strings.HasPrefix(r.Path, repo), "path %q must be under repo root %q", r.Path, repo) + } +} + +func TestSkillsInstall_ProjectFlagOutsideRepoFallsBackToCWD(t *testing.T) { + home := t.TempDir() + setTestHome(t, home) + + outsideRepo := t.TempDir() + resolvedOutside, err := filepath.EvalSymlinks(outsideRepo) + require.NoError(t, err) + t.Chdir(resolvedOutside) + + out, err := executeCommand(newRootCommand(), "skills", "install", "--harness", "claude", "--project") + require.NoError(t, err, "output: %s", out) + + wantPath := claudeSkillPath(resolvedOutside) + assert.Contains(t, out, wantPath) + assert.FileExists(t, wantPath) +} diff --git a/cmd/agentsview/transport.go b/cmd/agentsview/transport.go index ccf14dfb5..7d1b4fb39 100644 --- a/cmd/agentsview/transport.go +++ b/cmd/agentsview/transport.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "log" "net" "os" "strconv" @@ -440,7 +441,15 @@ func newService( "opening db: %w", err, ) } - cleanup := func() { d.Close() } + closeVectorSearcher := installDirectVectorSearcher(cfg, d) + cleanup := func() { + if closeVectorSearcher != nil { + if cerr := closeVectorSearcher(); cerr != nil { + log.Printf("close vectors.db: %v", cerr) + } + } + d.Close() + } // engine is nil — CLI reads don't need it, and Sync // is handled via the HTTP daemon when one is running. return service.NewDirectBackend(d, nil), cleanup, nil diff --git a/cmd/agentsview/write_lock.go b/cmd/agentsview/write_lock.go index 64a1809e5..284d93aeb 100644 --- a/cmd/agentsview/write_lock.go +++ b/cmd/agentsview/write_lock.go @@ -11,6 +11,13 @@ import ( const writeOwnerLockFile = "db.write.lock" +// vectorsWriteLockFile guards direct (non-daemon) writes to vectors.db — +// `embeddings build/activate/retire` take it via tryAcquireNamedLock so two +// concurrent direct-mode invocations cannot race each other. It is separate +// from writeOwnerLockFile because the two resources (sessions.db vs +// vectors.db) are written independently. +const vectorsWriteLockFile = "vectors.write.lock" + type writeOwnerLock struct { path string lock *flock.Flock @@ -33,18 +40,27 @@ func acquireWriteOwnerLock( } func tryAcquireWriteOwnerLock(dataDir string) (*writeOwnerLock, error) { + return tryAcquireNamedLock(dataDir, writeOwnerLockFile) +} + +// tryAcquireNamedLock acquires an exclusive flock named filename inside +// dataDir. It backs both tryAcquireWriteOwnerLock (db.write.lock, guarding +// direct sessions.db writes) and the embeddings CLI's direct path +// (vectors.write.lock, guarding direct vectors.db writes), so two direct +// (non-daemon) writers targeting the same resource cannot race each other. +// OS flock semantics release the lock when the owning process exits or +// crashes, which is the direct-writer recovery path after stale runtime +// records are ignored. +func tryAcquireNamedLock(dataDir, filename string) (*writeOwnerLock, error) { if err := os.MkdirAll(dataDir, 0o700); err != nil { return nil, fmt.Errorf("creating data dir for write lock: %w", err) } - // OS flock semantics release this lock when a daemon process exits or - // crashes, which is the direct-writer recovery path after stale runtime - // records are ignored. - path := writeOwnerLockPath(dataDir) + path := filepath.Join(dataDir, filename) lock := flock.New(path) locked, err := lock.TryLock() if err != nil { - return nil, fmt.Errorf("acquiring sqlite write-owner lock %s: %w", path, err) + return nil, fmt.Errorf("acquiring write lock %s: %w", path, err) } if !locked { return nil, writeOwnerLockHeldError{path: path} @@ -57,7 +73,7 @@ func (l *writeOwnerLock) Close() error { return nil } if err := l.lock.Unlock(); err != nil { - return fmt.Errorf("releasing sqlite write-owner lock %s: %w", l.path, err) + return fmt.Errorf("releasing write lock %s: %w", l.path, err) } return nil } @@ -68,8 +84,7 @@ type writeOwnerLockHeldError struct { func (e writeOwnerLockHeldError) Error() string { return fmt.Sprintf( - "sqlite archive is already owned by another agentsview process "+ - "(write-owner lock %s is held); run `agentsview serve stop`, "+ + "write lock %s is held by another process; run `agentsview serve stop`, "+ "wait for the daemon idle timeout, or retry after the offline "+ "operation finishes", e.path, diff --git a/docs/commands.md b/docs/commands.md index a8a686b2b..961c46177 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -831,6 +831,12 @@ agentsview session search # content search across sessions agentsview session usage # token usage and cost estimate ``` +`session search` supports substring (default), `--regex`, `--fts`, +`--semantic`, and `--hybrid` modes. Semantic and hybrid results can be scoped +with `--scope top|all|subordinate` (default `all`) to include or exclude +sidechain and subagent content — see +[Semantic Search](/semantic-search/#scoping-results-scope). + Structured response commands accept `--format json`; `--json` is a short alias for that scripting mode. `session export` and `session watch` are the exceptions: they stream raw bytes and NDJSON respectively, so they reject @@ -857,6 +863,21 @@ alias to show sessions active in the last 15 minutes; combine either flag with ______________________________________________________________________ +### `agentsview embeddings` + +Manage the local semantic search embedding index. Requires `[vector]` to be +enabled in config. See [Semantic Search](/semantic-search/) for full +documentation, including configuration and the search surface. + +```bash +agentsview embeddings build # build or refresh the index +agentsview embeddings list # list embedding generations +agentsview embeddings activate # activate a generation +agentsview embeddings retire # retire a generation +``` + +______________________________________________________________________ + ### `agentsview mcp` Run a read-only Model Context Protocol server for assistant clients that can @@ -937,6 +958,31 @@ warning to stderr. ______________________________________________________________________ +### `agentsview skills` + +Install or list the bundled skill files that teach coding-agent harnesses +(Claude Code, Codex, and other `.agents/skills` readers) to search AgentsView +history. See +[Semantic Search](/semantic-search/#skills-for-coding-agents) for what the +skill does and when to re-run it. + +```bash +agentsview skills install [--harness claude|agents] [--project] [--force] +agentsview skills list [--project] [--format json] +``` + +`install` renders the embedded `agentsview-finding-history` skill for each +`--harness` (default both) and writes `SKILL.md` under +`~/.claude/skills/agentsview-finding-history/` and/or +`~/.agents/skills/agentsview-finding-history/`, or under `.claude/skills/` / +`.agents/skills/` at the current git root with `--project`. It overwrites an +unmodified generated file, refuses a hand-edited or foreign file unless +`--force` is passed, and exits non-zero on any refusal. `list` reports +HARNESS, LEVEL, STATE (`missing`, `current`, `stale`, `modified`, `foreign`), +and PATH for every harness. + +______________________________________________________________________ + ### `agentsview help` Print usage information. diff --git a/docs/configuration.md b/docs/configuration.md index 6877c648c..f94c381dd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,6 +17,7 @@ AgentsView stores all persistent data under a single directory, defaulting to ``` ~/.agentsview/ ├── sessions.db # SQLite database (WAL mode) +├── vectors.db # Semantic-search vector index (when [vector] is enabled) ├── config.toml # Configuration file ├── config.toml.lock # Serializes concurrent config writers ├── db.write.lock # Per-data-dir SQLite write-owner lock @@ -74,6 +75,7 @@ daemon_idle_timeout = "20m" | `disable_update_check` | Disable the automatic update check (see [Privacy](#privacy-and-telemetry)) | | `[pg]` | PostgreSQL sync configuration — see [PostgreSQL Sync](/pg-sync/) | | `[duckdb]` | DuckDB mirror configuration — see [DuckDB Mirror](/duckdb/) | +| `[vector]` | Opt-in semantic-search index; model settings live in `[vector.embeddings]`, named endpoints in `[vector.embeddings.servers.]`, embedding schedule in `[vector.embed]` — see [Semantic Search](/semantic-search/#enabling-vector) for every key | | `[[remote_hosts]]` | Remote machines synced by a bare `agentsview sync` — see [CLI Reference](/commands/#agentsview-sync) | | `[automated]` | Custom automated-session patterns — see [Automated Session Detection](#automated-session-detection) | | `[custom_model_pricing]` | Per-model price overrides for usage reports — see [Custom Model Pricing](/token-usage/#custom-model-pricing) | diff --git a/docs/index.md b/docs/index.md index 78e01e3c3..a6146b1c0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -202,6 +202,9 @@ to a shared database for team or multi-machine setups. Full-text search across all message content. Find that one conversation where you discussed a specific function, error message, or design decision — even months later. + Opt-in [semantic search](/semantic-search/) matches by + meaning when you don't remember the exact words, and every + match cites the conversation unit it came from. - **Recent Edits** diff --git a/docs/mcp.md b/docs/mcp.md index d9cefd60d..53be51cb7 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -39,14 +39,24 @@ For local desktop-style MCP clients, use stdio: Restart or reload your MCP client after adding the server. Once connected, the client will see these tools: -| Tool | Purpose | -| ---------------------- | -------------------------------------------------- | -| `search_sessions` | Full-text search across recorded sessions | -| `list_sessions` | List recent or filtered sessions | -| `get_session_overview` | Fetch metadata and a compact message preview | -| `get_messages` | Read paginated message bodies from one session | -| `search_content` | Exact string or regex search over raw session text | -| `get_usage_summary` | Aggregate token and cost usage | +| Tool | Purpose | +| ---------------------- | ------------------------------------------------------------------ | +| `search_sessions` | Full-text search across recorded sessions | +| `list_sessions` | List recent or filtered sessions | +| `get_session_overview` | Fetch metadata and a compact message preview | +| `get_messages` | Read paginated message bodies from one session | +| `search_content` | Substring, regex, semantic, or hybrid search over raw session text | +| `get_usage_summary` | Aggregate token and cost usage | + +`search_content` accepts a `mode` of `substring` (default), `regex`, `semantic`, +or `hybrid`, plus a `scope` of `top`, `all` (default), or `subordinate` that is +only valid with the semantic and hybrid modes. The `semantic` and `hybrid` modes +need the opt-in [semantic search](/semantic-search/) index on the local SQLite +archive; without it they return a "not available" error. In every mode, each +match carries a conversation-unit citation: an `ordinal_range` of `[start, end]` +ordinals around the match, plus `subordinate`, `relationship`, +`parent_session_id`, and `is_sidechain` fields that flag hits from sidechain +runs and subagent or fork sessions. ## Daemon-Backed Reads diff --git a/docs/quickstart.md b/docs/quickstart.md index 32f65295b..94f95d923 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -278,3 +278,8 @@ Once running, the web UI provides: - **Activity reporting** with concurrency, agent-minutes, cost, and session rows - **Session export** to standalone HTML, markdown export links for agent handoff, or GitHub Gist + +Beyond full-text search, opt-in semantic search lets +`agentsview session search --semantic` (or `--hybrid`) match session content by +meaning, backed by a local or hosted embeddings endpoint. See +[Semantic Search](/semantic-search/) for setup. diff --git a/docs/semantic-search-internals.md b/docs/semantic-search-internals.md new file mode 100644 index 000000000..0297675b6 --- /dev/null +++ b/docs/semantic-search-internals.md @@ -0,0 +1,525 @@ +--- +title: Semantic Search Internals +description: Architecture and invariants behind the vector index — storage, generations, build pipeline, concurrency, and search path +--- + +This page documents the internal design of [Semantic Search](/semantic-search/) +for maintainers extending or debugging the vector index. It assumes the +user-facing behavior described there and does not repeat configuration or CLI +usage. + +## Storage layout + +`vectors.db` is a separate SQLite database beside the main archive +(`sessions.db`), not a set of tables inside it. Two things follow from that: + +- **It survives a parser-change resync.** A resync rebuilds and atomically swaps + `sessions.db`; `vectors.db` is untouched by the swap. The next mirror + refresh re-derives identities against the new archive, so unchanged + documents keep their vectors and only genuinely changed content re-embeds. +- **It's self-contained.** The mirror copies message content into `vectors.db` + rather than joining back to the archive, so the vector store never needs the + archive open to serve a query, and `vectors.db` can be deleted and rebuilt + (`embeddings build --full-rebuild`) without touching `sessions.db`. + +Tables inside the archive DB were rejected: that would tie vector writes to the +archive's write path and lock, and complicate the resync-swap story with +special-casing during the swap instead of a plain re-scan afterward. + +## Unit model: user documents and runs + +The index does not embed every message individually. The embeddable universe — +`role IN ('user','assistant')`, non-system, non-system-prefixed (per +`SystemPrefixSQL`), from non-trashed sessions — is reduced by +`db.ScanEmbeddableUnits` into **unit documents**: + +- **User documents**: one document per embeddable user message. +- **Run documents**: a maximal sequence of contiguous embeddable assistant + messages within one session, bounded by embeddable user rows, session edges, + and `is_sidechain` transitions (a contiguous sidechain block forms its own + run and never mixes with non-sidechain messages). Member texts are joined in + ordinal order with a single blank line (`"\n\n"`); no role labels, markers, + or metadata are injected into the embedded text. + +The boundary term is deliberately "embeddable user row", not "human turn": user +rows the system-prefix filter excludes (interruptions, task notifications, +command wrappers, continuation banners) are invisible to the reducer and do not +split runs — there is no separate "does this row split" detector to get wrong. A +run of one message degenerates to per-message behavior. + +Run grouping exists because per-message embedding diluted retrieval: most +assistant messages are short, procedural narration ("Let me check the file") +that is context-poor as a standalone semantic unit, so per-message vectors were +mostly near-duplicate fragments of long work stretches. Grouped between human +turns, roughly 1.1M assistant messages collapse into ~44k runs (~25x fewer +assistant-side documents), so a "reconstruct this design decision" query matches +narrative, not fragments. + +### Subordinate classification + +A unit is **subordinate** when any of the following holds: + +- its members have `is_sidechain = 1` (a sidechain transition always closes the + run first, so every member of one run shares a single value); +- its session's `relationship_type` is `subagent` or `fork`; +- its session is parent-linked (`parent_session_id <> ''`) with any relationship + type other than `continuation` — defensive, covering empty or unknown types. + +Continuations are deliberately top-level, deviating from the sidebar's +child-session convention: embedding cares about content provenance, a +continuation is the same human-driven conversation, its replayed banner is +already excluded as system-prefixed, and its new content is unique. Forks follow +the existing child convention because their prefix replays parent content — +deduplication happens by downranking, not exclusion. Subordinate units stay in +the index and stay searchable; they are penalized and annotated at search time +(see [Search path](#search-path)), never hidden by default. + +### The `vector_messages` mirror table + +One row per unit. Columns: `doc_key` (primary key), `session_id`, `source_uuid` +(the unit's first member's), `ordinal` (the unit's first member's ordinal — the +retained unique `(session_id, ordinal)` index makes this the slot invariant: one +unit per starting ordinal), `ordinal_end` (the last member's ordinal; equal to +`ordinal` for user documents), `subordinate`, `offsets`, `content` (the unit's +joined text), `content_hash` (sha256 of content, kit's revision column), +`embed_gen`. + +`offsets` is a JSON array with one entry per member message in ordinal order — +`[{"o": , "r": , "b": }, ...]` — ends implied +by the next entry or the content length. Rune offsets map kit chunk windows back +to member messages (anchoring); byte offsets slice snippets without re-decoding. +User documents store `[]`, so consumers parse one shape unconditionally. + +### `doc_key` scheme + +`internal/vector/mirror.go` builds `doc_key` from the unit's first member: + +- `u::` (user document) or + `r::` (run document) when the first member has a + `source_uuid` — with a `#` occurrence suffix when more than one message + in a session shares the same `source_uuid`. `n` is a 1-based counter + assigned in `(session_id, ordinal)` scan order and shared across unit kinds, + so it's deterministic across resyncs. +- `o::` or `ro::` otherwise (legacy + parsed data with no per-message UUID). + +`session_id` and `source_uuid` are percent-escaped before joining — a custom +escape that only encodes `%`, `:`, and `#` as `%XX`, not a general URL-encoder — +so a literal colon, hash, or percent sign inside either component can't be +mistaken for one of the key's own delimiters, and an occurrence-suffix-shaped +`source_uuid` can't collide with a real occurrence suffix. + +Keying a run on its *first* member makes run identity stable at the active tail: +a run that grows a trailing message keeps its `doc_key` — its `content_hash` +changes and the run re-embeds, which is the intended cost. A new user turn +landing mid-run after a resync splits the run: the second half becomes a new +document and the old one shrinks; the mirror's reconciliation and two-phase +eviction handle both. + +UUID-keyed rows survive ordinal renumbering (e.g. from a resync) as a cheap +`ordinal`/`content_hash` update with no re-embed. Ordinal-keyed rows become a +new document whenever their ordinal shifts, and re-embed — an accepted cost that +only affects data parsed before per-message UUIDs existed. + +## Mirror schema versioning + +`vector_meta` carries a `mirror_schema_version` key (currently `"3"`). It covers +both the mirror's DDL shape and its document-identity scheme — what one +`vector_messages` row *means* — and is bumped whenever either changes in a way +old rows cannot simply be read as-is. History: `"2"` added the +`ordinal_end`/`subordinate`/`offsets` columns while still holding one row per +message; `"3"` switched document identity to run-grouped units with no DDL +change. + +On a mismatch — including the key being absent while any mirror state already +exists: + +- **Write path** (daemon, CLI build): `Open` drops every mirror-state table in + `vectors.db` — `vector_messages`, `vector_meta`, and every kit-owned + `message_vectors*` table, including vec0 tables left behind by retired or + abandoned generations — recreates the current schema, and restamps the + version, so the next build takes the existing first-ever full-build path. + `embeddings activate` and `retire` also open read-write on their direct + (no-daemon) path (`directGenerationAction` in + `cmd/agentsview/embeddings.go`), so against a mismatched `vectors.db` they + trigger the same reset and then fail with "generation not found", the reset + having removed every generation. `vectors.db` is disposable by design; + `sessions.db` is never reset this way. +- **Read path** (read-only `Open`: CLI reads, direct-install search): `Open` + succeeds without touching any table, but every subsequent `Search`, + `StaleActive`, `Generations`, or `ResolveMessageUnits` call fails closed + with the typed sentinel `vector.ErrMirrorVersionMismatch` ("vector index was + built by an incompatible version: run `agentsview embeddings build`") rather + than risk misreading rows shaped by a different scheme. The search wiring + maps the sentinel onto `ErrSemanticUnavailable`, so it surfaces exactly like + the stale-fingerprint gate — semantic search stays wired and returns + rebuild-required (HTTP 501) instead of silently unwiring. + +The mirror version and the generation fingerprint (next section) are two +independent gates, and both are required: the version resets incompatible mirror +*state*, while the fingerprint cuts a new generation when the embedding +*configuration or scheme* changes even if the mirror were somehow current. + +## Generations and fingerprints + +The vector index moves through kit's generation lifecycle: **building → active → +retired**. A generation's fingerprint is derived from `model` + `dimension` + +the params map +`{max_input_chars, doc_unit_scheme: "run_v1", chunk_overlap_chars}` +(`vectorGeneration` in `cmd/agentsview/embeddings.go`), plus `input_suffix` when +configured — an empty suffix is omitted from the map rather than included as +`""`, so configs written before the key existed keep their fingerprints. +`chunk_overlap_chars` is computed by `vector.ChunkOverlap` — +`max_input_chars * 15 / 100` — the same function `Open` uses for kit's +`SplitOptions`, so the split behavior and its fingerprint can never drift apart. +Changing any input — the model, the dimension, the chunking cap, the input +suffix, the overlap formula, or the document-unit scheme — produces a different +fingerprint and cuts a new generation. Which +`[vector.embeddings.servers.]` entry encoded a document is deliberately +*not* a fingerprint input: every server serves the same globally-configured +model, so their vectors are interchangeable and a build may switch servers +(`embeddings build --using `) without invalidating the generation. + +- `embeddings build` (incremental): mirror refresh, then fill whatever the + active generation is missing. +- `embeddings build --full-rebuild`: if the target fingerprint differs from the + active generation's, cuts a new generation and fills it fully, activating on + clean completion; if the fingerprint is unchanged (e.g. rebuilding after a + content-only change), it resets and refills the *existing* active generation + in place — clearing its vectors, chunks, and stamps but keeping the + generation row — rather than cutting a new one. +- The staleness gate checked at query time is exactly this: the active + generation's stored fingerprint no longer matches the fingerprint computed + from the current `[vector.embeddings]` config. + +## Chunking and anchoring + +Unit content is chunked by kit's `Split` with `MaxRunes = max_input_chars` +(default 8192) and `Overlap = ChunkOverlap(max_input_chars)` — 15% of the cap: +1228 runes at the default, 375 at a 2500 cap. + +A hit on a run document is **anchored** to one member message: the member whose +rune span contains the matched chunk's center rune, +`chunk_start + len(chunk_runes)/2`. The center uses the chunk's *actual* rune +length, not `MaxRunes`, so a short final chunk anchors at its true center. Each +member owns only its own text span — the `"\n\n"` separator before the next +member belongs to the gap between spans — so a center falling inside a separator +anchors the earlier member, while a center exactly at a member's first rune +anchors that member. The chunk window is reproduced deterministically from the +mirrored content via kit's `Hit.ChunkIndex` plus the same `SplitOptions` +(`chunkWindow` in `internal/vector/search.go` mirrors `kitvec.Split`'s +arithmetic and is cross-checked against it in tests). + +A run hit's snippet is the intersection of the chunk's rune window with the +anchor member's span — always a substring of the anchor message's own text, so +the db layer's snippet centering can locate it inside the anchor message's +content. A stale `ChunkIndex` whose re-split window misses the member entirely +falls back to the anchor member's whole span; user documents snippet the whole +matched chunk, which is already message-local. + +## Build pipeline + +### Mirror refresh (scan) + +Before a fill, the mirror is reconciled against the archive's embeddable +universe: new identities are inserted, `ordinal`/`content_hash` updated on +existing ones, and identities no longer present removed. + +Removal is two-phase, and the ordering matters: deleting only the +`vector_messages` row would leave its vectors occupying KNN slots (the query +path filters them from hits, but the slots themselves are never reclaimed). +Removal always deletes the document's vectors first, then the mirror row — and, +within a single scan, a row that's merely displaced (for example a duplicate +`source_uuid` shifting occurrence) is parked at a negative sentinel ordinal +instead of being deleted outright, so a same-scan reinsert under the same +`doc_key` survives via upsert and keeps its `embed_gen` rather than +re-embedding. + +### Fill and skip-and-stamp + +Fill embeds every pending document (content changed, or never embedded, for the +active generation). Within each scan page, up to `concurrency` (the building +server's config, default 4) documents are split and encoded in parallel; saves +into `vectors.db` stay serialized on one goroutine, preserving the single-writer +model. Requests ask for `encoding_format: "base64"` (raw little-endian float32 +bytes, ~4x smaller than JSON float arrays); the encoder accepts either response +shape, and a server that rejects the field downgrades the encoder to plain float +requests for its lifetime. A document whose encode call fails with a permanent +error — a 400, 413, or 422 whose error body describes the input itself, e.g. a +token/context-length overflow or a content-policy rejection — is not retried in +that fill or the next one: it's stamped for the generation with no vectors at +its current `content_hash`, which marks it non-pending. It's logged (doc key +plus the underlying error) and counted in the build summary's skipped count, but +there is no separate poison list or periodic retry — the only way it embeds +again is if the document's content itself changes later (a new `content_hash`, +so a new pending row). Every other failure — 5xx, network errors, timeouts, 429, +and any 4xx that looks like an auth, route, model, or media-type problem rather +than a rejection of this document — aborts the fill and is retried on the next +scheduled build, so a config mistake can't silently stamp the whole corpus as +embedded-with-no-vectors. + +### Scope (`include_automated`) + +Whether automated sessions are in the embeddable universe is stored in +`vector_meta` (`scope_include_automated`). Changing it — in config or via the +one-off `--include-automated` flag — forces a full mirror *reconciliation*, not +a re-embed: it inserts or removes rows to match the new scope, but documents +that stay in scope and are unchanged keep their existing stamps. + +## Concurrency and locking + +`vectors.db` follows the archive's single-writer model, with its own lock file +(`vectors.write.lock`) separate from the archive's `db.write.lock` so fills +never contend with archive writes: + +- With a writable daemon running, `embeddings build`/`activate`/`retire` proxy + to it over HTTP; the daemon holds `vectors.write.lock` for its lifetime and + serializes all builds through one in-process `Manager`. +- Without a daemon, the CLI takes the same `vectors.write.lock` itself and runs + the build in-process. + +The after-sync scheduler debounces sync-completion signals about 30s before +triggering a build, and never blocks sync on embedding. A build already in +progress causes a new trigger to be dropped, not queued — the pending state is +left set so the next debounce or backstop tick picks it up. A periodic backstop +(`backstop_interval`, default 24h) runs a full reconciliation independent of +sync activity, to catch stragglers from crashes or transient encode failures; if +a backstop tick lands while a build is already running, it's remembered so the +*next* build, not the next 24h tick, carries the full-reconciliation flag. + +Generation activation always happens under the single writer. Search opens +`vectors.db` read-only from any process, with no locking. + +## Search path + +- **Active generation only.** Search never falls back to a building or retired + generation — if only a building generation exists, it hard-errors with a + progress percentage rather than silently querying partial data. +- **Hits are unit-level, anchored to a message.** A semantic hit resolves to + session + `ordinal_range` (the matched unit's span) + an anchor ordinal + a + snippet. The existing required `ordinal` field is kept and redefined as the + anchor ordinal — backward compatible, since for user documents and + one-message runs it is exactly the old per-message value. `ordinal_range`, + `subordinate`, and the lineage fields (`relationship`, `parent_session_id`, + `is_sidechain`) are carried by every mode: semantic/hybrid unit rows take + theirs from the mirror unit, lexical rows and hybrid unit-less rows from the + structural derivation described in + [Conversation-unit citations](#conversation-unit-citations). `--around` and + the context-cursor flow anchor on the anchor ordinal — the message-window + APIs are unchanged on every backend. +- **`scope` governs unit visibility and supersedes `include_children`.** + `scope=top|all|subordinate` (default `all`) filters each leg's hits before + the RRF merge and before the limit. The hybrid FTS leg fetches additional + rank-ordered batches until it holds the fusion depth `k` of surviving + entries (capped at `maxHybridFTSBatches`), so scope discards and same-unit + collapse do not starve it; the semantic (KNN) leg cannot page, so scoped or + collapse-heavy searches can still under-fill past those caps even when more + matches exist deeper in the ranking. In semantic/hybrid modes the + sidebar-child session exclusion is lifted (`semanticSessionScopeSubquery`) — + both hybrid legs must see the same universe for fusion to be sound — and an + explicit `include_children` is accepted but superseded. Subagent/fork-typed + and parent-linked sessions are also exempted from the one-shot + (`user_message_count <= 1`) exclusion in these modes, because a delegated + session structurally has exactly one "user" message (the task prompt) and + the default gate would silently exclude ~98% of subagent sessions, hollowing + out `scope=all`; root sessions with no parent link keep the one-shot + exclusion. All other session filters (project, agent, machine, dates, + automated) apply in every mode, and FTS-only, substring, and regex modes + keep today's `include_children` and one-shot semantics unchanged. +- **The subordinate penalty has exactly one implementation.** `rrfMerge` in + `internal/db` fuses rank-ordered legs with reciprocal rank fusion (rank + constant 60) and shifts subordinate units' effective rank by +5 — a + rank-based adjustment, not a hard tier or score multiplier, since RRF ranks + are the only scale comparable across legs. The merge is a local + implementation rather than kit's `Merge` because kit has no per-hit + rank-offset hook for the subordinate penalty (upstreamable later). + Semantic-only search routes its single ranked list through the same merge as + a one-leg fusion, so `--semantic` downranks subordinate hits identically to + `--hybrid` (matches still carry the searcher's own cosine scores; only the + order changes). +- **Hybrid fuses at unit granularity, with an FTS anchor override.** The FTS leg + stays message-granularity (exact strings, commands, filenames) over the same + embeddable-universe predicate `ScanEmbeddableUnits` uses. Each FTS message + hit is resolved to its containing unit via + `VectorSearcher.ResolveMessageUnits` — a point lookup on the mirror's unique + `(session_id, ordinal)` index: seek the greatest unit `ordinal <= x` for the + session, verify `x <= ordinal_end`. Units within a session never overlap, so + no extra index is needed. Hits on the same unit fuse under one key; when the + FTS leg contributes, the exact matched message becomes the hit's anchor + regardless of chunk center. An FTS hit with no containing unit keeps a + message-granularity fusion key and survives fusion on its own, carrying the + structurally derived subordinate flag through scope filtering and the fusion + penalty — the same classification lexical mode gives the anchor (see + [Conversation-unit citations](#conversation-unit-citations)). +- **Metadata filters post-filter the vector leg, with over-fetch.** Vector KNN + doesn't know about `--project`/`--agent`/`--date*`, so the vector leg + over-fetches `max(limit × 4, 200)` candidates, then filters and truncates to + the requested limit. At small corpora or narrow filters this can return + fewer than `--limit` results even though more exist — a known v1 tradeoff + (see [Limitations](/semantic-search/#limitations)). + +## Conversation-unit citations + +Every content-search match — every mode, every backend — carries a +conversation-unit citation: `OrdinalRange [2]int` with `json:"ordinal_range"`, +always present, never omitempty (`[ordinal, ordinal]` when the anchor is its own +unit; the array form deliberately avoids the omitempty-integer trap, so a unit +starting at ordinal 0 still serializes its start), plus the +`subordinate`/`relationship`/`parent_session_id`/`is_sidechain` lineage fields +(which keep `omitempty` — false/empty means top-level/no-lineage, +unambiguously). Row cardinality stays mode-specific — lexical +(substring/regex/FTS) returns one row per matching source row with unchanged +snippets and pagination, semantic one row per embedded unit, hybrid one row per +unit with the FTS-anchor override untouched — only the citation metadata is +uniform. The HTTP response serializes `db.ContentMatch` directly; the MCP +`contentMatch` mirror struct carries the same fields; the CLI renders +`#start-end @anchor` for multi-message ranges plus a `sub` marker, in any mode. + +### Per-mode provenance + +`ordinal_range` always means "conversation unit", not always "embedding unit": + +- **Semantic and hybrid unit rows** carry the embedded unit's span from the + vectors.db mirror — embedding identity, including build scope. +- **Lexical rows and hybrid unit-less rows** carry a structurally derived unit + computed from the messages/sessions tables only. Deterministic, never + depends on whether a vector index exists or is fresh — lexical output must + not flicker with index state. + +Derived and embedded spans coincide except where embedding scope diverges from +structure: sessions excluded from the build (`include_automated = false`) and +messages newer than the last mirror refresh. There is deliberately no provenance +discriminator field on the wire — the mode implies it. + +Hybrid unit-less rows (FTS hits whose message resolves to no mirror unit) are +classified **before** scope filtering and the RRF merge, so `scope` exclusion, +the subordinate rank penalty, and annotation treat a unit-less sidechain hit +exactly as lexical mode classifies the same anchor, instead of always passing it +as top-level. + +### Derived-unit rules + +The structural rules mirror `ScanEmbeddableUnits`'s reducer, so derived spans +equal embedding-unit spans on in-scope data. An **embeddable user row** is +`role = 'user' AND is_system = 0` with content not system-prefixed (the dialect +`SystemPrefixSQL` predicate); an **embeddable assistant row** is the same with +`role = 'assistant'` — the prefix predicate constrains only user rows, so a +system-prefixed assistant row stays embeddable and derives its run span. For an +anchor message row at ordinal `o` (`tool_input`/`tool_result` matches anchor on +the tool call's message row): + +1. Embeddable user row → `[o, o]` (user messages are their own units). +1. Embeddable assistant row → the maximal stretch of embeddable assistant rows + containing `o`, bounded exclusively by the nearest embeddable user row on + either side, the session edges, and the nearest embeddable assistant row + whose `is_sidechain` differs (runs never mix sidechain values). The + endpoints are the first and last **member** ordinals — the span may cover + non-member ordinals in between (system rows inside a run), exactly like the + reducer's `runUnit`. +1. Anything else (system rows, system-prefixed user rows, other roles) → + `[o, o]`: the row belongs to no conversation unit, so the citation is the + message itself. + +`tool_result_events` matches locate their anchor message row with a post-scan +secondary lookup, never a join — the events branches join only `sessions`, and +an inner join would drop matches whose anchor row is missing, changing +cardinality. An orphan event with no locatable anchor row falls back to `[o, o]` +with `is_sidechain` false; session lineage still applies. + +Automation gating is deliberately ignored: derivation is structural, so matches +inside automated sessions still get real ranges. The invariant, pinned by the +reducer-equivalence test in `internal/db/unit_range_test.go`: derivation at any +member ordinal of any unit produced by +`ScanEmbeddableUnits(include_automated = true)` returns exactly that unit's +`[Ordinal, OrdinalEnd]`. + +`subordinate` for derived rows uses the reducer's formula: session-subordinate +(`relationship_type IN ('subagent','fork')`, or parent-linked with +`relationship_type <> 'continuation'`) OR the anchor row's `is_sidechain`. + +### Seam architecture and batching + +Derivation is a pure Go pass in `internal/db` (`DeriveUnitRanges`) over a +backend-neutral seam, `UnitBoundsQuerier`, with two batched methods: +`NearestUserBoundaries` (the nearest exclusive embeddable-user boundaries around +each probe) and `RunExtents` (the first/last member ordinals of the anchor's +same-sidechain run within an exclusive interval). `internal/db`, +`internal/postgres`, and `internal/duckdb` each implement the seam with their +own dialect SQL; `internal/db` never references the other stores, and co-located +parity and reducer-parity tests in each package pin identical observable output +across backends. + +- **Shared resolvers, SQL-only backends.** `ResolveUserBoundaries` and + `ResolveRunExtents` own the orchestration — probe dedup, chunking at the + dialect's bind-variable limit, boundary resolution, alignment and invariant + checks — so each backend supplies only batched SQL (one statement per chunk + of batched correlated point lookups, never one per probe). +- **Post-scan, O(page).** The lexical search SQL before the LIMIT is untouched; + anchor classification and session lineage are fetched after truncation, and + derivation runs over the returned page only. Rule-1, rule-3, and missing + anchors resolve locally with no query. +- **Dense/sparse boundary flows.** Rule-2 (embeddable assistant) anchors are + deduplicated by `(session, ordinal, sidechain)`. `NearestUserBoundaries` + runs only on session-dense pages — at least `UnitBoundsFlowFactor` probes + per distinct session on average — where pre-fetched user bounds pay for + themselves by pruning stop scans and splitting probe groups at unit + boundaries; sparse pages probe with sentinel bounds and lean on + `RunExtents`' built-in user-row stops (real bounds are an optimization, + never a correctness requirement). +- **Median-representative run sharing.** Probes with the same + `(session, bounds, sidechain)` group key that land in the same run have + identical extents, so the first `RunExtents` round queries one + representative per group — the group's ordinal median, since page anchors + cluster in hot runs — and hands its extent to every group sibling the extent + covers (sound because a same-sidechain member inside the extent provably + belongs to that run). Siblings in other runs resolve in one second batch, so + a page costs at most one boundary statement and two `RunExtents` rounds + regardless of how its anchors spread. Twenty hits in one monologue cost one + probe. +- **Cost.** Measured overhead on the gated content-search benchmarks is + sub-millisecond-scale on a 50-hit page; the benchmarks live in `internal/db` + and are CI-gated. If real-corpus profiling ever shows meaningful cost, the + remedy is an explicit opt-out — citations must never silently self-disable + based on index or corpus state. + +## Error taxonomy + +Two sentinel errors carry every semantic/hybrid failure across CLI, HTTP, and +MCP: + +| Sentinel | Meaning | HTTP | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | ---- | +| `ErrSemanticUnavailable` | Not enabled/configured, index never finished a build, still building, stale (fingerprint mismatch), or built by an incompatible mirror version | 501 | +| `ErrSemanticTransient` | Embeddings endpoint unreachable or timed out at query time — retryable | 503 | + +`vector.ErrMirrorVersionMismatch` is not a third sentinel at this layer: the +search wiring translates it onto `ErrSemanticUnavailable`, preserving its +rebuild-required message, so callers see the same 501 family as the +stale-fingerprint case. + +The distinction matters for callers: 501 means the feature will not work until +something is configured or built; 503 means it should work and is worth +retrying. CLI and MCP surface the same cause-specific remediation text described +in the [user-facing error taxonomy](/semantic-search/#error-taxonomy). + +## Skill generation + +`internal/skills` renders the `agentsview-finding-history` skill (see +[Skills for coding agents](/semantic-search/#skills-for-coding-agents)) from a +single embedded template, `internal/skills/templates/finding-history.md.tmpl` +via `go:embed` — the same pattern `internal/web` uses for the frontend — with no +per-harness copies checked in. `Render` fills in a harness-specific delegation +phrase (whether the harness can dispatch a search subagent or must run the +bounded probes itself) and inserts a `generated-by` header — carrying the CLI +version and a sha256 hash of the pure template render — as a YAML comment on +line two, just inside the frontmatter fence, so the file still begins with `---` +and frontmatter-based skill discovery keeps working. Staleness and tamper +detection are hash-authoritative, not version-authoritative: `Classify` compares +a file's recorded hash against its own body hash to detect modification, and +against a fresh render's hash to detect staleness, and never consults the +version string, because dev builds all report version `"dev"` and would +otherwise be indistinguishable from one another. There is deliberately no Claude +Code plugin/marketplace packaging: that would tie distribution to one harness's +install mechanism, whereas the goal is a single `SKILL.md` artifact that any +`.agents/skills`-reading harness can consume the same way, installed directly by +the `agentsview` binary rather than a separate package manager. diff --git a/docs/semantic-search.md b/docs/semantic-search.md new file mode 100644 index 000000000..473ae4b73 --- /dev/null +++ b/docs/semantic-search.md @@ -0,0 +1,512 @@ +--- +title: Semantic Search +description: Vector (semantic) search over session messages, plus hybrid search and cursor-based context retrieval +--- + +AgentsView can index user and assistant message content into a local vector +store and search it by meaning instead of exact terms, alongside the existing +substring/regex/FTS5 content search. This is an opt-in feature backed by an +OpenAI-compatible embeddings endpoint — a local [Ollama](https://ollama.com) +model or a hosted API. + +For the architecture behind this page — storage layout, generations, +concurrency, and the search path — see +[Semantic Search Internals](/semantic-search-internals/). + +!!! note "SQLite only" + + Semantic and hybrid search require the local SQLite archive. + [PostgreSQL sync](/pg-sync/) and the [DuckDB mirror](/duckdb/) do not support a + vector backend yet, so `--semantic`/`--hybrid` against `--pg` or a DuckDB-backed + server return the same "not available" error described below. + +## Enabling `[vector]` + +Semantic search is disabled by default. Add a `[vector]` section to +`~/.agentsview/config.toml`: + +```toml +[vector] +enabled = true # default false; everything below is opt-in +# db_path defaults to /vectors.db +include_automated = false # default; automated sessions (e.g. roborev) are not embedded -- set true to include + +[vector.embeddings] +model = "nomic-embed-text" +dimension = 768 # every returned vector must have this length +max_input_chars = 8192 # per-chunk rune cap (default 8192) +# input_suffix = "<|endoftext|>" # appended to every embedded text; default empty (see below) +default_server = "local" # server used for query encoding and unnamed builds + +[vector.embeddings.servers.local] +endpoint = "http://localhost:11434/v1" # OpenAI-compatible base URL; "/embeddings" is appended +api_key_env = "OPENAI_API_KEY" # name of an env var holding the key; omit for anonymous access +batch_size = 32 # inputs per HTTP call (default 32) +concurrency = 4 # documents embedded in parallel during a build (default 4) +timeout = "30s" # per-HTTP-call timeout (default "30s") +max_retries = 3 # attempts on 429/5xx/network errors; 4xx fails fast (default 3) + +[vector.embed] +run_after_sync = true # daemon embeds deltas after each sync, debounced ~30s (default true) +backstop_interval = "24h" # periodic full reconciliation scan; negative disables (default "24h") +``` + +`model`, `dimension`, and at least one `[vector.embeddings.servers.]` +entry with an `endpoint` are required once `enabled = true`; `agentsview` fails +fast with an actionable message if any is missing or a duration field doesn't +parse. Restart the daemon (or run a CLI command) after editing the file. + +### Named embeddings servers + +Model identity — `model`, `dimension`, `max_input_chars`, `input_suffix` — is +global: every server in the `servers` table must serve that same model, so +vectors produced by any of them are interchangeable and land in the same +generation. What varies per server is transport and capacity: `endpoint`, +`api_key_env`, `timeout`, `max_retries`, `batch_size`, and `concurrency`. + +This split exists so you can encode search queries against a fast local server +while offloading bulk index builds to a bigger remote machine: + +```toml +[vector.embeddings] +model = "qwen3-embedding-4b" +dimension = 2560 +input_suffix = "<|endoftext|>" +default_server = "local" + +[vector.embeddings.servers.local] # laptop llama.cpp: low latency for queries +endpoint = "http://127.0.0.1:30000/v1" + +[vector.embeddings.servers.build-box] # remote GPU box: high throughput for builds +endpoint = "http://build-box:30000/v1" +timeout = "300s" +concurrency = 6 +``` + +`default_server` names the server used for search-time query encoding and for +any build that doesn't select one; with a single server defined it is implicit, +with more than one it is required. +`agentsview embeddings build --using build-box` runs one build against a +different server without touching the default. Because the model identity is +global, the server choice is not part of the generation fingerprint — a build +started on one server can be topped up incrementally from another. + +One caveat: the same model served at different quantizations (say F16 on one +box, Q8 on another) produces slightly different vectors for the same text. They +live in the same embedding space and search still works, but for bit-identical +vectors serve the same weights everywhere. + +`concurrency` bounds how many documents a build embeds in parallel. Builds are +usually round-trip-bound rather than compute-bound — especially against a remote +endpoint — so a few requests in flight at once multiply throughput. Servers that +process one request at a time simply queue the extras; raise the value if your +endpoint has spare parallel capacity, or set it to 1 to send one request at a +time. Responses are requested in the compact base64 encoding automatically (with +a transparent fallback for servers that reject or ignore `encoding_format`), +which cuts response transfer roughly 4x on slow links. + +`input_suffix` is appended verbatim to every text sent to the endpoint — +documents at build time and queries at search time — for models that expect a +terminator the serving layer does not add. The main example is Qwen3-Embedding +served by llama.cpp, which is benchmarked with `<|endoftext|>` appended to each +input. The suffix is part of the generation fingerprint, so changing it +(including setting it for the first time) re-embeds the whole archive on the +next build. + +The first scheduled build that `run_after_sync` triggers after enabling +`[vector]` embeds the entire existing archive, not just deltas, since the mirror +starts out empty and every document counts as pending. For a hosted embeddings +API that is a real cost event, so run `agentsview embeddings build` directly at +a time of your choosing if you want to control when that initial cost lands, +rather than letting the debounced after-sync scheduler trigger it on its own. +The same cost event can recur on upgrade: when a new agentsview version changes +the index's internal mirror schema or document-identity scheme, the next +writable open resets the mirror, and with `run_after_sync = true` the next sync +automatically re-embeds the entire archive against the configured endpoint. + +By default, `include_automated = false` keeps automated sessions (e.g. roborev) +out of the embedding index entirely, mirroring session search's default +exclusion of those sessions from results. This matters most for a large archive +dominated by automated sessions: embedding content that search already hides by +default just adds embedding API cost and dilutes semantic ranking with results +nobody is searching for. Because a session that was never embedded has no vector +to match, `session search --semantic --include-automated` still returns no +semantic hits for automated sessions unless the index was built with +`include_automated = true` (or a one-off `embeddings build --include-automated`, +see below). Changing `include_automated` between builds — in config or via the +flag — triggers a full mirror reconciliation on the next build: it removes +now-out-of-scope rows (and their vectors) or picks up newly-in-scope sessions, +without re-embedding documents that were already in scope and unchanged. + +### Ollama quickstart + +```bash +# Pull an embeddings model once. +ollama pull nomic-embed-text + +# Ollama serves an OpenAI-compatible endpoint at /v1; no API key needed. +``` + +```toml +[vector] +enabled = true + +[vector.embeddings] +model = "nomic-embed-text" +dimension = 768 + +[vector.embeddings.servers.local] +endpoint = "http://localhost:11434/v1" +``` + +The encoder POSTs to `/embeddings` with an OpenAI-style +`{"model": ..., "input": [...]}` body and expects +`{"data": [{"index": 0, "embedding": [...]}]}` back — this matches Ollama's +`/v1/embeddings` route as well as OpenAI and most self-hosted OpenAI-compatible +servers. A response whose embedding length doesn't match `dimension` is +rejected. + +## What gets embedded: units, not messages + +The index embeds **unit documents**, not individual messages: + +- Every embeddable user message (non-system, not system-prefixed) is its own + document. +- Assistant messages between those user messages are concatenated — in order, + separated by blank lines — into one **run** document per stretch of work. A + run captures a whole narrative arc (analysis, tool narration, conclusions) + instead of scattering it across hundreds of short fragments. + +This matters for both quality and cost. Most assistant messages are short, +procedural narration that is meaningless as a standalone search hit; grouped +into runs, roughly 1.1 million assistant messages collapse into ~44k documents — +about 25x fewer assistant-side documents to embed and rank. Long documents are +chunked at `max_input_chars` runes (default 8192) with a 15% overlap between +consecutive chunks (1228 runes at the default; 375 at a 2500 cap), so an initial +build sends several times fewer encode requests than a per-message scheme would. + +Content from sidechains and delegated (subagent/fork) sessions is embedded too, +but classified **subordinate**: still searchable, annotated in results, and +ranked below top-level human-driven work. The +[`--scope` flag](#scoping-results-scope) controls whether you see it. + +## Building the index + +```bash +agentsview embeddings build # incremental: refresh + fill whatever's missing +agentsview embeddings build --yes # skip confirmation prompts +agentsview embeddings build --full-rebuild --yes # re-embeds every document +agentsview embeddings build --backstop # force a full mirror reconciliation scan +agentsview embeddings build --include-automated # embed automated sessions for this build only +agentsview embeddings build --using build-box # encode against a named server instead of the default +``` + +`--using ` selects which `[vector.embeddings.servers.]` entry the +build encodes against; without it the build uses `default_server`. A mistyped +name fails immediately, before anything starts. + +`--include-automated` overrides `[vector].include_automated` for this one build; +it does not change the config file. Bare `--include-automated` embeds automated +sessions, and `--include-automated=false` force-excludes them even if the config +default is `true`. It is meant for a one-off build, not scheduled ones: the +after-sync scheduler and periodic backstop always build from the config value, +so mixing the flag with a different config default flips the index's scope back +and forth on every other build, forcing a full mirror reconciliation each time. +Set `include_automated = true` in `config.toml` instead if you want automated +sessions embedded on every build. + +`embeddings build` mirrors the embeddable universe (user documents and assistant +runs, see [What gets embedded](#what-gets-embedded-units-not-messages)) into +`vectors.db`, then fills whatever the active generation is missing. +`--full-rebuild` re-embeds every document: if the target fingerprint (derived +from `model`, `dimension`, `max_input_chars`, `input_suffix` when set, the +document-unit scheme, and the derived chunk overlap) differs from the active +generation, it cuts a new **generation**; if the fingerprint is unchanged, it +instead resets and refills the active generation in place rather than cutting a +new one. It prompts for confirmation with a live count of embeddable unit +documents unless `--yes` is passed. Progress prints every ~2 seconds while a +build runs, and a summary line reports documents embedded, chunks, skipped, and +stale counts on completion. + +When a writable local daemon is running, `build`/`activate`/`retire` proxy to it +over HTTP so the daemon remains the sole writer of `vectors.db`; without a +daemon, the CLI takes a dedicated `vectors.write.lock` in the data directory and +runs the build in-process. If [`run_after_sync`](#enabling-vector) is enabled, +the daemon also embeds sync deltas automatically on a debounce, so a manual +`build` is mainly for the initial index or a `--full-rebuild`. + +```bash +agentsview embeddings list +``` + +```text +ID STATE MODEL DIM EMBEDDED MISSING FINGERPRINT +1 active nomic-embed-text 768 482 0 3f2a9c1e0b7d +``` + +Generations move through **building → active → retired**. A first build +activates automatically once it reaches full coverage. + +```bash +agentsview embeddings activate [--force] +agentsview embeddings retire [--force] +``` + +`activate` on a generation with incomplete coverage, or `retire` on the +currently active generation, is refused unless `--force` is passed. + +## Searching: `session search --semantic` / `--hybrid` + +`--semantic` and `--hybrid` are new content-search modes alongside +`--regex`/`--fts`, mutually exclusive with each other and with the substring +default: + +```bash +agentsview session search "database connection pooling" --semantic --limit 10 +agentsview session search "flaky test" --hybrid --project myapp +``` + +- `--semantic` ranks by cosine similarity against the query's embedding. +- `--hybrid` fuses the semantic ranking with an FTS5 ranking of the same corpus + using reciprocal rank fusion, so exact-term matches and meaning-based + matches both surface. +- Both modes are restricted to the `messages` source — the same restriction + `--fts` already has — since only user/assistant message content is embedded + (never raw tool_input/tool_result rows or system messages). Passing `--in` + with any other source is rejected. +- All the usual filters apply: `--project`, `--agent`, `--machine`, `--date*`, + etc. Metadata filters are applied *after* the vector leg over-fetches + candidates (4x the requested limit) — see [Limitations](#limitations). In + these two modes [`--scope`](#scoping-results-scope) replaces + `--include-children` for deciding whether delegated-session content appears. +- Results are a single ranked page: `--cursor` is rejected for + `--semantic`/`--hybrid` with a clear error, since RRF and cosine ranking + don't have a stable offset to page from. Every match carries a `score` field + (cosine similarity, or the RRF score for hybrid); substring/regex/fts + matches leave `score` unset. +- An empty query pattern (`""`) returns no matches rather than an error, on + every mode. + +Human output shows the score inline: + +```text +abc123 #42 score=0.87 myapp message + ...ideas for pooling database connections across worker threads... +``` + +### Hit shape: ranges and anchors + +Every content-search match, in every mode, cites a *conversation unit* — a user +message, or a run of assistant messages between user turns — anchored to one +specific message inside it: + +- `ordinal` is the **anchor**: the exact matched message, same as every other + release. For user messages and single-message units it's the message's own + ordinal. +- `ordinal_range` is `[start, end]` — the conversation unit containing the + anchor. It is always present, never omitted: a single-message unit + serializes `[ordinal, ordinal]`, and a unit starting at ordinal 0 still + serializes its start. +- `subordinate`, `relationship`, `parent_session_id`, and `is_sidechain` carry + the hit's lineage in every mode: whether it came from a sidechain or a + delegated (subagent/fork) session, and which parent session to corroborate + against. These stay `omitempty`; a missing key unambiguously means top-level + / no lineage. + +What the range *means* depends on the mode: + +- **Semantic hits and hybrid unit hits** carry the embedded unit's span from the + vector index — the identity of the document that actually matched. +- **Substring, regex, and FTS matches** (and hybrid hits whose message has no + embedded unit) carry a **structurally derived** unit computed from the + archive's messages alone, using the same user-message/assistant-run rules + the index uses. Lexical citations therefore need no vector index and never + change with index state. Derived and embedded spans coincide except where + the index's scope diverges from structure: sessions excluded from the build + (`include_automated = false`) and messages newer than the last index + refresh. + +Lexical row cardinality is unchanged: substring/regex/FTS still return one row +per matching source row, with the same snippets — the range and lineage fields +are additive metadata on each row. + +Human output renders a multi-message unit as `#- @` and +marks subordinate hits with `sub`; both can appear in any mode: + +```text +def456 #12-40 @19 sub score=0.71 myapp message + ...decided to key runs on the first member so tail growth is cheap... +``` + +### Scoping results: `--scope` + +`--scope top|all|subordinate` (HTTP/MCP: `scope`) controls whether subordinate +content — sidechain runs and subagent/fork session content — appears in semantic +and hybrid results: + +- `all` (default): everything is searchable; subordinate hits are downranked + below top-level hits of similar relevance and annotated, never hidden. +- `top`: only top-level, human-driven conversation. Use this when reconstructing + decisions — delegated sessions repeat their parent's instructions and can + drown out the conversation where the decision was actually made. +- `subordinate`: only sidechain/delegated content, e.g. to find what a subagent + actually did. + +`--scope` is only valid with `--semantic`/`--hybrid` (other modes reject it) and +supersedes `--include-children` there: child sessions are always visible to +these modes so that `scope` alone governs what you see. Subagent/fork-typed and +parent-linked sessions are also exempt from the default one-shot exclusion in +these modes — a subagent session structurally has exactly one "user" message +(its task prompt), so the one-shot gate would otherwise hide nearly all of them. +Substring, regex, and FTS modes keep the existing `--include-children` and +one-shot behavior unchanged. + +### Inline context: `--context N` + +```bash +agentsview session search "database connection pooling" --semantic --context 2 +``` + +Every match gets `N` messages of context before and after it in the same +response — `context_before`/`context_after` arrays in JSON, indented +`role: content` lines around the match in human output. This works with every +search mode and costs one extra windowed query per hit. Values above 10 are +rejected with `context: maximum is 10` rather than silently clamped. Context +messages are secret-redacted by default, same as `--reveal` governs for the +match snippet itself. + +## Cursor-follow: from a hit to its surrounding conversation + +Every content-search match — regardless of mode — returns a +`(session_id, ordinal)` cursor. Use `session messages --around` to pull a window +of the conversation around that ordinal without re-running the search: + +```bash +agentsview session messages --around 42 --before 5 --after 5 +agentsview session messages --around 42 --role user,assistant +``` + +- `--around ` centers a window on that message; `--before`/`--after` + default to 5 and require `--around`. `--around` is mutually exclusive with + `--from`/`--direction`. +- `--role` filters to a comma-separated role list (e.g. `user,assistant`). With + a role filter, `--before`/`--after` count *filtered* messages, not raw + ordinals — the anchor message is always included regardless of its role. +- The response reports the window's first/last ordinals, so you can keep paging + forward with + `agentsview session messages --from --role user,assistant` to + walk the rest of the session's user/assistant history. There is no + unpaginated "give me everything" mode. +- `--before`/`--after` are clamped so the total window never exceeds the + server's message-page limit (1000 messages); an oversized request is + silently capped rather than rejected. + +The typical workflow: run `session search --semantic ""`, take the +`session_id`/`ordinal` off a hit, then +`session messages --around ` to read what led up to it and +what followed. For a hit whose unit spans a multi-message run, `ordinal` is the +anchor — the member the matched text belongs to — so centering `--around` on it +lands in the right part of the run; widen `--before`/`--after` toward the ends +of `ordinal_range` to read the whole stretch. + +## Error taxonomy + +| Situation | Message | +| ------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[vector]` not enabled | `vector search is not enabled: set [vector] enabled = true in config.toml` (from `agentsview embeddings ...`) | +| No `VectorSearcher` wired (index never built, or PG/DuckDB backend) | `semantic search not available: enable [vector] in config.toml and run 'agentsview embeddings build'` | +| Only a building generation exists | same message, plus `: index is building: N% complete` | +| Active generation's fingerprint no longer matches config (model, dimension, or chunking changed) | same message, plus `: index is stale (embedding config changed): run 'agentsview embeddings build --full-rebuild'` | +| Index was built by an incompatible agentsview version (mirror schema mismatch) | same message, plus `` : vector index was built by an incompatible version: run `agentsview embeddings build` `` | +| `--scope` with a lexical mode (or without `--semantic`/`--hybrid`) | CLI: `--scope requires --semantic or --hybrid`; HTTP/MCP: `scope is only supported for semantic and hybrid search modes` | +| Embeddings endpoint unreachable or timed out | `[vector.embeddings] request: ...` (the underlying transport error) | +| Embeddings endpoint returned non-200 | `[vector.embeddings] status : ` | +| `--in` names a source other than `messages` with `--semantic`/`--hybrid` | CLI: `--semantic searches messages only; drop --in` (or `--hybrid ...`); HTTP/MCP: `search: semantic search only supports the messages source (got "...")` | +| `--cursor` with `--semantic`/`--hybrid` | `semantic search returns a single ranked page; cursor pagination is not supported` | + +Over HTTP (`GET /api/v1/search/content`) and MCP (`search_content`), the "not +available" family of errors maps to HTTP `501 Not Implemented` and the matching +MCP tool error, carrying the same remediation text. + +## Limitations + +- **Metadata filters post-filter the vector leg.** `--semantic`/`--hybrid` + over-fetch candidates from the vector index (4x the requested limit, or a + fixed minimum if that's larger), then drop hits whose session fails + `--project`/`--agent`/`--date*`/etc., then truncate to the requested limit. + At small corpus sizes or with a narrow filter, this can return fewer than + `--limit` results even though more exist. A narrow `--scope` (and, in + hybrid, matches concentrated in one long run) can likewise return fewer than + `--limit` even when more matches exist deeper in the ranking. This is a + known v1 tradeoff, not a bug. +- **Legacy no-`source_uuid` rows re-embed on ordinal shifts.** Each embedded + document is keyed by its first message's stable per-message UUID when the + parser recorded one, or by `(session_id, ordinal)` when it didn't. + UUID-keyed documents survive ordinal renumbering (e.g. from a resync) as a + cheap metadata update with no re-embed; ordinal-keyed documents are treated + as new and re-embedded when their ordinal shifts. This only affects older + parsed data predating per-message UUIDs and is an accepted cost rather than + a bug. +- **The active run re-embeds as it grows.** A run document is keyed on its first + message, so a session's trailing run keeps its identity as new assistant + messages append — but its content changes, so each build re-embeds the + current tail. That is the intended cost of grouping; finished runs never + re-embed. +- **SQLite only.** PostgreSQL sync and the DuckDB mirror have no vector backend; + `--semantic`/`--hybrid` against `--pg` or a DuckDB-backed server return the + "not available" error (HTTP 501) described above. `pgvector` support is a + possible follow-up. +- **No frontend integration.** The web UI's command palette and in-session + search remain FTS-only; semantic and hybrid search are CLI/HTTP/MCP-only in + this release. +- **The index embeds message `content` verbatim.** Like `--fts`, it only draws + from the `messages` source, so raw tool_input/tool_result rows are never + candidates. System messages are handled more strictly, though: `--fts` still + includes them unless the caller passes `--exclude-system`, while + `--semantic`/`--hybrid` always exclude system messages from the index with + no flag to opt back in. But anything a parser rendered *into* a + user/assistant message's content is embedded with it: thinking text + flattened inline as `[Thinking]...[/Thinking]` markers, and tool-call + summaries some parsers render into assistant content, are all ordinary message + text to the index. Run documents concatenate that per-message text unchanged + — no role labels or markers are injected between members. + +## Skills for coding agents + +`agentsview skills install` writes a bundled skill file that teaches a +coding-agent harness the search workflow described on this page: when to reach +for `--hybrid` versus `--fts`, how to react to the +[error taxonomy](#error-taxonomy), and how to walk from a hit into its +surrounding conversation with +[`session messages --around`](#cursor-follow-from-a-hit-to-its-surrounding-conversation). + +```bash +agentsview skills install # both harnesses, user level +agentsview skills install --harness claude # one harness only +agentsview skills install --project # install under the current git root +agentsview skills list # show install state per harness +``` + +| `--harness` | Target | +| ----------- | -------------------------------------------------------------------------------------------------------------------------- | +| `claude` | `~/.claude/skills/agentsview-finding-history/SKILL.md` | +| `agents` | `$HOME/.agents/skills/agentsview-finding-history/SKILL.md` — the open convention Codex reads (per Codex's own skills docs) | + +`--project` swaps the base from the home directory to the current git root (or +the working directory itself outside a repo), writing to `.claude/skills/...` +and `.agents/skills/...` instead. + +Every rendered file carries a `generated-by` header with a content hash, written +as a YAML comment just inside the frontmatter fence so the file still starts +with `---` and harnesses keep discovering it. `install` overwrites a file whose +hash still matches its header (unmodified since the last install) but refuses a +file that was hand-edited or was never generated by `agentsview`, printing which +paths it refused and exiting non-zero; pass `--force` to overwrite anyway. +Re-run `agentsview skills install` after upgrading `agentsview` to pick up skill +content changes — the header records the CLI version for humans, but the content +hash, not the version, decides whether a reinstall is a no-op. + +`agentsview skills list [--project] [--format json]` reports each harness's +install state — `missing`, `current`, `stale` (unmodified but older than the +current render), `modified`, or `foreign` (no header) — without writing +anything. diff --git a/docs/session-api.md b/docs/session-api.md index 626f1add6..1fb68f08c 100644 --- a/docs/session-api.md +++ b/docs/session-api.md @@ -226,6 +226,7 @@ One-shot and automated sessions are excluded by default. Use the | `--date-from` | `date_from` | `YYYY-MM-DD` | | `--date-to` | `date_to` | `YYYY-MM-DD` | | `--active-since` | `active_since` | RFC3339 timestamp | +| `--since` | `active_since` | Relative — `Nh` hours, `Nd` days, `Nw` weeks, `Nm` calendar months (not minutes), `Ny` years — or `YYYY-MM-DD`; resolved against now and mutually exclusive with `--active-since` | | `--resume` | `active_since` | CLI shortcut for sessions active in the last 15 minutes | | `--active` | `active_since` | Alias for `--resume` | | `--min-messages` | `min_messages` | int | @@ -274,6 +275,7 @@ Return a window of messages. Response shape matches ```bash agentsview session messages [--from N] [--limit N] [--direction asc|desc] +agentsview session messages --around N [--before N] [--after N] [--role user,assistant] ``` `--from` is pointer-valued at the service layer: omitting it means @@ -281,6 +283,22 @@ agentsview session messages [--from N] [--limit N] [--direction asc|desc] page" for descending; an explicit `--from 0` means "start at ordinal 0" in both directions. `--direction` is validated to `asc` or `desc`. +Window and role flags (see +[Semantic Search](/semantic-search/#cursor-follow-from-a-hit-to-its-surrounding-conversation) +for the cursor-follow workflow they support): + +| Flag | HTTP param | Notes | +|------------|------------|--------------------------------------------------------------| +| `--around` | `around` | Center a window on this ordinal; mutually exclusive with `--from`/`--direction` | +| `--before` | `before` | Messages before the anchor (default 5); requires `--around` | +| `--after` | `after` | Messages after the anchor (default 5); requires `--around` | +| `--role` | `roles` | Comma-separated roles to include, e.g. `user,assistant` | + +With a `--role` filter, `--before`/`--after` count filtered messages; +the anchor message is always included. Responses report the window's +`first_ordinal`/`last_ordinal` so callers can continue paging with +`--from `. + ```json { "messages": [ @@ -519,6 +537,7 @@ agentsview session search [flags] "session_id": "abc-123", "project": "myapp", "ordinal": 17, + "ordinal_range": [12, 24], "location": "tool_result", "tool_name": "Bash", "snippet": "...connecting to db with token ***REDACTED***..." @@ -535,6 +554,10 @@ default; opt back in with `--include-one-shot`, |-----------------------|---------------------|--------------------------------------------------------| | `--regex` | `mode=regex` | Treat pattern as an RE2 regex | | `--fts` | `mode=fts` | Tokenized FTS5 search; messages-only | +| `--semantic` | `mode=semantic` | Vector search over user/assistant messages; messages-only — see [Semantic Search](/semantic-search/) | +| `--hybrid` | `mode=hybrid` | Semantic + FTS reciprocal rank fusion; messages-only — see [Semantic Search](/semantic-search/) | +| `--scope` | `scope` | `top`, `all` (default), or `subordinate` — semantic/hybrid only; supersedes `include_children` in those modes | +| `--context` | `context` | int — N messages of context before/after each match (max 10) | | `--in` | `in` | Comma-separated: `messages,tool_input,tool_result` (default all) | | `--exclude-system` | `exclude_system` | Drop system messages from the scan | | `--reveal` | `reveal` | Show full secret values (localhost-only; warning to stderr) | @@ -547,17 +570,38 @@ default; opt back in with `--include-one-shot`, | `--date-from` | `date_from` | `YYYY-MM-DD` | | `--date-to` | `date_to` | `YYYY-MM-DD` | | `--active-since` | `active_since` | RFC3339 timestamp | +| `--since` | `active_since` | Relative — `Nh` hours, `Nd` days, `Nw` weeks, `Nm` calendar months (not minutes), `Ny` years — or `YYYY-MM-DD`; resolved against now and mutually exclusive with `--active-since` | | `--include-children` | `include_children` | bool | | `--include-automated` | `include_automated` | bool | | `--include-one-shot` | `include_one_shot` | bool | | `--limit` | `limit` | int; default 50, max 500 | | `--cursor` | `cursor` | int — pagination cursor from a previous response | -`--regex` and `--fts` are mutually exclusive. `--fts` is the -fastest mode on large archives but only searches message bodies; -substring (the default) and regex modes also walk -`tool_calls.input_json`, `tool_calls.result_content`, and the -`tool_result_events` rows. +`--regex`, `--fts`, `--semantic`, and `--hybrid` are mutually +exclusive. `--fts` is the fastest mode on large archives but only +searches message bodies; substring (the default) and regex modes +also walk `tool_calls.input_json`, `tool_calls.result_content`, +and the `tool_result_events` rows. `--semantic` and `--hybrid` +require an embedding index and return a single ranked page +(`--cursor` is rejected) — see [Semantic Search](/semantic-search/) +for setup, scoring, and limitations. + +Every match, in every mode, carries the conversation-unit +citation described in +[Hit shape](/semantic-search/#hit-shape-ranges-and-anchors): +`ordinal_range` — `[start, end]` of the conversation unit +containing the match, always present, `[ordinal, ordinal]` when +the match is its own unit — plus the lineage fields +`subordinate`, `relationship`, `parent_session_id`, and +`is_sidechain`. `ordinal` stays the anchor (the exact matched +message) in every mode. Only the lineage fields are `omitempty`: +a missing key unambiguously means top-level with no lineage, +while `ordinal_range` is never omitted, even at `[0, 0]`. +`score` is the one field only `--semantic`/`--hybrid` emit. +`--scope` is rejected outside `--semantic`/`--hybrid`; in those +modes it supersedes `--include-children`, and +subagent/fork-typed or parent-linked sessions are exempt from +the default one-shot exclusion. Snippets carry ~60 characters of context on each side of the match, snapped to rune boundaries. Any substring that matches diff --git a/docs/zensical.toml b/docs/zensical.toml index a00f553f1..f1a8ff160 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -25,6 +25,8 @@ nav = [ {"Session Export" = "session-export.md"}, {"Stats" = "stats.md"}, {"Session API" = "session-api.md"}, + {"Semantic Search" = "semantic-search.md"}, + {"Semantic Search Internals" = "semantic-search-internals.md"}, {"Configuration" = "configuration.md"}, {"Remote Access" = "remote-access.md"}, {"PostgreSQL Sync" = "pg-sync.md"}, diff --git a/frontend/src/lib/api/generated/index.ts b/frontend/src/lib/api/generated/index.ts index cd18c0142..13189439f 100644 --- a/frontend/src/lib/api/generated/index.ts +++ b/frontend/src/lib/api/generated/index.ts @@ -18,6 +18,7 @@ export type { AgentsResponse } from './models/AgentsResponse'; export type { AgentTotal } from './models/AgentTotal'; export type { ApiErrorResponse } from './models/ApiErrorResponse'; export type { ApplyWorktreeMappingsResponse } from './models/ApplyWorktreeMappingsResponse'; +export type { BatchDeleteInputBody } from './models/BatchDeleteInputBody'; export type { BranchesResponse } from './models/BranchesResponse'; export type { BulkStarInputBody } from './models/BulkStarInputBody'; export type { CacheStats } from './models/CacheStats'; @@ -33,11 +34,17 @@ export type { DbAgentInfo } from './models/DbAgentInfo'; export type { DbAgentSummary } from './models/DbAgentSummary'; export type { DbAnalyticsSummary } from './models/DbAnalyticsSummary'; export type { DbBranchInfo } from './models/DbBranchInfo'; +export type { DbCacheHitRatioDistribution } from './models/DbCacheHitRatioDistribution'; export type { DbCallTiming } from './models/DbCallTiming'; export type { DbCategoryTotal } from './models/DbCategoryTotal'; +export type { DbCodeAttribution } from './models/DbCodeAttribution'; +export type { DbCodeAttributionSource } from './models/DbCodeAttributionSource'; export type { DbContentMatch } from './models/DbContentMatch'; +export type { DbCursorAttributionMetrics } from './models/DbCursorAttributionMetrics'; +export type { DbCursorConversationCount } from './models/DbCursorConversationCount'; export type { DbDailyUsageEntry } from './models/DbDailyUsageEntry'; export type { DbDistributionBucket } from './models/DbDistributionBucket'; +export type { DbDistributionBucketV1 } from './models/DbDistributionBucketV1'; export type { DbHeatmapEntry } from './models/DbHeatmapEntry'; export type { DbHeatmapLevels } from './models/DbHeatmapLevels'; export type { DbHeatmapResponse } from './models/DbHeatmapResponse'; @@ -46,6 +53,7 @@ export type { DbHourOfWeekResponse } from './models/DbHourOfWeekResponse'; export type { DbInsight } from './models/DbInsight'; export type { DbMessage } from './models/DbMessage'; export type { DbModelBreakdown } from './models/DbModelBreakdown'; +export type { DbPeakContextDistribution } from './models/DbPeakContextDistribution'; export type { DbPercentiles } from './models/DbPercentiles'; export type { DbPinnedMessage } from './models/DbPinnedMessage'; export type { DbProjectAnalytics } from './models/DbProjectAnalytics'; @@ -57,12 +65,15 @@ export type { DbQualitySignalTotals } from './models/DbQualitySignalTotals'; export type { DbRecentEdit } from './models/DbRecentEdit'; export type { DbRecentEditFile } from './models/DbRecentEditFile'; export type { DbRecentEditsResult } from './models/DbRecentEditsResult'; +export type { DbScopedDistribution } from './models/DbScopedDistribution'; +export type { DbScopedDistributionPair } from './models/DbScopedDistributionPair'; export type { DbSearchResult } from './models/DbSearchResult'; export type { DbSecretFindingRow } from './models/DbSecretFindingRow'; export type { DbSession } from './models/DbSession'; export type { DbSessionActivityBucket } from './models/DbSessionActivityBucket'; export type { DbSessionActivityResponse } from './models/DbSessionActivityResponse'; export type { DbSessionShapeResponse } from './models/DbSessionShapeResponse'; +export type { DbSessionStats } from './models/DbSessionStats'; export type { DbSessionTiming } from './models/DbSessionTiming'; export type { DbSidebarSessionIndex } from './models/DbSidebarSessionIndex'; export type { DbSidebarSessionIndexRow } from './models/DbSidebarSessionIndexRow'; @@ -82,6 +93,22 @@ export type { DbSkillsAnalyticsResponse } from './models/DbSkillsAnalyticsRespon export type { DbSkillTrendEntry } from './models/DbSkillTrendEntry'; export type { DbSkillUsage } from './models/DbSkillUsage'; export type { DbStats } from './models/DbStats'; +export type { DbStatsAdoption } from './models/DbStatsAdoption'; +export type { DbStatsAgentPortfolio } from './models/DbStatsAgentPortfolio'; +export type { DbStatsArchetypes } from './models/DbStatsArchetypes'; +export type { DbStatsCacheEconomics } from './models/DbStatsCacheEconomics'; +export type { DbStatsDistributions } from './models/DbStatsDistributions'; +export type { DbStatsFilters } from './models/DbStatsFilters'; +export type { DbStatsModelMix } from './models/DbStatsModelMix'; +export type { DbStatsOutcomes } from './models/DbStatsOutcomes'; +export type { DbStatsOutcomeStats } from './models/DbStatsOutcomeStats'; +export type { DbStatsPercentiles } from './models/DbStatsPercentiles'; +export type { DbStatsTemporal } from './models/DbStatsTemporal'; +export type { DbStatsToolMix } from './models/DbStatsToolMix'; +export type { DbStatsTotals } from './models/DbStatsTotals'; +export type { DbStatsVelocity } from './models/DbStatsVelocity'; +export type { DbStatsWindow } from './models/DbStatsWindow'; +export type { DbTemporalHourlyUTCEntry } from './models/DbTemporalHourlyUTCEntry'; export type { DbToolAgentBreakdown } from './models/DbToolAgentBreakdown'; export type { DbToolCall } from './models/DbToolCall'; export type { DbToolCategoryCount } from './models/DbToolCategoryCount'; @@ -102,8 +129,19 @@ export type { DbVelocityBreakdown } from './models/DbVelocityBreakdown'; export type { DbVelocityOverview } from './models/DbVelocityOverview'; export type { DbVelocityResponse } from './models/DbVelocityResponse'; export type { DbWorktreeProjectMapping } from './models/DbWorktreeProjectMapping'; +export type { DuckdbPushDiagnostics } from './models/DuckdbPushDiagnostics'; export type { DuckdbPushResult } from './models/DuckdbPushResult'; +export type { DuckdbPushSessionCounts } from './models/DuckdbPushSessionCounts'; +export type { EmbeddingsBuildResponse } from './models/EmbeddingsBuildResponse'; +export type { EmbeddingsGenerationActionRequest } from './models/EmbeddingsGenerationActionRequest'; +export type { EmbeddingsGenerationsResponse } from './models/EmbeddingsGenerationsResponse'; export type { EmptyTrashResponse } from './models/EmptyTrashResponse'; +export type { ExportEffectiveModelRate } from './models/ExportEffectiveModelRate'; +export type { ExportPricingBlock } from './models/ExportPricingBlock'; +export type { ExportPricingFallback } from './models/ExportPricingFallback'; +export type { ExportProjectIdentity } from './models/ExportProjectIdentity'; +export type { ExportProjectMapEntry } from './models/ExportProjectMapEntry'; +export type { FillStats } from './models/FillStats'; export type { FormFile } from './models/FormFile'; export type { GenerateInsightRequest } from './models/GenerateInsightRequest'; export type { GithubConfigResponse } from './models/GithubConfigResponse'; @@ -125,7 +163,9 @@ export type { ProjectsResponse } from './models/ProjectsResponse'; export type { ProjectTotal } from './models/ProjectTotal'; export type { PublishResponse } from './models/PublishResponse'; export type { RemoteSyncRequest } from './models/RemoteSyncRequest'; +export type { RemotesyncTargetSet } from './models/RemotesyncTargetSet'; export type { RenameRequest } from './models/RenameRequest'; +export type { ResolveSessionIDsResponse } from './models/ResolveSessionIDsResponse'; export type { ResumeRequest } from './models/ResumeRequest'; export type { ResumeResponse } from './models/ResumeResponse'; export type { SearchResponse } from './models/SearchResponse'; @@ -147,7 +187,9 @@ export type { SetGithubConfigResponse } from './models/SetGithubConfigResponse'; export type { SettingsResponse } from './models/SettingsResponse'; export type { SettingsUpdateRequest } from './models/SettingsUpdateRequest'; export type { StarredResponse } from './models/StarredResponse'; +export type { SyncAnomalyStats } from './models/SyncAnomalyStats'; export type { SyncProgress } from './models/SyncProgress'; +export type { SyncSanitizeStats } from './models/SyncSanitizeStats'; export type { SyncStatusResponse } from './models/SyncStatusResponse'; export type { SyncSyncStats } from './models/SyncSyncStats'; export { TerminalConfigBody } from './models/TerminalConfigBody'; @@ -157,6 +199,11 @@ export type { UnsupportedUsage } from './models/UnsupportedUsage'; export type { UpdateCheckResponse } from './models/UpdateCheckResponse'; export type { UploadSessionResponse } from './models/UploadSessionResponse'; export type { UsageSummaryResponse } from './models/UsageSummaryResponse'; +export type { VectorBuildRequest } from './models/VectorBuildRequest'; +export type { VectorBuildResult } from './models/VectorBuildResult'; +export type { VectorBuildStatus } from './models/VectorBuildStatus'; +export type { VectorGenerationInfo } from './models/VectorGenerationInfo'; +export type { VectorRefreshStats } from './models/VectorRefreshStats'; export type { VersionInfo } from './models/VersionInfo'; export type { WorktreeMappingRequest } from './models/WorktreeMappingRequest'; export type { WorktreeMappingsResponse } from './models/WorktreeMappingsResponse'; @@ -165,6 +212,7 @@ export { ActivityService } from './services/ActivityService'; export { AnalyticsService } from './services/AnalyticsService'; export { AssetsService } from './services/AssetsService'; export { ConfigService } from './services/ConfigService'; +export { EmbeddingsService } from './services/EmbeddingsService'; export { HealthService } from './services/HealthService'; export { ImportService } from './services/ImportService'; export { InsightsService } from './services/InsightsService'; @@ -173,6 +221,7 @@ export { OpenersService } from './services/OpenersService'; export { PinsService } from './services/PinsService'; export { PushService } from './services/PushService'; export { RecentEditsService } from './services/RecentEditsService'; +export { RemoteSyncService } from './services/RemoteSyncService'; export { SearchService } from './services/SearchService'; export { SecretsService } from './services/SecretsService'; export { SessionsService } from './services/SessionsService'; diff --git a/frontend/src/lib/api/generated/models/ActivityReport.ts b/frontend/src/lib/api/generated/models/ActivityReport.ts index fd1923026..dc3a76878 100644 --- a/frontend/src/lib/api/generated/models/ActivityReport.ts +++ b/frontend/src/lib/api/generated/models/ActivityReport.ts @@ -4,6 +4,8 @@ /* eslint-disable */ import type { ActivityPeak } from './ActivityPeak'; import type { ActivityTotals } from './ActivityTotals'; +import type { ExportPricingBlock } from './ExportPricingBlock'; +import type { ExportProjectMapEntry } from './ExportProjectMapEntry'; export type ActivityReport = { as_of: string | null; bucket_count: number; @@ -19,8 +21,11 @@ export type ActivityReport = { intervals: any[] | null; partial: boolean; peak: ActivityPeak; + pricing?: ExportPricingBlock; + projects: Record; range_end: string; range_start: string; + schema_version?: number; timezone: string; totals: ActivityTotals; }; diff --git a/frontend/src/lib/api/generated/models/BatchDeleteInputBody.ts b/frontend/src/lib/api/generated/models/BatchDeleteInputBody.ts index bc9a24774..679b0c29c 100644 --- a/frontend/src/lib/api/generated/models/BatchDeleteInputBody.ts +++ b/frontend/src/lib/api/generated/models/BatchDeleteInputBody.ts @@ -8,3 +8,4 @@ export type BatchDeleteInputBody = { */ session_ids: Array; }; + diff --git a/frontend/src/lib/api/generated/models/ConfigRemoteHost.ts b/frontend/src/lib/api/generated/models/ConfigRemoteHost.ts index 510464957..f6d0e8569 100644 --- a/frontend/src/lib/api/generated/models/ConfigRemoteHost.ts +++ b/frontend/src/lib/api/generated/models/ConfigRemoteHost.ts @@ -6,6 +6,8 @@ export type ConfigRemoteHost = { host: string; interval?: number; port?: number; + transport?: string; + url?: string; user?: string; }; diff --git a/frontend/src/lib/api/generated/models/DbCacheHitRatioDistribution.ts b/frontend/src/lib/api/generated/models/DbCacheHitRatioDistribution.ts new file mode 100644 index 000000000..e5035e902 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbCacheHitRatioDistribution.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbCacheHitRatioDistribution = { + buckets: any[] | null; + overall: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbCodeAttribution.ts b/frontend/src/lib/api/generated/models/DbCodeAttribution.ts new file mode 100644 index 000000000..441662682 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbCodeAttribution.ts @@ -0,0 +1,8 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbCodeAttribution = { + sources?: any[] | null; +}; + diff --git a/frontend/src/lib/api/generated/models/DbCodeAttributionSource.ts b/frontend/src/lib/api/generated/models/DbCodeAttributionSource.ts new file mode 100644 index 000000000..9f2812e27 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbCodeAttributionSource.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbCursorAttributionMetrics } from './DbCursorAttributionMetrics'; +export type DbCodeAttributionSource = { + metrics?: DbCursorAttributionMetrics; + provider: string; + scope: string; + status: string; + warnings?: any[] | null; +}; + diff --git a/frontend/src/lib/api/generated/models/DbContentMatch.ts b/frontend/src/lib/api/generated/models/DbContentMatch.ts index 024cf19fa..7d0b437e2 100644 --- a/frontend/src/lib/api/generated/models/DbContentMatch.ts +++ b/frontend/src/lib/api/generated/models/DbContentMatch.ts @@ -4,12 +4,20 @@ /* eslint-disable */ export type DbContentMatch = { agent: string; + context_after?: any[] | null; + context_before?: any[] | null; + is_sidechain?: boolean; location: string; ordinal: number; + ordinal_range: any[] | null; + parent_session_id?: string; project: string; + relationship?: string; role: string; + score?: number; session_id: string; snippet: string; + subordinate?: boolean; timestamp: string; tool_name?: string; }; diff --git a/frontend/src/lib/api/generated/models/DbCursorAttributionMetrics.ts b/frontend/src/lib/api/generated/models/DbCursorAttributionMetrics.ts new file mode 100644 index 000000000..015bec702 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbCursorAttributionMetrics.ts @@ -0,0 +1,20 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbCursorAttributionMetrics = { + ai_authored_pct: number; + blank_lines_added: number; + blank_lines_deleted: number; + composer_lines_added: number; + composer_lines_deleted: number; + conversation_counts?: any[] | null; + human_lines_added: number; + human_lines_deleted: number; + lines_added: number; + lines_deleted: number; + scored_commits: number; + tab_lines_added: number; + tab_lines_deleted: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbCursorConversationCount.ts b/frontend/src/lib/api/generated/models/DbCursorConversationCount.ts new file mode 100644 index 000000000..7852ef680 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbCursorConversationCount.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbCursorConversationCount = { + count: number; + mode: string; + model: string; +}; + diff --git a/frontend/src/lib/api/generated/models/DbDailyUsageEntry.ts b/frontend/src/lib/api/generated/models/DbDailyUsageEntry.ts index 72bbf937e..6204debf2 100644 --- a/frontend/src/lib/api/generated/models/DbDailyUsageEntry.ts +++ b/frontend/src/lib/api/generated/models/DbDailyUsageEntry.ts @@ -3,15 +3,15 @@ /* tslint:disable */ /* eslint-disable */ export type DbDailyUsageEntry = { - agentBreakdowns?: any[] | null; + agentBreakdowns: any[] | null; cacheCreationTokens: number; cacheReadTokens: number; date: string; inputTokens: number; - modelBreakdowns?: any[] | null; + modelBreakdowns: any[] | null; modelsUsed: any[] | null; outputTokens: number; - projectBreakdowns?: any[] | null; + projectBreakdowns: any[] | null; totalCost: number; }; diff --git a/frontend/src/lib/api/generated/models/DbDistributionBucketV1.ts b/frontend/src/lib/api/generated/models/DbDistributionBucketV1.ts new file mode 100644 index 000000000..db94c4af2 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbDistributionBucketV1.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbDistributionBucketV1 = { + count: number; + edge: any[] | null; +}; + diff --git a/frontend/src/lib/api/generated/models/DbPeakContextDistribution.ts b/frontend/src/lib/api/generated/models/DbPeakContextDistribution.ts new file mode 100644 index 000000000..47d3ef397 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbPeakContextDistribution.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbScopedDistribution } from './DbScopedDistribution'; +export type DbPeakContextDistribution = { + claude_only: boolean; + null_count: number; + scope_all: DbScopedDistribution; + scope_human: DbScopedDistribution; +}; + diff --git a/frontend/src/lib/api/generated/models/DbScopedDistribution.ts b/frontend/src/lib/api/generated/models/DbScopedDistribution.ts new file mode 100644 index 000000000..5306e3b6f --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbScopedDistribution.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbScopedDistribution = { + buckets: any[] | null; + mean: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbScopedDistributionPair.ts b/frontend/src/lib/api/generated/models/DbScopedDistributionPair.ts new file mode 100644 index 000000000..3dfde1228 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbScopedDistributionPair.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbScopedDistribution } from './DbScopedDistribution'; +export type DbScopedDistributionPair = { + scope_all: DbScopedDistribution; + scope_human: DbScopedDistribution; +}; + diff --git a/frontend/src/lib/api/generated/models/DbSessionStats.ts b/frontend/src/lib/api/generated/models/DbSessionStats.ts new file mode 100644 index 000000000..7904bd64c --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbSessionStats.ts @@ -0,0 +1,39 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbCodeAttribution } from './DbCodeAttribution'; +import type { DbStatsAdoption } from './DbStatsAdoption'; +import type { DbStatsAgentPortfolio } from './DbStatsAgentPortfolio'; +import type { DbStatsArchetypes } from './DbStatsArchetypes'; +import type { DbStatsCacheEconomics } from './DbStatsCacheEconomics'; +import type { DbStatsDistributions } from './DbStatsDistributions'; +import type { DbStatsFilters } from './DbStatsFilters'; +import type { DbStatsModelMix } from './DbStatsModelMix'; +import type { DbStatsOutcomes } from './DbStatsOutcomes'; +import type { DbStatsOutcomeStats } from './DbStatsOutcomeStats'; +import type { DbStatsTemporal } from './DbStatsTemporal'; +import type { DbStatsToolMix } from './DbStatsToolMix'; +import type { DbStatsTotals } from './DbStatsTotals'; +import type { DbStatsVelocity } from './DbStatsVelocity'; +import type { DbStatsWindow } from './DbStatsWindow'; +export type DbSessionStats = { + adoption?: DbStatsAdoption; + agent_portfolio: DbStatsAgentPortfolio; + archetypes: DbStatsArchetypes; + cache_economics?: DbStatsCacheEconomics; + code_attribution?: DbCodeAttribution; + distributions: DbStatsDistributions; + filters: DbStatsFilters; + generated_at: string; + model_mix: DbStatsModelMix; + outcome_stats?: DbStatsOutcomeStats; + outcomes?: DbStatsOutcomes; + schema_version: number; + temporal: DbStatsTemporal; + tool_mix: DbStatsToolMix; + totals: DbStatsTotals; + velocity: DbStatsVelocity; + window: DbStatsWindow; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsAdoption.ts b/frontend/src/lib/api/generated/models/DbStatsAdoption.ts new file mode 100644 index 000000000..752ae43cf --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsAdoption.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsAdoption = { + claude_only: boolean; + distinct_skills: number; + plan_mode_rate: number; + subagents_per_session: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsAgentPortfolio.ts b/frontend/src/lib/api/generated/models/DbStatsAgentPortfolio.ts new file mode 100644 index 000000000..89eef1929 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsAgentPortfolio.ts @@ -0,0 +1,15 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsAgentPortfolio = { + by_messages: Record; + by_messages_human: Record; + by_sessions: Record; + by_sessions_human: Record; + by_tokens: Record; + by_tokens_human: Record; + primary: string; + primary_human: string; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsArchetypes.ts b/frontend/src/lib/api/generated/models/DbStatsArchetypes.ts new file mode 100644 index 000000000..6a6254c16 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsArchetypes.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsArchetypes = { + automation: number; + deep: number; + marathon: number; + primary: string; + primary_human: string; + quick: number; + standard: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsCacheEconomics.ts b/frontend/src/lib/api/generated/models/DbStatsCacheEconomics.ts new file mode 100644 index 000000000..5db2f6a0e --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsCacheEconomics.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbCacheHitRatioDistribution } from './DbCacheHitRatioDistribution'; +export type DbStatsCacheEconomics = { + cache_hit_ratio: DbCacheHitRatioDistribution; + claude_only: boolean; + dollars_saved_vs_uncached: number; + dollars_spent: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsDistributions.ts b/frontend/src/lib/api/generated/models/DbStatsDistributions.ts new file mode 100644 index 000000000..0cbbd3735 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsDistributions.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbPeakContextDistribution } from './DbPeakContextDistribution'; +import type { DbScopedDistributionPair } from './DbScopedDistributionPair'; +export type DbStatsDistributions = { + duration_minutes: DbScopedDistributionPair; + peak_context_tokens: DbPeakContextDistribution; + tools_per_turn: DbScopedDistributionPair; + user_messages: DbScopedDistributionPair; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsFilters.ts b/frontend/src/lib/api/generated/models/DbStatsFilters.ts new file mode 100644 index 000000000..85c7295ff --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsFilters.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsFilters = { + agent: string; + projects_excluded: any[] | null; + projects_included?: any[] | null; + timezone: string; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsModelMix.ts b/frontend/src/lib/api/generated/models/DbStatsModelMix.ts new file mode 100644 index 000000000..63a104900 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsModelMix.ts @@ -0,0 +1,8 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsModelMix = { + by_tokens: Record; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsOutcomeStats.ts b/frontend/src/lib/api/generated/models/DbStatsOutcomeStats.ts new file mode 100644 index 000000000..97df9125b --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsOutcomeStats.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsOutcomeStats = { + commits: number; + files_changed: number; + loc_added: number; + loc_removed: number; + prs_merged?: number; + prs_opened?: number; + repos_active: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsOutcomes.ts b/frontend/src/lib/api/generated/models/DbStatsOutcomes.ts new file mode 100644 index 000000000..4bc2e5430 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsOutcomes.ts @@ -0,0 +1,15 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsOutcomes = { + avg_edit_churn: number; + claude_only: boolean; + compactions_per_session: number; + failure: number; + grade_distribution: Record; + success: number; + tool_retry_rate: number; + unknown: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsPercentiles.ts b/frontend/src/lib/api/generated/models/DbStatsPercentiles.ts new file mode 100644 index 000000000..69bfda25e --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsPercentiles.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsPercentiles = { + mean: number; + p50: number; + p90: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsTemporal.ts b/frontend/src/lib/api/generated/models/DbStatsTemporal.ts new file mode 100644 index 000000000..efece42ce --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsTemporal.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsTemporal = { + hourly_utc: any[] | null; + reporter_timezone: string; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsToolMix.ts b/frontend/src/lib/api/generated/models/DbStatsToolMix.ts new file mode 100644 index 000000000..863fa6e2e --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsToolMix.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsToolMix = { + by_category: Record; + total_calls: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsTotals.ts b/frontend/src/lib/api/generated/models/DbStatsTotals.ts new file mode 100644 index 000000000..fc44ef79d --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsTotals.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsTotals = { + messages_total: number; + sessions_all: number; + sessions_automation: number; + sessions_human: number; + sessions_subagent: number; + user_messages_total: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsVelocity.ts b/frontend/src/lib/api/generated/models/DbStatsVelocity.ts new file mode 100644 index 000000000..0b12a1e8c --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsVelocity.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DbStatsPercentiles } from './DbStatsPercentiles'; +export type DbStatsVelocity = { + first_response_seconds: DbStatsPercentiles; + messages_per_active_hour: number; + turn_cycle_seconds: DbStatsPercentiles; +}; + diff --git a/frontend/src/lib/api/generated/models/DbStatsWindow.ts b/frontend/src/lib/api/generated/models/DbStatsWindow.ts new file mode 100644 index 000000000..a16368a15 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbStatsWindow.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbStatsWindow = { + days: number; + since: string; + until: string; +}; + diff --git a/frontend/src/lib/api/generated/models/DbTemporalHourlyUTCEntry.ts b/frontend/src/lib/api/generated/models/DbTemporalHourlyUTCEntry.ts new file mode 100644 index 000000000..eacf1bb47 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DbTemporalHourlyUTCEntry.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DbTemporalHourlyUTCEntry = { + sessions: number; + ts: string; + user_messages: number; +}; + diff --git a/frontend/src/lib/api/generated/models/DbTopSession.ts b/frontend/src/lib/api/generated/models/DbTopSession.ts index 2c23f350b..11808f97d 100644 --- a/frontend/src/lib/api/generated/models/DbTopSession.ts +++ b/frontend/src/lib/api/generated/models/DbTopSession.ts @@ -3,9 +3,9 @@ /* tslint:disable */ /* eslint-disable */ export type DbTopSession = { + active_duration_min: number; display_name?: string; duration_min: number; - active_duration_min: number; ended_at?: string; first_message: string | null; id: string; diff --git a/frontend/src/lib/api/generated/models/DuckdbPushDiagnostics.ts b/frontend/src/lib/api/generated/models/DuckdbPushDiagnostics.ts new file mode 100644 index 000000000..0eb0821f9 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DuckdbPushDiagnostics.ts @@ -0,0 +1,16 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { DuckdbPushSessionCounts } from './DuckdbPushSessionCounts'; +export type DuckdbPushDiagnostics = { + CandidateSessions: DuckdbPushSessionCounts; + Cutoff: string; + DeletedStaleSessions: number; + Full: boolean; + LastPushAt: string; + LocalSessions: DuckdbPushSessionCounts; + PushedSessions: DuckdbPushSessionCounts; + SkippedUnchangedSessions: DuckdbPushSessionCounts; +}; + diff --git a/frontend/src/lib/api/generated/models/DuckdbPushResult.ts b/frontend/src/lib/api/generated/models/DuckdbPushResult.ts index ed04ac91d..5c4cf1b6a 100644 --- a/frontend/src/lib/api/generated/models/DuckdbPushResult.ts +++ b/frontend/src/lib/api/generated/models/DuckdbPushResult.ts @@ -2,7 +2,9 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ +import type { DuckdbPushDiagnostics } from './DuckdbPushDiagnostics'; export type DuckdbPushResult = { + Diagnostics: DuckdbPushDiagnostics; Duration: number; Errors: number; MessagesPushed: number; diff --git a/frontend/src/lib/api/generated/models/DuckdbPushSessionCounts.ts b/frontend/src/lib/api/generated/models/DuckdbPushSessionCounts.ts new file mode 100644 index 000000000..dc470a262 --- /dev/null +++ b/frontend/src/lib/api/generated/models/DuckdbPushSessionCounts.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type DuckdbPushSessionCounts = { + ByAgent: Record; + Total: number; +}; + diff --git a/frontend/src/lib/api/generated/models/EmbeddingsBuildResponse.ts b/frontend/src/lib/api/generated/models/EmbeddingsBuildResponse.ts new file mode 100644 index 000000000..1a8f08e1d --- /dev/null +++ b/frontend/src/lib/api/generated/models/EmbeddingsBuildResponse.ts @@ -0,0 +1,8 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EmbeddingsBuildResponse = { + started: boolean; +}; + diff --git a/frontend/src/lib/api/generated/models/EmbeddingsGenerationActionRequest.ts b/frontend/src/lib/api/generated/models/EmbeddingsGenerationActionRequest.ts new file mode 100644 index 000000000..0a6ab6cdf --- /dev/null +++ b/frontend/src/lib/api/generated/models/EmbeddingsGenerationActionRequest.ts @@ -0,0 +1,8 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EmbeddingsGenerationActionRequest = { + force?: boolean; +}; + diff --git a/frontend/src/lib/api/generated/models/EmbeddingsGenerationsResponse.ts b/frontend/src/lib/api/generated/models/EmbeddingsGenerationsResponse.ts new file mode 100644 index 000000000..9d0fa2a28 --- /dev/null +++ b/frontend/src/lib/api/generated/models/EmbeddingsGenerationsResponse.ts @@ -0,0 +1,8 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type EmbeddingsGenerationsResponse = { + generations: any[] | null; +}; + diff --git a/frontend/src/lib/api/generated/models/ExportEffectiveModelRate.ts b/frontend/src/lib/api/generated/models/ExportEffectiveModelRate.ts new file mode 100644 index 000000000..8a4cafbdf --- /dev/null +++ b/frontend/src/lib/api/generated/models/ExportEffectiveModelRate.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ExportEffectiveModelRate = { + cache_read_cost_per_mtok: number; + cache_write_cost_per_mtok: number; + cost_source: string; + input_cost_per_mtok: number; + matched_pattern: string | null; + output_cost_per_mtok: number; +}; + diff --git a/frontend/src/lib/api/generated/models/ExportPricingBlock.ts b/frontend/src/lib/api/generated/models/ExportPricingBlock.ts new file mode 100644 index 000000000..ee5d06305 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ExportPricingBlock.ts @@ -0,0 +1,18 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { ExportEffectiveModelRate } from './ExportEffectiveModelRate'; +import type { ExportPricingFallback } from './ExportPricingFallback'; +export type ExportPricingBlock = { + cost_source: string; + custom_override_count: number; + digest: string; + effective_row_count: number; + fallback: ExportPricingFallback; + latest_row_updated_at: string | null; + models: Record; + source: string; + table_version: string; +}; + diff --git a/frontend/src/lib/api/generated/models/ExportPricingFallback.ts b/frontend/src/lib/api/generated/models/ExportPricingFallback.ts new file mode 100644 index 000000000..aa4f52e37 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ExportPricingFallback.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ExportPricingFallback = { + models: any[] | null; + used: boolean; +}; + diff --git a/frontend/src/lib/api/generated/models/ExportProjectIdentity.ts b/frontend/src/lib/api/generated/models/ExportProjectIdentity.ts new file mode 100644 index 000000000..7776ef374 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ExportProjectIdentity.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ExportProjectIdentity = { + key: string; + key_source: string; + machine_local?: boolean; + normalized_remote?: string; + root_path?: string; +}; + diff --git a/frontend/src/lib/api/generated/models/ExportProjectMapEntry.ts b/frontend/src/lib/api/generated/models/ExportProjectMapEntry.ts new file mode 100644 index 000000000..ee9409b2f --- /dev/null +++ b/frontend/src/lib/api/generated/models/ExportProjectMapEntry.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { ExportProjectIdentity } from './ExportProjectIdentity'; +export type ExportProjectMapEntry = { + identity: ExportProjectIdentity; + resolution: string; +}; + diff --git a/frontend/src/lib/api/generated/models/FillStats.ts b/frontend/src/lib/api/generated/models/FillStats.ts new file mode 100644 index 000000000..b68e7839c --- /dev/null +++ b/frontend/src/lib/api/generated/models/FillStats.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type FillStats = { + Chunks: number; + Documents: number; + Skipped: number; + Stale: number; +}; + diff --git a/frontend/src/lib/api/generated/models/RemotesyncTargetSet.ts b/frontend/src/lib/api/generated/models/RemotesyncTargetSet.ts new file mode 100644 index 000000000..52f50a6bf --- /dev/null +++ b/frontend/src/lib/api/generated/models/RemotesyncTargetSet.ts @@ -0,0 +1,9 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type RemotesyncTargetSet = { + dirs: Record; + extra_files?: any[] | null; +}; + diff --git a/frontend/src/lib/api/generated/models/ResolveSessionIDsResponse.ts b/frontend/src/lib/api/generated/models/ResolveSessionIDsResponse.ts new file mode 100644 index 000000000..17e798298 --- /dev/null +++ b/frontend/src/lib/api/generated/models/ResolveSessionIDsResponse.ts @@ -0,0 +1,8 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type ResolveSessionIDsResponse = { + ids: any[] | null; +}; + diff --git a/frontend/src/lib/api/generated/models/ServiceMessageList.ts b/frontend/src/lib/api/generated/models/ServiceMessageList.ts index 66da8b225..062ba4637 100644 --- a/frontend/src/lib/api/generated/models/ServiceMessageList.ts +++ b/frontend/src/lib/api/generated/models/ServiceMessageList.ts @@ -4,6 +4,8 @@ /* eslint-disable */ export type ServiceMessageList = { count: number; + first_ordinal?: number; + last_ordinal?: number; messages: any[] | null; }; diff --git a/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonDelta.ts b/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonDelta.ts index 0240aef16..bec5a2d04 100644 --- a/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonDelta.ts +++ b/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonDelta.ts @@ -22,3 +22,4 @@ export type ServiceUsagePairwiseComparisonDelta = { totalTokensDelta: number; totalTokensDeltaRatio: number | null; }; + diff --git a/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonResponse.ts b/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonResponse.ts index 7e4e0e5af..547a60532 100644 --- a/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonResponse.ts +++ b/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonResponse.ts @@ -9,3 +9,4 @@ export type ServiceUsagePairwiseComparisonResponse = { left: ServiceUsagePairwiseComparisonSide; right: ServiceUsagePairwiseComparisonSide; }; + diff --git a/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonSide.ts b/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonSide.ts index 0f8f746c7..3c8d28439 100644 --- a/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonSide.ts +++ b/frontend/src/lib/api/generated/models/ServiceUsagePairwiseComparisonSide.ts @@ -13,3 +13,4 @@ export type ServiceUsagePairwiseComparisonSide = { totalCost: number; totalTokens: number; }; + diff --git a/frontend/src/lib/api/generated/models/SyncAnomalyStats.ts b/frontend/src/lib/api/generated/models/SyncAnomalyStats.ts new file mode 100644 index 000000000..986588ab5 --- /dev/null +++ b/frontend/src/lib/api/generated/models/SyncAnomalyStats.ts @@ -0,0 +1,15 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { SyncSanitizeStats } from './SyncSanitizeStats'; +export type SyncAnomalyStats = { + gen_metadata_without_usage_by_agent?: Record; + gen_metadata_without_usage_total?: number; + malformed_lines_by_agent?: Record; + malformed_lines_total?: number; + sanitize?: SyncSanitizeStats; + unknown_schema_sessions_by_agent?: Record; + unknown_schema_sessions_total?: number; +}; + diff --git a/frontend/src/lib/api/generated/models/SyncProgress.ts b/frontend/src/lib/api/generated/models/SyncProgress.ts index 1b199e698..74a325cd9 100644 --- a/frontend/src/lib/api/generated/models/SyncProgress.ts +++ b/frontend/src/lib/api/generated/models/SyncProgress.ts @@ -3,6 +3,8 @@ /* tslint:disable */ /* eslint-disable */ export type SyncProgress = { + bytes_done?: number; + bytes_total?: number; current_project?: string; detail?: string; hint?: string; diff --git a/frontend/src/lib/api/generated/models/SyncSanitizeStats.ts b/frontend/src/lib/api/generated/models/SyncSanitizeStats.ts new file mode 100644 index 000000000..6eea733ba --- /dev/null +++ b/frontend/src/lib/api/generated/models/SyncSanitizeStats.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type SyncSanitizeStats = { + control_chars_stripped?: number; + model_clamped?: number; + role_coerced?: number; + timestamps_blanked?: number; + tokens_clamped?: number; +}; + diff --git a/frontend/src/lib/api/generated/models/SyncSyncStats.ts b/frontend/src/lib/api/generated/models/SyncSyncStats.ts index 594867bad..77aca274c 100644 --- a/frontend/src/lib/api/generated/models/SyncSyncStats.ts +++ b/frontend/src/lib/api/generated/models/SyncSyncStats.ts @@ -2,8 +2,10 @@ /* istanbul ignore file */ /* tslint:disable */ /* eslint-disable */ +import type { SyncAnomalyStats } from './SyncAnomalyStats'; export type SyncSyncStats = { aborted?: boolean; + anomalies?: SyncAnomalyStats; failed: number; orphaned_copied?: number; skipped: number; diff --git a/frontend/src/lib/api/generated/models/UnsupportedUsage.ts b/frontend/src/lib/api/generated/models/UnsupportedUsage.ts index 47778f884..521ae0dc9 100644 --- a/frontend/src/lib/api/generated/models/UnsupportedUsage.ts +++ b/frontend/src/lib/api/generated/models/UnsupportedUsage.ts @@ -5,3 +5,4 @@ export type UnsupportedUsage = { kind: string; }; + diff --git a/frontend/src/lib/api/generated/models/UsageSummaryResponse.ts b/frontend/src/lib/api/generated/models/UsageSummaryResponse.ts index e689974e7..5002f3fc8 100644 --- a/frontend/src/lib/api/generated/models/UsageSummaryResponse.ts +++ b/frontend/src/lib/api/generated/models/UsageSummaryResponse.ts @@ -6,6 +6,8 @@ import type { CacheStats } from './CacheStats'; import type { Comparison } from './Comparison'; import type { DbUsageSessionCounts } from './DbUsageSessionCounts'; import type { DbUsageTotals } from './DbUsageTotals'; +import type { ExportPricingBlock } from './ExportPricingBlock'; +import type { ExportProjectMapEntry } from './ExportProjectMapEntry'; import type { UnsupportedUsage } from './UnsupportedUsage'; export type UsageSummaryResponse = { agentTotals: any[] | null; @@ -14,9 +16,13 @@ export type UsageSummaryResponse = { daily: any[] | null; from: string; modelTotals: any[] | null; + pricing?: ExportPricingBlock; projectTotals: any[] | null; + projects: Record; + schema_version?: number; sessionCounts: DbUsageSessionCounts; to: string; totals: DbUsageTotals; unsupportedUsage?: UnsupportedUsage; }; + diff --git a/frontend/src/lib/api/generated/models/VectorBuildRequest.ts b/frontend/src/lib/api/generated/models/VectorBuildRequest.ts new file mode 100644 index 000000000..1b039ecef --- /dev/null +++ b/frontend/src/lib/api/generated/models/VectorBuildRequest.ts @@ -0,0 +1,11 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type VectorBuildRequest = { + backstop?: boolean; + full_rebuild?: boolean; + include_automated?: boolean; + using?: string; +}; + diff --git a/frontend/src/lib/api/generated/models/VectorBuildResult.ts b/frontend/src/lib/api/generated/models/VectorBuildResult.ts new file mode 100644 index 000000000..3a195e570 --- /dev/null +++ b/frontend/src/lib/api/generated/models/VectorBuildResult.ts @@ -0,0 +1,13 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { FillStats } from './FillStats'; +import type { VectorRefreshStats } from './VectorRefreshStats'; +export type VectorBuildResult = { + Activated: boolean; + Fill: FillStats; + Fingerprint: string; + Refresh: VectorRefreshStats; +}; + diff --git a/frontend/src/lib/api/generated/models/VectorBuildStatus.ts b/frontend/src/lib/api/generated/models/VectorBuildStatus.ts new file mode 100644 index 000000000..1bb65ba21 --- /dev/null +++ b/frontend/src/lib/api/generated/models/VectorBuildStatus.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { VectorBuildResult } from './VectorBuildResult'; +export type VectorBuildStatus = { + done: number; + last_error?: string; + last_result?: VectorBuildResult; + phase?: string; + running: boolean; + total: number; +}; + diff --git a/frontend/src/lib/api/generated/models/VectorGenerationInfo.ts b/frontend/src/lib/api/generated/models/VectorGenerationInfo.ts new file mode 100644 index 000000000..52fa9e59b --- /dev/null +++ b/frontend/src/lib/api/generated/models/VectorGenerationInfo.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type VectorGenerationInfo = { + dimension: number; + embedded: number; + fingerprint: string; + id: number; + missing: number; + model: string; + state: string; +}; + diff --git a/frontend/src/lib/api/generated/models/VectorRefreshStats.ts b/frontend/src/lib/api/generated/models/VectorRefreshStats.ts new file mode 100644 index 000000000..6c74d3c41 --- /dev/null +++ b/frontend/src/lib/api/generated/models/VectorRefreshStats.ts @@ -0,0 +1,10 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +export type VectorRefreshStats = { + Deleted: number; + Unchanged: number; + Upserted: number; +}; + diff --git a/frontend/src/lib/api/generated/models/VersionInfo.ts b/frontend/src/lib/api/generated/models/VersionInfo.ts index c8109c321..7821ed816 100644 --- a/frontend/src/lib/api/generated/models/VersionInfo.ts +++ b/frontend/src/lib/api/generated/models/VersionInfo.ts @@ -11,3 +11,4 @@ export type VersionInfo = { read_only?: boolean; version: string; }; + diff --git a/frontend/src/lib/api/generated/services/AnalyticsService.ts b/frontend/src/lib/api/generated/services/AnalyticsService.ts index b0416c8d8..325db5d07 100644 --- a/frontend/src/lib/api/generated/services/AnalyticsService.ts +++ b/frontend/src/lib/api/generated/services/AnalyticsService.ts @@ -708,7 +708,7 @@ export class AnalyticsService { */ agent?: string, /** - * Filter by model + * Comma-separated model filter */ model?: string, /** diff --git a/frontend/src/lib/api/generated/services/EmbeddingsService.ts b/frontend/src/lib/api/generated/services/EmbeddingsService.ts new file mode 100644 index 000000000..dad95702b --- /dev/null +++ b/frontend/src/lib/api/generated/services/EmbeddingsService.ts @@ -0,0 +1,166 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { EmbeddingsBuildResponse } from '../models/EmbeddingsBuildResponse'; +import type { EmbeddingsGenerationActionRequest } from '../models/EmbeddingsGenerationActionRequest'; +import type { EmbeddingsGenerationsResponse } from '../models/EmbeddingsGenerationsResponse'; +import type { VectorBuildRequest } from '../models/VectorBuildRequest'; +import type { VectorBuildStatus } from '../models/VectorBuildStatus'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { OpenAPI } from '../core/OpenAPI'; +import { request as __request } from '../core/request'; +export class EmbeddingsService { + /** + * Start an embeddings build + * @returns EmbeddingsBuildResponse OK + * @throws ApiError + */ + public static postApiV1EmbeddingsBuild({ + requestBody, + }: { + requestBody: VectorBuildRequest, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/embeddings/build', + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * List embedding generations + * @returns EmbeddingsGenerationsResponse OK + * @throws ApiError + */ + public static getApiV1EmbeddingsGenerations(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/embeddings/generations', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Activate an embedding generation + * @returns void + * @throws ApiError + */ + public static postApiV1EmbeddingsGenerationsIdActivate({ + id, + requestBody, + }: { + /** + * Generation ordinal ID + */ + id: number, + requestBody: EmbeddingsGenerationActionRequest, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/embeddings/generations/{id}/activate', + path: { + 'id': id, + }, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Retire an embedding generation + * @returns void + * @throws ApiError + */ + public static postApiV1EmbeddingsGenerationsIdRetire({ + id, + requestBody, + }: { + /** + * Generation ordinal ID + */ + id: number, + requestBody: EmbeddingsGenerationActionRequest, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/embeddings/generations/{id}/retire', + path: { + 'id': id, + }, + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } + /** + * Embeddings build status + * @returns VectorBuildStatus OK + * @throws ApiError + */ + public static getApiV1EmbeddingsStatus(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/embeddings/status', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } +} diff --git a/frontend/src/lib/api/generated/services/MetadataService.ts b/frontend/src/lib/api/generated/services/MetadataService.ts index 71c9e4976..7350a7650 100644 --- a/frontend/src/lib/api/generated/services/MetadataService.ts +++ b/frontend/src/lib/api/generated/services/MetadataService.ts @@ -4,6 +4,7 @@ /* eslint-disable */ import type { AgentsResponse } from '../models/AgentsResponse'; import type { BranchesResponse } from '../models/BranchesResponse'; +import type { DbSessionStats } from '../models/DbSessionStats'; import type { DbStats } from '../models/DbStats'; import type { MachinesResponse } from '../models/MachinesResponse'; import type { ProjectsResponse } from '../models/ProjectsResponse'; @@ -173,6 +174,82 @@ export class MetadataService { }, }); } + /** + * Get session stats + * @returns DbSessionStats OK + * @throws ApiError + */ + public static getApiV1SessionStats({ + since, + until, + agent, + includeProject, + excludeProject, + timezone, + includeGitOutcomes, + includeGithubOutcomes, + }: { + /** + * Start of window + */ + since?: string, + /** + * End of window + */ + until?: string, + /** + * Filter by agent + */ + agent?: string, + /** + * Restrict to these projects + */ + includeProject?: any[] | null, + /** + * Exclude these projects + */ + excludeProject?: any[] | null, + /** + * IANA timezone name + */ + timezone?: string, + /** + * Include git-derived outcome stats + */ + includeGitOutcomes?: boolean, + /** + * Include GitHub PR outcome stats + */ + includeGithubOutcomes?: boolean, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/session-stats', + query: { + 'since': since, + 'until': until, + 'agent': agent, + 'include_project': includeProject, + 'exclude_project': excludeProject, + 'timezone': timezone, + 'include_git_outcomes': includeGitOutcomes, + 'include_github_outcomes': includeGithubOutcomes, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } /** * Get stats * @returns DbStats OK diff --git a/frontend/src/lib/api/generated/services/RemoteSyncService.ts b/frontend/src/lib/api/generated/services/RemoteSyncService.ts new file mode 100644 index 000000000..13eb9e5ca --- /dev/null +++ b/frontend/src/lib/api/generated/services/RemoteSyncService.ts @@ -0,0 +1,33 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { RemotesyncTargetSet } from '../models/RemotesyncTargetSet'; +import type { CancelablePromise } from '../core/CancelablePromise'; +import { OpenAPI } from '../core/OpenAPI'; +import { request as __request } from '../core/request'; +export class RemoteSyncService { + /** + * Resolve remote sync targets + * @returns RemotesyncTargetSet OK + * @throws ApiError + */ + public static getApiV1RemoteSyncTargets(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/remote-sync/targets', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } +} diff --git a/frontend/src/lib/api/generated/services/SearchService.ts b/frontend/src/lib/api/generated/services/SearchService.ts index 90b04932c..f9c90ec9a 100644 --- a/frontend/src/lib/api/generated/services/SearchService.ts +++ b/frontend/src/lib/api/generated/services/SearchService.ts @@ -74,6 +74,8 @@ export class SearchService { public static getApiV1SearchContent({ pattern, mode, + scope, + xAgentsViewSearchIntent, _in, excludeSystem, reveal, @@ -91,6 +93,7 @@ export class SearchService { includeOneShot, limit, cursor, + context, }: { /** * Pattern to search for @@ -99,7 +102,15 @@ export class SearchService { /** * Search mode */ - mode?: 'substring' | 'regex' | 'fts', + mode?: 'substring' | 'regex' | 'fts' | 'semantic' | 'hybrid', + /** + * Semantic/hybrid result scope: top, all, or subordinate (default all) + */ + scope?: 'top' | 'all' | 'subordinate', + /** + * Required for semantic/hybrid GET searches + */ + xAgentsViewSearchIntent?: string, /** * Comma-separated content sources */ @@ -168,13 +179,21 @@ export class SearchService { * Pagination cursor */ cursor?: number, + /** + * Include N messages of context before and after each match (max 10) + */ + context?: number, }): CancelablePromise { return __request(OpenAPI, { method: 'GET', url: '/api/v1/search/content', + headers: { + 'X-AgentsView-Search-Intent': xAgentsViewSearchIntent, + }, query: { 'pattern': pattern, 'mode': mode, + 'scope': scope, 'in': _in, 'exclude_system': excludeSystem, 'reveal': reveal, @@ -192,6 +211,7 @@ export class SearchService { 'include_one_shot': includeOneShot, 'limit': limit, 'cursor': cursor, + 'context': context, }, errors: { 400: `Bad Request`, diff --git a/frontend/src/lib/api/generated/services/SessionsService.ts b/frontend/src/lib/api/generated/services/SessionsService.ts index e0031d095..e34224384 100644 --- a/frontend/src/lib/api/generated/services/SessionsService.ts +++ b/frontend/src/lib/api/generated/services/SessionsService.ts @@ -13,6 +13,7 @@ import type { OpenSessionResponse } from '../models/OpenSessionResponse'; import type { OrdinalsResponse } from '../models/OrdinalsResponse'; import type { PublishResponse } from '../models/PublishResponse'; import type { RenameRequest } from '../models/RenameRequest'; +import type { ResolveSessionIDsResponse } from '../models/ResolveSessionIDsResponse'; import type { ResumeRequest } from '../models/ResumeRequest'; import type { ResumeResponse } from '../models/ResumeResponse'; import type { ServiceMessageList } from '../models/ServiceMessageList'; @@ -50,6 +51,46 @@ export class SessionsService { }, }); } + /** + * Resolve session IDs + * @returns ResolveSessionIDsResponse OK + * @throws ApiError + */ + public static getApiV1SessionIdsResolve({ + partial, + limit, + }: { + /** + * Session ID substring + */ + partial: string, + /** + * Maximum number of matching IDs + */ + limit?: number, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/api/v1/session-ids/resolve', + query: { + 'partial': partial, + 'limit': limit, + }, + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } /** * List sessions * @returns ServiceSessionList OK @@ -228,6 +269,36 @@ export class SessionsService { }, }); } + /** + * Batch delete sessions + * @returns void + * @throws ApiError + */ + public static postApiV1SessionsBatchDelete({ + requestBody, + }: { + requestBody: BatchDeleteInputBody, + }): CancelablePromise { + return __request(OpenAPI, { + method: 'POST', + url: '/api/v1/sessions/batch-delete', + body: requestBody, + mediaType: 'application/json', + errors: { + 400: `Bad Request`, + 401: `Unauthorized`, + 403: `Forbidden`, + 404: `Not Found`, + 409: `Conflict`, + 422: `Unprocessable Entity`, + 500: `Internal Server Error`, + 501: `Not Implemented`, + 502: `Bad Gateway`, + 503: `Service Unavailable`, + 504: `Gateway Timeout`, + }, + }); + } /** * List sidebar sessions * @returns DbSidebarSessionIndex OK @@ -708,6 +779,10 @@ export class SessionsService { limit, direction, from, + around, + before, + after, + roles, }: { /** * Session ID @@ -725,6 +800,22 @@ export class SessionsService { * Starting message ordinal */ from?: number, + /** + * Center a symmetric window on this ordinal (mutually exclusive with from/direction) + */ + around?: number, + /** + * Messages before the around anchor (default 5) + */ + before?: number, + /** + * Messages after the around anchor (default 5) + */ + after?: number, + /** + * Comma-separated roles to include, e.g. user,assistant + */ + roles?: string, }): CancelablePromise { return __request(OpenAPI, { method: 'GET', @@ -736,6 +827,10 @@ export class SessionsService { 'limit': limit, 'direction': direction, 'from': from, + 'around': around, + 'before': before, + 'after': after, + 'roles': roles, }, errors: { 400: `Bad Request`, @@ -1200,34 +1295,4 @@ export class SessionsService { }, }); } - /** - * Batch delete sessions - * @returns void - * @throws ApiError - */ - public static postApiV1SessionsBatchDelete({ - requestBody, - }: { - requestBody: BatchDeleteInputBody, - }): CancelablePromise { - return __request(OpenAPI, { - method: 'POST', - url: '/api/v1/sessions/batch-delete', - body: requestBody, - mediaType: 'application/json', - errors: { - 400: `Bad Request`, - 401: `Unauthorized`, - 403: `Forbidden`, - 404: `Not Found`, - 409: `Conflict`, - 422: `Unprocessable Entity`, - 500: `Internal Server Error`, - 501: `Not Implemented`, - 502: `Bad Gateway`, - 503: `Service Unavailable`, - 504: `Gateway Timeout`, - }, - }); - } } diff --git a/frontend/src/lib/components/activity/Breakdowns.test.ts b/frontend/src/lib/components/activity/Breakdowns.test.ts index 09910146a..3719c813b 100644 --- a/frontend/src/lib/components/activity/Breakdowns.test.ts +++ b/frontend/src/lib/components/activity/Breakdowns.test.ts @@ -42,6 +42,7 @@ function makeReport(): Report { by_agent: [], by_session: [], intervals: [], + projects: {}, } as Report; } diff --git a/frontend/src/lib/components/activity/SummaryCards.test.ts b/frontend/src/lib/components/activity/SummaryCards.test.ts index 1a44d921c..60e28bf31 100644 --- a/frontend/src/lib/components/activity/SummaryCards.test.ts +++ b/frontend/src/lib/components/activity/SummaryCards.test.ts @@ -41,6 +41,7 @@ function makeReport(totals: Partial = {}): Report { by_agent: [], by_session: [], intervals: [], + projects: {}, } as Report; } diff --git a/go.mod b/go.mod index d32a5084e..5e1ecd143 100644 --- a/go.mod +++ b/go.mod @@ -24,12 +24,13 @@ require ( github.com/testcontainers/testcontainers-go v0.43.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.43.0 github.com/tidwall/gjson v1.19.0 - go.kenn.io/kit v0.1.7 + go.kenn.io/kit v0.2.1 golang.org/x/mod v0.37.0 golang.org/x/perf v0.0.0-20260615155930-9e4b9ddef5b6 golang.org/x/sync v0.21.0 golang.org/x/sys v0.46.0 golang.org/x/term v0.44.0 + modernc.org/sqlite v1.53.0 ) require ( @@ -38,6 +39,7 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect github.com/apache/arrow-go/v18 v18.5.1 // indirect + github.com/asg017/sqlite-vec-go-bindings v0.1.6 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect @@ -75,6 +77,7 @@ require ( github.com/klauspost/crc32 v1.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect + github.com/mattn/go-isatty v0.0.21 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -86,6 +89,7 @@ require ( github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pascaldekloe/name v1.0.0 // indirect @@ -93,6 +97,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/posthog/posthog-go v1.12.6 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.6.0 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect @@ -121,4 +126,7 @@ require ( golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gopkg.in/ini.v1 v1.67.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index d347c22a3..cdacec16c 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4 github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= +github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww= +github.com/asg017/sqlite-vec-go-bindings v0.1.6/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -95,6 +97,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -130,6 +134,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= +github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-sqlite3 v1.14.47 h1:jOBI62gS7nKeZv+as1oGEy0+1qISgXwH/QBlR6KbfIo= github.com/mattn/go-sqlite3 v1.14.47/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= @@ -164,6 +170,8 @@ github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modelcontextprotocol/go-sdk v1.6.1 h1:0zOSupjKUxPKSocPT1Wtago+mUHU2/uZ4xSOY0FGReU= github.com/modelcontextprotocol/go-sdk v1.6.1/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -181,6 +189,8 @@ github.com/posthog/posthog-go v1.12.6 h1:N+FrKWY6DOuDhV2OMgvtKAKDYGTdtS9/nuvr0BT github.com/posthog/posthog-go v1.12.6/go.mod h1:xsVOW9YImilUcazwPNEq4PJDqEZf2KeCS758zXjwkPg= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -238,8 +248,8 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= -go.kenn.io/kit v0.1.7 h1:VT1IzeHnzAqW5MCjwyZzQwVJaXB+7Qm4fhDJDL/fr0g= -go.kenn.io/kit v0.1.7/go.mod h1:FqRfVTTGAiCf4sxqK0sjx5OVkjz2+6bFtNl/uB2Z3wk= +go.kenn.io/kit v0.2.1 h1:gbWJ7IrqX5gVe8ZeHd5fzkTz8QCNzik/mPf6EuY0Grk= +go.kenn.io/kit v0.2.1/go.mod h1:T9hrly8meqRPeUoFFCaYYhGK3Bp5gCIEUoJ5mplei0I= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= @@ -298,5 +308,33 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/config/config.go b/internal/config/config.go index e3dfcec10..3809a8eda 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -101,6 +101,269 @@ type DuckDBConfig struct { ExcludeProjects []string `toml:"exclude_projects" json:"exclude_projects,omitempty"` } +// VectorConfig holds settings for the optional local semantic-search +// vector index (embeddings + vectors.db). +type VectorConfig struct { + Enabled bool `toml:"enabled" json:"enabled"` + DBPath string `toml:"db_path" json:"db_path,omitempty"` + // IncludeAutomated controls whether automated (e.g. roborev) sessions' + // messages are embedded into the vector index, mirroring the + // IncludeAutomated convention search already uses to exclude those + // sessions from results by default. Default false: automated sessions + // are excluded from embedding, since they otherwise dominate a large + // archive's index with content that search already hides by default. + // `embeddings build --include-automated` can override this for a + // one-off build; see that flag's help for the scheduled-build caveat. + IncludeAutomated bool `toml:"include_automated" json:"include_automated"` + Embeddings VectorEmbeddingsConfig `toml:"embeddings" json:"embeddings"` + Embed VectorEmbedConfig `toml:"embed" json:"embed"` +} + +// VectorEmbeddingsConfig describes the embedding space — the model identity +// every server must share — and the named servers that can encode it. +// +// Model identity (model, dimension, max_input_chars, input_suffix) is +// deliberately global rather than per-server: it joins the generation +// fingerprint, and query vectors are only comparable to stored document +// vectors from the same space. Servers differ only in transport and +// capacity, so a build run on any server produces vectors every other +// server's queries can search. +type VectorEmbeddingsConfig struct { + Model string `toml:"model" json:"model"` + Dimension int `toml:"dimension" json:"dimension"` + // MaxInputChars caps the rune length of each chunk sent for + // embedding. Default 8192. + MaxInputChars int `toml:"max_input_chars" json:"max_input_chars"` + // InputSuffix is appended verbatim to every text sent for embedding + // (documents and queries alike). Some models expect a terminator the + // serving layer does not add — e.g. Qwen3-Embedding under llama.cpp + // wants "<|endoftext|>" appended client-side. Changing it cuts a new + // vector generation. Default empty. + InputSuffix string `toml:"input_suffix" json:"input_suffix,omitempty"` + // DefaultServer names the server used for search-time query encoding + // and for builds that don't select one (scheduled builds, and + // `embeddings build` without --using). Optional when exactly one + // server is defined. + DefaultServer string `toml:"default_server" json:"default_server,omitempty"` + // Servers is the set of named OpenAI-compatible endpoints that serve + // Model, keyed by the name `embeddings build --using ` selects. + Servers map[string]VectorEmbeddingsServerConfig `toml:"servers" json:"servers"` +} + +// VectorEmbeddingsServerConfig is one named embeddings server: transport +// and capacity settings only; the model identity lives on +// VectorEmbeddingsConfig. +type VectorEmbeddingsServerConfig struct { + Endpoint string `toml:"endpoint" json:"endpoint"` + // APIKeyEnv names the environment variable holding the API key. + // Empty means anonymous access. + APIKeyEnv string `toml:"api_key_env" json:"api_key_env,omitempty"` + // BatchSize is the number of inputs sent per HTTP call. Default 32. + BatchSize int `toml:"batch_size" json:"batch_size"` + // Concurrency is the number of documents embedded in parallel during a + // build against this server. Sequential requests leave a build + // round-trip-bound against remote endpoints, so the default is 4; + // servers that process one request at a time simply queue the extras. + Concurrency int `toml:"concurrency" json:"concurrency"` + // Timeout is a parseable duration string applied to each HTTP + // call. Default "30s". + Timeout string `toml:"timeout" json:"timeout"` + // MaxRetries is the maximum total attempts on 429/5xx/network errors + // (4xx fails fast); 0 means one attempt. Default 3. + MaxRetries int `toml:"max_retries" json:"max_retries"` +} + +// ResolvedDefaultServer returns the server name used when no explicit +// choice is made: default_server when set, otherwise the only defined +// server, otherwise "". +func (c VectorEmbeddingsConfig) ResolvedDefaultServer() string { + if c.DefaultServer != "" { + return c.DefaultServer + } + if len(c.Servers) == 1 { + for name := range c.Servers { + return name + } + } + return "" +} + +// Server resolves name to a defined embeddings server; "" means the +// default. The resolved name is returned alongside the server so callers +// can report which server a build actually used. +func (c VectorEmbeddingsConfig) Server(name string) (string, VectorEmbeddingsServerConfig, error) { + if name == "" { + name = c.ResolvedDefaultServer() + } + s, ok := c.Servers[name] + if !ok { + return "", VectorEmbeddingsServerConfig{}, fmt.Errorf( + "[vector.embeddings] no server named %q; define it under [vector.embeddings.servers.%s] (have: %s)", + name, name, strings.Join(sortedServerNames(c.Servers), ", ")) + } + return name, s, nil +} + +// normalizedEmbeddingsServers fills each named server's unset transport +// fields with their defaults (batch_size 32, concurrency 4, timeout "30s", +// max_retries 3). meta.IsDefined distinguishes "unset" (apply the default) +// from an explicit zero — an explicit max_retries = 0 disables retries, and +// an explicit zero batch_size/concurrency stays zero so validation rejects +// it instead of silently substituting the default. +func normalizedEmbeddingsServers( + servers map[string]VectorEmbeddingsServerConfig, meta toml.MetaData, +) map[string]VectorEmbeddingsServerConfig { + out := make(map[string]VectorEmbeddingsServerConfig, len(servers)) + for name, s := range servers { + if !meta.IsDefined("vector", "embeddings", "servers", name, "batch_size") { + s.BatchSize = 32 + } + if !meta.IsDefined("vector", "embeddings", "servers", name, "concurrency") { + s.Concurrency = 4 + } + if s.Timeout == "" { + s.Timeout = "30s" + } + if !meta.IsDefined("vector", "embeddings", "servers", name, "max_retries") { + s.MaxRetries = 3 + } + out[name] = s + } + return out +} + +// sortedServerNames returns the configured server names in sorted order, +// for deterministic error messages and validation. +func sortedServerNames(servers map[string]VectorEmbeddingsServerConfig) []string { + names := make([]string, 0, len(servers)) + for name := range servers { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// VectorEmbedConfig configures when the daemon runs embedding work. +type VectorEmbedConfig struct { + // RunAfterSync enables debounced embedding of sync deltas. + // Defaults to true when unset; read it via RunAfterSyncEnabled. + RunAfterSync *bool `toml:"run_after_sync" json:"run_after_sync,omitempty"` + // BackstopInterval is a parseable duration string for a periodic + // full rescan. Default "24h"; a negative duration disables it. + BackstopInterval string `toml:"backstop_interval" json:"backstop_interval"` +} + +// Validate checks the vector config for internal consistency. It is a +// no-op when the section is disabled. +func (c VectorConfig) Validate() error { + if !c.Enabled { + return nil + } + if c.Embeddings.Model == "" { + return fmt.Errorf("[vector.embeddings] model is required when [vector] is enabled") + } + if c.Embeddings.Dimension <= 0 { + return fmt.Errorf("[vector.embeddings] dimension must be greater than 0 when [vector] is enabled") + } + if c.Embeddings.MaxInputChars <= 0 { + return fmt.Errorf( + "[vector.embeddings] max_input_chars must be greater than 0, got %d", + c.Embeddings.MaxInputChars) + } + if err := c.Embeddings.validateServers(); err != nil { + return err + } + backstop, err := time.ParseDuration(c.Embed.BackstopInterval) + if err != nil { + return fmt.Errorf("[vector.embed] invalid backstop_interval %q: %w", c.Embed.BackstopInterval, err) + } + if backstop == 0 { + return fmt.Errorf( + "[vector.embed] backstop_interval must not be zero; " + + "use a negative value to disable or omit for the 24h default") + } + return nil +} + +// ResolvedDBPath returns DBPath if set, else /vectors.db. +func (c VectorConfig) ResolvedDBPath(dataDir string) string { + if c.DBPath != "" { + return c.DBPath + } + return filepath.Join(dataDir, "vectors.db") +} + +// APIKey reads the API key from the environment variable named by +// APIKeyEnv. Returns "" when APIKeyEnv is unset. +func (c VectorEmbeddingsServerConfig) APIKey() string { + if c.APIKeyEnv == "" { + return "" + } + return os.Getenv(c.APIKeyEnv) +} + +// validateServers checks the named-servers section: at least one server, an +// unambiguous default, and per-server transport settings that parse. +func (c VectorEmbeddingsConfig) validateServers() error { + if len(c.Servers) == 0 { + return fmt.Errorf( + "[vector.embeddings] at least one server is required when [vector] is enabled; " + + "define one under [vector.embeddings.servers.]") + } + if c.DefaultServer == "" && len(c.Servers) > 1 { + return fmt.Errorf( + "[vector.embeddings] default_server is required when more than one server is defined (have: %s)", + strings.Join(sortedServerNames(c.Servers), ", ")) + } + if c.DefaultServer != "" { + if _, ok := c.Servers[c.DefaultServer]; !ok { + return fmt.Errorf( + "[vector.embeddings] default_server %q is not a defined server (have: %s)", + c.DefaultServer, strings.Join(sortedServerNames(c.Servers), ", ")) + } + } + for _, name := range sortedServerNames(c.Servers) { + if err := c.Servers[name].validate(name); err != nil { + return err + } + } + return nil +} + +// validate checks one named server's transport settings. +func (c VectorEmbeddingsServerConfig) validate(name string) error { + section := fmt.Sprintf("[vector.embeddings.servers.%s]", name) + if c.Endpoint == "" { + return fmt.Errorf("%s endpoint is required", section) + } + if c.BatchSize <= 0 { + return fmt.Errorf("%s batch_size must be greater than 0, got %d", section, c.BatchSize) + } + if c.Concurrency <= 0 { + return fmt.Errorf("%s concurrency must be greater than 0, got %d", section, c.Concurrency) + } + if c.MaxRetries < 0 { + return fmt.Errorf("%s max_retries must be >= 0, got %d", section, c.MaxRetries) + } + timeout, err := time.ParseDuration(c.Timeout) + if err != nil { + return fmt.Errorf("%s invalid timeout %q: %w", section, c.Timeout, err) + } + if timeout <= 0 { + return fmt.Errorf("%s timeout must be greater than 0, got %q", section, c.Timeout) + } + return nil +} + +// RunAfterSyncEnabled reports whether embedding should run after sync, +// defaulting to true when RunAfterSync is unset. +func (c VectorEmbedConfig) RunAfterSyncEnabled() bool { + if c.RunAfterSync == nil { + return true + } + return *c.RunAfterSync +} + // AutomatedConfig holds user-supplied additions to the // automated-session classifier. Parse-only; all semantic // normalization (trim, dedupe, length cap, built-in overlap @@ -172,6 +435,7 @@ type Config struct { DefaultPG string `json:"default_pg,omitempty" toml:"default_pg"` PGTargets map[string]PGConfig `json:"-" toml:"-"` DuckDB DuckDBConfig `json:"duckdb,omitempty" toml:"duckdb"` + Vector VectorConfig `json:"vector,omitempty" toml:"vector"` Automated AutomatedConfig `json:"automated,omitempty" toml:"automated"` Agent map[string]AgentConfig `json:"agent,omitempty" toml:"agent"` WriteTimeout time.Duration `json:"-" toml:"-"` @@ -406,6 +670,14 @@ func Default() (Config, error) { EventsCoalesceInterval: 10 * time.Second, DaemonIdleTimeout: 20 * time.Minute, Agent: map[string]AgentConfig{}, + Vector: VectorConfig{ + Embeddings: VectorEmbeddingsConfig{ + MaxInputChars: 8192, + }, + Embed: VectorEmbedConfig{ + BackstopInterval: "24h", + }, + }, }, nil } @@ -680,6 +952,7 @@ func (c *Config) applyConfigTOML(data string) error { DefaultPG string `toml:"default_pg"` PG PGConfig `toml:"pg"` DuckDB DuckDBConfig `toml:"duckdb"` + Vector VectorConfig `toml:"vector"` Automated AutomatedConfig `toml:"automated"` Agent map[string]AgentConfig `toml:"agent"` EventsCoalesceInterval time.Duration `toml:"events_coalesce_interval"` @@ -793,6 +1066,42 @@ func (c *Config) applyConfigTOML(data string) error { if file.DuckDB.ExcludeProjects != nil && c.DuckDB.ExcludeProjects == nil { c.DuckDB.ExcludeProjects = file.DuckDB.ExcludeProjects } + if file.Vector.Enabled { + c.Vector.Enabled = true + } + if file.Vector.DBPath != "" { + c.Vector.DBPath = file.Vector.DBPath + } + // IsDefined distinguishes "unset" (keep the default false) from an + // explicit include_automated = false, matching the other vector + // section fields' treatment even though both currently agree. + if meta.IsDefined("vector", "include_automated") { + c.Vector.IncludeAutomated = file.Vector.IncludeAutomated + } + if file.Vector.Embeddings.Model != "" { + c.Vector.Embeddings.Model = file.Vector.Embeddings.Model + } + if file.Vector.Embeddings.Dimension != 0 { + c.Vector.Embeddings.Dimension = file.Vector.Embeddings.Dimension + } + if meta.IsDefined("vector", "embeddings", "max_input_chars") { + c.Vector.Embeddings.MaxInputChars = file.Vector.Embeddings.MaxInputChars + } + if file.Vector.Embeddings.InputSuffix != "" { + c.Vector.Embeddings.InputSuffix = file.Vector.Embeddings.InputSuffix + } + if file.Vector.Embeddings.DefaultServer != "" { + c.Vector.Embeddings.DefaultServer = file.Vector.Embeddings.DefaultServer + } + if len(file.Vector.Embeddings.Servers) > 0 { + c.Vector.Embeddings.Servers = normalizedEmbeddingsServers(file.Vector.Embeddings.Servers, meta) + } + if file.Vector.Embed.RunAfterSync != nil { + c.Vector.Embed.RunAfterSync = file.Vector.Embed.RunAfterSync + } + if file.Vector.Embed.BackstopInterval != "" { + c.Vector.Embed.BackstopInterval = file.Vector.Embed.BackstopInterval + } // IsDefined distinguishes "unset" (leave default 10s) from an // explicit "0s" (disable coalescing). Checking != 0 would silently // ignore the latter. @@ -1286,6 +1595,9 @@ func finalize(cfg *Config) error { if cfg.DaemonIdleTimeout < 0 { return fmt.Errorf("invalid daemon_idle_timeout: %s", cfg.DaemonIdleTimeout) } + if err := cfg.Vector.Validate(); err != nil { + return err + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d7a930ee9..7f736df39 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -378,6 +378,7 @@ func TestLoadPFlags_AppliesExplicitFlags(t *testing.T) { } func TestLoad_NilFlagSet(t *testing.T) { + setupTestEnv(t) cfg, err := Load(nil) require.NoError(t, err) diff --git a/internal/config/config_vector_test.go b/internal/config/config_vector_test.go new file mode 100644 index 000000000..6830c1d30 --- /dev/null +++ b/internal/config/config_vector_test.go @@ -0,0 +1,458 @@ +package config + +import ( + "maps" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// validVectorConfig returns a VectorConfig with every field enabled +// and populated with values that pass Validate, so each table case +// below can mutate exactly the field under test. +func validVectorConfig() VectorConfig { + return VectorConfig{ + Enabled: true, + Embeddings: VectorEmbeddingsConfig{ + Model: "nomic-embed-text", + Dimension: 768, + MaxInputChars: 8192, + DefaultServer: "local", + Servers: map[string]VectorEmbeddingsServerConfig{ + "local": { + Endpoint: "http://localhost:11434/v1", + Timeout: "30s", + BatchSize: 32, + Concurrency: 4, + MaxRetries: 3, + }, + "remote": { + Endpoint: "http://build-box:30000/v1", + Timeout: "300s", + BatchSize: 32, + Concurrency: 6, + MaxRetries: 3, + }, + }, + }, + Embed: VectorEmbedConfig{ + BackstopInterval: "24h", + }, + } +} + +// mutateServer applies fn to the named server entry, working around map +// values not being addressable. +func mutateServer(c *VectorConfig, name string, fn func(*VectorEmbeddingsServerConfig)) { + s := c.Embeddings.Servers[name] + fn(&s) + c.Embeddings.Servers[name] = s +} + +func TestVectorConfigValidate(t *testing.T) { + tests := []struct { + name string + mutate func(*VectorConfig) + wantErr string + }{ + { + name: "disabled is valid even with empty fields", + mutate: func(c *VectorConfig) { *c = VectorConfig{} }, + }, + { + name: "enabled missing model", + mutate: func(c *VectorConfig) { c.Embeddings.Model = "" }, + wantErr: "model is required", + }, + { + name: "enabled missing dimension", + mutate: func(c *VectorConfig) { c.Embeddings.Dimension = 0 }, + wantErr: "dimension", + }, + { + name: "enabled negative dimension", + mutate: func(c *VectorConfig) { c.Embeddings.Dimension = -1 }, + wantErr: "dimension", + }, + { + name: "enabled zero max_input_chars", + mutate: func(c *VectorConfig) { c.Embeddings.MaxInputChars = 0 }, + wantErr: "max_input_chars", + }, + { + name: "enabled negative max_input_chars", + mutate: func(c *VectorConfig) { c.Embeddings.MaxInputChars = -1 }, + wantErr: "max_input_chars", + }, + { + name: "enabled with no servers", + mutate: func(c *VectorConfig) { c.Embeddings.Servers = nil }, + wantErr: "at least one server", + }, + { + name: "multiple servers without default_server", + mutate: func(c *VectorConfig) { + c.Embeddings.DefaultServer = "" + }, + wantErr: "default_server is required", + }, + { + name: "default_server names an undefined server", + mutate: func(c *VectorConfig) { + c.Embeddings.DefaultServer = "nope" + }, + wantErr: `default_server "nope" is not a defined server`, + }, + { + name: "single server needs no default_server", + mutate: func(c *VectorConfig) { + delete(c.Embeddings.Servers, "remote") + c.Embeddings.DefaultServer = "" + }, + }, + { + name: "server missing endpoint", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.Endpoint = "" }) + }, + wantErr: "[vector.embeddings.servers.local] endpoint is required", + }, + { + name: "server zero batch_size", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.BatchSize = 0 }) + }, + wantErr: "batch_size", + }, + { + name: "server negative batch_size", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.BatchSize = -1 }) + }, + wantErr: "batch_size", + }, + { + name: "server zero concurrency", + mutate: func(c *VectorConfig) { + mutateServer(c, "remote", func(s *VectorEmbeddingsServerConfig) { s.Concurrency = 0 }) + }, + wantErr: "[vector.embeddings.servers.remote] concurrency", + }, + { + name: "server negative max_retries", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.MaxRetries = -1 }) + }, + wantErr: "max_retries", + }, + { + name: "server zero max_retries disables retries and is valid", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.MaxRetries = 0 }) + }, + }, + { + name: "server bad timeout", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.Timeout = "not-a-duration" }) + }, + wantErr: "timeout", + }, + { + name: "server zero timeout", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.Timeout = "0s" }) + }, + wantErr: "timeout", + }, + { + name: "server negative timeout", + mutate: func(c *VectorConfig) { + mutateServer(c, "local", func(s *VectorEmbeddingsServerConfig) { s.Timeout = "-1s" }) + }, + wantErr: "timeout", + }, + { + name: "enabled bad backstop interval", + mutate: func(c *VectorConfig) { c.Embed.BackstopInterval = "not-a-duration" }, + wantErr: "backstop_interval", + }, + { + name: "enabled explicit zero backstop interval is invalid", + mutate: func(c *VectorConfig) { c.Embed.BackstopInterval = "0s" }, + wantErr: "use a negative value to disable", + }, + { + name: "enabled negative backstop interval disables and is valid", + mutate: func(c *VectorConfig) { c.Embed.BackstopInterval = "-1s" }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := validVectorConfig() + tt.mutate(&cfg) + err := cfg.Validate() + if tt.wantErr == "" { + assert.NoError(t, err) + return + } + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} + +func TestVectorEmbeddingsServerResolution(t *testing.T) { + c := validVectorConfig().Embeddings + + name, server, err := c.Server("") + require.NoError(t, err) + assert.Equal(t, "local", name, "empty name resolves to default_server") + assert.Equal(t, "http://localhost:11434/v1", server.Endpoint) + + name, server, err = c.Server("remote") + require.NoError(t, err) + assert.Equal(t, "remote", name) + assert.Equal(t, "http://build-box:30000/v1", server.Endpoint) + + _, _, err = c.Server("nope") + require.Error(t, err) + assert.Contains(t, err.Error(), `no server named "nope"`) + assert.Contains(t, err.Error(), "local, remote", "error lists the defined servers") + + c.DefaultServer = "" + delete(c.Servers, "remote") + name, _, err = c.Server("") + require.NoError(t, err) + assert.Equal(t, "local", name, "a single server is the implicit default") +} + +func TestVectorConfigDefaults(t *testing.T) { + cfg, err := Default() + require.NoError(t, err) + + assert.Equal(t, 8192, cfg.Vector.Embeddings.MaxInputChars) + assert.Empty(t, cfg.Vector.Embeddings.Servers) + assert.True(t, cfg.Vector.Embed.RunAfterSyncEnabled(), + "run_after_sync defaults to true when unset") + + disabled := false + cfg.Vector.Embed.RunAfterSync = &disabled + assert.False(t, cfg.Vector.Embed.RunAfterSyncEnabled(), + "explicit false overrides the default") + + assert.Equal(t, filepath.Join(cfg.DataDir, "vectors.db"), + cfg.Vector.ResolvedDBPath(cfg.DataDir), "falls back to /vectors.db") + + cfg.Vector.DBPath = "/custom/path/vec.db" + assert.Equal(t, "/custom/path/vec.db", cfg.Vector.ResolvedDBPath(cfg.DataDir), + "explicit db_path overrides the fallback") +} + +func TestVectorConfigAPIKeyEnv(t *testing.T) { + server := VectorEmbeddingsServerConfig{} + assert.Equal(t, "", server.APIKey(), "no env var configured") + + server.APIKeyEnv = "AGENTSVIEW_TEST_VECTOR_API_KEY" + assert.Equal(t, "", server.APIKey(), "configured env var not set in environment") + + t.Setenv("AGENTSVIEW_TEST_VECTOR_API_KEY", "secret-123") + assert.Equal(t, "secret-123", server.APIKey()) +} + +// minimalServers returns the smallest valid servers table for TOML load +// tests, as the raw map shape loadMinimalWithConfig marshals. +func minimalServers() map[string]any { + return map[string]any{ + "local": map[string]any{ + "endpoint": "http://localhost:11434/v1", + }, + } +} + +// TestVectorConfigTOMLLoad exercises the full config-file load path so the +// default-merge logic in applyConfigTOML (not just the section types in +// isolation) is covered, including per-server defaults and the ability to +// explicitly override a zero-value field like max_retries. +func TestVectorConfigTOMLLoad(t *testing.T) { + t.Run("unset server fields keep defaults, explicit zero overrides", func(t *testing.T) { + cfg := loadMinimalWithConfig(t, map[string]any{ + "vector": map[string]any{ + "enabled": true, + "embeddings": map[string]any{ + "model": "nomic-embed-text", + "dimension": 768, + "servers": map[string]any{ + "local": map[string]any{ + "endpoint": "http://localhost:11434/v1", + "max_retries": 0, + }, + }, + }, + }, + }) + require.True(t, cfg.Vector.Enabled) + server := cfg.Vector.Embeddings.Servers["local"] + assert.Equal(t, "http://localhost:11434/v1", server.Endpoint) + assert.Equal(t, 32, server.BatchSize, "unset batch_size keeps default") + assert.Equal(t, 4, server.Concurrency, "unset concurrency keeps default") + assert.Equal(t, "30s", server.Timeout, "unset timeout keeps default") + assert.Equal(t, 0, server.MaxRetries, "explicit max_retries=0 overrides default") + assert.Equal(t, 8192, cfg.Vector.Embeddings.MaxInputChars, "unset max_input_chars keeps default") + assert.Equal(t, "24h", cfg.Vector.Embed.BackstopInterval, "unset backstop_interval keeps default") + assert.False(t, cfg.Vector.IncludeAutomated, "unset include_automated keeps the false default") + assert.Empty(t, cfg.Vector.Embeddings.InputSuffix, "unset input_suffix defaults to empty") + }) + + t.Run("named servers with default_server load and resolve", func(t *testing.T) { + cfg := loadMinimalWithConfig(t, map[string]any{ + "vector": map[string]any{ + "enabled": true, + "embeddings": map[string]any{ + "model": "qwen3-embedding-4b", + "dimension": 2560, + "input_suffix": "<|endoftext|>", + "default_server": "local", + "servers": map[string]any{ + "local": map[string]any{ + "endpoint": "http://127.0.0.1:30000/v1", + }, + "remote": map[string]any{ + "endpoint": "http://build-box:30000/v1", + "timeout": "300s", + "concurrency": 6, + }, + }, + }, + }, + }) + assert.Equal(t, "<|endoftext|>", cfg.Vector.Embeddings.InputSuffix) + assert.Equal(t, "local", cfg.Vector.Embeddings.DefaultServer) + + name, server, err := cfg.Vector.Embeddings.Server("") + require.NoError(t, err) + assert.Equal(t, "local", name) + assert.Equal(t, "http://127.0.0.1:30000/v1", server.Endpoint) + + _, remote, err := cfg.Vector.Embeddings.Server("remote") + require.NoError(t, err) + assert.Equal(t, "300s", remote.Timeout, "per-server timeout override") + assert.Equal(t, 6, remote.Concurrency, "per-server concurrency override") + assert.Equal(t, 32, remote.BatchSize, "unset per-server batch_size keeps default") + }) + + t.Run("include_automated true is loaded", func(t *testing.T) { + cfg := loadMinimalWithConfig(t, map[string]any{ + "vector": map[string]any{ + "enabled": true, + "include_automated": true, + "embeddings": map[string]any{ + "model": "nomic-embed-text", + "dimension": 768, + "servers": minimalServers(), + }, + }, + }) + assert.True(t, cfg.Vector.IncludeAutomated) + }) + + t.Run("enabled without servers fails to load", func(t *testing.T) { + err := loadMinimalErrWithConfig(t, map[string]any{ + "vector": map[string]any{ + "enabled": true, + "embeddings": map[string]any{ + "model": "nomic-embed-text", + "dimension": 768, + }, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "at least one server") + }) + + t.Run("multiple servers without default_server fails to load", func(t *testing.T) { + servers := minimalServers() + servers["remote"] = map[string]any{"endpoint": "http://build-box:30000/v1"} + err := loadMinimalErrWithConfig(t, map[string]any{ + "vector": map[string]any{ + "enabled": true, + "embeddings": map[string]any{ + "model": "nomic-embed-text", + "dimension": 768, + "servers": servers, + }, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "default_server is required") + }) + + t.Run("disabled section with no fields loads fine", func(t *testing.T) { + cfg := loadMinimalWithConfig(t, map[string]any{ + "vector": map[string]any{}, + }) + assert.False(t, cfg.Vector.Enabled) + }) + + t.Run("explicit zero/negative operational overrides fail to load", func(t *testing.T) { + tests := []struct { + name string + server map[string]any + embed map[string]any + wantErr string + }{ + { + name: "explicit zero batch_size", + server: map[string]any{"batch_size": 0}, + wantErr: "batch_size", + }, + { + name: "explicit negative batch_size", + server: map[string]any{"batch_size": -1}, + wantErr: "batch_size", + }, + { + name: "explicit zero concurrency", + server: map[string]any{"concurrency": 0}, + wantErr: "concurrency", + }, + { + name: "explicit negative max_retries", + server: map[string]any{"max_retries": -1}, + wantErr: "max_retries", + }, + { + name: "explicit zero timeout", + server: map[string]any{"timeout": "0s"}, + wantErr: "timeout", + }, + { + name: "explicit zero backstop_interval", + embed: map[string]any{"backstop_interval": "0s"}, + wantErr: "use a negative value to disable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + local := map[string]any{"endpoint": "http://localhost:11434/v1"} + maps.Copy(local, tt.server) + vector := map[string]any{ + "enabled": true, + "embeddings": map[string]any{ + "model": "nomic-embed-text", + "dimension": 768, + "servers": map[string]any{"local": local}, + }, + } + if tt.embed != nil { + vector["embed"] = tt.embed + } + err := loadMinimalErrWithConfig(t, map[string]any{"vector": vector}) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } + }) +} diff --git a/internal/db/db.go b/internal/db/db.go index 170c6cb1e..9bb3edcef 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -379,6 +379,9 @@ type DB struct { checkpointMu sync.Mutex checkpointStop chan struct{} checkpointDone chan struct{} + + vectorMu sync.RWMutex + vectorSearcher VectorSearcher } // Reader exposes guarded read-only query operations. It intentionally does @@ -633,9 +636,23 @@ func (db *DB) SetCursorSecret(secret []byte) { } // makeDSN builds a SQLite connection string with shared pragmas. +// +// Both branches emit a file: URI. mattn/go-sqlite3 forwards the `_`-prefixed +// pragma params either way, but it only honors mode=ro when the DSN carries +// the file: scheme — a bare path silently opens read-write, so the ro +// contract depends on the prefix. +// +// The path component is percent-encoded (slashes kept intact): SQLite +// percent-decodes URI paths and splits params at `?`, so a raw path +// containing `%`, `?`, or `#` would be misparsed — e.g. a literal "%41" in a +// directory name would silently open a different file. +// +// _journal_mode=WAL is set only on writable DSNs (mirroring vectorDSN): +// PRAGMA journal_mode=WAL is a write, so with mode=ro honored it would fail +// outright on a database left in a non-WAL journal mode. Read-only +// connections just adopt whatever journal mode the file already has. func makeDSN(path string, readOnly bool) string { params := url.Values{} - params.Set("_journal_mode", "WAL") params.Set("_busy_timeout", "5000") params.Set("_foreign_keys", "ON") params.Set("_mmap_size", "268435456") @@ -643,9 +660,11 @@ func makeDSN(path string, readOnly bool) string { if readOnly { params.Set("mode", "ro") } else { + params.Set("_journal_mode", "WAL") params.Set("_synchronous", "NORMAL") } - return path + "?" + params.Encode() + escaped := (&url.URL{Path: path}).EscapedPath() + return "file:" + escaped + "?" + params.Encode() } // Open creates or opens a SQLite database at the given path. @@ -2525,15 +2544,18 @@ func (db *DB) Close() error { db.connMu.Unlock() db.mu.Unlock() + // Close the writer last: SQLite checkpoints and removes the WAL when + // the final connection closes, and the reader pool is mode=ro so its + // close cannot perform that checkpoint. var errs []error - if w != nil && w != r { - errs = append(errs, w.Close()) + for _, p := range retired { + errs = append(errs, p.Close()) } if r != nil { errs = append(errs, r.Close()) } - for _, p := range retired { - errs = append(errs, p.Close()) + if w != nil && w != r { + errs = append(errs, w.Close()) } return errors.Join(errs...) } @@ -2552,13 +2574,19 @@ func (db *DB) CloseConnections() error { db.connMu.Lock() defer db.connMu.Unlock() - errs := []error{ - db.rawWriter().Close(), - db.rawReader().Close(), - } + // Close the writer last: SQLite checkpoints and removes the WAL when + // the final connection closes, and the reader pool is mode=ro so its + // close cannot perform that checkpoint. Callers rename or delete the + // WAL file after this returns, so a skipped checkpoint would lose + // every write still sitting in the log. + var errs []error for _, p := range db.retired { errs = append(errs, p.Close()) } + errs = append(errs, + db.rawReader().Close(), + db.rawWriter().Close(), + ) db.retired = nil return errors.Join(errs...) } diff --git a/internal/db/messages.go b/internal/db/messages.go index 57788eaa8..c98a8f27d 100644 --- a/internal/db/messages.go +++ b/internal/db/messages.go @@ -8,9 +8,11 @@ import ( "errors" "fmt" "log" + "slices" "strings" "time" "unicode" + "unicode/utf8" "go.kenn.io/agentsview/internal/parser" ) @@ -179,6 +181,165 @@ func (db *DB) GetMessages( return msgs, nil } +// MessageWindow parameterises GetMessagesWindow. Exactly one retrieval +// mode: Around non-nil = symmetric window; otherwise linear from/limit. +type MessageWindow struct { + From *int + Limit int + Asc bool + Around *int + Before int // used only with Around; default handled by caller + After int + Roles []string // empty = all roles +} + +// GetMessagesWindow returns messages for a session using either linear +// pagination (mirroring GetMessages, optionally role-filtered) or a +// symmetric window centered on an ordinal (Around/Before/After). Around +// mode always includes the anchor row even when its own role is excluded +// by Roles; the before/after counts are taken after applying the role +// filter, so they count role-matching messages rather than raw ordinal +// distance from the anchor. +func (db *DB) GetMessagesWindow( + ctx context.Context, sessionID string, w MessageWindow, +) ([]Message, error) { + if w.Around != nil { + return db.getMessagesAroundAnchor(ctx, sessionID, w) + } + from := 0 + if w.From != nil { + from = *w.From + } + if len(w.Roles) == 0 { + return db.GetMessages(ctx, sessionID, from, w.Limit, w.Asc) + } + return db.getMessagesLinearRoleFiltered( + ctx, sessionID, from, w.Limit, w.Asc, w.Roles, + ) +} + +// getMessagesLinearRoleFiltered is GetMessages plus an "AND role IN (...)" +// predicate, used when MessageWindow.Roles is non-empty. +func (db *DB) getMessagesLinearRoleFiltered( + ctx context.Context, + sessionID string, from, limit int, asc bool, roles []string, +) ([]Message, error) { + if limit <= 0 || limit > MaxMessageLimit { + limit = DefaultMessageLimit + } + dir := "ASC" + op := ">=" + if !asc { + dir = "DESC" + op = "<=" + } + roleClause, roleArgs := roleFilterClause(roles) + query := fmt.Sprintf(` + SELECT %s + FROM messages + WHERE session_id = ? AND ordinal %s ?%s + ORDER BY ordinal %s + LIMIT ?`, selectMessageCols, op, roleClause, dir) + args := append([]any{sessionID, from}, roleArgs...) + args = append(args, limit) + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("querying role-filtered messages: %w", err) + } + defer rows.Close() + msgs, err := scanMessages(rows) + if err != nil { + return nil, err + } + if err := db.attachToolCalls(ctx, msgs); err != nil { + return nil, err + } + return msgs, nil +} + +// getMessagesAroundAnchor implements MessageWindow's Around mode: three +// queries (before/anchor/after) merged into one ascending slice. The +// anchor query has no role predicate so the anchor row is always present; +// before/after apply the role filter (when set) before taking Before/After +// rows, so the counts reflect role-matching messages, not raw ordinals. +func (db *DB) getMessagesAroundAnchor( + ctx context.Context, sessionID string, w MessageWindow, +) ([]Message, error) { + anchor := *w.Around + beforeLimit := max(w.Before, 0) + afterLimit := max(w.After, 0) + roleClause, roleArgs := roleFilterClause(w.Roles) + + beforeQuery := fmt.Sprintf(` + SELECT %s FROM messages + WHERE session_id = ? AND ordinal < ?%s + ORDER BY ordinal DESC LIMIT ?`, selectMessageCols, roleClause) + beforeArgs := append([]any{sessionID, anchor}, roleArgs...) + beforeArgs = append(beforeArgs, beforeLimit) + before, err := db.queryMessageRows(ctx, beforeQuery, beforeArgs...) + if err != nil { + return nil, fmt.Errorf("querying before-window messages: %w", err) + } + slices.Reverse(before) + + anchorQuery := fmt.Sprintf(` + SELECT %s FROM messages WHERE session_id = ? AND ordinal = ?`, + selectMessageCols) + anchorMsgs, err := db.queryMessageRows(ctx, anchorQuery, sessionID, anchor) + if err != nil { + return nil, fmt.Errorf("querying anchor message: %w", err) + } + + afterQuery := fmt.Sprintf(` + SELECT %s FROM messages + WHERE session_id = ? AND ordinal > ?%s + ORDER BY ordinal ASC LIMIT ?`, selectMessageCols, roleClause) + afterArgs := append([]any{sessionID, anchor}, roleArgs...) + afterArgs = append(afterArgs, afterLimit) + after, err := db.queryMessageRows(ctx, afterQuery, afterArgs...) + if err != nil { + return nil, fmt.Errorf("querying after-window messages: %w", err) + } + + msgs := make([]Message, 0, len(before)+len(anchorMsgs)+len(after)) + msgs = append(msgs, before...) + msgs = append(msgs, anchorMsgs...) + msgs = append(msgs, after...) + if err := db.attachToolCalls(ctx, msgs); err != nil { + return nil, err + } + return msgs, nil +} + +// queryMessageRows runs query and scans the resulting message rows, +// without attaching tool calls (callers batch that across the merged set). +func (db *DB) queryMessageRows( + ctx context.Context, query string, args ...any, +) ([]Message, error) { + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanMessages(rows) +} + +// roleFilterClause returns an "AND role IN (...)" clause and its bind +// args for the given roles, or ("", nil) when roles is empty. +func roleFilterClause(roles []string) (string, []any) { + if len(roles) == 0 { + return "", nil + } + placeholders := make([]string, len(roles)) + args := make([]any, len(roles)) + for i, r := range roles { + placeholders[i] = "?" + args[i] = r + } + return " AND role IN (" + strings.Join(placeholders, ",") + ")", args +} + // GetAllMessages returns all messages for a session ordered by ordinal. func (db *DB) GetAllMessages( ctx context.Context, sessionID string, @@ -202,6 +363,327 @@ func (db *DB) GetAllMessages( return msgs, nil } +// EmbeddableUnit is one embedding document: a single embeddable user +// message, or a run of contiguous embeddable assistant messages. +type EmbeddableUnit struct { + SessionID string + Kind string // "user" | "run" + SourceUUID string // first member's source_uuid ("" legacy) + Ordinal int // first member's ordinal (ordinal_start) + OrdinalEnd int // last member's ordinal (== Ordinal for user docs) + Subordinate bool + Content string // members joined with "\n\n" + Offsets []UnitOffset // one per member; nil for user docs +} + +// UnitOffset locates one member message inside a run's joined content. +type UnitOffset struct { + Ordinal int `json:"o"` + RuneStart int `json:"r"` + ByteStart int `json:"b"` +} + +// ScanEmbeddableUnits streams the embeddable universe — user/assistant +// messages that are not is_system and not system-prefixed (per +// SystemPrefixSQL), from non-trashed sessions — reducing contiguous runs of +// embeddable assistant messages between embeddable user messages into single +// EmbeddableUnit "run" documents; each embeddable user message is emitted as +// its own "user" unit. Units are emitted in (session_id, ordinal) order of +// their first member, closing any open run whenever an embeddable user row, +// a session boundary, or an is_sidechain transition is reached, and finally +// at the end of the scan. +// +// since != "" restricts the scan to sessions with ended_at >= since (RFC3339 +// or RFC3339Nano) for incremental refresh, comparing parsed timestamps +// rather than raw strings via SQLite's datetime() so mixed fractional-second +// precision doesn't produce a wrong ordering (see optionalSinceClause); "" +// scans every session. includeAutomated=false additionally excludes +// automated sessions (sessions.is_automated = 1) using the exact predicate +// sessionFilterPredicates' ExcludeAutomated scope applies +// (automatedScopePredicate("human", ...)), so the embedding index's default +// scope matches session search's default exclusion of automated sessions. +// maxEnded returns the maximum sessions.ended_at seen across the scanned +// rows (as its original raw string), or "" when the scan produced no rows. +// +// Because the SQL predicates already exclude non-embeddable user rows +// (is_system, system-prefixed) from the stream entirely, "does this row +// split the run" has no separate detector to get wrong: an embeddable user +// row always closes any open run and emits its own unit, and an excluded +// user row is simply invisible to the reducer, which is exactly the desired +// "does not split" behavior. +// +// A unit is Subordinate when its session is a subagent or fork session, or +// has a parent session and is not an explicit continuation of it, or (for a +// run) its members are is_sidechain -- a sidechain transition always closes +// the run first, so every member of one run shares a single is_sidechain +// value. +func (db *DB) ScanEmbeddableUnits( + ctx context.Context, since string, includeAutomated bool, + fn func(EmbeddableUnit) error, +) (maxEnded string, err error) { + preds := []string{ + "m.role IN ('user', 'assistant')", + "m.is_system = 0", + "s.deleted_at IS NULL", + SystemPrefixSQL("m.content", "m.role"), + } + if !includeAutomated { + preds = append(preds, automatedScopePredicate("human", "s.is_automated")) + } + + query := ` + SELECT m.session_id, m.role, m.source_uuid, m.ordinal, m.content, + m.is_sidechain, s.relationship_type, s.parent_session_id, + s.ended_at + FROM messages m + JOIN sessions s ON s.id = m.session_id + WHERE ` + strings.Join(preds, "\n\t\t AND ") + ` + ` + optionalSinceClause(since) + ` + ORDER BY m.session_id, m.ordinal` + + args := []any{} + if since != "" { + args = append(args, since) + } + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return "", fmt.Errorf("scanning embeddable units: %w", err) + } + defer rows.Close() + + red := &unitReducer{fn: fn} + maxEnded, err = reduceUnitRows(rows, red) + if err != nil { + return "", err + } + if err := red.finish(); err != nil { + return "", err + } + return maxEnded, nil +} + +// reduceUnitRows scans every row of an open ScanEmbeddableUnits query into +// red, tracking the chronologically latest sessions.ended_at seen across +// them. It does not flush red's final open run -- callers must call +// red.finish() once scanning completes. +func reduceUnitRows(rows *sql.Rows, red *unitReducer) (maxEnded string, err error) { + for rows.Next() { + var row unitRow + var relationshipType string + var parentSessionID, ended sql.NullString + if err := rows.Scan( + &row.sessionID, &row.role, &row.sourceUUID, &row.ordinal, + &row.content, &row.sidechain, &relationshipType, + &parentSessionID, &ended, + ); err != nil { + return "", fmt.Errorf("scanning embeddable unit row: %w", err) + } + if ended.Valid && endedAfter(ended.String, maxEnded) { + maxEnded = ended.String + } + row.subordinateSession = isSubordinateSession(relationshipType, parentSessionID) + if err := red.push(row); err != nil { + return "", err + } + } + if err := rows.Err(); err != nil { + return "", fmt.Errorf("iterating embeddable units: %w", err) + } + return maxEnded, nil +} + +// isSubordinateSession reports whether every unit produced from a session is +// subordinate: a subagent or fork session, or any session with a parent that +// is not an explicit continuation of it. A session with no parent (and not +// itself a subagent/fork) is top-level. +func isSubordinateSession( + relationshipType string, parentSessionID sql.NullString, +) bool { + if relationshipType == "subagent" || relationshipType == "fork" { + return true + } + hasParent := parentSessionID.Valid && parentSessionID.String != "" + return hasParent && relationshipType != "continuation" +} + +// unitRow is one scanned ScanEmbeddableUnits row, carrying the +// session-level subordinate classification alongside the per-message fields +// needed to build either a user doc or a run member. +type unitRow struct { + sessionID string + role string + sourceUUID string + ordinal int + content string + sidechain bool + subordinateSession bool +} + +// unitReducer accumulates ScanEmbeddableUnits rows (already ordered by +// session_id, ordinal) into EmbeddableUnit documents, emitting each through +// fn as soon as it closes. Rows must be pushed in stream order; finish must +// be called once after the last row to flush any run still open at the end +// of the scan. +type unitReducer struct { + fn func(EmbeddableUnit) error + + haveSession bool + sessionID string + run []unitRow +} + +// push feeds one row into the reducer, emitting a user unit immediately or +// accumulating an assistant row into the open run. It closes any open run +// first whenever the row starts a new session, is a user row, or (for an +// assistant row) has an is_sidechain value different from the open run's. +func (r *unitReducer) push(row unitRow) error { + newSession := r.haveSession && row.sessionID != r.sessionID + if err := r.closeRunIf(newSession); err != nil { + return err + } + r.haveSession = true + r.sessionID = row.sessionID + + if row.role == "user" { + if err := r.closeRun(); err != nil { + return err + } + return r.fn(userUnit(row)) + } + + sidechainFlip := len(r.run) > 0 && row.sidechain != r.run[0].sidechain + if err := r.closeRunIf(sidechainFlip); err != nil { + return err + } + r.run = append(r.run, row) + return nil +} + +// closeRunIf closes the open run when cond is true; it is a no-op otherwise. +func (r *unitReducer) closeRunIf(cond bool) error { + if !cond { + return nil + } + return r.closeRun() +} + +// finish flushes any run left open at the end of the scan. +func (r *unitReducer) finish() error { + return r.closeRun() +} + +func (r *unitReducer) closeRun() error { + if len(r.run) == 0 { + return nil + } + unit := runUnit(r.run) + r.run = nil + return r.fn(unit) +} + +// userUnit builds the single-member "user" unit for an embeddable user row. +func userUnit(row unitRow) EmbeddableUnit { + return EmbeddableUnit{ + SessionID: row.sessionID, + Kind: "user", + SourceUUID: row.sourceUUID, + Ordinal: row.ordinal, + OrdinalEnd: row.ordinal, + Subordinate: row.subordinateSession || row.sidechain, + Content: row.content, + } +} + +// runUnit joins a closed run's members with "\n\n" into one "run" unit, +// recording each member's rune/byte offset into the joined content. The +// separator is ASCII, so its rune and byte lengths are equal. +func runUnit(members []unitRow) EmbeddableUnit { + const sep = "\n\n" + first := members[0] + var b strings.Builder + offsets := make([]UnitOffset, len(members)) + runeStart, byteStart := 0, 0 + for i, m := range members { + if i > 0 { + b.WriteString(sep) + runeStart += len(sep) + byteStart += len(sep) + } + offsets[i] = UnitOffset{ + Ordinal: m.ordinal, RuneStart: runeStart, ByteStart: byteStart, + } + b.WriteString(m.content) + runeStart += utf8.RuneCountInString(m.content) + byteStart += len(m.content) + } + return EmbeddableUnit{ + SessionID: first.sessionID, + Kind: "run", + SourceUUID: first.sourceUUID, + Ordinal: first.ordinal, + OrdinalEnd: members[len(members)-1].ordinal, + Subordinate: first.subordinateSession || first.sidechain, + Content: b.String(), + Offsets: offsets, + } +} + +// optionalSinceClause returns the AND clause restricting the embeddable scan +// to sessions with ended_at >= since (or ended_at IS NULL), or "" when since +// is unset. It compares via SQLite's datetime() rather than raw string +// ordering: RFC3339Nano's variable fractional-second precision (e.g. +// ended_at values are sometimes stored with milliseconds, sometimes +// without) makes lexicographic comparison wrong, since "...00.123Z" sorts +// before "...00Z". datetime() truncates to second granularity, so a session +// whose true ended_at is a few hundred milliseconds before since may be +// re-scanned; that overlap is harmless because Refresh's upserts are +// idempotent. +// +// A NULL ended_at (a session still in progress, or one whose parser never +// set it) always matches: excluding it would make its messages invisible to +// every incremental scan until a full (since="") rebuild happens to catch +// it, even though re-scanning an unchanged session is just a cheap no-op +// mirror upsert. A legacy empty-string ended_at (the pre-NULLIF-migration +// "unset" sentinel this repo's other read queries guard against, e.g. +// sessions.go's COALESCE(NULLIF(ended_at, ""), ...) chains) must be treated +// the same way via NULLIF(s.ended_at, ""): without it, "" is neither NULL +// nor >= since, so a changed legacy session would never be rescanned again +// once any watermark exists. +func optionalSinceClause(since string) string { + if since == "" { + return "" + } + return "AND (NULLIF(s.ended_at, '') IS NULL OR " + + "datetime(NULLIF(s.ended_at, '')) >= datetime(?))" +} + +// endedAfter reports whether candidate is chronologically after current, +// comparing parsed RFC3339/RFC3339Nano timestamps rather than raw strings +// so variable fractional-second precision can't produce a wrong ordering. +// An empty current is always considered older. Falls back to a +// lexicographic comparison if either value fails to parse. +func endedAfter(candidate, current string) bool { + if current == "" { + return true + } + c, errC := parseEndedAt(candidate) + cur, errCur := parseEndedAt(current) + if errC != nil || errCur != nil { + return candidate > current + } + return c.After(cur) +} + +// parseEndedAt parses an ended_at value, trying RFC3339Nano (which also +// accepts plain RFC3339) first and falling back to strict RFC3339. +func parseEndedAt(s string) (time.Time, error) { + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + return time.Parse(time.RFC3339, s) +} + // insertMessagesTx batch-inserts messages within an existing // transaction. Returns a slice of message IDs parallel to the // input msgs slice. The caller must hold db.mu. diff --git a/internal/db/messages_units_test.go b/internal/db/messages_units_test.go new file mode 100644 index 000000000..9cde33d85 --- /dev/null +++ b/internal/db/messages_units_test.go @@ -0,0 +1,703 @@ +package db + +import ( + "context" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// scanUnits runs ScanEmbeddableUnits and collects every emitted unit in +// stream order, failing the test on error. +func scanUnits( + t *testing.T, d *DB, since string, includeAutomated bool, +) ([]EmbeddableUnit, string) { + t.Helper() + var got []EmbeddableUnit + maxEnded, err := d.ScanEmbeddableUnits( + context.Background(), since, includeAutomated, + func(u EmbeddableUnit) error { + got = append(got, u) + return nil + }) + require.NoError(t, err) + return got, maxEnded +} + +// TestScanEmbeddableUnitsUserAssistantAlternation asserts that alternating +// user/assistant messages produce one "user" unit per user message and one +// "run" unit per contiguous span of assistant messages, joined with "\n\n" +// and carrying the first/last member's ordinal. +func TestScanEmbeddableUnitsUserAssistantAlternation(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-1", Ordinal: 0, Role: "user", + Content: "u0", ContentLength: 2, Timestamp: tsZero, + SourceUUID: "uuid-u0", + }, + Message{ + SessionID: "sess-1", Ordinal: 1, Role: "assistant", + Content: "a1", ContentLength: 2, Timestamp: tsZeroS1, + SourceUUID: "uuid-a1", + }, + Message{ + SessionID: "sess-1", Ordinal: 2, Role: "assistant", + Content: "a2", ContentLength: 2, Timestamp: tsZeroS2, + }, + Message{ + SessionID: "sess-1", Ordinal: 3, Role: "user", + Content: "u3", ContentLength: 2, Timestamp: tsHour1, + }, + Message{ + SessionID: "sess-1", Ordinal: 4, Role: "assistant", + Content: "a4", ContentLength: 2, Timestamp: tsHour1, + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 4) + assert.Equal(t, EmbeddableUnit{ + SessionID: "sess-1", Kind: "user", SourceUUID: "uuid-u0", + Ordinal: 0, OrdinalEnd: 0, Content: "u0", + }, got[0]) + + assert.Equal(t, "run", got[1].Kind) + assert.Equal(t, "uuid-a1", got[1].SourceUUID) + assert.Equal(t, 1, got[1].Ordinal) + assert.Equal(t, 2, got[1].OrdinalEnd) + assert.Equal(t, "a1\n\na2", got[1].Content) + require.Len(t, got[1].Offsets, 2) + + assert.Equal(t, EmbeddableUnit{ + SessionID: "sess-1", Kind: "user", + Ordinal: 3, OrdinalEnd: 3, Content: "u3", + }, got[2]) + + assert.Equal(t, "run", got[3].Kind) + assert.Equal(t, 4, got[3].Ordinal) + assert.Equal(t, 4, got[3].OrdinalEnd) + require.Len(t, got[3].Offsets, 1) +} + +// TestScanEmbeddableUnitsSystemPrefixedUserRowDoesNotSplitRun asserts that a +// system-prefixed user row (excluded from the embeddable universe by +// SystemPrefixSQL) is invisible to the reducer and therefore does not split +// the assistant run around it. +func TestScanEmbeddableUnitsSystemPrefixedUserRowDoesNotSplitRun(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-1", Ordinal: 0, Role: "assistant", + Content: "a0", ContentLength: 2, Timestamp: tsZero, + }, + Message{ + SessionID: "sess-1", Ordinal: 1, Role: "user", + Content: " x", ContentLength: 22, + Timestamp: tsZeroS1, + }, + Message{ + SessionID: "sess-1", Ordinal: 2, Role: "assistant", + Content: "a2", ContentLength: 2, Timestamp: tsZeroS2, + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 1) + assert.Equal(t, "run", got[0].Kind) + assert.Equal(t, 0, got[0].Ordinal) + assert.Equal(t, 2, got[0].OrdinalEnd) + assert.Equal(t, "a0\n\na2", got[0].Content) +} + +// TestScanEmbeddableUnitsIsSystemUserRowDoesNotSplitButPlainUserRowDoes +// asserts that an is_system=1 user row (also excluded from the embeddable +// universe) does not split a run, while a plain embeddable user row does. +func TestScanEmbeddableUnitsIsSystemUserRowDoesNotSplitButPlainUserRowDoes( + t *testing.T, +) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-1", Ordinal: 0, Role: "assistant", + Content: "a0", ContentLength: 2, Timestamp: tsZero, + }, + Message{ + SessionID: "sess-1", Ordinal: 1, Role: "user", + Content: "system flag set", ContentLength: 15, + Timestamp: tsZeroS1, IsSystem: true, + }, + Message{ + SessionID: "sess-1", Ordinal: 2, Role: "assistant", + Content: "a2", ContentLength: 2, Timestamp: tsZeroS2, + }, + Message{ + SessionID: "sess-1", Ordinal: 3, Role: "user", + Content: "u3", ContentLength: 2, Timestamp: tsHour1, + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 2) + assert.Equal(t, "run", got[0].Kind) + assert.Equal(t, 0, got[0].Ordinal) + assert.Equal(t, 2, got[0].OrdinalEnd) + assert.Equal(t, "user", got[1].Kind) + assert.Equal(t, 3, got[1].Ordinal) +} + +// TestScanEmbeddableUnitsSidechainTransitionSplitsRun asserts that a +// transition in is_sidechain between consecutive assistant messages closes +// the open run and starts a new one, and that a run whose members are +// is_sidechain is marked Subordinate even in an otherwise top-level session. +func TestScanEmbeddableUnitsSidechainTransitionSplitsRun(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-1", Ordinal: 0, Role: "assistant", + Content: "a0", ContentLength: 2, Timestamp: tsZero, + }, + Message{ + SessionID: "sess-1", Ordinal: 1, Role: "assistant", + Content: "a1", ContentLength: 2, Timestamp: tsZeroS1, + IsSidechain: true, + }, + Message{ + SessionID: "sess-1", Ordinal: 2, Role: "assistant", + Content: "a2", ContentLength: 2, Timestamp: tsZeroS2, + IsSidechain: true, + }, + Message{ + SessionID: "sess-1", Ordinal: 3, Role: "assistant", + Content: "a3", ContentLength: 2, Timestamp: tsHour1, + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 3) + + assert.Equal(t, 0, got[0].Ordinal) + assert.Equal(t, 0, got[0].OrdinalEnd) + assert.False(t, got[0].Subordinate) + + assert.Equal(t, 1, got[1].Ordinal) + assert.Equal(t, 2, got[1].OrdinalEnd) + assert.True(t, got[1].Subordinate, + "a run whose members are is_sidechain must be marked subordinate") + + assert.Equal(t, 3, got[2].Ordinal) + assert.Equal(t, 3, got[2].OrdinalEnd) + assert.False(t, got[2].Subordinate) +} + +// TestScanEmbeddableUnitsSubordinateClassification covers the session-level +// Subordinate rule: subagent/fork sessions are always subordinate, a +// continuation with a parent is top-level, a parent-linked session with an +// empty relationship_type is subordinate, and a session with neither a +// parent nor a relationship is top-level. +func TestScanEmbeddableUnitsSubordinateClassification(t *testing.T) { + tests := []struct { + name string + relationshipType string + parentSessionID *string + wantSubordinate bool + }{ + {"SubagentIsSubordinate", "subagent", nil, true}, + {"ForkIsSubordinate", "fork", nil, true}, + { + "ContinuationWithParentIsTopLevel", "continuation", + Ptr("parent-1"), false, + }, + { + "ParentLinkedWithEmptyRelationshipIsSubordinate", "", + Ptr("parent-1"), true, + }, + {"NoParentNoRelationshipIsTopLevel", "", nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.RelationshipType = tt.relationshipType + s.ParentSessionID = tt.parentSessionID + }) + insertMessages(t, d, Message{ + SessionID: "sess-1", Ordinal: 0, Role: "user", + Content: "u0", ContentLength: 2, Timestamp: tsZero, + }) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 1) + assert.Equal(t, tt.wantSubordinate, got[0].Subordinate) + }) + } +} + +// TestScanEmbeddableUnitsOffsetsMultiByteContent asserts that member offsets +// into a run's joined content are computed in rune and byte units that +// correctly account for multi-byte UTF-8 characters, and that each offset +// locates the start of its member's own text within Content. +func TestScanEmbeddableUnitsOffsetsMultiByteContent(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + first := "héllo…" + second := "world" + insertMessages(t, d, + Message{ + SessionID: "sess-1", Ordinal: 0, Role: "assistant", + Content: first, ContentLength: len(first), Timestamp: tsZero, + }, + Message{ + SessionID: "sess-1", Ordinal: 1, Role: "assistant", + Content: second, ContentLength: len(second), Timestamp: tsZeroS1, + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 1) + unit := got[0] + require.Len(t, unit.Offsets, 2) + + assert.Equal(t, 0, unit.Offsets[0].RuneStart) + assert.Equal(t, 0, unit.Offsets[0].ByteStart) + assert.True(t, strings.HasPrefix( + unit.Content[unit.Offsets[0].ByteStart:], first, + )) + + wantSecondRuneStart := utf8.RuneCountInString(first) + utf8.RuneCountInString("\n\n") + wantSecondByteStart := len(first) + len("\n\n") + assert.Equal(t, wantSecondRuneStart, unit.Offsets[1].RuneStart) + assert.Equal(t, wantSecondByteStart, unit.Offsets[1].ByteStart) + assert.True(t, strings.HasPrefix( + unit.Content[unit.Offsets[1].ByteStart:], second, + )) + + assert.Equal(t, + utf8.RuneCountInString(unit.Content[:unit.Offsets[1].ByteStart]), + unit.Offsets[1].RuneStart, + "RuneStart must equal the rune count of everything preceding it in Content") +} + +// TestScanEmbeddableUnitsSingleMessageRunDegenerates asserts that a run +// consisting of a single assistant message has Ordinal == OrdinalEnd and a +// single zero-based offset. +func TestScanEmbeddableUnitsSingleMessageRunDegenerates(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, Message{ + SessionID: "sess-1", Ordinal: 5, Role: "assistant", + Content: "solo", ContentLength: 4, Timestamp: tsZero, + SourceUUID: "uuid-solo", + }) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 1) + assert.Equal(t, "run", got[0].Kind) + assert.Equal(t, "uuid-solo", got[0].SourceUUID) + assert.Equal(t, 5, got[0].Ordinal) + assert.Equal(t, got[0].Ordinal, got[0].OrdinalEnd) + assert.Equal(t, "solo", got[0].Content) + require.Len(t, got[0].Offsets, 1) + assert.Equal(t, UnitOffset{Ordinal: 5, RuneStart: 0, ByteStart: 0}, + got[0].Offsets[0]) +} + +// TestScanEmbeddableUnitsMixedFractionalPrecisionSinceAndMaxEnded asserts +// the since filter and the returned maxEnded watermark compare mixed +// fractional-second ended_at precision chronologically, not +// lexicographically: a raw string comparison ranks "...01Z" above +// "...01.500Z" because '.' sorts below 'Z', so a buggy implementation would +// both wrongly exclude a since-eligible fractional row and wrongly report an +// earlier whole-second row as the max. +func TestScanEmbeddableUnitsMixedFractionalPrecisionSinceAndMaxEnded(t *testing.T) { + d := testDB(t) + + seed := func(id, endedAt string) { + insertSession(t, d, id, "proj", func(s *Session) { + s.EndedAt = Ptr(endedAt) + }) + insertMessages(t, d, Message{ + SessionID: id, Ordinal: 0, Role: "user", + Content: id + " content", ContentLength: len(id) + len(" content"), + Timestamp: tsZero, + }) + } + + seed("too-old", "2024-01-01T00:00:00Z") + seed("frac-after-since", "2024-01-01T00:00:01.500Z") + seed("whole-second-max-trap", "2024-01-01T00:00:05Z") + seed("true-max-fractional", "2024-01-01T00:00:05.900Z") + + got, maxEnded := scanUnits(t, d, "2024-01-01T00:00:01Z", true) + + var ids []string + for _, u := range got { + ids = append(ids, u.SessionID) + } + assert.NotContains(t, ids, "too-old", + "a session ended before since must be excluded") + assert.Contains(t, ids, "frac-after-since") + assert.Contains(t, ids, "whole-second-max-trap") + assert.Contains(t, ids, "true-max-fractional") + assert.Equal(t, "2024-01-01T00:00:05.900Z", maxEnded, + "maxEnded must be the chronologically latest ended_at") +} + +// TestScanEmbeddableUnitsExcludesAutomatedByDefault asserts an automated +// session's units are excluded when includeAutomated is false (the embedding +// index's default scope, mirroring session search's default exclusion of +// automated sessions) and included when true. +func TestScanEmbeddableUnitsExcludesAutomatedByDefault(t *testing.T) { + d := testDB(t) + insertSession(t, d, "human-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, Message{ + SessionID: "human-sess", Ordinal: 0, Role: "user", + Content: "human content", ContentLength: len("human content"), + Timestamp: tsZero, + }) + + insertSession(t, d, "auto-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.IsAutomated = true + }) + insertMessages(t, d, Message{ + SessionID: "auto-sess", Ordinal: 0, Role: "user", + Content: "automated content", ContentLength: len("automated content"), + Timestamp: tsZero, + }) + + tests := []struct { + name string + includeAutomated bool + want []string + }{ + {"ExcludesAutomatedByDefault", false, []string{"human-sess"}}, + {"IncludesAutomatedWhenOptedIn", true, []string{"auto-sess", "human-sess"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _ := scanUnits(t, d, "", tt.includeAutomated) + var ids []string + for _, u := range got { + ids = append(ids, u.SessionID) + } + assert.ElementsMatch(t, tt.want, ids) + }) + } +} + +// TestScanEmbeddableUnitsFiltersRolesAndPrefixes seeds one session with a +// mix of user/assistant/tool/system-role messages plus a system-prefixed and +// an is_system user message, and asserts only the clean user/assistant rows +// contribute units, with maxEnded reporting the session's ended_at. +func TestScanEmbeddableUnitsFiltersRolesAndPrefixes(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "sess-1", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-1", Ordinal: 0, Role: "user", + Content: "hello there", ContentLength: len("hello there"), + Timestamp: tsZero, + }, + Message{ + SessionID: "sess-1", Ordinal: 1, Role: "assistant", + Content: "hi back", ContentLength: len("hi back"), + Timestamp: tsZeroS1, + }, + Message{ + SessionID: "sess-1", Ordinal: 2, Role: "tool", + Content: "tool output", ContentLength: len("tool output"), + Timestamp: tsZeroS2, + }, + Message{ + SessionID: "sess-1", Ordinal: 3, Role: "system", + Content: "system note", ContentLength: len("system note"), + Timestamp: tsHour1, + }, + Message{ + SessionID: "sess-1", Ordinal: 4, Role: "user", + Content: "This session is being continued from a previous one", + ContentLength: 10, Timestamp: tsHour1, + }, + Message{ + SessionID: "sess-1", Ordinal: 5, Role: "user", + Content: "is_system flag set", ContentLength: 19, + Timestamp: tsHour1, IsSystem: true, + }, + ) + + got, maxEnded := scanUnits(t, d, "", true) + + require.Len(t, got, 2) + assert.Equal(t, EmbeddableUnit{ + SessionID: "sess-1", Kind: "user", Ordinal: 0, OrdinalEnd: 0, + Content: "hello there", + }, got[0]) + assert.Equal(t, "run", got[1].Kind) + assert.Equal(t, 1, got[1].Ordinal) + assert.Equal(t, 1, got[1].OrdinalEnd) + assert.Equal(t, "hi back", got[1].Content) + assert.Equal(t, tsHour1, maxEnded) +} + +// TestScanEmbeddableUnitsSinceFiltersOlderSessions asserts that since +// restricts the scan to sessions whose ended_at is >= since, excluding an +// older session entirely. +func TestScanEmbeddableUnitsSinceFiltersOlderSessions(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "old-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsZero) + }) + insertMessages(t, d, Message{ + SessionID: "old-sess", Ordinal: 0, Role: "user", + Content: "old content", ContentLength: len("old content"), + Timestamp: tsZero, + }) + + insertSession(t, d, "new-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsMidYear) + }) + insertMessages(t, d, Message{ + SessionID: "new-sess", Ordinal: 0, Role: "user", + Content: "new content", ContentLength: len("new content"), + Timestamp: tsMidYear, + }) + + got, maxEnded := scanUnits(t, d, tsHour1, true) + + require.Len(t, got, 1) + assert.Equal(t, "new-sess", got[0].SessionID) + assert.Equal(t, tsMidYear, maxEnded) +} + +// TestScanEmbeddableUnitsSinceIncludesNullEndedAtSessions asserts that a +// session whose ended_at is NULL (still in progress, or never set by its +// parser) is not silently excluded from an incremental (since-watermark) +// scan — only a full rescan (since="") previously caught it, leaving its +// messages invisible to the embedding index until then. A session that +// genuinely ended before since must still be excluded. +func TestScanEmbeddableUnitsSinceIncludesNullEndedAtSessions(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "open-sess", "proj") // EndedAt left NULL + insertMessages(t, d, Message{ + SessionID: "open-sess", Ordinal: 0, Role: "user", + Content: "still running", ContentLength: len("still running"), + Timestamp: tsZero, + }) + + insertSession(t, d, "old-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsZero) + }) + insertMessages(t, d, Message{ + SessionID: "old-sess", Ordinal: 0, Role: "user", + Content: "old content", ContentLength: len("old content"), + Timestamp: tsZero, + }) + + got, _ := scanUnits(t, d, tsHour1, true) + + var ids []string + for _, u := range got { + ids = append(ids, u.SessionID) + } + assert.Contains(t, ids, "open-sess", + "a NULL ended_at session must still be visible to an incremental scan") + assert.NotContains(t, ids, "old-sess", + "a session that genuinely ended before since must still be excluded") +} + +// TestScanEmbeddableUnitsSinceIncludesEmptyStringEndedAtSessions asserts +// that a session whose ended_at is the legacy empty-string sentinel (not +// NULL, but never populated by an older parser run) behaves the same as a +// NULL ended_at in an incremental scan: it must not be excluded once any +// refresh watermark exists, and it must never become the reported maxEnded +// watermark, since "" is not a valid timestamp to persist. +func TestScanEmbeddableUnitsSinceIncludesEmptyStringEndedAtSessions(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "legacy-sess", "proj", func(s *Session) { + s.EndedAt = Ptr("") + }) + insertMessages(t, d, Message{ + SessionID: "legacy-sess", Ordinal: 0, Role: "user", + Content: "legacy content", ContentLength: len("legacy content"), + Timestamp: tsZero, + }) + + insertSession(t, d, "old-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsZero) + }) + insertMessages(t, d, Message{ + SessionID: "old-sess", Ordinal: 0, Role: "user", + Content: "old content", ContentLength: len("old content"), + Timestamp: tsZero, + }) + + got, maxEnded := scanUnits(t, d, tsHour1, true) + + var ids []string + for _, u := range got { + ids = append(ids, u.SessionID) + } + assert.Contains(t, ids, "legacy-sess", + "a legacy empty-string ended_at session must still be visible to "+ + "an incremental scan") + assert.NotContains(t, ids, "old-sess", + "a session that genuinely ended before since must still be excluded") + assert.Empty(t, maxEnded, + "an empty-string ended_at must never become the refresh watermark") +} + +// TestScanEmbeddableUnitsEmptyReturnsEmptyWatermark asserts that scanning an +// archive with no embeddable messages returns an empty maxEnded and never +// emits a unit. +func TestScanEmbeddableUnitsEmptyReturnsEmptyWatermark(t *testing.T) { + d := testDB(t) + + got, maxEnded := scanUnits(t, d, "", true) + assert.Empty(t, got) + assert.Empty(t, maxEnded) +} + +// TestScanEmbeddableUnitsOrdersBySessionThenOrdinal asserts units stream in +// (session_id, ordinal) order of their first member across multiple +// sessions, regardless of insertion order. +func TestScanEmbeddableUnitsOrdersBySessionThenOrdinal(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "sess-b", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertSession(t, d, "sess-a", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-b", Ordinal: 0, Role: "user", + Content: "b0", ContentLength: 2, Timestamp: tsZero, + }, + Message{ + SessionID: "sess-a", Ordinal: 1, Role: "assistant", + Content: "a1", ContentLength: 2, Timestamp: tsZeroS1, + SourceUUID: "uuid-a1", + }, + Message{ + SessionID: "sess-a", Ordinal: 0, Role: "user", + Content: "a0", ContentLength: 2, Timestamp: tsZero, + SourceUUID: "uuid-a0", + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 3) + assert.Equal(t, "sess-a", got[0].SessionID) + assert.Equal(t, 0, got[0].Ordinal) + assert.Equal(t, "uuid-a0", got[0].SourceUUID) + assert.Equal(t, "sess-a", got[1].SessionID) + assert.Equal(t, 1, got[1].Ordinal) + assert.Equal(t, "uuid-a1", got[1].SourceUUID) + assert.Equal(t, "sess-b", got[2].SessionID) + assert.Equal(t, 0, got[2].Ordinal) +} + +// TestScanEmbeddableUnitsExcludesTrashedSessions asserts that units +// belonging to a soft-deleted (trashed) session never stream, since a +// trashed session's content should not be indexed for semantic search. +func TestScanEmbeddableUnitsExcludesTrashedSessions(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "trashed-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, Message{ + SessionID: "trashed-sess", Ordinal: 0, Role: "user", + Content: "trashed content", ContentLength: len("trashed content"), + Timestamp: tsZero, + }) + require.NoError(t, d.SoftDeleteSession("trashed-sess")) + + insertSession(t, d, "live-sess", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, Message{ + SessionID: "live-sess", Ordinal: 0, Role: "user", + Content: "live content", ContentLength: len("live content"), + Timestamp: tsZero, + }) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 1) + assert.Equal(t, "live-sess", got[0].SessionID) +} + +// TestScanEmbeddableUnitsSessionChangeClosesOpenRun asserts that a run left +// open at the end of one session is closed and emitted before any unit from +// the next session, even though both sessions end in an open assistant run. +func TestScanEmbeddableUnitsSessionChangeClosesOpenRun(t *testing.T) { + d := testDB(t) + insertSession(t, d, "sess-a", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertSession(t, d, "sess-b", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + }) + insertMessages(t, d, + Message{ + SessionID: "sess-a", Ordinal: 0, Role: "assistant", + Content: "a0", ContentLength: 2, Timestamp: tsZero, + }, + Message{ + SessionID: "sess-a", Ordinal: 1, Role: "assistant", + Content: "a1", ContentLength: 2, Timestamp: tsZeroS1, + }, + Message{ + SessionID: "sess-b", Ordinal: 0, Role: "assistant", + Content: "b0", ContentLength: 2, Timestamp: tsZero, + }, + ) + + got, _ := scanUnits(t, d, "", true) + + require.Len(t, got, 2) + assert.Equal(t, "sess-a", got[0].SessionID) + assert.Equal(t, 0, got[0].Ordinal) + assert.Equal(t, 1, got[0].OrdinalEnd) + assert.Equal(t, "sess-b", got[1].SessionID) + assert.Equal(t, 0, got[1].Ordinal) +} diff --git a/internal/db/messages_window_test.go b/internal/db/messages_window_test.go new file mode 100644 index 000000000..4d04b2112 --- /dev/null +++ b/internal/db/messages_window_test.go @@ -0,0 +1,157 @@ +package db + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// windowMsgSpec describes one seeded message's role, keyed by ordinal. +type windowMsgSpec struct { + ordinal int + role string +} + +// seedWindowMessages seeds a session with 12 messages (ordinals 0..11) with a +// mix of user/assistant/system roles, used across the GetMessagesWindow +// tests. Layout: +// +// 0 user, 1 assistant, 2 user, 3 assistant, 4 system, 5 user, +// 6 assistant, 7 user, 8 assistant, 9 system, 10 user, 11 assistant +func seedWindowMessages(t *testing.T, d *DB, sessionID string) { + t.Helper() + insertSession(t, d, sessionID, "proj") + specs := []windowMsgSpec{ + {0, "user"}, {1, "assistant"}, {2, "user"}, {3, "assistant"}, + {4, "system"}, {5, "user"}, {6, "assistant"}, {7, "user"}, + {8, "assistant"}, {9, "system"}, {10, "user"}, {11, "assistant"}, + } + msgs := make([]Message, 0, len(specs)) + for _, sp := range specs { + content := "msg" + msgs = append(msgs, Message{ + SessionID: sessionID, + Ordinal: sp.ordinal, + Role: sp.role, + Content: content, + ContentLength: len(content), + IsSystem: sp.role == "system", + }) + } + insertMessages(t, d, msgs...) +} + +func ordinalsOf(msgs []Message) []int { + out := make([]int, len(msgs)) + for i, m := range msgs { + out[i] = m.Ordinal + } + return out +} + +func TestGetMessagesWindow_AroundMidSession(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sMid") + + anchor := 6 + msgs, err := d.GetMessagesWindow(ctx, "sMid", MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{4, 5, 6, 7, 8}, ordinalsOf(msgs), + "unfiltered window should return anchor +/- 2 ordinals ascending") +} + +func TestGetMessagesWindow_RoleFilterCountsFilteredMessages(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sRoleCount") + + anchor := 6 + msgs, err := d.GetMessagesWindow(ctx, "sRoleCount", MessageWindow{ + Around: &anchor, Before: 2, After: 2, + Roles: []string{"user", "assistant"}, + }) + require.NoError(t, err) + // Ordinal 4 (system) sits between 3 and 5, so the 2 role-filtered + // messages before the anchor are ordinals 3 and 5, not 4 and 5. + assert.Equal(t, []int{3, 5, 6, 7, 8}, ordinalsOf(msgs), + "before/after counts should count role-filtered messages, not raw ordinals") +} + +func TestGetMessagesWindow_AnchorIncludedEvenWhenRoleFiltered(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sAnchorFiltered") + + anchor := 4 // role "system", excluded by the role filter + msgs, err := d.GetMessagesWindow(ctx, "sAnchorFiltered", MessageWindow{ + Around: &anchor, Before: 1, After: 1, + Roles: []string{"user", "assistant"}, + }) + require.NoError(t, err) + require.Equal(t, []int{3, 4, 5}, ordinalsOf(msgs), + "anchor must be included even though its own role is filtered out") + assert.Equal(t, "system", msgs[1].Role) +} + +func TestGetMessagesWindow_AroundOrdinalZeroHasNoBefore(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sFirst") + + anchor := 0 + msgs, err := d.GetMessagesWindow(ctx, "sFirst", MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{0, 1, 2}, ordinalsOf(msgs), + "no before rows exist above the first ordinal") +} + +func TestGetMessagesWindow_AroundLastOrdinalHasNoAfter(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sLast") + + anchor := 11 + msgs, err := d.GetMessagesWindow(ctx, "sLast", MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{9, 10, 11}, ordinalsOf(msgs), + "no after rows exist below the last ordinal") +} + +func TestGetMessagesWindow_LinearModeWithRoles(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sLinearRoles") + + msgs, err := d.GetMessagesWindow(ctx, "sLinearRoles", MessageWindow{ + Limit: 100, Asc: true, Roles: []string{"user"}, + }) + require.NoError(t, err) + assert.Equal(t, []int{0, 2, 5, 7, 10}, ordinalsOf(msgs), + "linear mode should apply the role filter like the around mode") +} + +func TestGetMessagesWindow_EmptyRolesEquivalentToGetMessages(t *testing.T) { + d := testDB(t) + ctx := context.Background() + seedWindowMessages(t, d, "sEquiv") + + direct, err := d.GetMessages(ctx, "sEquiv", 3, 5, true) + require.NoError(t, err) + + from := 3 + windowed, err := d.GetMessagesWindow(ctx, "sEquiv", MessageWindow{ + From: &from, Limit: 5, Asc: true, + }) + require.NoError(t, err) + assert.Equal(t, direct, windowed, + "empty Roles should behave identically to GetMessages") +} diff --git a/internal/db/project_identity.go b/internal/db/project_identity.go index 77c0b1723..d42bc2f9e 100644 --- a/internal/db/project_identity.go +++ b/internal/db/project_identity.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "slices" "sort" "strings" @@ -625,6 +626,12 @@ func discoverLegacyLocalGitIdentity(root string) (string, map[string]string) { if !filepath.IsAbs(root) { return "", nil } + // Skip macOS automounter namespaces: probing them wakes + // automountd/opendirectoryd for paths that virtually never exist + // locally (see export.IsAutomountNamespacePath). + if export.IsAutomountNamespacePath(runtime.GOOS, filepath.Clean(root)) { + return "", nil + } resolved, err := filepath.EvalSymlinks(filepath.Clean(root)) if err != nil { return "", nil diff --git a/internal/db/query_dialect.go b/internal/db/query_dialect.go index 594764855..a12eac262 100644 --- a/internal/db/query_dialect.go +++ b/internal/db/query_dialect.go @@ -527,12 +527,7 @@ func sessionFilterPredicates( scope := normalizeAutomatedScope(f.AutomatedScope, f.ExcludeAutomated) oneShotPred := "" if f.ExcludeOneShot { - pred := q("user_message_count") + " > 1" - if scope != "human" { - pred = "(" + q("user_message_count") + " > 1 OR " + - q("is_automated") + " = " + - b.dialect.trueLiteral + ")" - } + pred := oneShotPredicate(f, b, q, scope) if f.IncludeChildren { oneShotPred = pred } else { @@ -577,6 +572,34 @@ func sessionFilterPredicates( return preds, oneShotPred } +// oneShotPredicate builds the ExcludeOneShot predicate: sessions with a +// single user message are dropped unless automated (outside "human" scope) +// or, when ChildExemptOneShot is set (semantic/hybrid content-search scope +// only), the session is a child — nearly all non-automated subagent +// transcripts carry exactly one user message, so without the carve-out the +// one-shot gate would hide the subordinate units the Scope filter governs. +// With ChildExemptOneShot false the emitted SQL is byte-identical to the +// historical predicate. +func oneShotPredicate( + f SessionFilter, b *QueryBuilder, q func(string) string, scope string, +) string { + conds := []string{q("user_message_count") + " > 1"} + if scope != "human" { + conds = append(conds, + q("is_automated")+" = "+b.dialect.trueLiteral) + } + if f.ChildExemptOneShot { + conds = append(conds, + q("relationship_type")+" IN ("+ + b.dialect.SidebarChildRelationshipsSQL()+")", + q("parent_session_id")+" <> ''") + } + if len(conds) == 1 { + return conds[0] + } + return "(" + strings.Join(conds, " OR ") + ")" +} + // buildSessionBaseFilter returns a WHERE clause and args containing the base // predicates (message_count > 0, deleted_at IS NULL) plus user-facing filter // predicates (project, machine, agent, date, etc.) WITHOUT the relationship_type diff --git a/internal/db/read_only_test.go b/internal/db/read_only_test.go index fc59d57c8..646f2fcca 100644 --- a/internal/db/read_only_test.go +++ b/internal/db/read_only_test.go @@ -124,6 +124,87 @@ func TestOpenReadOnlyExistingDBDoesNotWrite(t *testing.T) { assert.Equal(t, before.ModTime(), after.ModTime()) } +// TestOpenReadOnlyReaderRefusesWritesAtSQLiteLevel pins the read-only +// contract below the Go-level requireWritable guard: mattn/go-sqlite3 only +// honors mode=ro when the DSN carries a file: URI prefix, so a bare-path DSN +// silently handed out writable reader handles. A write attempted directly on +// the reader pool must fail inside SQLite itself. +func TestOpenReadOnlyReaderRefusesWritesAtSQLiteLevel(t *testing.T) { + path := createClosedTestDB(t, tempDBPath(t, "sessions.db"), nil) + readonly := openReadOnlyTestDB(t, path) + + _, err := readonly.rawReader().Exec( + `INSERT INTO stats (key, value) VALUES ('ro_probe', 1)`) + require.Error(t, err, + "a read-only reader connection must refuse writes") + assert.Contains(t, err.Error(), "readonly", + "the refusal must be SQLite's readonly-database error, got: %v", err) +} + +// TestOpenPathWithSpecialCharacters pins makeDSN's path escaping: SQLite +// percent-decodes file: URI paths and splits params at `?`, so a directory +// name containing a space and a literal %-hex sequence ("%41") would, raw, +// be decoded to a different path ("weArd dir") and fail to open. Both the +// writable and read-only opens must escape the path, and the read-only +// reader must still refuse writes. +func TestOpenPathWithSpecialCharacters(t *testing.T) { + dir := filepath.Join(t.TempDir(), "we%41rd dir") + require.NoError(t, os.MkdirAll(dir, 0o755)) + path := filepath.Join(dir, "sessions.db") + + rw, err := Open(path) + require.NoError(t, err, + "writable Open must succeed on a path with %% and space") + require.NoError(t, rw.SetSyncState("special_path_probe", "x")) + require.NoError(t, rw.Close()) + + _, err = os.Stat(path) + require.NoError(t, err, + "the database file must exist at the literal path, not a decoded one") + + readonly := openReadOnlyTestDB(t, path) + got, err := readonly.GetSyncState("special_path_probe") + require.NoError(t, err) + assert.Equal(t, "x", got) + + _, err = readonly.rawReader().Exec( + `INSERT INTO stats (key, value) VALUES ('ro_probe', 1)`) + require.Error(t, err, + "a read-only reader connection must refuse writes") + assert.Contains(t, err.Error(), "readonly", + "the refusal must be SQLite's readonly-database error, got: %v", err) +} + +// TestOpenReadOnlyNonWALJournalMode pins that a current-schema database left +// in a non-WAL journal mode still opens read-only: the ro DSN must not carry +// _journal_mode=WAL, because PRAGMA journal_mode=WAL is a write and fails on +// a mode=ro connection. The reader adopts the file's DELETE journal mode and +// still refuses writes. +func TestOpenReadOnlyNonWALJournalMode(t *testing.T) { + path := createClosedTestDB(t, tempDBPath(t, "sessions.db"), func(d *DB) { + require.NoError(t, d.SetSyncState("journal_probe", "delete-mode")) + }) + execRawSQLite(t, path, "PRAGMA journal_mode=DELETE") + _, err := os.Stat(path + "-wal") + require.ErrorIs(t, err, os.ErrNotExist, + "test setup: DELETE journal mode must have removed the WAL file") + + readonly := openReadOnlyTestDB(t, path) + assert.True(t, readonly.ReadOnly()) + + got, err := readonly.GetSyncState("journal_probe") + require.NoError(t, err) + assert.Equal(t, "delete-mode", got) + + require.ErrorIs(t, readonly.SetSyncState("journal_probe", "x"), ErrReadOnly) + _, err = readonly.rawReader().Exec( + `INSERT INTO stats (key, value) VALUES ('ro_probe', 1)`) + require.Error(t, err, + "a read-only reader connection must refuse writes") + assert.Contains(t, err.Error(), "readonly", + "the refusal must be SQLite's readonly-database error, got: %v", err) +} + func TestOpenReadOnlyWriteMethodsReturnErrReadOnly(t *testing.T) { pricing := testModelPricing("model-a") path := createClosedTestDB(t, tempDBPath(t, "sessions.db"), func(d *DB) { diff --git a/internal/db/search.go b/internal/db/search.go index 547d16530..fcaa94fd0 100644 --- a/internal/db/search.go +++ b/internal/db/search.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "regexp" + "sort" + "strconv" "strings" ) @@ -99,10 +101,50 @@ func systemPrefixSQL( )) } parts = append(parts, goalContextPrefixSQL(trimmed, dialect)) - return "NOT (" + roleCol + " = 'user' AND (" + + guard := "" + if dialect == systemPrefixSQLite { + guard = systemPrefixFirstCPGuardSQL(contentCol) + " AND " + } + return "NOT (" + roleCol + " = 'user' AND " + guard + "(" + strings.Join(parts, " OR ") + "))" } +// systemPrefixFirstCPGuardSQL builds a cheap prefilter implied by every +// prefix branch of systemPrefixSQL: for any branch to match, the raw +// content's first code point must be a trimmable whitespace character or the +// first character of one of the known prefixes. unicode() returns the first +// code point as an integer (NULL for empty content, COALESCEd to 0, which is +// never in the set), so rows with ordinary content skip the repeated +// LTRIM/prefix chain after one integer IN test. The guard is AND'ed inside +// the NOT(...), so a false guard reproduces exactly the all-branches-false +// result. SQLite-only for now: PG (ascii) and DuckDB (unicode) analogues +// need their own empty-string audits before the other dialects adopt it. +func systemPrefixFirstCPGuardSQL(contentCol string) string { + seen := make(map[rune]bool) + var cps []int + add := func(r rune) { + if !seen[r] { + seen[r] = true + cps = append(cps, int(r)) + } + } + for _, p := range SystemMsgPrefixes { + add([]rune(p)[0]) + } + add([]rune(legacyGoalContextPrefix)[0]) + add([]rune(codexInternalContextTagPrefix)[0]) + for _, r := range systemPrefixTrimCutset { + add(r) + } + sort.Ints(cps) + items := make([]string, len(cps)) + for i, cp := range cps { + items[i] = strconv.Itoa(cp) + } + return "COALESCE(unicode(" + contentCol + "), 0) IN (" + + strings.Join(items, ", ") + ")" +} + func systemPrefixSQLTrimmed(contentCol string) string { return "LTRIM(" + contentCol + ", ' \t\n\v\f\r" + "\u0085\u00A0\u1680" + diff --git a/internal/db/search_content.go b/internal/db/search_content.go index dc9265f52..850e99119 100644 --- a/internal/db/search_content.go +++ b/internal/db/search_content.go @@ -7,6 +7,7 @@ import ( "fmt" "regexp" "slices" + "strconv" "strings" "unicode" "unicode/utf8" @@ -27,7 +28,7 @@ const ( // include-children / one-shot / orphan logic is shared, not reimplemented. type ContentSearchFilter struct { Pattern string - Mode string // "substring" (default) | "regex" | "fts" + Mode string // "substring" (default) | "regex" | "fts" | "semantic" | "hybrid" Sources []string // subset of {"messages","tool_input","tool_result"} ExcludeSystem bool @@ -37,6 +38,14 @@ type ContentSearchFilter struct { // GitBranch is a branchListSep-joined list of opaque (project, branch) tokens (EncodeBranchFilterToken). GitBranch string + // Scope governs unit visibility for modes "semantic" and "hybrid": + // "top" drops subordinate units (sidechain runs, subagent/fork + // sessions), "subordinate" keeps only them, and "all" (or "") keeps + // both. In those modes it supersedes IncludeChildren, which the other + // modes keep honoring; validation happens at the API/CLI boundary and + // an unknown value here is a SearchInputError. + Scope string + // RevealSecrets returns raw snippets. It defaults false so snippets are // secret-redacted unless a caller (the localhost-gated reveal path) // explicitly opts out; a forgotten flag fails safe. @@ -60,6 +69,31 @@ type ContentMatch struct { Ordinal int `json:"ordinal"` Timestamp string `json:"timestamp"` Snippet string `json:"snippet"` + // Score is the searcher's relevance score for "semantic"/"hybrid" modes, + // nil for the other modes which have no comparable ranking signal. + Score *float64 `json:"score,omitempty"` + // OrdinalRange is always present: [start, end] of the conversation unit + // containing the anchor; [ordinal, ordinal] when the anchor is its own + // unit. Ordinal stays the anchor in every mode. + OrdinalRange [2]int `json:"ordinal_range"` + // Subordinate marks a match whose unit is classified subordinate + // (sidechain run, or subagent/fork session), in every mode. + Subordinate bool `json:"subordinate,omitempty"` + // Relationship and ParentSessionID carry the matched session's lineage + // and Sidechain the anchor message's is_sidechain flag, populated in + // every mode (enrichSemanticHits for semantic/hybrid, + // deriveLexicalUnits for substring/regex/fts). + Relationship string `json:"relationship,omitempty"` + ParentSessionID string `json:"parent_session_id,omitempty"` + Sidechain bool `json:"is_sidechain,omitempty"` + // ContextBefore and ContextAfter hold the N messages immediately before + // and after this match's ordinal when the caller requested inline + // context (ContentSearchRequest.Context > 0). Populated by + // directBackend.SearchContent, not by the store itself; nil when + // context was not requested. The anchor message (this match's own + // ordinal) is excluded from both slices. + ContextBefore []Message `json:"context_before,omitempty"` + ContextAfter []Message `json:"context_after,omitempty"` } // ContentSearchPage is a page of matches with an optional next cursor. @@ -79,16 +113,15 @@ func searchInputErrorf(format string, a ...any) error { return &SearchInputError{Msg: fmt.Sprintf(format, a...)} } -// sessionScopeSubquery returns "session_id IN (SELECT id FROM sessions -// WHERE )" plus its args, reusing the session -// filter machinery. The Limit/Cursor on the inner filter are irrelevant -// (no LIMIT in a SELECT id subquery), so they are left unset. -func sessionScopeSubquery(f ContentSearchFilter) (string, []any) { - // Mirror session list: one-shot and automated sessions are excluded by - // default, and IncludeOneShot/IncludeAutomated opt them back in. - // Comprehensive secret coverage comes from the secrets subsystem - // (scanned over every session at sync), not from search defaults. - sf := SessionFilter{ +// contentSessionFilter maps a ContentSearchFilter's session-scoping fields to +// a SessionFilter. Mirroring session list: one-shot and automated sessions +// are excluded by default, and IncludeOneShot/IncludeAutomated opt them back +// in. Comprehensive secret coverage comes from the secrets subsystem +// (scanned over every session at sync), not from search defaults. Shared by +// sessionScopeSubquery (substring/regex/fts) and the semantic-mode +// allowed-session-id lookup so the mapping cannot drift between them. +func contentSessionFilter(f ContentSearchFilter) SessionFilter { + return SessionFilter{ Project: f.Project, ExcludeProject: f.ExcludeProject, Machine: f.Machine, GitBranch: f.GitBranch, Agent: f.Agent, Date: f.Date, DateFrom: f.DateFrom, DateTo: f.DateTo, @@ -97,7 +130,36 @@ func sessionScopeSubquery(f ContentSearchFilter) (string, []any) { ExcludeAutomated: !f.IncludeAutomated, IncludeChildren: f.IncludeChildren, } - where, args := buildSessionFilter(sf) +} + +// sessionScopeSubquery returns "session_id IN (SELECT id FROM sessions +// WHERE )" plus its args, reusing the session +// filter machinery. The Limit/Cursor on the inner filter are irrelevant +// (no LIMIT in a SELECT id subquery), so they are left unset. +func sessionScopeSubquery(f ContentSearchFilter) (string, []any) { + where, args := buildSessionFilter(contentSessionFilter(f)) + return "session_id IN (SELECT id FROM sessions WHERE " + where + ")", args +} + +// semanticContentSessionFilter maps a ContentSearchFilter for the +// semantic/hybrid session scope: the shared contentSessionFilter mapping +// plus the child one-shot exemption (SessionFilter.ChildExemptOneShot) — +// child sessions must not be dropped by the one-shot gate in these modes, +// while top-level one-shots keep today's exclusion. +func semanticContentSessionFilter(f ContentSearchFilter) SessionFilter { + sf := contentSessionFilter(f) + sf.ChildExemptOneShot = true + return sf +} + +// semanticSessionScopeSubquery is sessionScopeSubquery minus the +// sidebar-child exclusion: semantic/hybrid unit visibility is governed by +// Scope (which supersedes IncludeChildren), so the hybrid FTS leg must see +// the same universe the vector leg does — every other predicate (project, +// agent, dates, automated, one-shot for top-level sessions) still applies +// to each session's own row. +func semanticSessionScopeSubquery(f ContentSearchFilter) (string, []any) { + where, args := buildSessionBaseFilter(semanticContentSessionFilter(f)) return "session_id IN (SELECT id FROM sessions WHERE " + where + ")", args } @@ -111,6 +173,17 @@ func (db *DB) SearchContent( if f.Pattern == "" { return ContentSearchPage{}, nil } + + // Semantic and hybrid validate and default Sources themselves (messages + // only) ahead of the substring/regex/fts source-set default just below, + // which fills in tool_input/tool_result that neither mode supports. + switch f.Mode { + case "semantic": + return db.searchContentSemantic(ctx, f) + case "hybrid": + return db.searchContentHybrid(ctx, f) + } + if len(f.Sources) == 0 { f.Sources = []string{"messages", "tool_input", "tool_result"} } @@ -244,9 +317,11 @@ func (db *DB) searchContentSubstring( } // scanContentMatches runs query and assembles a ContentSearchPage, treating -// the (Limit+1)-th row as the cursor sentinel. The query's final column is the -// full source field; makeSnippet derives the (windowed, redacted) snippet from -// it so redaction sees whole secrets rather than a pre-truncated window. +// the (Limit+1)-th row as the cursor sentinel. The body column is the full +// source field; makeSnippet derives the (windowed, redacted) snippet from it +// so redaction sees whole secrets rather than a pre-truncated window. The +// returned page then gets its derived unit ranges and lineage assigned by +// the shared deriveLexicalUnits pass (post-truncation, O(page)). func (db *DB) scanContentMatches( ctx context.Context, query string, args []any, limit, cursor int, makeSnippet func(body string) string, @@ -271,11 +346,21 @@ func (db *DB) scanContentMatches( if err := rows.Err(); err != nil { return ContentSearchPage{}, err } + // Close the cursor before deriving units (exhausting Next already + // auto-closed it; this keeps the release explicit): deriveLexicalUnits + // issues new queries, which must never wait on a connection this cursor + // would otherwise still pin. + if err := rows.Close(); err != nil { + return ContentSearchPage{}, fmt.Errorf("closing content matches: %w", err) + } page := ContentSearchPage{Matches: out} if len(out) > limit { page.Matches = out[:limit] page.NextCursor = cursor + limit } + if err := db.deriveLexicalUnits(ctx, page.Matches); err != nil { + return ContentSearchPage{}, err + } return page, nil } @@ -328,19 +413,29 @@ func (db *DB) searchContentRegex( if err := rows.Err(); err != nil { return ContentSearchPage{}, err } + // Close the candidate cursor before deriving units: the loop breaks out + // with rows still open once Limit+1 matches are collected, and + // deriveLexicalUnits issues new queries that could otherwise block on a + // constrained connection pool while this cursor pins a connection. + if err := rows.Close(); err != nil { + return ContentSearchPage{}, fmt.Errorf("closing regex candidates: %w", err) + } page := ContentSearchPage{Matches: out} if len(out) > f.Limit { page.Matches = out[:f.Limit] page.NextCursor = f.Cursor + f.Limit } + if err := db.deriveLexicalUnits(ctx, page.Matches); err != nil { + return ContentSearchPage{}, err + } return page, nil } // regexCandidateRows returns full-body rows for the selected sources, // LIKE-prefiltered by lit when non-empty, ordered for stable paging. // Each branch selects: session_id, project, agent, location, role, -// tool_name, ordinal, ts AS ts, body, sort_ts. -// The outer query projects the first 9 columns by name. +// tool_name, ordinal, ts AS ts, body, sort_ts, src, row_id. The outer +// query projects the first 9 columns by name. func (db *DB) regexCandidateRows( ctx context.Context, f ContentSearchFilter, lit string, ) (*sql.Rows, error) { @@ -519,6 +614,11 @@ func literalPrefix(pattern string) string { return prefix } +// errFTSUnavailable is returned by the "fts" and "hybrid" content-search +// modes when messages_fts is missing or unusable (e.g. the fts5 module +// failed to load), so both modes report the same capability gate. +var errFTSUnavailable = errors.New("search: full-text search is unavailable") + // searchContentFTS uses messages_fts for fast tokenized matching over // message content only. The caller (service/CLI) guarantees Sources is // messages-only for fts mode. @@ -530,7 +630,7 @@ func (db *DB) searchContentFTS( // as invalid user input (400). With FTS present, the only SQLITE_ERROR the // MATCH query can raise comes from a malformed pattern. if !db.HasFTS() { - return ContentSearchPage{}, errors.New("search: full-text search is unavailable") + return ContentSearchPage{}, errFTSUnavailable } scope, scopeArgs := sessionScopeSubquery(f) sysPred := "1=1" @@ -612,3 +712,751 @@ func classifyFTSError(err error) error { } return err } + +// semanticOverfetchMin floors the candidate count requested from the +// VectorSearcher (k = max(f.Limit*4, semanticOverfetchMin)): session-scope +// filtering may drop some of the searcher's top hits, so more are fetched +// than will ultimately be returned. +const semanticOverfetchMin = 200 + +// validateSemanticSources returns a SearchInputError unless f.Sources is +// empty or exactly {"messages"}: semantic (and hybrid) search only indexes +// message content, mirroring the --fts messages-only restriction enforced +// upstream for fts mode. +func validateSemanticSources(f ContentSearchFilter) error { + for _, s := range f.Sources { + if s != "messages" { + return searchInputErrorf( + "search: semantic search only supports the messages source (got %q)", s) + } + } + return nil +} + +// ValidateSemanticFilter applies the input validation shared by modes +// "semantic" and "hybrid": sources must be empty or exactly {"messages"}, +// and cursor pagination is rejected because both modes return a single +// ranked page rather than an offset-paged result set. It is exported so the +// PostgreSQL and DuckDB backends, which lack a VectorSearcher seam and +// always report ErrSemanticUnavailable for these modes, can run the same +// validation before that capability gate: an invalid request (bad cursor, +// wrong source) must return the same 400 SearchInputError on every backend +// rather than a 501 on backends that check capability first (backend parity, +// see AGENTS.md). +func ValidateSemanticFilter(f ContentSearchFilter) error { + if err := validateSemanticSources(f); err != nil { + return err + } + if f.Cursor != 0 { + return searchInputErrorf( + "semantic search returns a single ranked page; cursor pagination is not supported") + } + switch f.Scope { + case "", "top", "all", "subordinate": + default: + return searchInputErrorf( + "search: invalid scope %q (valid: top, all, subordinate)", f.Scope) + } + return nil +} + +// scopeExcludes reports whether a unit with the given subordinate flag +// falls outside the requested scope: "top" excludes subordinate units, +// "subordinate" excludes top-level ones, and ""/"all" exclude nothing. +// Scope filtering runs on each leg's hits before the RRF merge (and before +// the limit), so a scoped search still fills up to Limit from the +// over-fetched candidates instead of returning a post-truncation remnant. +func scopeExcludes(scope string, subordinate bool) bool { + switch scope { + case "top": + return subordinate + case "subordinate": + return !subordinate + default: + return false + } +} + +// searchContentSemantic runs mode "semantic": it over-fetches ranked hits +// from the wired VectorSearcher, keeps hits whose session passes the +// filter's metadata scope (loaded with one query over the hit session IDs; +// the sidebar-child exclusion is lifted — f.Scope governs subordinate-unit +// visibility instead, dropping hits scopeExcludes rules out), +// routes the surviving ranking through the same RRF merge hybrid uses as a +// one-leg fusion (so subordinate units are penalized identically; matches +// still carry the searcher's own scores), enriches surviving (session_id, +// ordinal) pairs with session/message metadata in one query, and returns +// them in the fused order, truncated to f.Limit. +func (db *DB) searchContentSemantic( + ctx context.Context, f ContentSearchFilter, +) (ContentSearchPage, error) { + if err := ValidateSemanticFilter(f); err != nil { + return ContentSearchPage{}, err + } + searcher := db.getVectorSearcher() + if searcher == nil { + return ContentSearchPage{}, ErrSemanticUnavailable + } + + k := max(f.Limit*4, semanticOverfetchMin) + hits, err := searcher.SemanticSearch(ctx, f.Pattern, k) + if err != nil { + return ContentSearchPage{}, err + } + if len(hits) == 0 { + return ContentSearchPage{}, nil + } + + allowed, err := db.semanticAllowedSessionIDs(ctx, f, uniqueSessionIDs(hits)) + if err != nil { + return ContentSearchPage{}, err + } + surviving := make([]VectorHit, 0, len(hits)) + for _, h := range hits { + if allowed[h.SessionID] && !scopeExcludes(f.Scope, h.Subordinate) { + surviving = append(surviving, h) + } + } + if len(surviving) == 0 { + return ContentSearchPage{}, nil + } + surviving = applySubordinatePenalty(surviving) + + meta, err := db.enrichSemanticHits(ctx, surviving) + if err != nil { + return ContentSearchPage{}, err + } + + out := make([]ContentMatch, 0, min(len(surviving), f.Limit)) + for _, h := range surviving { + info, ok := meta[semanticHitKey{h.SessionID, h.Ordinal}] + if !ok { + continue + } + score := float64(h.Score) + out = append(out, ContentMatch{ + SessionID: h.SessionID, + Project: info.project, + Agent: info.agent, + Location: "message", + Role: info.role, + Ordinal: h.Ordinal, + OrdinalRange: [2]int{h.OrdinalStart, h.OrdinalEnd}, + Subordinate: h.Subordinate, + Relationship: info.relationshipType, + ParentSessionID: info.parentSessionID, + Sidechain: info.isSidechain, + Timestamp: info.timestamp, + Snippet: f.semanticSnippet(info.content, h.Snippet), + Score: &score, + }) + if len(out) >= f.Limit { + break + } + } + return ContentSearchPage{Matches: out}, nil +} + +// unitFusionKey identifies one embedding unit across the hybrid search's +// legs: the mirror's unique (session_id, ordinal_start) pair. The vector leg +// derives it from a VectorHit, the FTS leg from a resolved UnitRef, so hits +// on the same unit fuse. +func unitFusionKey(sessionID string, ordinalStart int) string { + return "u\x00" + sessionID + "\x00" + strconv.Itoa(ordinalStart) +} + +// messageFusionKey identifies an FTS hit with no containing unit at message +// granularity, so an exact-string hit outside the embeddable universe never +// vanishes from the fused result. The "m" prefix keeps it disjoint from +// unitFusionKey's space. +func messageFusionKey(sessionID string, ordinal int) string { + return "m\x00" + sessionID + "\x00" + strconv.Itoa(ordinal) +} + +// unitRanked is one leg entry for rrfMerge: a fusion key plus the unit's +// subordinate flag. +type unitRanked struct { + Key string + Subordinate bool +} + +// mergedUnit is one fused rrfMerge result. +type mergedUnit struct { + unit unitRanked + score float64 +} + +// rrfMerge fuses per-leg unit rankings (best first) with reciprocal-rank +// fusion, penalizing subordinate units by shifting their effective rank +// (rank+5 against a rank constant of 60). Semantic-only search routes its +// single ranked list through this same merge as a one-leg fusion, so the +// penalty applies identically in both modes. Ties break deterministically by +// ascending key; limit > 0 truncates the fused list. Each leg's entries must +// already be deduplicated by Key — both callers dedup via their display-map +// seen-checks — since a repeated key within one leg would accumulate score +// twice. This is a local merge rather than kitvec.Merge because kit's Merge +// has no per-hit rank-offset hook for the subordinate penalty; upstreaming +// such a hook would let this collapse onto kit's implementation later. +func rrfMerge(legs [][]unitRanked, limit int) []mergedUnit { + const rankConstant = 60 + const subordinatePenalty = 5 + scores := make(map[string]float64) + var units []unitRanked + for _, leg := range legs { + for i, u := range leg { + rank := i + 1 + if u.Subordinate { + rank += subordinatePenalty + } + if _, seen := scores[u.Key]; !seen { + units = append(units, u) + } + scores[u.Key] += 1.0 / float64(rankConstant+rank) + } + } + merged := make([]mergedUnit, len(units)) + for i, u := range units { + merged[i] = mergedUnit{unit: u, score: scores[u.Key]} + } + slices.SortFunc(merged, func(a, b mergedUnit) int { + if a.score != b.score { + if a.score > b.score { + return -1 + } + return 1 + } + return strings.Compare(a.unit.Key, b.unit.Key) + }) + if limit > 0 && len(merged) > limit { + merged = merged[:limit] + } + return merged +} + +// applySubordinatePenalty reorders rank-ordered semantic hits through +// rrfMerge as a one-leg fusion, so mode "semantic" penalizes subordinate +// units exactly like mode "hybrid" (one implementation, no hybrid-only +// special case). Hits keep their own scores; only the order changes. A +// duplicate fusion key (two hits on the same unit) keeps its best-ranked +// hit. +func applySubordinatePenalty(hits []VectorHit) []VectorHit { + leg := make([]unitRanked, 0, len(hits)) + byKey := make(map[string]VectorHit, len(hits)) + for _, h := range hits { + key := unitFusionKey(h.SessionID, h.OrdinalStart) + if _, dup := byKey[key]; dup { + continue + } + leg = append(leg, unitRanked{Key: key, Subordinate: h.Subordinate}) + byKey[key] = h + } + merged := rrfMerge([][]unitRanked{leg}, 0) + out := make([]VectorHit, 0, len(merged)) + for _, m := range merged { + out = append(out, byKey[m.unit.Key]) + } + return out +} + +// hybridDisplay carries what one fused unit needs for presentation: the +// anchor (session, ordinal) the match reports and enriches by, the unit's +// ordinal span and subordinate flag (structurally derived for a unit-less +// message-granularity FTS hit, see classifyUnitlessHybridHits), plus the +// leg's raw (unredacted) approximate snippet text used only to center the +// redacted window. +type hybridDisplay struct { + sessionID string + ordinal int + ordinalStart int + ordinalEnd int + subordinate bool + snippet string +} + +// hybridLeg is one rank-ordered fusion leg: entries for rrfMerge plus each +// key's display info. +type hybridLeg struct { + ranked []unitRanked + display map[string]hybridDisplay +} + +// searchContentHybrid runs mode "hybrid": lexical (FTS) and semantic (vector) +// rankings are each over-fetched to k, the vector leg is filtered down to +// sessions passing the filter's metadata scope (the FTS leg filters in SQL), +// FTS message hits are resolved to their containing units, and the two +// rank-ordered leg lists are fused at unit granularity with rrfMerge. +// Returned matches are enriched with session/message metadata via the same +// lookup semantic search uses, ordered by fused score descending, truncated +// to f.Limit. +func (db *DB) searchContentHybrid( + ctx context.Context, f ContentSearchFilter, +) (ContentSearchPage, error) { + if err := ValidateSemanticFilter(f); err != nil { + return ContentSearchPage{}, err + } + searcher := db.getVectorSearcher() + if searcher == nil { + return ContentSearchPage{}, ErrSemanticUnavailable + } + if !db.HasFTS() { + return ContentSearchPage{}, errFTSUnavailable + } + + k := max(f.Limit*4, semanticOverfetchMin) + vecLeg, err := db.hybridVectorLeg(ctx, f, searcher, k) + if err != nil { + return ContentSearchPage{}, err + } + ftsLeg, err := db.hybridFTSLeg(ctx, f, searcher, k) + if err != nil { + return ContentSearchPage{}, err + } + if len(vecLeg.ranked) == 0 && len(ftsLeg.ranked) == 0 { + return ContentSearchPage{}, nil + } + + merged := rrfMerge([][]unitRanked{vecLeg.ranked, ftsLeg.ranked}, f.Limit) + return db.enrichHybridMatches(ctx, f, merged, vecLeg.display, ftsLeg.display) +} + +// hybridVectorLeg over-fetches k semantic unit hits, drops any whose session +// fails the filter's metadata scope (the same child-exclusion-lifted lookup +// searchContentSemantic uses) or whose subordinate flag falls outside +// f.Scope, and returns the survivors as a rank-ordered fusion leg keyed by +// unit. Both filters run before the merge so reciprocal-rank fusion only +// ranks eligible units and a scoped search can still fill the limit. +func (db *DB) hybridVectorLeg( + ctx context.Context, f ContentSearchFilter, searcher VectorSearcher, k int, +) (hybridLeg, error) { + leg := hybridLeg{display: make(map[string]hybridDisplay)} + hits, err := searcher.SemanticSearch(ctx, f.Pattern, k) + if err != nil { + return hybridLeg{}, err + } + if len(hits) == 0 { + return leg, nil + } + allowed, err := db.semanticAllowedSessionIDs(ctx, f, uniqueSessionIDs(hits)) + if err != nil { + return hybridLeg{}, err + } + for _, h := range hits { + if !allowed[h.SessionID] || scopeExcludes(f.Scope, h.Subordinate) { + continue + } + key := unitFusionKey(h.SessionID, h.OrdinalStart) + if _, seen := leg.display[key]; seen { + continue + } + leg.ranked = append(leg.ranked, unitRanked{Key: key, Subordinate: h.Subordinate}) + leg.display[key] = hybridDisplay{ + sessionID: h.SessionID, ordinal: h.Ordinal, snippet: h.Snippet, + ordinalStart: h.OrdinalStart, ordinalEnd: h.OrdinalEnd, + subordinate: h.Subordinate, + } + } + return leg, nil +} + +// maxHybridFTSBatches caps how many k-row FTS batches hybridFTSLeg fetches. +// It bounds the worst-case work when discard dominates — many rows collapsing +// into one unit, or a narrow f.Scope dropping most rows — while letting the +// leg keep paging past discarded rows instead of under-filling after the +// first batch. The residual is documented: a leg needing survivors deeper +// than maxHybridFTSBatches x k rows can still under-fill. +const maxHybridFTSBatches = 4 + +// hybridFTSLeg runs a rank-ordered FTS query over the embedded universe +// (role user/assistant, is_system = 0, system-prefix excluded -- the same +// predicate ScanEmbeddableUnits uses), scoped in SQL to sessions passing +// the child-exclusion-lifted filter (semanticSessionScopeSubquery, so both +// hybrid legs see the same universe), resolves each message hit to its +// containing unit, drops units outside f.Scope, and returns up to k hits +// as a rank-ordered fusion leg. A hit inside a unit adopts the unit's fusion +// key and subordinate flag while keeping its own message ordinal as the +// anchor and its FTS snippet for display (the FTS-anchor override); several +// hits in one unit collapse to the best-ranked one. A hit with no containing +// unit keeps a message-granularity key and survives fusion on its own, with +// its range and subordinate flag structurally derived before the scope +// filter and the merge (classifyUnitlessHybridHits), so it is excluded and +// penalized exactly like lexical mode classifies the same anchor. +// +// Rows are fetched in rank-ordered batches of k with OFFSET continuation: +// collapse and scope filtering can discard most of a batch, so the leg keeps +// fetching until it holds k entries, the stream is exhausted, or +// maxHybridFTSBatches is hit. The display seen-check dedups across batches; +// earlier batches rank better, so the best-ranked hit per unit always wins. +func (db *DB) hybridFTSLeg( + ctx context.Context, f ContentSearchFilter, searcher VectorSearcher, k int, +) (hybridLeg, error) { + leg := hybridLeg{display: make(map[string]hybridDisplay, k)} + for batch := range maxHybridFTSBatches { + hits, err := db.fetchHybridFTSBatch(ctx, f, k, batch*k) + if err != nil { + return hybridLeg{}, err + } + if err := db.appendHybridFTSHits(ctx, searcher, f.Scope, hits, &leg); err != nil { + return hybridLeg{}, err + } + if len(hits) < k || len(leg.ranked) >= k { + break + } + } + return leg, nil +} + +// fetchHybridFTSBatch fetches one rank-ordered batch of at most k FTS message +// rows for hybridFTSLeg, starting at offset. The ORDER BY carries m.id as a +// deterministic tiebreak so OFFSET continuation is stable across batches when +// ranks tie. +func (db *DB) fetchHybridFTSBatch( + ctx context.Context, f ContentSearchFilter, k, offset int, +) ([]hybridDisplay, error) { + scope, scopeArgs := semanticSessionScopeSubquery(f) + query := fmt.Sprintf(` + SELECT m.session_id, m.ordinal, + snippet(messages_fts, 0, '', '', '...', 32) AS snip + FROM messages_fts f JOIN messages m ON m.id = f.rowid + WHERE messages_fts MATCH ? AND m.role IN ('user','assistant') + AND m.is_system = 0 AND %s + AND m.%s + ORDER BY f.rank, m.id LIMIT ? OFFSET ?`, + SystemPrefixSQL("m.content", "m.role"), scope) + + args := []any{PrepareFTSQuery(f.Pattern)} + args = append(args, scopeArgs...) + args = append(args, k, offset) + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return nil, classifyFTSError(fmt.Errorf("hybrid search fts leg: %w", err)) + } + defer rows.Close() + + var hits []hybridDisplay + for rows.Next() { + var hit hybridDisplay + if err := rows.Scan(&hit.sessionID, &hit.ordinal, &hit.snippet); err != nil { + return nil, fmt.Errorf("scan hybrid fts hit: %w", err) + } + hits = append(hits, hit) + } + if err := rows.Err(); err != nil { + return nil, err + } + return hits, nil +} + +// appendHybridFTSHits resolves one batch of FTS message hits to their +// containing units, classifies unit-less hits structurally (range and +// subordinate flag, so the scope filter and the fusion penalty treat them +// exactly like lexical mode), and accumulates the survivors into leg: hits +// outside scope are dropped, and a unit already seen (within or across +// batches) keeps its earlier, better-ranked entry. +func (db *DB) appendHybridFTSHits( + ctx context.Context, searcher VectorSearcher, scope string, + hits []hybridDisplay, leg *hybridLeg, +) error { + if len(hits) == 0 { + return nil + } + refs := make([]MessageRef, len(hits)) + for i, hit := range hits { + refs[i] = MessageRef{SessionID: hit.sessionID, Ordinal: hit.ordinal} + } + units, err := searcher.ResolveMessageUnits(ctx, refs) + if err != nil { + return fmt.Errorf("resolving fts hits to units: %w", err) + } + if len(units) != len(refs) { + return fmt.Errorf( + "resolving fts hits to units: got %d units for %d refs", len(units), len(refs)) + } + + keys := make([]string, len(hits)) + var unitless []int + for i := range hits { + hit := &hits[i] + keys[i] = messageFusionKey(hit.sessionID, hit.ordinal) + hit.ordinalStart, hit.ordinalEnd = hit.ordinal, hit.ordinal + if units[i].DocKey == "" { + unitless = append(unitless, i) + continue + } + keys[i] = unitFusionKey(units[i].SessionID, units[i].OrdinalStart) + hit.ordinalStart = units[i].OrdinalStart + hit.ordinalEnd = units[i].OrdinalEnd + hit.subordinate = units[i].Subordinate + } + if err := db.classifyUnitlessHybridHits(ctx, hits, unitless); err != nil { + return err + } + + for i, hit := range hits { + if scopeExcludes(scope, hit.subordinate) { + continue + } + if _, seen := leg.display[keys[i]]; seen { + continue + } + leg.ranked = append(leg.ranked, unitRanked{Key: keys[i], Subordinate: hit.subordinate}) + leg.display[keys[i]] = hit + } + return nil +} + +// enrichHybridMatches looks up session/message metadata for the fused units +// (reusing enrichSemanticHits' CTE join) and assembles the final page in +// fused-score order. When the FTS leg contributed to a unit, its display +// wins: the match anchors on the FTS-matched message's ordinal and centers +// on the FTS snippet (the vector leg's chunk anchor may be a different run +// member). Either way the returned snippet itself is built (and redacted) +// from the anchor message's full content via semanticSnippet, the same +// guarantee mode "semantic" gives. Unit-less FTS rows arrive with their +// derived range and subordinate flag already assigned pre-merge +// (classifyUnitlessHybridHits), so no derivation runs here. +func (db *DB) enrichHybridMatches( + ctx context.Context, f ContentSearchFilter, merged []mergedUnit, + vecDisplay, ftsDisplay map[string]hybridDisplay, +) (ContentSearchPage, error) { + displays := make([]hybridDisplay, len(merged)) + asHits := make([]VectorHit, len(merged)) + for i, m := range merged { + d, ok := ftsDisplay[m.unit.Key] + if !ok { + d = vecDisplay[m.unit.Key] + } + displays[i] = d + asHits[i] = VectorHit{SessionID: d.sessionID, Ordinal: d.ordinal} + } + meta, err := db.enrichSemanticHits(ctx, asHits) + if err != nil { + return ContentSearchPage{}, err + } + + out := make([]ContentMatch, 0, len(merged)) + for i, m := range merged { + d := displays[i] + info, ok := meta[semanticHitKey{d.sessionID, d.ordinal}] + if !ok { + continue + } + score := m.score + out = append(out, ContentMatch{ + SessionID: d.sessionID, + Project: info.project, + Agent: info.agent, + Location: "message", + Role: info.role, + Ordinal: d.ordinal, + OrdinalRange: [2]int{d.ordinalStart, d.ordinalEnd}, + Subordinate: d.subordinate, + Relationship: info.relationshipType, + ParentSessionID: info.parentSessionID, + Sidechain: info.isSidechain, + Timestamp: info.timestamp, + Snippet: f.semanticSnippet(info.content, d.snippet), + Score: &score, + }) + } + return ContentSearchPage{Matches: out}, nil +} + +// uniqueSessionIDs returns the distinct session IDs referenced by hits. +// Order is irrelevant: the result only feeds an IN (...) clause. +func uniqueSessionIDs(hits []VectorHit) []string { + seen := make(map[string]bool, len(hits)) + ids := make([]string, 0, len(hits)) + for _, h := range hits { + if !seen[h.SessionID] { + seen[h.SessionID] = true + ids = append(ids, h.SessionID) + } + } + return ids +} + +// semanticAllowedSessionIDs runs one query per maxSQLVars-sized chunk of ids +// returning the subset that pass the ContentSearchFilter's metadata scope +// (project, agent, date range, one-shot/automated, ...), reusing the same +// SessionFilter mapping sessionScopeSubquery uses so the two paths cannot +// drift apart. Like semanticSessionScopeSubquery it deliberately omits the +// sidebar-child exclusion and exempts child sessions from the one-shot +// gate (semanticContentSessionFilter): in semantic/hybrid modes Scope +// supersedes IncludeChildren, so subordinate units stay visible to the +// vector leg. Chunking keeps each query's bind count under SQLite's +// 999-variable limit: a semantic overfetch can surface hits from thousands +// of distinct sessions, well past a single IN clause's budget. +func (db *DB) semanticAllowedSessionIDs( + ctx context.Context, f ContentSearchFilter, ids []string, +) (map[string]bool, error) { + if len(ids) == 0 { + return nil, nil + } + where, filterArgs := buildSessionBaseFilter(semanticContentSessionFilter(f)) + query := "SELECT id FROM sessions WHERE " + where + " AND id IN " + + allowed := make(map[string]bool, len(ids)) + err := queryChunked(ids, func(chunk []string) error { + placeholders, chunkArgs := inPlaceholders(chunk) + args := make([]any, 0, len(filterArgs)+len(chunkArgs)) + args = append(args, filterArgs...) + args = append(args, chunkArgs...) + + rows, err := db.getReader().QueryContext(ctx, query+placeholders, args...) + if err != nil { + return fmt.Errorf("semantic search session scope: %w", err) + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return fmt.Errorf("scan semantic session id: %w", err) + } + allowed[id] = true + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + return rows.Close() + }) + if err != nil { + return nil, err + } + return allowed, nil +} + +// semanticHitKey identifies a (session_id, ordinal) pair for enrichment +// lookup. +type semanticHitKey struct { + sessionID string + ordinal int +} + +// semanticHitInfo is the session/message metadata enrichSemanticHits attaches +// to a surviving hit. content is the message's full, un-truncated content: +// semantic/hybrid snippets are built from it (see semanticSnippet) rather +// than from the searcher's pre-truncated chunk/snippet text, so secret +// redaction sees the same whole-body context the substring/regex/fts paths +// give it instead of a fragment that can split a secret at the truncation +// boundary. relationshipType, parentSessionID, and isSidechain carry the +// hit's lineage — joined here from sessions.db (the vector mirror does not +// store lineage per hit); isSidechain is the ANCHOR ordinal's message flag. +type semanticHitInfo struct { + project, agent, role, timestamp, content string + relationshipType, parentSessionID string + isSidechain bool +} + +// enrichHitsChunk is the max hits enrichSemanticHits binds per VALUES CTE +// query. Each hit binds 2 params (session_id, ordinal), so this halves the +// shared maxSQLVars chunk to keep 2*chunk within SQLite's 999-variable limit. +const enrichHitsChunk = maxSQLVars / 2 + +// enrichSemanticHits looks up session/message metadata for hits' (session_id, +// ordinal) pairs via a "WITH hits(session_id, ordinal) AS (VALUES ...)" CTE +// joined to messages/sessions, one query per enrichHitsChunk-sized slice of +// hits (a semantic overfetch can carry thousands of hits, well past what one +// VALUES clause can bind). SQLite versions without row-value IN support over +// VALUES rule out "(session_id, ordinal) IN (VALUES ...)"; the CTE join form +// works everywhere. +func (db *DB) enrichSemanticHits( + ctx context.Context, hits []VectorHit, +) (map[semanticHitKey]semanticHitInfo, error) { + out := make(map[semanticHitKey]semanticHitInfo, len(hits)) + for start := 0; start < len(hits); start += enrichHitsChunk { + chunk := hits[start:min(start+enrichHitsChunk, len(hits))] + + values := make([]string, len(chunk)) + args := make([]any, 0, len(chunk)*2) + for i, h := range chunk { + values[i] = "(?, ?)" + args = append(args, h.SessionID, h.Ordinal) + } + query := "WITH hits(session_id, ordinal) AS (VALUES " + + strings.Join(values, ", ") + ") " + + "SELECT m.session_id, s.project, s.agent, m.role, m.ordinal, " + + "COALESCE(m.timestamp, ''), m.content, " + + "COALESCE(s.relationship_type, ''), " + + "COALESCE(s.parent_session_id, ''), m.is_sidechain " + + "FROM hits h " + + "JOIN messages m ON m.session_id = h.session_id AND m.ordinal = h.ordinal " + + "JOIN sessions s ON s.id = m.session_id" + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("semantic search enrich: %w", err) + } + for rows.Next() { + var key semanticHitKey + var info semanticHitInfo + if err := rows.Scan(&key.sessionID, &info.project, &info.agent, + &info.role, &key.ordinal, &info.timestamp, &info.content, + &info.relationshipType, &info.parentSessionID, + &info.isSidechain); err != nil { + rows.Close() + return nil, fmt.Errorf("scan semantic hit: %w", err) + } + out[key] = info + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return out, nil +} + +// snippetTruncationMarkers are the elision markers left by the two sources of +// approximate snippet text semantic/hybrid modes locate within a message's +// full content: the vector index's trailing unicode ellipsis +// (internal/vector's truncateRunes) and FTS5 snippet()'s literal "..." marker +// (used at both ends), configured as fetchHybridFTSBatch's 5th snippet() +// argument. +var snippetTruncationMarkers = []string{"...", "…"} + +// approxSnippetSpan locates approx (a searcher-provided chunk/snippet or +// FTS snippet() fragment, possibly elided at one or both ends) within the +// message's full content, returning the byte span to center a redacted +// window on. approx is trimmed of elision markers first since those markers +// are not literal substrings of content. Returns ok=false when approx cannot +// be located verbatim (e.g. content changed since the snippet was derived), +// leaving the caller to fall back to some other span -- content itself is +// always what gets redacted, so a miss here only affects centering, not the +// redaction guarantee. +func approxSnippetSpan(content, approx string) (start, end int, ok bool) { + trimmed := strings.TrimSpace(approx) + for _, marker := range snippetTruncationMarkers { + trimmed = strings.TrimSpace(strings.TrimSuffix(trimmed, marker)) + trimmed = strings.TrimSpace(strings.TrimPrefix(trimmed, marker)) + } + if trimmed == "" { + return 0, 0, false + } + off := strings.Index(content, trimmed) + if off < 0 { + return 0, 0, false + } + return off, off + len(trimmed), true +} + +// semanticSnippet builds the returned snippet for "semantic" and "hybrid" +// matches from the message's full content, not from the searcher's +// pre-truncated approx (chunk or FTS snippet() text): redaction +// (buildSnippet -> secrets.RedactWindow) must see the whole message so a +// secret straddling approx's truncation boundary cannot leak a fragment that +// full-content redaction would otherwise catch. approx is used only to +// center the window; when it cannot be located in content, FTSSnippetRange +// centers on the query pattern instead, and failing that on the start of +// content -- content is still what gets redacted either way. +func (f ContentSearchFilter) semanticSnippet(content, approx string) string { + if start, end, ok := approxSnippetSpan(content, approx); ok { + return f.buildSnippet(content, start, end) + } + start, end := FTSSnippetRange(f.Pattern, content) + return f.buildSnippet(content, start, end) +} diff --git a/internal/db/search_content_bench_test.go b/internal/db/search_content_bench_test.go new file mode 100644 index 000000000..cbe7c64c6 --- /dev/null +++ b/internal/db/search_content_bench_test.go @@ -0,0 +1,169 @@ +package db + +import ( + "context" + "fmt" + "testing" +) + +// Pre-change baseline for content-search page fetches. CI's bench-gate +// workflow will compare these against a candidate once per-match citation +// derivation lands on top of SearchContent (see the conversation-unit +// citations design), so a regression in the derivation cost shows up as an +// ns/op or allocs/op delta on a PR instead of shipping silently. +// +// The corpus is built once per benchmark (outside the timed loop) to stress +// exactly what citation derivation has to walk: long assistant monologues, +// system rows breaking up a run without ending it, and sidechain stretches +// that do end a run. See seedContentSearchBench for the exact shape. +const ( + benchContentSessions = 40 + benchContentMessages = 300 + // benchContentRunStart and benchContentRunEnd bound the one long + // assistant run per session (inclusive ordinals). + benchContentRunStart = 10 + benchContentRunEnd = 260 + // benchContentSegment is both the system-row cadence inside the run and + // the sidechain-stretch width: every 50th ordinal inside the run (not at + // its start or end) is a system row, and the assistant messages between + // consecutive system rows alternate is_sidechain, so the run contains + // multiple contiguous sidechain stretches rather than one flat run. + benchContentSegment = 50 +) + +// seedContentSearchBench builds a corpus that stresses the citation +// derivation: long assistant runs (the monologue case), sidechain +// stretches, system rows inside runs, and a term that matches broadly. +// 40 sessions x 300 messages; in each session ordinals 10..260 form one +// assistant run (with a system row every 50 ordinals inside it), the rest +// alternate user/assistant. Every assistant message contains "needle"; +// IN-RUN assistant messages additionally contain "runneedle", so the +// rank-ordered FTS benchmark can pin its page to the long-run region +// instead of filling with outside-run hits. +func seedContentSearchBench(b *testing.B, d *DB) { + b.Helper() + for i := range benchContentSessions { + sessionID := fmt.Sprintf("bench-search-%03d", i) + if err := d.UpsertSession(Session{ + ID: sessionID, Project: "bench", Machine: "local", Agent: "claude", + // MessageCount > 0 so buildSessionFilter's base + // "message_count > 0" predicate keeps the session; UserMessageCount + // > 1 so the one-shot/automated exclusion in contentSessionFilter + // does not drop it either. + MessageCount: benchContentMessages, + UserMessageCount: 2, + }); err != nil { + b.Fatalf("seed session %s: %v", sessionID, err) + } + msgs := benchContentSearchMessages(sessionID) + if err := d.InsertMessages(msgs); err != nil { + b.Fatalf("seed messages for %s: %v", sessionID, err) + } + } +} + +// benchContentSearchMessages builds the benchContentMessages-message +// timeline for one session per seedContentSearchBench's shape. +func benchContentSearchMessages(sessionID string) []Message { + msgs := make([]Message, 0, benchContentMessages) + for i := range benchContentMessages { + msgs = append(msgs, benchContentSearchMessage(sessionID, i)) + } + return msgs +} + +// benchContentSearchMessage builds the single message at ordinal in +// sessionID: a system row or sidechain-tagged assistant turn inside the +// long run (benchContentRunStart..benchContentRunEnd), or an alternating +// user/assistant message outside it. Every assistant message contains +// "needle". +func benchContentSearchMessage(sessionID string, ordinal int) Message { + ts := fmt.Sprintf("2026-06-%02dT10:00:00Z", 1+ordinal%28) + if ordinal >= benchContentRunStart && ordinal <= benchContentRunEnd { + return benchContentRunMessage(sessionID, ordinal, ts) + } + if ordinal%2 == 1 { + content := fmt.Sprintf( + "assistant reply %d contains needle outside the monologue run", ordinal, + ) + return Message{ + SessionID: sessionID, Ordinal: ordinal, Role: "assistant", + Content: content, Timestamp: ts, Model: "claude-bench-model", + ContentLength: len(content), + } + } + content := fmt.Sprintf("user message %d asking an unrelated question", ordinal) + return Message{ + SessionID: sessionID, Ordinal: ordinal, Role: "user", + Content: content, Timestamp: ts, ContentLength: len(content), + } +} + +// benchContentRunMessage builds one message inside the long assistant run: +// a system row on interior segment boundaries, otherwise an assistant +// message tagged is_sidechain for alternating benchContentSegment-wide +// stretches. +func benchContentRunMessage(sessionID string, ordinal int, ts string) Message { + offset := ordinal - benchContentRunStart + runLen := benchContentRunEnd - benchContentRunStart + if offset > 0 && offset < runLen && offset%benchContentSegment == 0 { + content := fmt.Sprintf( + "system notice at ordinal %d inside the assistant run", ordinal, + ) + return Message{ + SessionID: sessionID, Ordinal: ordinal, Role: "system", + Content: content, Timestamp: ts, IsSystem: true, + ContentLength: len(content), + } + } + content := fmt.Sprintf( + "assistant monologue turn %d contains needle and runneedle for the search benchmark", + ordinal, + ) + return Message{ + SessionID: sessionID, Ordinal: ordinal, Role: "assistant", + Content: content, Timestamp: ts, Model: "claude-bench-model", + IsSidechain: (offset/benchContentSegment)%2 == 1, + ContentLength: len(content), + } +} + +// BenchmarkSearchContentSubstringPage measures a full 50-hit substring +// content-search page over benchContentSessions sessions, each with a +// "needle" in every assistant message -- the same broad-match, worst-case +// shape citation derivation has to walk per match. +func BenchmarkSearchContentSubstringPage(b *testing.B) { + d := testDB(b) + seedContentSearchBench(b, d) + f := ContentSearchFilter{Pattern: "needle", Limit: 50} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + page, err := d.SearchContent(context.Background(), f) + if err != nil || len(page.Matches) != 50 { + b.Fatalf("search: %v (%d matches, want a full 50-hit page)", err, len(page.Matches)) + } + } +} + +// BenchmarkSearchContentFTSPage is BenchmarkSearchContentSubstringPage's +// FTS-mode counterpart, over the identical corpus. It searches the run-only +// term "runneedle": FTS orders by rank, so a broad term would fill the +// 50-hit page with outside-run hits and never exercise the long-run +// derivation shape this benchmark exists to measure. +func BenchmarkSearchContentFTSPage(b *testing.B) { + d := testDB(b) + if !d.HasFTS() { + b.Skip("fts5 not available") + } + seedContentSearchBench(b, d) + f := ContentSearchFilter{Pattern: "runneedle", Mode: "fts", Limit: 50} + b.ReportAllocs() + b.ResetTimer() + for range b.N { + page, err := d.SearchContent(context.Background(), f) + if err != nil || len(page.Matches) != 50 { + b.Fatalf("search: %v (%d matches, want a full 50-hit page)", err, len(page.Matches)) + } + } +} diff --git a/internal/db/search_content_chunk_test.go b/internal/db/search_content_chunk_test.go new file mode 100644 index 000000000..afea585c6 --- /dev/null +++ b/internal/db/search_content_chunk_test.go @@ -0,0 +1,97 @@ +package db + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSemanticAllowedSessionIDsOverSQLiteVarLimit forces the reader pool's +// SQLite bind-variable limit down to 999 (mirroring +// forceReaderVarLimit's rationale in activityreport_test.go: some builds +// compile against SQLite's older 999 default), then asks +// semanticAllowedSessionIDs to scope a candidate set of 1002 session IDs — a +// count a deep semantic overfetch (thousands of hits from distinct +// sessions) can plausibly produce and single-shot IN (...) query would +// exceed. It must chunk the query and still return exactly the real, +// filter-passing sessions. +func TestSemanticAllowedSessionIDsOverSQLiteVarLimit(t *testing.T) { + d := testDB(t) + ctx := context.Background() + forceReaderVarLimit(t, d, 999) + + // Guard: prove the lowered limit is live on the pool, so a setup that + // failed to constrain it cannot mask the regression checked below. + overLimitPh, overLimitArgs := inPlaceholders(make([]string, 1001)) + _, probeErr := d.getReader().QueryContext( + ctx, "SELECT 1 WHERE '' IN "+overLimitPh, overLimitArgs...) + require.Error(t, probeErr, "reader variable limit was not constrained") + + insertSession(t, d, "real-1", "proj") + insertSession(t, d, "real-2", "proj") + + ids := []string{"real-1", "real-2"} + for i := range 1000 { + ids = append(ids, fmt.Sprintf("fake-%d", i)) + } + + f := ContentSearchFilter{IncludeOneShot: true, IncludeAutomated: true} + allowed, err := d.semanticAllowedSessionIDs(ctx, f, ids) + require.NoError(t, err) + + assert.True(t, allowed["real-1"]) + assert.True(t, allowed["real-2"]) + assert.Len(t, allowed, 2, "no nonexistent id should appear in the result") +} + +// TestEnrichSemanticHitsOverSQLiteVarLimit forces the reader pool's SQLite +// bind-variable limit down to 999, then asks enrichSemanticHits to enrich +// 1002 (session_id, ordinal) hits — each binding 2 params in the VALUES CTE, +// so 2004 total, well past a single query's budget. It must chunk and still +// resolve exactly the hits with a real backing message/session row. +func TestEnrichSemanticHitsOverSQLiteVarLimit(t *testing.T) { + d := testDB(t) + ctx := context.Background() + forceReaderVarLimit(t, d, 999) + + overLimitPh, overLimitArgs := inPlaceholders(make([]string, 1001)) + _, probeErr := d.getReader().QueryContext( + ctx, "SELECT 1 WHERE '' IN "+overLimitPh, overLimitArgs...) + require.Error(t, probeErr, "reader variable limit was not constrained") + + insertSession(t, d, "real-sess", "proj") + insertMessages(t, d, + Message{ + SessionID: "real-sess", Ordinal: 0, Role: "user", + Content: "hello there", ContentLength: len("hello there"), + Timestamp: tsZero, + }, + Message{ + SessionID: "real-sess", Ordinal: 1, Role: "assistant", + Content: "hi back", ContentLength: len("hi back"), + Timestamp: tsZeroS1, + }, + ) + + hits := []VectorHit{ + {SessionID: "real-sess", Ordinal: 0}, + {SessionID: "real-sess", Ordinal: 1}, + } + for i := range 1000 { + hits = append(hits, VectorHit{ + SessionID: fmt.Sprintf("no-such-session-%d", i), Ordinal: i, + }) + } + + meta, err := d.enrichSemanticHits(ctx, hits) + require.NoError(t, err) + + require.Len(t, meta, 2) + assert.Equal(t, "hello there", + meta[semanticHitKey{"real-sess", 0}].content) + assert.Equal(t, "hi back", + meta[semanticHitKey{"real-sess", 1}].content) +} diff --git a/internal/db/search_content_hybrid_test.go b/internal/db/search_content_hybrid_test.go new file mode 100644 index 000000000..a549b134d --- /dev/null +++ b/internal/db/search_content_hybrid_test.go @@ -0,0 +1,666 @@ +package db + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSearchContentHybridNoSearcherUnavailable pins the same capability gate +// as "semantic": hybrid needs a wired VectorSearcher regardless of FTS +// availability, and reports ErrSemanticUnavailable when none is wired. +func TestSearchContentHybridNoSearcherUnavailable(t *testing.T) { + d := testDB(t) + assert.False(t, d.HasSemantic(), "HasSemantic before wiring a searcher") + + _, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "hybrid", + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSemanticUnavailable), + "expected ErrSemanticUnavailable, got %v", err) +} + +// TestSearchContentHybridCursorRejected pins the shared semantic/hybrid +// validation: cursor pagination is rejected before the capability check. +func TestSearchContentHybridCursorRejected(t *testing.T) { + d := testDB(t) + d.SetVectorSearcher(&fakeVectorSearcher{}) + + _, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "hybrid", Cursor: 1, + }) + require.Error(t, err) + var inputErr *SearchInputError + assert.True(t, errors.As(err, &inputErr), + "expected *SearchInputError, got %T: %v", err, err) +} + +// TestSearchContentHybridFTSUnavailable pins the FTS-missing capability gate: +// hybrid additionally requires db.HasFTS() and mirrors mode "fts"'s error +// when the messages_fts table is gone. +func TestSearchContentHybridFTSUnavailable(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + d.SetVectorSearcher(&fakeVectorSearcher{}) + _, err := d.getWriter().Exec("DROP TABLE IF EXISTS messages_fts") + require.NoError(t, err, "drop messages_fts") + + _, err = d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "hybrid", + }) + require.Error(t, err) + assert.True(t, errors.Is(err, errFTSUnavailable), + "expected errFTSUnavailable, got %v", err) +} + +// TestSearchContentHybridBothLegsOutrankSingleLeg pins reciprocal-rank +// fusion's core guarantee: a document ranked top by both the vector and FTS +// legs must fuse to a higher score than a document appearing in only one leg, +// and must sort first. +func TestSearchContentHybridBothLegsOutrankSingleLeg(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "both", "proj", [][2]string{ + {"user", "needle in a haystack"}, + }) + seedSearchSession(t, d, "vec-only", "proj", [][2]string{ + {"user", "totally unrelated content"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "both", Ordinal: 0, Score: 0.9, Snippet: "needle in a haystack"}, + {SessionID: "vec-only", Ordinal: 0, Score: 0.8, Snippet: "totally unrelated content"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "needle", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 2, "matches") + + require.Equal(t, "both", page.Matches[0].SessionID, "double-leg hit ranks first") + require.Equal(t, "vec-only", page.Matches[1].SessionID, "single-leg hit ranks second") + require.NotNil(t, page.Matches[0].Score, "Score") + require.NotNil(t, page.Matches[1].Score, "Score") + assert.Greater(t, *page.Matches[0].Score, *page.Matches[1].Score, + "double-leg fused score must exceed single-leg fused score") +} + +// TestSearchContentHybridVectorOnlyAndFTSOnlyBothAppear pins RRF's union +// semantics: a hit found only by the vector leg and a hit found only by the +// FTS leg must both survive the fusion, not just the leg-overlapping ones. +func TestSearchContentHybridVectorOnlyAndFTSOnlyBothAppear(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "fts-only", "proj", [][2]string{ + {"user", "needle in a haystack"}, + }) + seedSearchSession(t, d, "vec-only", "proj", [][2]string{ + {"user", "totally unrelated content"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "vec-only", Ordinal: 0, Score: 0.9, Snippet: "totally unrelated content"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "needle", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 2, "matches") + + var ids []string + for _, m := range page.Matches { + ids = append(ids, m.SessionID) + } + assert.ElementsMatch(t, []string{"fts-only", "vec-only"}, ids) +} + +// TestSearchContentHybridScoresStrictlyDescending pins that fused RRF scores +// order the page strictly descending, with no inversions or unexpected ties +// across a mix of a double-leg hit and single-leg (vector-only) hits. +func TestSearchContentHybridScoresStrictlyDescending(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "both", "proj", [][2]string{ + {"user", "needle in a haystack"}, + }) + seedSearchSession(t, d, "vec2", "proj", [][2]string{ + {"user", "other stuff entirely"}, + }) + seedSearchSession(t, d, "vec3", "proj", [][2]string{ + {"user", "yet more stuff"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "both", Ordinal: 0, Score: 0.9, Snippet: "needle in a haystack"}, + {SessionID: "vec2", Ordinal: 0, Score: 0.8, Snippet: "other stuff entirely"}, + {SessionID: "vec3", Ordinal: 0, Score: 0.7, Snippet: "yet more stuff"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "needle", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 3, "matches") + + for i := 1; i < len(page.Matches); i++ { + require.NotNil(t, page.Matches[i-1].Score, "Score at %d", i-1) + require.NotNil(t, page.Matches[i].Score, "Score at %d", i) + assert.Greater(t, *page.Matches[i-1].Score, *page.Matches[i].Score, + "scores must be strictly descending at index %d", i) + } +} + +// TestSearchContentHybridProjectFilterConstrainsBothLegs pins that the +// session-scope filter narrows both legs: the FTS leg filters in SQL, the +// vector leg is filtered post-hoc before the merge, and a session outside the +// requested project must be dropped from the fused result even though it +// matches both legs' raw searches. +func TestSearchContentHybridProjectFilterConstrainsBothLegs(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "in-scope", "alpha", [][2]string{ + {"user", "needle in a haystack"}, + }) + seedSearchSession(t, d, "out-of-scope", "beta", [][2]string{ + {"user", "needle in another haystack"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "in-scope", Ordinal: 0, Score: 0.9, Snippet: "needle in a haystack"}, + {SessionID: "out-of-scope", Ordinal: 0, Score: 0.8, Snippet: "needle in another haystack"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "needle", Mode: "hybrid", Limit: 50, Project: "alpha", + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 1, "matches after project filter") + assert.Equal(t, "in-scope", page.Matches[0].SessionID, "surviving session") +} + +// TestSearchContentHybridRedactsSecretPastChunkTruncation mirrors the +// semantic-mode regression (TestSearchContentSemanticRedactsSecretPastChunkTruncation) +// for hybrid: the vector leg's chunk snippet is truncated mid-PEM-body, before +// the "-----END" marker the PEM rule requires to fire. Redacting that +// fragment in isolation would miss the secret entirely; hybrid must redact +// against the message's full content the same way semantic mode does. +func TestSearchContentHybridRedactsSecretPastChunkTruncation(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + pem := "-----BEGIN RSA PRIVATE KEY-----\n" + + strings.Repeat("MIIBSECRETKEYMATERIAL0123456789ABCDEF\n", 5) + + "-----END RSA PRIVATE KEY-----" + content := "needle deploy with this attached key " + pem + " ok" + seedSearchSession(t, d, "s1", "proj", [][2]string{ + {"user", content}, + }) + + cut := strings.Index(content, "MIIBSECRETKEYMATERIAL") + len("MIIBSECRETKEYMATERIAL") + 3 + require.Less(t, cut, strings.Index(content, "-----END"), + "test setup: cut must land before the END marker") + truncatedSnippet := content[:cut] + "…" + + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 0, Score: 0.9, Snippet: truncatedSnippet}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "needle", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 1, "matches") + assert.NotContains(t, page.Matches[0].Snippet, "SECRETKEYMATERIAL", + "hybrid snippet leaked key material truncated out of the vector chunk") + assert.Contains(t, page.Matches[0].Snippet, "needle", + "snippet lost the matched context") +} + +// mergedKeys projects a merged result to its keys in rank order. +func mergedKeys(merged []mergedUnit) []string { + keys := make([]string, len(merged)) + for i, m := range merged { + keys[i] = m.unit.Key + } + return keys +} + +// TestRRFMergeBothLegsOutrankSingleLeg pins RRF's core guarantee at the +// merge level: a unit ranked by both legs scores the sum of its per-leg +// reciprocal ranks and outranks a unit seen by only one leg. +func TestRRFMergeBothLegsOutrankSingleLeg(t *testing.T) { + merged := rrfMerge([][]unitRanked{ + {{Key: "a"}, {Key: "b"}}, + {{Key: "a"}}, + }, 0) + require.Equal(t, []string{"a", "b"}, mergedKeys(merged)) + assert.InDelta(t, 2.0/61.0, merged[0].score, 1e-12, "double-leg score") + assert.InDelta(t, 1.0/62.0, merged[1].score, 1e-12, "single-leg score") +} + +// TestRRFMergeSubordinatePenaltyAcrossLegs pins the rank+5 penalty: a +// subordinate unit at leg rank 1 must fall below a top-level unit at the +// same rank in the other leg. +func TestRRFMergeSubordinatePenaltyAcrossLegs(t *testing.T) { + merged := rrfMerge([][]unitRanked{ + {{Key: "sub", Subordinate: true}}, + {{Key: "top"}}, + }, 0) + require.Equal(t, []string{"top", "sub"}, mergedKeys(merged)) + assert.InDelta(t, 1.0/61.0, merged[0].score, 1e-12) + assert.InDelta(t, 1.0/66.0, merged[1].score, 1e-12, "subordinate uses rank+5") +} + +// TestRRFMergeOneLegSubordinatePenaltyReorders pins the one-leg (semantic- +// only) fusion contract: a subordinate unit ranked immediately above a +// top-level unit drops below it after the merge. +func TestRRFMergeOneLegSubordinatePenaltyReorders(t *testing.T) { + merged := rrfMerge([][]unitRanked{{ + {Key: "sub", Subordinate: true}, + {Key: "top"}, + }}, 0) + assert.Equal(t, []string{"top", "sub"}, mergedKeys(merged)) +} + +// TestRRFMergeDeterministicTieBreak pins tie handling: a subordinate unit at +// rank 1 (effective rank 6) scores exactly like a top-level unit at rank 6, +// and the tie breaks by ascending key, not map iteration order. +func TestRRFMergeDeterministicTieBreak(t *testing.T) { + leg := []unitRanked{ + {Key: "zzz", Subordinate: true}, + {Key: "m2"}, {Key: "m3"}, {Key: "m4"}, {Key: "m5"}, + {Key: "aaa"}, + } + for range 20 { + merged := rrfMerge([][]unitRanked{leg}, 0) + require.Equal(t, + []string{"m2", "m3", "m4", "m5", "aaa", "zzz"}, mergedKeys(merged)) + } +} + +// TestRRFMergeLimitHonored pins truncation: limit > 0 caps the merged list +// at the top-scored units. +func TestRRFMergeLimitHonored(t *testing.T) { + merged := rrfMerge([][]unitRanked{ + {{Key: "a"}, {Key: "b"}, {Key: "c"}}, + }, 2) + assert.Equal(t, []string{"a", "b"}, mergedKeys(merged)) +} + +// TestSearchContentHybridFTSHitInsideRunFusesWithFTSAnchor is the +// end-to-end unit-fusion test: an FTS-matched message inside a run must fuse +// with the run's semantic hit into ONE result whose anchor ordinal is the +// FTS-matched message (overriding the vector leg's chunk anchor) and whose +// snippet centers on the FTS-matched text. +func TestSearchContentHybridFTSHitInsideRunFusesWithFTSAnchor(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "s1", "proj", [][2]string{ + {"user", "the question"}, + {"assistant", "first step of the answer"}, + {"assistant", "second step mentions zebra"}, + }) + // The run [1,2] anchors its semantic hit at ordinal 1; FTS matches + // ordinal 2. + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 1, OrdinalStart: 1, OrdinalEnd: 2, + Score: 0.9, Snippet: "first step of the answer"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 1, + "the run's semantic hit and its FTS-matched member must fuse into one result") + m := page.Matches[0] + assert.Equal(t, "s1", m.SessionID) + assert.Equal(t, 2, m.Ordinal, "anchor overridden to the FTS-matched message") + assert.Equal(t, "assistant", m.Role, "role of the FTS-matched message") + assert.Contains(t, m.Snippet, "zebra", "FTS snippet wins for display") + require.NotNil(t, m.Score) + assert.InDelta(t, 2.0/61.0, *m.Score, 1e-9, + "fused score: rank 1 in both legs") +} + +// TestSearchContentHybridNoUnitFTSHitKeepsMessageGranularity pins the +// no-unit escape hatch: an FTS hit on a message with no containing unit +// (outside the mirror) survives fusion under its own message-granularity key +// with its own ordinal, and carries the structurally derived unit range +// rather than a self-range: the "uncovered" hit sits in a two-message +// assistant run, so its range must span the run even though the mirror knows +// nothing about the session. +func TestSearchContentHybridNoUnitFTSHitKeepsMessageGranularity(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "uncovered", "proj", [][2]string{ + {"user", "irrelevant lead-in"}, + {"assistant", "tool output mentions zebra"}, + {"assistant", "further elaboration on the output"}, + }) + seedSearchSession(t, d, "covered", "proj", [][2]string{ + {"user", "unrelated content"}, + }) + // The searcher's mirror knows only the "covered" session, so the + // resolver returns a zero UnitRef for the "uncovered" FTS hit. + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "covered", Ordinal: 0, Score: 0.9, Snippet: "unrelated content"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 2, "the unit-less FTS hit must not vanish") + + byID := map[string]ContentMatch{} + for _, m := range page.Matches { + byID[m.SessionID] = m + } + uncovered, ok := byID["uncovered"] + require.True(t, ok, "no-unit FTS hit survives") + assert.Equal(t, 1, uncovered.Ordinal, "message-granularity ordinal kept") + assert.Equal(t, [2]int{1, 2}, uncovered.OrdinalRange, + "unit-less hit gets the derived run range, not a self-range") + assert.False(t, uncovered.Subordinate, + "unit-less top-level non-sidechain hit stays non-subordinate") + assert.Contains(t, uncovered.Snippet, "zebra") +} + +// seedUnitlessSidechainFixture seeds one top-level session ("side") whose +// matching assistant message is a sidechain row (ordinals 1-2 form one +// sidechain run) and one plain top-level session ("plain") matching once. +// "side" repeats the term so bm25 ranks it above "plain" in the FTS leg. The +// returned searcher's mirror knows neither session, so both FTS hits are +// unit-less and their classification must be structurally derived. +func seedUnitlessSidechainFixture(t *testing.T, d *DB) *fakeVectorSearcher { + t.Helper() + insertSession(t, d, "side", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + }) + require.NoError(t, d.ReplaceSessionMessages("side", []Message{ + {SessionID: "side", Ordinal: 0, Role: "user", + Content: "the question", Timestamp: "2026-05-20T12:00:00Z"}, + {SessionID: "side", Ordinal: 1, Role: "assistant", IsSidechain: true, + Content: "zebra zebra zebra zebra", Timestamp: "2026-05-20T12:00:01Z"}, + {SessionID: "side", Ordinal: 2, Role: "assistant", IsSidechain: true, + Content: "sidechain elaboration", Timestamp: "2026-05-20T12:00:02Z"}, + })) + seedSearchSession(t, d, "plain", "proj", [][2]string{ + {"user", "zebra appears once here"}, + }) + return &fakeVectorSearcher{} +} + +// TestSearchContentHybridUnitlessSidechainClassifiedSubordinate pins the +// pre-merge classification of unit-less FTS hits: a hit anchored in a +// sidechain run (no mirror unit) must carry subordinate=true and the derived +// sidechain-run range on the wire — matching what lexical mode emits for the +// same anchor — and must be rank-penalized below an equal top-level hit at +// the default (all) scope even though the FTS leg ranks it first. +func TestSearchContentHybridUnitlessSidechainClassifiedSubordinate(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + d.SetVectorSearcher(seedUnitlessSidechainFixture(t, d)) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 2) + assert.Equal(t, "plain", page.Matches[0].SessionID, + "subordinate penalty must drop the higher-FTS-ranked sidechain hit below top-level") + side := page.Matches[1] + require.Equal(t, "side", side.SessionID) + assert.True(t, side.Subordinate, "unit-less sidechain hit classified subordinate") + assert.True(t, side.Sidechain, "anchor message is_sidechain") + assert.Equal(t, 1, side.Ordinal, "message-granularity anchor kept") + assert.Equal(t, [2]int{1, 2}, side.OrdinalRange, + "derived sidechain-run range, not a self-range") + + // Lexical parity: mode "fts" must classify the same anchor identically. + lexical, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "fts", Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent fts") + var found bool + for _, m := range lexical.Matches { + if m.SessionID == "side" && m.Ordinal == 1 { + found = true + assert.Equal(t, side.Subordinate, m.Subordinate, "Subordinate parity") + assert.Equal(t, side.Sidechain, m.Sidechain, "Sidechain parity") + assert.Equal(t, side.OrdinalRange, m.OrdinalRange, "OrdinalRange parity") + } + } + require.True(t, found, "lexical fts must also match the sidechain anchor") +} + +// TestSearchContentHybridUnitlessSidechainScopeFiltering pins that scope +// filtering sees the derived classification of unit-less hits: scope=top +// excludes the sidechain hit, scope=subordinate keeps only it. +func TestSearchContentHybridUnitlessSidechainScopeFiltering(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + d.SetVectorSearcher(seedUnitlessSidechainFixture(t, d)) + + top, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Scope: "top", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid scope=top") + require.Len(t, top.Matches, 1, "scope=top excludes the unit-less sidechain hit") + assert.Equal(t, "plain", top.Matches[0].SessionID) + + sub, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Scope: "subordinate", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid scope=subordinate") + require.Len(t, sub.Matches, 1, "scope=subordinate keeps only the sidechain hit") + assert.Equal(t, "side", sub.Matches[0].SessionID) + assert.True(t, sub.Matches[0].Subordinate) +} + +// TestSearchContentHybridFTSLegSubordinateUnitPenalized pins the FTS-side +// subordinate flag: an FTS hit resolving to a subordinate unit is penalized +// in the merge, while a unit-less top-level FTS hit is not — so the +// lower-FTS-ranked top-level message overtakes the subordinate unit. +func TestSearchContentHybridFTSLegSubordinateUnitPenalized(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + // "subd" repeats the term so bm25 ranks it above "plain" in the FTS leg. + seedSearchSession(t, d, "subd", "proj", [][2]string{ + {"assistant", "zebra zebra zebra zebra"}, + }) + seedSearchSession(t, d, "plain", "proj", [][2]string{ + {"user", "zebra appears once here"}, + }) + // The vector leg is empty; the subordinate unit is known only to the + // resolver. + d.SetVectorSearcher(&fakeVectorSearcher{units: []UnitRef{ + {DocKey: "r:subd:0", SessionID: "subd", + OrdinalStart: 0, OrdinalEnd: 0, Subordinate: true}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 2) + assert.Equal(t, "plain", page.Matches[0].SessionID, + "unpenalized message-granularity hit must overtake the subordinate unit") + assert.Equal(t, "subd", page.Matches[1].SessionID) +} + +// TestSearchContentHybridMatchCarriesUnitRangeAndLineage pins the hybrid +// surface for run-grouped units: a fused FTS-in-run hit exposes the +// containing unit's ordinal range and subordinate flag (from the resolved +// UnitRef) plus the anchor's lineage, while Ordinal stays the FTS-overridden +// anchor ordinal. +func TestSearchContentHybridMatchCarriesUnitRangeAndLineage(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + insertSession(t, d, "top", "proj", func(s *Session) { + s.UserMessageCount = 2 + }) + insertSession(t, d, "child", "proj", func(s *Session) { + s.UserMessageCount = 2 + s.ParentSessionID = Ptr("top") + s.RelationshipType = "subagent" + }) + require.NoError(t, d.ReplaceSessionMessages("child", []Message{ + {SessionID: "child", Ordinal: 0, Role: "user", + Content: "the question", Timestamp: "2026-05-20T12:00:00Z"}, + {SessionID: "child", Ordinal: 1, Role: "assistant", IsSidechain: true, + Content: "first step of the answer", Timestamp: "2026-05-20T12:00:01Z"}, + {SessionID: "child", Ordinal: 2, Role: "assistant", IsSidechain: true, + Content: "second step mentions zebra", Timestamp: "2026-05-20T12:00:02Z"}, + })) + // The run [1,2] anchors its semantic hit at ordinal 1; FTS matches + // ordinal 2 and overrides the anchor. + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "child", Ordinal: 1, OrdinalStart: 1, OrdinalEnd: 2, + Subordinate: true, Score: 0.9, Snippet: "first step of the answer"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 1, "the two legs must fuse into one result") + m := page.Matches[0] + assert.Equal(t, 2, m.Ordinal, "Ordinal stays the FTS-overridden anchor") + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "OrdinalRange spans the containing unit") + assert.True(t, m.Subordinate, "Subordinate carries the unit flag") + assert.Equal(t, "subagent", m.Relationship) + assert.Equal(t, "top", m.ParentSessionID) + assert.True(t, m.Sidechain, "anchor message is_sidechain") +} + +// TestSearchContentHybridFTSLegCollapseRefillsFromDeeperRanks pins the +// batched FTS leg against unit collapse: when more than k (the +// semanticOverfetchMin=200 fusion depth) rank-ordered FTS rows all fall +// inside ONE run-unit, they collapse to a single leg entry, and a match in a +// different unit ranked below all of them must still be fetched and returned +// rather than being cut off by the first batch's window. +func TestSearchContentHybridFTSLegCollapseRefillsFromDeeperRanks(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + // One run-unit spanning 205 assistant messages, each repeating the term + // so bm25 ranks every one of them above the single-occurrence session. + const runLen = 205 + runMsgs := make([][2]string, runLen) + for i := range runMsgs { + runMsgs[i] = [2]string{"assistant", "zebra zebra zebra zebra"} + } + seedSearchSession(t, d, "bigrun", "proj", runMsgs) + seedSearchSession(t, d, "other", "proj", [][2]string{ + {"user", "zebra appears once in a much longer unrelated sentence"}, + }) + // The vector leg is empty; the resolver knows the whole run as one unit. + d.SetVectorSearcher(&fakeVectorSearcher{units: []UnitRef{ + {DocKey: "r:bigrun:0", SessionID: "bigrun", + OrdinalStart: 0, OrdinalEnd: runLen - 1}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Limit: 10, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 2, + "the lower-ranked unit past the collapsed run must be fetched") + ids := []string{page.Matches[0].SessionID, page.Matches[1].SessionID} + assert.ElementsMatch(t, []string{"bigrun", "other"}, ids) +} + +// TestSearchContentHybridFTSLegScopeExcludedRowsRefill pins the batched FTS +// leg against scope discard: with scope=subordinate, when the first k FTS +// rows are all top-level (and so all dropped), a subordinate match ranked +// below them must still be fetched and returned. +func TestSearchContentHybridFTSLegScopeExcludedRowsRefill(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + // 205 top-level sessions would be slow; one top-level session with 205 + // matching messages fills the first batch the same way, since each + // message is its own unit-less row classified top-level (non-sidechain, + // no subordinate lineage). + const topLen = 205 + topMsgs := make([][2]string, topLen) + for i := range topMsgs { + topMsgs[i] = [2]string{"user", "zebra zebra zebra zebra"} + } + seedSearchSession(t, d, "toplots", "proj", topMsgs) + seedSubagentSession(t, d, "sub", "toplots", "proj", [][2]string{ + {"user", "zebra appears once in a much longer subagent sentence"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{units: []UnitRef{ + {DocKey: "u:sub:0", SessionID: "sub", + OrdinalStart: 0, OrdinalEnd: 0, Subordinate: true}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "hybrid", Scope: "subordinate", Limit: 10, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 1, + "the subordinate match past the excluded top-level rows must be fetched") + assert.Equal(t, "sub", page.Matches[0].SessionID) +} + +// TestSearchContentHybridVectorOnlyMatchCarriesUnitRange pins the +// vector-leg display path: a unit only the semantic leg found keeps its +// chunk anchor and still exposes the unit range and subordinate flag. +func TestSearchContentHybridVectorOnlyMatchCarriesUnitRange(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedSearchSession(t, d, "s1", "proj", [][2]string{ + {"user", "the question"}, + {"assistant", "first step of the answer"}, + {"assistant", "second step of the answer"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 2, OrdinalStart: 1, OrdinalEnd: 2, + Score: 0.9, Snippet: "second step of the answer"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "nomatchinfts", Mode: "hybrid", Limit: 50, + }) + require.NoError(t, err, "SearchContent hybrid") + require.Len(t, page.Matches, 1, "vector-only hit survives fusion") + m := page.Matches[0] + assert.Equal(t, 2, m.Ordinal, "vector leg keeps its chunk anchor") + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange) + assert.False(t, m.Subordinate) +} diff --git a/internal/db/search_content_scope_test.go b/internal/db/search_content_scope_test.go new file mode 100644 index 000000000..33698e92e --- /dev/null +++ b/internal/db/search_content_scope_test.go @@ -0,0 +1,343 @@ +package db + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// seedSubagentSession inserts a subagent child session (a sidebar-child +// relationship, excluded by default from session lists) with the given +// messages for scope-filter tests. +func seedSubagentSession( + t *testing.T, d *DB, id, parent, project string, msgs [][2]string, +) { + t.Helper() + insertSession(t, d, id, project, func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + s.ParentSessionID = Ptr(parent) + s.RelationshipType = "subagent" + }) + var out []Message + for i, rc := range msgs { + out = append(out, Message{ + SessionID: id, Ordinal: i, Role: rc[0], + Content: rc[1], Timestamp: "2026-05-20T12:00:0" + itoa(i) + "Z", + }) + } + require.NoError(t, d.ReplaceSessionMessages(id, out), "ReplaceSessionMessages") +} + +// seedScopeFixture seeds one top-level session ("top") and one subagent +// child ("sub"), both matching the FTS pattern "zebra", and returns a +// searcher whose hits cover both (the subagent hit flagged Subordinate). +func seedScopeFixture(t *testing.T, d *DB) *fakeVectorSearcher { + t.Helper() + seedSearchSession(t, d, "top", "proj", [][2]string{ + {"user", "zebra question at top level"}, + }) + seedSubagentSession(t, d, "sub", "top", "proj", [][2]string{ + {"user", "zebra prompt inside the subagent"}, + }) + return &fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "sub", Ordinal: 0, Subordinate: true, Score: 0.9, + Snippet: "zebra prompt inside the subagent"}, + {SessionID: "top", Ordinal: 0, Score: 0.5, + Snippet: "zebra question at top level"}, + }} +} + +func matchSessionIDs(page ContentSearchPage) []string { + ids := make([]string, 0, len(page.Matches)) + for _, m := range page.Matches { + ids = append(ids, m.SessionID) + } + return ids +} + +// requireHybridReady skips hybrid-mode subtests when FTS5 is unavailable. +func requireHybridReady(t *testing.T, d *DB, mode string) { + t.Helper() + if mode == "hybrid" && !d.HasFTS() { + t.Skip("fts5 not available") + } +} + +// TestSearchContentScopeDefaultAllIncludesSubordinate is the precedence +// rule's critical test: with the default scope ("all") a subagent-session +// unit IS returned by semantic and hybrid search even though +// IncludeChildren is false — include_children must not hide subordinate +// units from either leg in these modes. +func TestSearchContentScopeDefaultAllIncludesSubordinate(t *testing.T) { + for _, mode := range []string{"semantic", "hybrid"} { + t.Run(mode, func(t *testing.T) { + d := testDB(t) + requireHybridReady(t, d, mode) + d.SetVectorSearcher(seedScopeFixture(t, d)) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: mode, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + ids := matchSessionIDs(page) + assert.Contains(t, ids, "sub", + "subordinate unit must be visible with the default scope despite IncludeChildren=false") + assert.Contains(t, ids, "top") + }) + } +} + +// TestSearchContentScopeTopExcludesSubordinate pins scope=top: subordinate +// units are excluded entirely from both modes. +func TestSearchContentScopeTopExcludesSubordinate(t *testing.T) { + for _, mode := range []string{"semantic", "hybrid"} { + t.Run(mode, func(t *testing.T) { + d := testDB(t) + requireHybridReady(t, d, mode) + d.SetVectorSearcher(seedScopeFixture(t, d)) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: mode, Scope: "top", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + assert.Equal(t, []string{"top"}, matchSessionIDs(page)) + }) + } +} + +// TestSearchContentScopeSubordinateOnly pins scope=subordinate: only +// subordinate units are returned. +func TestSearchContentScopeSubordinateOnly(t *testing.T) { + for _, mode := range []string{"semantic", "hybrid"} { + t.Run(mode, func(t *testing.T) { + d := testDB(t) + requireHybridReady(t, d, mode) + d.SetVectorSearcher(seedScopeFixture(t, d)) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: mode, Scope: "subordinate", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + assert.Equal(t, []string{"sub"}, matchSessionIDs(page)) + }) + } +} + +// TestSearchContentScopeSupersedesIncludeChildren pins that explicit +// include_children (either value) does not change the semantic-mode unit +// universe: scope governs visibility, so false and true return the same +// session set. +func TestSearchContentScopeSupersedesIncludeChildren(t *testing.T) { + d := testDB(t) + searcher := seedScopeFixture(t, d) + d.SetVectorSearcher(searcher) + + withoutChildren, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "semantic", IncludeChildren: false, Limit: 50, + }) + require.NoError(t, err, "SearchContent IncludeChildren=false") + withChildren, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "semantic", IncludeChildren: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent IncludeChildren=true") + + assert.Equal(t, matchSessionIDs(withChildren), matchSessionIDs(withoutChildren), + "include_children must be superseded in semantic mode") + assert.Contains(t, matchSessionIDs(withoutChildren), "sub") +} + +// TestSearchContentScopeStillAppliesSessionFilters guards that lifting the +// child exclusion in semantic mode does not lift the other session +// predicates: project, automated, and one-shot filtering still drop hits. +func TestSearchContentScopeStillAppliesSessionFilters(t *testing.T) { + seedExtra := func(t *testing.T, d *DB) *fakeVectorSearcher { + t.Helper() + searcher := seedScopeFixture(t, d) + // ReplaceSessionMessages recomputes is_automated from the stored + // transcript, so the automated session must genuinely classify as + // automated: one user message with a recognized automation prefix. + autoContent := "You are a code reviewer. Find the zebra issue." + insertSession(t, d, "auto", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 1 + s.FirstMessage = Ptr(autoContent) + }) + require.NoError(t, d.ReplaceSessionMessages("auto", []Message{ + {SessionID: "auto", Ordinal: 0, Role: "user", + Content: autoContent, Timestamp: "2026-05-20T12:00:00Z"}, + })) + insertSession(t, d, "oneshot", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 1 + }) + require.NoError(t, d.ReplaceSessionMessages("oneshot", []Message{ + {SessionID: "oneshot", Ordinal: 0, Role: "user", + Content: "zebra one-shot", Timestamp: "2026-05-20T12:00:00Z"}, + })) + insertSession(t, d, "elsewhere", "otherproj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + }) + require.NoError(t, d.ReplaceSessionMessages("elsewhere", []Message{ + {SessionID: "elsewhere", Ordinal: 0, Role: "user", + Content: "zebra elsewhere", Timestamp: "2026-05-20T12:00:00Z"}, + })) + searcher.hits = append(searcher.hits, + VectorHit{SessionID: "auto", Ordinal: 0, Score: 0.4, Snippet: "zebra from automation"}, + VectorHit{SessionID: "oneshot", Ordinal: 0, Score: 0.3, Snippet: "zebra one-shot"}, + VectorHit{SessionID: "elsewhere", Ordinal: 0, Score: 0.2, Snippet: "zebra elsewhere"}, + ) + return searcher + } + + t.Run("defaults drop automated one-shot and other projects", func(t *testing.T) { + d := testDB(t) + d.SetVectorSearcher(seedExtra(t, d)) + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "semantic", Project: "proj", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + assert.ElementsMatch(t, []string{"top", "sub"}, matchSessionIDs(page)) + }) + + t.Run("opt-ins restore automated and one-shot", func(t *testing.T) { + d := testDB(t) + d.SetVectorSearcher(seedExtra(t, d)) + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "semantic", Project: "proj", + IncludeAutomated: true, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + assert.ElementsMatch(t, []string{"top", "sub", "auto", "oneshot"}, + matchSessionIDs(page)) + }) +} + +// TestSearchContentFTSModeIncludeChildrenUnchanged is the regression guard +// for the non-semantic paths: mode "fts" keeps today's include_children +// semantics — a subagent child is hidden by default and reachable only via +// IncludeChildren=true. +func TestSearchContentFTSModeIncludeChildrenUnchanged(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + d.SetVectorSearcher(seedScopeFixture(t, d)) + + base := ContentSearchFilter{ + Pattern: "zebra", Mode: "fts", Sources: []string{"messages"}, Limit: 50, + } + page, err := d.SearchContent(context.Background(), base) + require.NoError(t, err, "SearchContent fts default") + assert.Equal(t, []string{"top"}, matchSessionIDs(page), + "fts mode must keep excluding sidebar children by default") + + base.IncludeChildren = true + page, err = d.SearchContent(context.Background(), base) + require.NoError(t, err, "SearchContent fts include_children") + assert.ElementsMatch(t, []string{"top", "sub"}, matchSessionIDs(page)) +} + +// seedOneShotSubagentFixture seeds a normal top-level session ("top"), a +// one-shot subagent child ("sub1": exactly one user message, the shape +// nearly all non-automated subagent transcripts have), and a top-level +// one-shot ("solo"), all matching "zebra", plus a searcher covering all +// three (sub1 subordinate). +func seedOneShotSubagentFixture(t *testing.T, d *DB) *fakeVectorSearcher { + t.Helper() + seedSearchSession(t, d, "top", "proj", [][2]string{ + {"user", "zebra question at top level"}, + }) + insertSession(t, d, "sub1", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 1 + s.ParentSessionID = Ptr("top") + s.RelationshipType = "subagent" + }) + require.NoError(t, d.ReplaceSessionMessages("sub1", []Message{ + {SessionID: "sub1", Ordinal: 0, Role: "user", + Content: "zebra prompt for the subagent", Timestamp: "2026-05-20T12:00:00Z"}, + }), "ReplaceSessionMessages sub1") + insertSession(t, d, "solo", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 1 + }) + require.NoError(t, d.ReplaceSessionMessages("solo", []Message{ + {SessionID: "solo", Ordinal: 0, Role: "user", + Content: "zebra one-shot at top level", Timestamp: "2026-05-20T12:00:00Z"}, + }), "ReplaceSessionMessages solo") + return &fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "sub1", Ordinal: 0, Subordinate: true, Score: 0.9, + Snippet: "zebra prompt for the subagent"}, + {SessionID: "top", Ordinal: 0, Score: 0.5, + Snippet: "zebra question at top level"}, + {SessionID: "solo", Ordinal: 0, Score: 0.4, + Snippet: "zebra one-shot at top level"}, + }} +} + +// TestSearchContentOneShotSubagentVisibleInSemanticModes pins the child +// carve-out from the one-shot gate: a subagent session with exactly one +// user message IS returned by semantic and hybrid under default filters +// (scope=all), while a TOP-LEVEL one-shot stays excluded. +func TestSearchContentOneShotSubagentVisibleInSemanticModes(t *testing.T) { + for _, mode := range []string{"semantic", "hybrid"} { + t.Run(mode, func(t *testing.T) { + d := testDB(t) + requireHybridReady(t, d, mode) + d.SetVectorSearcher(seedOneShotSubagentFixture(t, d)) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: mode, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + ids := matchSessionIDs(page) + assert.Contains(t, ids, "sub1", + "one-shot subagent unit must survive the one-shot gate in %s mode", mode) + assert.Contains(t, ids, "top") + assert.NotContains(t, ids, "solo", + "top-level one-shot must keep being excluded by default") + }) + } +} + +// TestSearchContentFTSModeOneShotSubagentStillExcluded guards the untouched +// path: mode "fts" with default filters keeps excluding the one-shot +// subagent session (both as a child and as a one-shot). +func TestSearchContentFTSModeOneShotSubagentStillExcluded(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + d.SetVectorSearcher(seedOneShotSubagentFixture(t, d)) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "fts", Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent fts") + assert.Equal(t, []string{"top"}, matchSessionIDs(page), + "fts mode must keep today's one-shot and child exclusions") +} + +// TestSearchContentScopeInvalidRejected pins the db-side backstop: an +// unknown scope value is a SearchInputError for both modes. +func TestSearchContentScopeInvalidRejected(t *testing.T) { + for _, mode := range []string{"semantic", "hybrid"} { + t.Run(mode, func(t *testing.T) { + d := testDB(t) + d.SetVectorSearcher(&fakeVectorSearcher{}) + + _, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: mode, Scope: "bogus", + }) + require.Error(t, err) + var inputErr *SearchInputError + assert.True(t, errors.As(err, &inputErr), + "expected *SearchInputError, got %T: %v", err, err) + }) + } +} diff --git a/internal/db/search_content_semantic_test.go b/internal/db/search_content_semantic_test.go new file mode 100644 index 000000000..017cf1ce2 --- /dev/null +++ b/internal/db/search_content_semantic_test.go @@ -0,0 +1,465 @@ +package db + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeVectorSearcher is a canned VectorSearcher for db-layer semantic search +// tests: it returns a fixed, rank-ordered slice of hits (optionally trimmed +// to limit) or a fixed error, so tests can pin the db layer's handling of +// searcher output without a real embedding index. ResolveMessageUnits +// resolves against the explicit units list first, then against the units +// implied by hits (the real resolver reads the same mirror the hits come +// from), so hybrid tests fuse without wiring a real index. +type fakeVectorSearcher struct { + hits []VectorHit + units []UnitRef + err error + resolveErr error + calls int +} + +func (f *fakeVectorSearcher) SemanticSearch( + _ context.Context, _ string, limit int, +) ([]VectorHit, error) { + f.calls++ + if f.err != nil { + return nil, f.err + } + hits := f.hits + if limit > 0 && limit < len(hits) { + hits = hits[:limit] + } + return hits, nil +} + +func (f *fakeVectorSearcher) ResolveMessageUnits( + _ context.Context, refs []MessageRef, +) ([]UnitRef, error) { + if f.resolveErr != nil { + return nil, f.resolveErr + } + out := make([]UnitRef, len(refs)) + for i, ref := range refs { + out[i] = f.resolveRef(ref) + } + return out, nil +} + +func (f *fakeVectorSearcher) resolveRef(ref MessageRef) UnitRef { + for _, u := range f.units { + if u.SessionID == ref.SessionID && + ref.Ordinal >= u.OrdinalStart && ref.Ordinal <= u.OrdinalEnd { + return u + } + } + for _, h := range f.hits { + if h.SessionID == ref.SessionID && + ref.Ordinal >= h.OrdinalStart && ref.Ordinal <= h.OrdinalEnd { + return UnitRef{ + DocKey: fmt.Sprintf("fake:%s:%d", h.SessionID, h.OrdinalStart), + SessionID: h.SessionID, + OrdinalStart: h.OrdinalStart, + OrdinalEnd: h.OrdinalEnd, + Subordinate: h.Subordinate, + } + } + } + return UnitRef{} +} + +func TestSearchContentSemanticNoSearcherUnavailable(t *testing.T) { + d := testDB(t) + assert.False(t, d.HasSemantic(), "HasSemantic before wiring a searcher") + + _, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "semantic", + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrSemanticUnavailable), + "expected ErrSemanticUnavailable, got %v", err) +} + +func TestHasSemanticFlipsWithSetVectorSearcher(t *testing.T) { + d := testDB(t) + require.False(t, d.HasSemantic(), "HasSemantic before SetVectorSearcher") + + d.SetVectorSearcher(&fakeVectorSearcher{}) + assert.True(t, d.HasSemantic(), "HasSemantic after SetVectorSearcher") + + d.SetVectorSearcher(nil) + assert.False(t, d.HasSemantic(), "HasSemantic after clearing the searcher") +} + +func TestSearchContentSemanticRoutesAndPreservesRank(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "s1", "alpha", [][2]string{ + {"user", "hello world foo"}, + }) + seedSearchSession(t, d, "s2", "beta", [][2]string{ + {"user", "another message"}, + }) + // s2 ranks first despite sorting after "s1" alphabetically, so preserved + // order can only come from the searcher, not a re-sort by session id. + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s2", Ordinal: 0, Score: 0.9, Snippet: "another message"}, + {SessionID: "s1", Ordinal: 0, Score: 0.5, Snippet: "hello world foo"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "semantic", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 2, "matches") + + assert.Equal(t, "s2", page.Matches[0].SessionID, "rank order: s2 first") + assert.Equal(t, "s1", page.Matches[1].SessionID, "rank order: s1 second") + + m0 := page.Matches[0] + assert.Equal(t, "beta", m0.Project, "Project") + assert.Equal(t, "message", m0.Location, "Location") + require.NotNil(t, m0.Score, "Score") + assert.InDelta(t, 0.9, *m0.Score, 0.0001, "Score value") + assert.Equal(t, "another message", m0.Snippet, "Snippet") +} + +func TestSearchContentSemanticProjectFilterDropsNonMatching(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "s1", "alpha", [][2]string{ + {"user", "hello world foo"}, + }) + seedSearchSession(t, d, "s2", "beta", [][2]string{ + {"user", "another message"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 0, Score: 0.9, Snippet: "hello world foo"}, + {SessionID: "s2", Ordinal: 0, Score: 0.5, Snippet: "another message"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "semantic", Limit: 50, Project: "alpha", + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 1, "matches after project filter") + assert.Equal(t, "s1", page.Matches[0].SessionID, "surviving session") +} + +func TestSearchContentSemanticLimitTrims(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "s1", "alpha", [][2]string{{"user", "a"}}) + seedSearchSession(t, d, "s2", "alpha", [][2]string{{"user", "b"}}) + seedSearchSession(t, d, "s3", "alpha", [][2]string{{"user", "c"}}) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 0, Score: 0.9, Snippet: "a"}, + {SessionID: "s2", Ordinal: 0, Score: 0.8, Snippet: "b"}, + {SessionID: "s3", Ordinal: 0, Score: 0.7, Snippet: "c"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "x", Mode: "semantic", Limit: 2, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 2, "matches trimmed to limit") + assert.Equal(t, "s1", page.Matches[0].SessionID) + assert.Equal(t, "s2", page.Matches[1].SessionID) +} + +func TestSearchContentSemanticCursorRejected(t *testing.T) { + d := testDB(t) + d.SetVectorSearcher(&fakeVectorSearcher{}) + + _, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "semantic", Cursor: 1, + }) + require.Error(t, err) + var inputErr *SearchInputError + assert.True(t, errors.As(err, &inputErr), + "expected *SearchInputError, got %T: %v", err, err) +} + +func TestSearchContentSemanticToolInputSourceRejected(t *testing.T) { + d := testDB(t) + d.SetVectorSearcher(&fakeVectorSearcher{}) + + _, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "semantic", Sources: []string{"tool_input"}, + }) + require.Error(t, err) + var inputErr *SearchInputError + assert.True(t, errors.As(err, &inputErr), + "expected *SearchInputError, got %T: %v", err, err) +} + +func TestSearchContentSemanticMessagesSourceAllowed(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "s1", "alpha", [][2]string{{"user", "hello"}}) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 0, Score: 0.9, Snippet: "hello"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "hello", Mode: "semantic", Sources: []string{"messages"}, + }) + require.NoError(t, err, "SearchContent with explicit messages source") + require.Len(t, page.Matches, 1) +} + +// TestEnrichSemanticHitsCarriesLineage pins the enrichment join for +// run-anchored hits: relationship_type and parent_session_id come from the +// hit's session row and is_sidechain from the anchor ordinal's message row, +// while a top-level session yields empty lineage. +func TestEnrichSemanticHitsCarriesLineage(t *testing.T) { + d := testDB(t) + insertSession(t, d, "parent", "proj", func(s *Session) { + s.UserMessageCount = 2 + }) + insertSession(t, d, "child", "proj", func(s *Session) { + s.UserMessageCount = 2 + s.ParentSessionID = Ptr("parent") + s.RelationshipType = "subagent" + }) + require.NoError(t, d.ReplaceSessionMessages("parent", []Message{ + {SessionID: "parent", Ordinal: 0, Role: "user", + Content: "top-level question", Timestamp: "2026-05-20T12:00:00Z"}, + })) + require.NoError(t, d.ReplaceSessionMessages("child", []Message{ + {SessionID: "child", Ordinal: 0, Role: "user", + Content: "subagent prompt", Timestamp: "2026-05-20T12:00:01Z"}, + {SessionID: "child", Ordinal: 1, Role: "assistant", IsSidechain: true, + Content: "sidechain step one", Timestamp: "2026-05-20T12:00:02Z"}, + {SessionID: "child", Ordinal: 2, Role: "assistant", IsSidechain: true, + Content: "sidechain step two", Timestamp: "2026-05-20T12:00:03Z"}, + })) + + meta, err := d.enrichSemanticHits(context.Background(), []VectorHit{ + {SessionID: "child", Ordinal: 1, OrdinalStart: 1, OrdinalEnd: 2, + Subordinate: true, Score: 0.9}, + {SessionID: "parent", Ordinal: 0, OrdinalStart: 0, OrdinalEnd: 0, + Score: 0.5}, + }) + require.NoError(t, err) + + child, ok := meta[semanticHitKey{"child", 1}] + require.True(t, ok, "child hit enriched") + assert.Equal(t, "subagent", child.relationshipType) + assert.Equal(t, "parent", child.parentSessionID) + assert.True(t, child.isSidechain, "anchor message is_sidechain") + assert.Equal(t, "assistant", child.role) + + top, ok := meta[semanticHitKey{"parent", 0}] + require.True(t, ok, "parent hit enriched") + assert.Empty(t, top.relationshipType) + assert.Empty(t, top.parentSessionID) + assert.False(t, top.isSidechain) +} + +// TestSearchContentSemanticAnchorOrdinalHit pins that a run-anchored hit +// (anchor ordinal pointing at an assistant message inside the run, with the +// range and subordinate flag populated) enriches by the anchor ordinal: the +// match carries the anchor message's role and timestamp. +func TestSearchContentSemanticAnchorOrdinalHit(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "s1", "alpha", [][2]string{ + {"user", "the question"}, + {"assistant", "first step of the answer"}, + {"assistant", "second step of the answer"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 2, OrdinalStart: 1, OrdinalEnd: 2, + Score: 0.9, Snippet: "second step of the answer"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "answer", Mode: "semantic", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 1, "matches") + m := page.Matches[0] + assert.Equal(t, 2, m.Ordinal, "anchor ordinal") + assert.Equal(t, "assistant", m.Role, "anchor message role") + assert.Contains(t, m.Snippet, "second step") +} + +// TestSearchContentSemanticRedactsSecretPastChunkTruncation pins that +// semantic mode redacts against the message's full content, not the +// searcher's pre-truncated chunk snippet. The fake searcher's Snippet is cut +// off mid-PEM-body (before the "-----END" marker), mimicking a real chunk +// boundary or the 200-rune vector snippet truncation landing inside a +// secret. The PEM rule only fires on a BEGIN/END pair, so redacting the +// truncated snippet in isolation finds no match and ships the key material +// raw; redacting the full message content (which has both markers) must +// still catch and mask it. +func TestSearchContentSemanticRedactsSecretPastChunkTruncation(t *testing.T) { + d := testDB(t) + pem := "-----BEGIN RSA PRIVATE KEY-----\n" + + strings.Repeat("MIIBSECRETKEYMATERIAL0123456789ABCDEF\n", 5) + + "-----END RSA PRIVATE KEY-----" + content := "deploy with this attached key " + pem + " ok" + seedSearchSession(t, d, "s1", "proj", [][2]string{ + {"user", content}, + }) + + // Cut the chunk snippet well before the END marker so the raw fragment + // itself never contains a BEGIN/END pair. + cut := strings.Index(content, "MIIBSECRETKEYMATERIAL") + len("MIIBSECRETKEYMATERIAL") + 3 + require.Less(t, cut, strings.Index(content, "-----END"), + "test setup: cut must land before the END marker") + truncatedSnippet := content[:cut] + "…" + + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "s1", Ordinal: 0, Score: 0.9, Snippet: truncatedSnippet}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "attached key", Mode: "semantic", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 1, "matches") + assert.NotContains(t, page.Matches[0].Snippet, "SECRETKEYMATERIAL", + "semantic snippet leaked key material truncated out of the chunk") + assert.Contains(t, page.Matches[0].Snippet, "attached key", + "snippet lost the matched context") +} + +// TestSearchContentSemanticSubordinatePenaltyReorders pins the spec's +// one-leg fusion requirement: semantic-only results route through the same +// RRF merge hybrid uses, so a subordinate unit ranked (by cosine score) +// above a top-level unit drops below it, while each match keeps the +// searcher's own score. +func TestSearchContentSemanticSubordinatePenaltyReorders(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "subchain", "alpha", [][2]string{ + {"assistant", "sidechain answer text"}, + }) + seedSearchSession(t, d, "toplevel", "alpha", [][2]string{ + {"user", "top-level question text"}, + }) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "subchain", Ordinal: 0, Subordinate: true, + Score: 0.9, Snippet: "sidechain answer text"}, + {SessionID: "toplevel", Ordinal: 0, + Score: 0.5, Snippet: "top-level question text"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "answer", Mode: "semantic", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 2, "matches") + assert.Equal(t, "toplevel", page.Matches[0].SessionID, + "top-level unit must overtake the subordinate unit after the one-leg merge") + assert.Equal(t, "subchain", page.Matches[1].SessionID) + require.NotNil(t, page.Matches[0].Score) + assert.InDelta(t, 0.5, *page.Matches[0].Score, 0.0001, + "semantic mode keeps the searcher's score, not the fusion score") +} + +// TestSearchContentSemanticMatchCarriesUnitRangeAndLineage pins the API +// surface added for run-grouped units: a semantic match exposes the unit's +// ordinal range, subordinate flag, and lineage (relationship, parent session, +// anchor sidechain flag) while Ordinal stays the anchor ordinal; a top-level +// single-message unit leaves them all zero so its JSON is unchanged. +func TestSearchContentSemanticMatchCarriesUnitRangeAndLineage(t *testing.T) { + d := testDB(t) + insertSession(t, d, "parent", "proj", func(s *Session) { + s.UserMessageCount = 2 + }) + insertSession(t, d, "child", "proj", func(s *Session) { + s.UserMessageCount = 2 + s.ParentSessionID = Ptr("parent") + s.RelationshipType = "subagent" + }) + require.NoError(t, d.ReplaceSessionMessages("parent", []Message{ + {SessionID: "parent", Ordinal: 0, Role: "user", + Content: "top-level step question", Timestamp: "2026-05-20T12:00:00Z"}, + })) + require.NoError(t, d.ReplaceSessionMessages("child", []Message{ + {SessionID: "child", Ordinal: 0, Role: "user", + Content: "subagent prompt", Timestamp: "2026-05-20T12:00:01Z"}, + {SessionID: "child", Ordinal: 1, Role: "assistant", IsSidechain: true, + Content: "sidechain step one", Timestamp: "2026-05-20T12:00:02Z"}, + {SessionID: "child", Ordinal: 2, Role: "assistant", IsSidechain: true, + Content: "sidechain step two", Timestamp: "2026-05-20T12:00:03Z"}, + })) + d.SetVectorSearcher(&fakeVectorSearcher{hits: []VectorHit{ + {SessionID: "child", Ordinal: 1, OrdinalStart: 1, OrdinalEnd: 2, + Subordinate: true, Score: 0.9, Snippet: "sidechain step one"}, + {SessionID: "parent", Ordinal: 0, OrdinalStart: 0, OrdinalEnd: 0, + Score: 0.5, Snippet: "top-level step question"}, + }}) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "step", Mode: "semantic", Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 2, "matches") + byID := map[string]ContentMatch{} + for _, m := range page.Matches { + byID[m.SessionID] = m + } + + sub, ok := byID["child"] + require.True(t, ok, "subordinate run hit present") + assert.Equal(t, 1, sub.Ordinal, "Ordinal stays the anchor ordinal") + assert.Equal(t, [2]int{1, 2}, sub.OrdinalRange, "OrdinalRange spans the unit") + assert.True(t, sub.Subordinate, "Subordinate carries the unit flag") + assert.Equal(t, "subagent", sub.Relationship) + assert.Equal(t, "parent", sub.ParentSessionID) + assert.True(t, sub.Sidechain, "anchor message is_sidechain") + + data, err := json.Marshal(sub) + require.NoError(t, err) + for _, want := range []string{ + `"ordinal":1`, `"ordinal_range":[1,2]`, + `"subordinate":true`, `"relationship":"subagent"`, + `"parent_session_id":"parent"`, `"is_sidechain":true`, + } { + assert.Contains(t, string(data), want) + } + + top, ok := byID["parent"] + require.True(t, ok, "top-level hit present") + assert.Equal(t, [2]int{0, 0}, top.OrdinalRange) + assert.False(t, top.Subordinate) + assert.Empty(t, top.Relationship) + assert.Empty(t, top.ParentSessionID) + assert.False(t, top.Sidechain) +} + +// TestContentMatchJSONUnitFieldsOmittedForLexicalMatches guards the lexical +// modes' wire format: a substring match always carries an ordinal_range (here +// the derived [0,0] — a user row is its own conversation unit), while the +// omitempty lineage keys stay absent when zero-valued (top-level session, no +// sidechain), keeping FTS/substring/regex responses free of noise keys. +func TestContentMatchJSONUnitFieldsOmittedForLexicalMatches(t *testing.T) { + d := testDB(t) + seedSearchSession(t, d, "s1", "proj", [][2]string{ + {"user", "find the zebra here"}, + }) + + page, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "zebra", Mode: "substring", Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, page.Matches, 1) + + data, err := json.Marshal(page.Matches[0]) + require.NoError(t, err) + assert.Contains(t, string(data), `"ordinal_range":[0,0]`, + "lexical match carries the derived unit range (user row at ordinal 0)") + for _, key := range []string{ + "score", "ordinal_start", "ordinal_end", "subordinate", + "relationship", "parent_session_id", "is_sidechain", + } { + assert.NotContains(t, string(data), `"`+key+`"`, + "lexical match JSON must not grow semantic-only keys") + } +} diff --git a/internal/db/search_content_semantic_vector_test.go b/internal/db/search_content_semantic_vector_test.go new file mode 100644 index 000000000..c190ea126 --- /dev/null +++ b/internal/db/search_content_semantic_vector_test.go @@ -0,0 +1,118 @@ +// ABOUTME: end-to-end semantic search tests wiring a real internal/vector +// ABOUTME: index into db.SearchContent, pinning anchor-local snippet centering. +package db_test + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/dbtest" + "go.kenn.io/agentsview/internal/vector" + kitvec "go.kenn.io/kit/vector" +) + +// vectorIndexSearcher adapts a real *vector.Index to db.VectorSearcher for +// tests, mirroring the production searcherAdapter in cmd/agentsview without +// its staleness gate. +type vectorIndexSearcher struct { + ix *vector.Index + enc kitvec.EncodeFunc +} + +func (s vectorIndexSearcher) SemanticSearch( + ctx context.Context, query string, limit int, +) ([]db.VectorHit, error) { + hits, err := s.ix.Search(ctx, s.enc, query, limit) + if err != nil { + return nil, err + } + out := make([]db.VectorHit, len(hits)) + for i, h := range hits { + out[i] = db.VectorHit{ + SessionID: h.SessionID, + Ordinal: h.Ordinal, + OrdinalStart: h.OrdinalStart, + OrdinalEnd: h.OrdinalEnd, + Subordinate: h.Subordinate, + Score: h.Score, + Snippet: h.Snippet, + } + } + return out, nil +} + +func (s vectorIndexSearcher) ResolveMessageUnits( + ctx context.Context, refs []db.MessageRef, +) ([]db.UnitRef, error) { + return s.ix.ResolveMessageUnits(ctx, refs) +} + +// TestSearchContentSemanticCrossMemberChunkCentersOnAnchorMessage is the +// end-to-end regression test for run-chunk snippet mislocation: a run whose +// matched chunk spans two assistant messages must produce a ContentMatch +// whose snippet centers on the ANCHOR message's content. Before the fix, the +// vector layer returned the whole cross-member chunk as the snippet; the db +// layer could not locate that text inside the anchor message's content and +// fell back to centering on the query pattern (absent here), i.e. the start +// of the message — losing the matched region entirely. +func TestSearchContentSemanticCrossMemberChunkCentersOnAnchorMessage(t *testing.T) { + ctx := context.Background() + d := dbtest.OpenTestDB(t) + + memberA := "a short first assistant step" + // The distinctive matched text sits past the snippet window's 60-byte + // radius from the start of the anchor message, so a start-of-content + // fallback cannot accidentally include it. + memberB := strings.Repeat("background context sentence. ", 4) + + "the particles remain entangled across any distance" + msgs := []db.Message{ + dbtest.UserMsg("s1", 0, "please explain the experiment results"), + dbtest.AsstMsg("s1", 1, memberA), + dbtest.AsstMsg("s1", 2, memberB), + } + dbtest.SeedSessionWithMessages(t, d, "s1", "proj", msgs, + dbtest.WithMessageCounts(3, 2)) + + enc := func(_ context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, len(texts)) + for i, text := range texts { + if strings.Contains(text, "entangled") || strings.Contains(text, "quantum") { + out[i] = []float32{1, 0, 0} + } else { + out[i] = []float32{0, 1, 0} + } + } + return out, nil + } + + ix, err := vector.Open(ctx, filepath.Join(t.TempDir(), "vectors.db"), false, 4000) + require.NoError(t, err) + defer func() { require.NoError(t, ix.Close()) }() + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + _, err = ix.Build(ctx, d, enc, gen, vector.BuildOptions{}) + require.NoError(t, err) + + d.SetVectorSearcher(vectorIndexSearcher{ix: ix, enc: enc}) + + // The query shares no literal token with the anchor message, so a + // pattern-based fallback cannot rescue a mislocated snippet. + page, err := d.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "quantum superposition", Mode: "semantic", Limit: 10, + }) + require.NoError(t, err) + require.NotEmpty(t, page.Matches) + + m := page.Matches[0] + assert.Equal(t, "s1", m.SessionID) + assert.Equal(t, 2, m.Ordinal, + "anchor: the member containing the matched chunk's center") + assert.Contains(t, m.Snippet, "entangled", + "snippet must center on the anchor message's matched content") + assert.NotContains(t, m.Snippet, memberA, + "snippet must not carry text from a different run member") +} diff --git a/internal/db/search_content_test.go b/internal/db/search_content_test.go index 36824dc29..a7e55dbc7 100644 --- a/internal/db/search_content_test.go +++ b/internal/db/search_content_test.go @@ -3,6 +3,7 @@ package db import ( "context" "errors" + "fmt" "strings" "testing" "unicode/utf8" @@ -588,3 +589,332 @@ func TestFTSSnippetCentersOnPhrase(t *testing.T) { "fallback snippet not centered on first token") }) } + +// seedUnitSession inserts a session (lineage-configurable via opts) plus full +// Message rows, for derived-unit citation tests that need +// is_system/is_sidechain/tool fields beyond seedSearchSession's role/content +// pairs. SessionID and a per-ordinal timestamp are filled in when unset. +func seedUnitSession( + t *testing.T, d *DB, id string, opts func(*Session), msgs []Message, +) { + t.Helper() + insertSession(t, d, id, "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + if opts != nil { + opts(s) + } + }) + for i := range msgs { + msgs[i].SessionID = id + if msgs[i].Timestamp == "" { + msgs[i].Timestamp = fmt.Sprintf("2026-05-20T12:00:%02dZ", i) + } + } + require.NoError(t, d.ReplaceSessionMessages(id, msgs), + "ReplaceSessionMessages %s", id) +} + +// matchesByOrdinal indexes a page's matches by anchor ordinal, requiring the +// ordinals to be unique. +func matchesByOrdinal(t *testing.T, page ContentSearchPage) map[int]ContentMatch { + t.Helper() + out := make(map[int]ContentMatch, len(page.Matches)) + for _, m := range page.Matches { + _, dup := out[m.Ordinal] + require.False(t, dup, "duplicate match ordinal %d", m.Ordinal) + out[m.Ordinal] = m + } + return out +} + +// TestSearchContentSubstringDerivedRunRange pins the derived +// conversation-unit citation on substring rows: every match in one assistant +// run carries the run's full range (spanning a non-member system row), an +// embeddable user row and a system row are their own units, and ExcludeSystem +// changes nothing but which rows match. +func TestSearchContentSubstringDerivedRunRange(t *testing.T) { + d := testDB(t) + seedUnitSession(t, d, "run1", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "the RUNHIT question"}, + {Ordinal: 1, Role: "assistant", Content: "RUNHIT step one"}, + {Ordinal: 2, Role: "user", Content: "sys RUNHIT note", IsSystem: true}, + {Ordinal: 3, Role: "assistant", Content: "RUNHIT step two"}, + {Ordinal: 4, Role: "assistant", Content: "RUNHIT step three"}, + {Ordinal: 5, Role: "user", Content: "next question"}, + }) + got, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "RUNHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 5, "matches") + byOrd := matchesByOrdinal(t, got) + assert.Equal(t, [2]int{0, 0}, byOrd[0].OrdinalRange, "user row is its own unit") + assert.Equal(t, [2]int{2, 2}, byOrd[2].OrdinalRange, "system row is its own unit") + for _, o := range []int{1, 3, 4} { + m := byOrd[o] + assert.Equal(t, [2]int{1, 4}, m.OrdinalRange, "run member %d", o) + assert.Equal(t, o, m.Ordinal, "anchor ordinal %d", o) + assert.False(t, m.Subordinate, "top-level run member %d", o) + assert.False(t, m.Sidechain, "non-sidechain run member %d", o) + assert.Empty(t, m.Relationship, "top-level relationship %d", o) + assert.Empty(t, m.ParentSessionID, "top-level parent %d", o) + } + + // ExcludeSystem drops the system row but leaves the derived ranges of the + // surviving rows unchanged. + ex, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "RUNHIT", Mode: "substring", + Sources: []string{"messages"}, ExcludeSystem: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent ExcludeSystem") + require.Len(t, ex.Matches, 4, "ExcludeSystem matches") + exByOrd := matchesByOrdinal(t, ex) + assert.NotContains(t, exByOrd, 2, "system row excluded") + assert.Equal(t, [2]int{0, 0}, exByOrd[0].OrdinalRange) + for _, o := range []int{1, 3, 4} { + assert.Equal(t, [2]int{1, 4}, exByOrd[o].OrdinalRange, + "ExcludeSystem run member %d", o) + } +} + +// TestSearchContentSubstringSidechainRunSubordinate pins the sidechain rules: +// a sidechain run's members are Subordinate + Sidechain, and the sidechain +// flip bounds both the sidechain run and the following top-level run. +func TestSearchContentSubstringSidechainRunSubordinate(t *testing.T) { + d := testDB(t) + seedUnitSession(t, d, "side1", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "the question"}, + {Ordinal: 1, Role: "assistant", Content: "SIDEHIT step a", IsSidechain: true}, + {Ordinal: 2, Role: "assistant", Content: "SIDEHIT step b", IsSidechain: true}, + {Ordinal: 3, Role: "assistant", Content: "main MAINHIT answer"}, + }) + side, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "SIDEHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent sidechain") + require.Len(t, side.Matches, 2, "sidechain matches") + for _, m := range side.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "sidechain run range") + assert.True(t, m.Subordinate, "sidechain run is subordinate") + assert.True(t, m.Sidechain, "anchor sidechain flag") + assert.Empty(t, m.Relationship, "no session lineage") + } + main, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "MAINHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent main") + require.Len(t, main.Matches, 1, "main matches") + m := main.Matches[0] + assert.Equal(t, [2]int{3, 3}, m.OrdinalRange, + "sidechain flip bounds the top-level run") + assert.False(t, m.Subordinate, "top-level run") + assert.False(t, m.Sidechain, "top-level anchor") +} + +// TestSearchContentSubstringSubagentLineage pins session-level lineage on +// lexical rows: a match inside a subagent session is Subordinate with +// Relationship and ParentSessionID populated from the sessions join. +func TestSearchContentSubstringSubagentLineage(t *testing.T) { + d := testDB(t) + insertSession(t, d, "parent", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + }) + seedUnitSession(t, d, "child", func(s *Session) { + s.ParentSessionID = Ptr("parent") + s.RelationshipType = "subagent" + }, []Message{ + {Ordinal: 0, Role: "user", Content: "subagent prompt"}, + {Ordinal: 1, Role: "assistant", Content: "SUBHIT answer"}, + }) + got, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "SUBHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeChildren: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 1, "matches") + m := got.Matches[0] + assert.Equal(t, [2]int{1, 1}, m.OrdinalRange, "single-member run") + assert.True(t, m.Subordinate, "subagent session is subordinate") + assert.Equal(t, "subagent", m.Relationship, "Relationship") + assert.Equal(t, "parent", m.ParentSessionID, "ParentSessionID") + assert.False(t, m.Sidechain, "anchor not sidechain") +} + +// TestSearchContentToolDerivedRunRange pins derivation for tool_input and +// canonical tool_result rows: the anchor is the tool call's message row, so +// both locations carry the enclosing run's range while the wire Role stays +// the hard-coded "assistant". +func TestSearchContentToolDerivedRunRange(t *testing.T) { + d := testDB(t) + seedUnitSession(t, d, "tool1", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "the question"}, + {Ordinal: 1, Role: "assistant", Content: "running the tool", + ToolCalls: []ToolCall{{ + ToolName: "Bash", Category: "Bash", ToolUseID: "tu1", + InputJSON: `{"command":"TOOLHIT"}`, + ResultContent: "output RESHIT data", + }}}, + {Ordinal: 2, Role: "assistant", Content: "continuing the answer"}, + {Ordinal: 3, Role: "user", Content: "thanks"}, + }) + in, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "TOOLHIT", Mode: "substring", + Sources: []string{"tool_input"}, Limit: 50, + }) + require.NoError(t, err, "tool_input search") + require.Len(t, in.Matches, 1, "tool_input matches") + assert.Equal(t, "assistant", in.Matches[0].Role, "wire role stays assistant") + assert.Equal(t, 1, in.Matches[0].Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 2}, in.Matches[0].OrdinalRange, + "tool_input anchor classified from the real message row") + + res, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "RESHIT", Mode: "substring", + Sources: []string{"tool_result"}, Limit: 50, + }) + require.NoError(t, err, "tool_result search") + require.Len(t, res.Matches, 1, "tool_result matches") + assert.Equal(t, [2]int{1, 2}, res.Matches[0].OrdinalRange, + "canonical tool_result anchor classified from the real message row") +} + +// TestSearchContentToolAnchorUsesRealRowRole pins the role-sensitive anchor +// classification: a tool call hanging off a user-role message keeps the +// hard-coded "assistant" wire role, but derivation must classify the anchor +// by the REAL row's role — an embeddable user row is its own unit, never an +// assistant run member. +func TestSearchContentToolAnchorUsesRealRowRole(t *testing.T) { + d := testDB(t) + seedUnitSession(t, d, "toolu", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "prompt"}, + {Ordinal: 1, Role: "user", Content: "user-attached call", + ToolCalls: []ToolCall{{ + ToolName: "Bash", Category: "Bash", ToolUseID: "tuu", + InputJSON: `{"command":"UHIT"}`, + }}}, + {Ordinal: 2, Role: "assistant", Content: "assistant reply"}, + }) + got, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "UHIT", Mode: "substring", + Sources: []string{"tool_input"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 1, "matches") + m := got.Matches[0] + assert.Equal(t, "assistant", m.Role, "wire role stays assistant") + assert.Equal(t, [2]int{1, 1}, m.OrdinalRange, + "user-role anchor row is its own unit") +} + +// TestSearchContentToolResultEventsDerived pins the events branches: an +// orphaned event (no message row at its ordinal) still returns its match +// (row cardinality must not change) with the [o, o] fallback and session +// lineage, while an event whose message row sits inside a run gets the run's +// range via the post-scan anchor lookup. +func TestSearchContentToolResultEventsDerived(t *testing.T) { + d := testDB(t) + + insertSession(t, d, "boss", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + }) + insertSession(t, d, "evorph", "proj", func(s *Session) { + s.Agent = "claude" + s.UserMessageCount = 2 + s.ParentSessionID = Ptr("boss") + s.RelationshipType = "subagent" + }) + _, err := d.getWriter().Exec(`INSERT INTO tool_result_events + (session_id, tool_call_message_ordinal, tool_use_id, source, status, + content, content_length, timestamp, event_index) + VALUES ('evorph', 7, 'tux', 'stdout', 'success', + 'ORPHHIT event content', 21, '2026-05-20T12:00:00Z', 0)`) + require.NoError(t, err, "insert orphan event") + + seedUnitSession(t, d, "evrun", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "the question"}, + {Ordinal: 1, Role: "assistant", Content: "running", + ToolCalls: []ToolCall{{ + ToolName: "Bash", Category: "Bash", ToolUseID: "tu1", + InputJSON: `{"command":"x"}`, + ResultEvents: []ToolResultEvent{{ + ToolUseID: "tu1", Source: "stdout", Status: "success", + Content: "EVHIT streamed output", EventIndex: 0, + }}, + }}}, + {Ordinal: 2, Role: "assistant", Content: "wrapping up"}, + }) + + orph, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "ORPHHIT", Mode: "substring", + Sources: []string{"tool_result"}, IncludeChildren: true, Limit: 50, + }) + require.NoError(t, err, "orphan search") + require.Len(t, orph.Matches, 1, "orphaned event row must not be dropped") + m := orph.Matches[0] + assert.Equal(t, 7, m.Ordinal, "event ordinal") + assert.Equal(t, [2]int{7, 7}, m.OrdinalRange, "missing anchor falls back to [o, o]") + assert.False(t, m.Sidechain, "missing anchor has no sidechain flag") + assert.True(t, m.Subordinate, "session lineage still applies") + assert.Equal(t, "subagent", m.Relationship, "Relationship from sessions join") + assert.Equal(t, "boss", m.ParentSessionID, "ParentSessionID from sessions join") + + ev, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "EVHIT", Mode: "substring", + Sources: []string{"tool_result"}, Limit: 50, + }) + require.NoError(t, err, "event search") + require.Len(t, ev.Matches, 1, "event matches") + assert.Equal(t, 1, ev.Matches[0].Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 2}, ev.Matches[0].OrdinalRange, + "event with a message row inside a run gets the run's range") +} + +// TestSearchContentRegexDerivedRange spot-checks that regex mode routes +// through the shared derivation pass. +func TestSearchContentRegexDerivedRange(t *testing.T) { + d := testDB(t) + seedUnitSession(t, d, "rx1", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "the question"}, + {Ordinal: 1, Role: "assistant", Content: "RXHIT alpha"}, + {Ordinal: 2, Role: "assistant", Content: "RXHIT beta"}, + }) + got, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: `RXHIT [a-z]+`, Mode: "regex", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent regex") + require.Len(t, got.Matches, 2, "regex matches") + for _, m := range got.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "derived run range") + } +} + +// TestSearchContentFTSDerivedRange spot-checks that fts mode routes through +// the shared derivation pass. +func TestSearchContentFTSDerivedRange(t *testing.T) { + d := testDB(t) + if !d.HasFTS() { + t.Skip("fts5 not available") + } + seedUnitSession(t, d, "fx1", nil, []Message{ + {Ordinal: 0, Role: "user", Content: "the question"}, + {Ordinal: 1, Role: "assistant", Content: "ftshit alpha step"}, + {Ordinal: 2, Role: "assistant", Content: "ftshit beta step"}, + }) + got, err := d.SearchContent(context.Background(), ContentSearchFilter{ + Pattern: "ftshit", Mode: "fts", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent fts") + require.Len(t, got.Matches, 2, "fts matches") + for _, m := range got.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "derived run range") + assert.False(t, m.Subordinate, "top-level run") + } +} diff --git a/internal/db/search_content_units.go b/internal/db/search_content_units.go new file mode 100644 index 000000000..ffbd72964 --- /dev/null +++ b/internal/db/search_content_units.go @@ -0,0 +1,228 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// contentAnchorMeta is one match's anchor metadata: session lineage plus the +// anchor message row's classification columns (role, sidechain, embeddable). +// fillAnchorMeta resolves all of it post-truncation with one batched lookup — +// deliberately NOT in the search SQL, where extra columns would be evaluated +// for every candidate row before the LIMIT and carried through the sort. +// Rows whose message row does not exist are marked missing. +type contentAnchorMeta struct { + relationship string + parentSessionID string + role sql.NullString + sidechain sql.NullBool + embeddable sql.NullBool + missing bool +} + +// deriveLexicalUnits is the shared post-scan pass for the substring, regex, +// and fts modes: it fetches each row's anchor classification and session +// lineage with one batched lookup, derives each match's conversation-unit +// range (DeriveUnitRanges issues one batched statement per seam method for +// the whole page, deduplicating duplicate probes), and assigns OrdinalRange +// plus the lineage fields. matches is the already-truncated page, so the +// pass is O(page). +func (db *DB) deriveLexicalUnits( + ctx context.Context, matches []ContentMatch, +) error { + if len(matches) == 0 { + return nil + } + metas, err := db.fillAnchorMeta(ctx, matches) + if err != nil { + return err + } + anchors := buildContentUnitAnchors(matches, metas) + ranges, err := DeriveUnitRanges(ctx, db, anchors) + if err != nil { + return fmt.Errorf("deriving lexical units: %w", err) + } + for i := range matches { + matches[i].OrdinalRange = ranges[i] + matches[i].Relationship = metas[i].relationship + matches[i].ParentSessionID = metas[i].parentSessionID + matches[i].Sidechain = anchors[i].Sidechain + matches[i].Subordinate = anchors[i].Sidechain || + SubordinateSession(metas[i].relationship, metas[i].parentSessionID) + } + return nil +} + +// buildContentUnitAnchors maps scanned sidecars to DeriveUnitRanges anchors. +// A missing anchor row keeps zero-valued classification fields, so it +// resolves to [o, o] with Sidechain false (session lineage still applies). +func buildContentUnitAnchors( + matches []ContentMatch, metas []contentAnchorMeta, +) []UnitAnchor { + anchors := make([]UnitAnchor, len(matches)) + for i, m := range matches { + meta := metas[i] + anchors[i] = UnitAnchor{ + SessionID: m.SessionID, + Ordinal: m.Ordinal, + Role: meta.role.String, + Sidechain: meta.sidechain.Valid && meta.sidechain.Bool, + Embeddable: meta.embeddable.Valid && meta.embeddable.Bool, + Missing: meta.missing, + } + } + return anchors +} + +// fillAnchorMeta fetches anchor classification and session lineage for every +// page row via lookupAnchorMeta. Refs whose message row does not exist +// (tool_result_events orphans) are marked missing so derivation falls back +// to [o, o]; their session lineage is still populated via the sessions join. +// The result aligns 1:1 with matches. +func (db *DB) fillAnchorMeta( + ctx context.Context, matches []ContentMatch, +) ([]contentAnchorMeta, error) { + refs := make([]semanticHitKey, len(matches)) + for i := range matches { + refs[i] = semanticHitKey{matches[i].SessionID, matches[i].Ordinal} + } + found, err := db.lookupAnchorMeta(ctx, refs) + if err != nil { + return nil, err + } + metas := make([]contentAnchorMeta, len(matches)) + for i, ref := range refs { + got, ok := found[ref] + if !ok { + metas[i].missing = true + continue + } + got.missing = !got.role.Valid + metas[i] = got + } + return metas, nil +} + +// lookupAnchorMeta resolves anchor classification and session lineage for +// refs: one batched VALUES-CTE lookup per enrichHitsChunk distinct +// (session_id, ordinal) refs, never a per-row query. Refs whose session row +// is absent are omitted from the result map. +func (db *DB) lookupAnchorMeta( + ctx context.Context, refs []semanticHitKey, +) (map[semanticHitKey]contentAnchorMeta, error) { + seen := make(map[semanticHitKey]bool, len(refs)) + distinct := make([]semanticHitKey, 0, len(refs)) + for _, ref := range refs { + if !seen[ref] { + seen[ref] = true + distinct = append(distinct, ref) + } + } + found := make(map[semanticHitKey]contentAnchorMeta, len(distinct)) + for start := 0; start < len(distinct); start += enrichHitsChunk { + chunk := distinct[start:min(start+enrichHitsChunk, len(distinct))] + if err := db.lookupAnchorMetaChunk(ctx, chunk, found); err != nil { + return nil, err + } + } + return found, nil +} + +// lookupAnchorMetaChunk resolves one chunk of (session_id, ordinal) refs to +// session lineage plus the anchor message row's classification columns: +// role, sidechain, and the embeddable flag (is_system = 0 AND content not +// system-prefixed — SystemPrefixSQL constrains only user rows, exactly like +// the embedding reducer's predicate). messages is LEFT JOINed so a ref whose +// message row is absent (tool_result_events orphan) still resolves lineage; +// its classification columns come back NULL. +func (db *DB) lookupAnchorMetaChunk( + ctx context.Context, refs []semanticHitKey, + out map[semanticHitKey]contentAnchorMeta, +) error { + values := make([]string, len(refs)) + args := make([]any, 0, len(refs)*2) + for i, r := range refs { + values[i] = "(?, ?)" + args = append(args, r.sessionID, r.ordinal) + } + query := "WITH refs(session_id, ordinal) AS (VALUES " + + strings.Join(values, ", ") + ") " + + "SELECT r.session_id, r.ordinal, " + + "COALESCE(s.relationship_type,''), COALESCE(s.parent_session_id,''), " + + "m.role, m.is_sidechain, " + + "CASE WHEN m.is_system = 0 AND " + + SystemPrefixSQL("m.content", "m.role") + " THEN 1 ELSE 0 END " + + "FROM refs r " + + "JOIN sessions s ON s.id = r.session_id " + + "LEFT JOIN messages m ON m.session_id = r.session_id AND m.ordinal = r.ordinal" + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("looking up match anchors: %w", err) + } + defer rows.Close() + for rows.Next() { + var key semanticHitKey + var meta contentAnchorMeta + if err := rows.Scan(&key.sessionID, &key.ordinal, + &meta.relationship, &meta.parentSessionID, + &meta.role, &meta.sidechain, &meta.embeddable); err != nil { + return fmt.Errorf("scanning match anchor: %w", err) + } + out[key] = meta + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating match anchors: %w", err) + } + return nil +} + +// classifyUnitlessHybridHits assigns each unit-less hybrid FTS hit (the +// resolver returned no mirror unit; hits[i] for i in idxs) its structurally +// derived conversation-unit range and subordinate flag BEFORE scope filtering +// and fusion, so a unit-less sidechain (or subagent/fork-session) hit is +// excluded, included, and penalized exactly like lexical mode classifies the +// same anchor. One batched anchor lookup plus one DeriveUnitRanges pass +// covers all idxs — unit-less refs are rare (system rows or mirror lag), so +// the pre-merge leg stays cheap; mirror-unit rows keep their embedded span +// and resolver-provided subordinate flag untouched. +func (db *DB) classifyUnitlessHybridHits( + ctx context.Context, hits []hybridDisplay, idxs []int, +) error { + if len(idxs) == 0 { + return nil + } + refs := make([]semanticHitKey, len(idxs)) + for k, i := range idxs { + refs[k] = semanticHitKey{hits[i].sessionID, hits[i].ordinal} + } + metas, err := db.lookupAnchorMeta(ctx, refs) + if err != nil { + return err + } + anchors := make([]UnitAnchor, len(idxs)) + for k, ref := range refs { + meta := metas[ref] + anchors[k] = UnitAnchor{ + SessionID: ref.sessionID, + Ordinal: ref.ordinal, + Role: meta.role.String, + Sidechain: meta.sidechain.Valid && meta.sidechain.Bool, + Embeddable: meta.embeddable.Valid && meta.embeddable.Bool, + Missing: !meta.role.Valid, + } + } + ranges, err := DeriveUnitRanges(ctx, db, anchors) + if err != nil { + return fmt.Errorf("deriving hybrid unit-less ranges: %w", err) + } + for k, i := range idxs { + hits[i].ordinalStart, hits[i].ordinalEnd = ranges[k][0], ranges[k][1] + meta := metas[refs[k]] + hits[i].subordinate = anchors[k].Sidechain || + SubordinateSession(meta.relationship, meta.parentSessionID) + } + return nil +} diff --git a/internal/db/sessions.go b/internal/db/sessions.go index df8bbd014..3cfab35c9 100644 --- a/internal/db/sessions.go +++ b/internal/db/sessions.go @@ -460,25 +460,34 @@ type SessionFilter struct { ExcludeProject string // exclude sessions with this project name Machine string // GitBranch is a branchListSep-joined list of opaque (project, branch) tokens (EncodeBranchFilterToken). - GitBranch string - Agent string - Date string // exact date YYYY-MM-DD - DateFrom string // range start (inclusive) - DateTo string // range end (inclusive) - ActiveSince string // ISO-8601 timestamp; filters on most recent activity - MinMessages int // message_count >= N (0 = no filter) - MaxMessages int // message_count <= N (0 = no filter) - MinUserMessages int // user_message_count >= N (0 = no filter) - ExcludeOneShot bool // exclude sessions with user_message_count <= 1 - ExcludeAutomated bool // exclude sessions where is_automated = 1 - AutomatedScope string // "", "human", "all", or "automated" - IncludeChildren bool // include subagent sessions (for sidebar grouping) - IncludeOrphans bool // promote orphan child rows to sidebar roots - Outcome []string // filter by outcome values - HealthGrade []string // filter by health grade values - MinToolFailures *int // minimum tool_failure_signal_count - HasSecret bool // only sessions with current secret_leak_count > 0 - Starred bool // only sessions starred by the user + GitBranch string + Agent string + Date string // exact date YYYY-MM-DD + DateFrom string // range start (inclusive) + DateTo string // range end (inclusive) + ActiveSince string // ISO-8601 timestamp; filters on most recent activity + MinMessages int // message_count >= N (0 = no filter) + MaxMessages int // message_count <= N (0 = no filter) + MinUserMessages int // user_message_count >= N (0 = no filter) + ExcludeOneShot bool // exclude sessions with user_message_count <= 1 + // ChildExemptOneShot carves child sessions (a sidebar-child + // relationship_type or a non-empty parent_session_id) out of the + // ExcludeOneShot gate. Set only by the semantic/hybrid content-search + // session scope: nearly all non-automated subagent transcripts carry a + // single user message, so the one-shot gate would otherwise drop the + // subordinate units the Scope filter exists to govern. Top-level + // sessions keep the one-shot exclusion unchanged; every other caller + // (session list, substring/regex/fts search) leaves this false. + ChildExemptOneShot bool + ExcludeAutomated bool // exclude sessions where is_automated = 1 + AutomatedScope string // "", "human", "all", or "automated" + IncludeChildren bool // include subagent sessions (for sidebar grouping) + IncludeOrphans bool // promote orphan child rows to sidebar roots + Outcome []string // filter by outcome values + HealthGrade []string // filter by health grade values + MinToolFailures *int // minimum tool_failure_signal_count + HasSecret bool // only sessions with current secret_leak_count > 0 + Starred bool // only sessions starred by the user // SecretsRulesVersions limits HasSecret to sessions scanned by one of these // current scanner versions. Empty preserves raw DB semantics for tests and // direct store callers that explicitly want unversioned counts. diff --git a/internal/db/store.go b/internal/db/store.go index 08a77d446..d96ce9629 100644 --- a/internal/db/store.go +++ b/internal/db/store.go @@ -37,6 +37,7 @@ type Store interface { // Messages. GetMessages(ctx context.Context, sessionID string, from, limit int, asc bool) ([]Message, error) + GetMessagesWindow(ctx context.Context, sessionID string, w MessageWindow) ([]Message, error) GetAllMessages(ctx context.Context, sessionID string) ([]Message, error) GetSessionActivity(ctx context.Context, sessionID string) (*SessionActivityResponse, error) @@ -45,6 +46,7 @@ type Store interface { // Search. HasFTS() bool + HasSemantic() bool Search(ctx context.Context, f SearchFilter) (SearchPage, error) SearchSession(ctx context.Context, sessionID, query string) ([]int, error) SearchContent(ctx context.Context, f ContentSearchFilter) (ContentSearchPage, error) diff --git a/internal/db/unit_range.go b/internal/db/unit_range.go new file mode 100644 index 000000000..d816e4330 --- /dev/null +++ b/internal/db/unit_range.go @@ -0,0 +1,666 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" +) + +// UnitAnchor classifies one match's anchor message row. +type UnitAnchor struct { + SessionID string + Ordinal int + Role string // "user"/"assistant"/other; "" when Missing + Sidechain bool + Embeddable bool // is_system = 0 AND content not system-prefixed + Missing bool // anchor row absent (tool_result_events orphan) +} + +// UnitProbe asks for the nearest embeddable-user boundaries around Ordinal. +type UnitProbe struct { + SessionID string + Ordinal int +} + +// UnitBounds carries exclusive user boundaries; sentinel values when absent: +// Prev = -1, Next = UnitOrdinalMax. +type UnitBounds struct{ Prev, Next int } + +// ExtentProbe asks for the first/last member ordinals of the anchor's +// same-sidechain run within the exclusive interval (Lo, Hi). +type ExtentProbe struct { + SessionID string + Ordinal int + Lo, Hi int // sentinels as above + Sidechain bool +} + +// UnitOrdinalMax bounds Hi sentinels; ordinals are int32-safe on all +// backends (PG INTEGER). Exported so every backend seam shares the exact +// sentinel value. +const UnitOrdinalMax = 1<<31 - 1 + +// UnitBoundsQuerier is the backend seam. Both methods are BATCHED: +// NearestUserBoundaries groups probes per session and RunExtents dedups +// probes, and each chunk of sessions or probes costs one SQL statement — +// never one statement, or query, per probe. Results align 1:1 with probes. +// RunExtents' stop set includes the unit boundaries NearestUserBoundaries +// reports, so DeriveUnitRanges consults NearestUserBoundaries only on +// session-dense pages where pre-fetched bounds pay for themselves (see +// UnitBoundsFlowFactor). +type UnitBoundsQuerier interface { + NearestUserBoundaries(ctx context.Context, probes []UnitProbe) ([]UnitBounds, error) + RunExtents(ctx context.Context, probes []ExtentProbe) ([][2]int, error) +} + +// SubordinateSession reports the session-level subordinate classification +// (subagent/fork-typed, or parent-linked non-continuation). Exported +// wrapper over the existing isSubordinateSession logic so the PG and +// DuckDB packages compute the same subordinate flag. +func SubordinateSession(relationshipType, parentSessionID string) bool { + return isSubordinateSession(relationshipType, sql.NullString{ + String: parentSessionID, Valid: parentSessionID != "", + }) +} + +// DeriveUnitRanges applies the spec's rules 1-3 + missing-anchor fallback. +// Result aligns 1:1 with anchors. +// +// Rule-1 (embeddable user), rule-3 (system rows, other roles, non-embeddable +// rows), and missing anchors resolve to [o, o] with no queries. Embeddable +// assistant anchors resolve with at most TWO batched RunExtents calls +// covering every pending anchor — duplicate (session, ordinal, sidechain) +// anchors share a single probe, and anchors in the same run share one +// representative probe (see deriveProbeExtents) — so a page's query count +// stays constant no matter how its anchors spread across sessions and runs. +func DeriveUnitRanges( + ctx context.Context, q UnitBoundsQuerier, anchors []UnitAnchor, +) ([][2]int, error) { + out := make([][2]int, len(anchors)) + pending := classifyUnitAnchors(anchors, out) + if len(pending) == 0 { + return out, nil + } + keys, keyIdx := dedupUnitProbeKeys(anchors, pending) + extents, err := deriveProbeExtents(ctx, q, keys) + if err != nil { + return nil, err + } + for _, i := range pending { + out[i] = extents[keyIdx[unitProbeKeyOf(anchors[i])]] + } + return out, nil +} + +// unitProbeKey identifies one distinct rule-2 probe. Sidechain is part of the +// key defensively: anchors at the same ordinal always describe the same +// message row, but a mismatched flag must not silently share a result. +type unitProbeKey struct { + sessionID string + ordinal int + sidechain bool +} + +func unitProbeKeyOf(a UnitAnchor) unitProbeKey { + return unitProbeKey{ + sessionID: a.SessionID, ordinal: a.Ordinal, sidechain: a.Sidechain, + } +} + +// runDerivable reports whether an anchor needs rule-2 run derivation: an +// embeddable assistant row that was actually found. +func runDerivable(a UnitAnchor) bool { + return !a.Missing && a.Embeddable && a.Role == "assistant" +} + +// classifyUnitAnchors fills [o, o] for every anchor that resolves locally +// (rules 1/3 and missing anchors) and returns the indexes of the anchors +// that need run derivation. +func classifyUnitAnchors(anchors []UnitAnchor, out [][2]int) []int { + var pending []int + for i, a := range anchors { + if runDerivable(a) { + pending = append(pending, i) + continue + } + out[i] = [2]int{a.Ordinal, a.Ordinal} + } + return pending +} + +// dedupUnitProbeKeys collects the distinct probe keys of the pending anchors +// in first-seen order and returns them with a key -> slot lookup. +func dedupUnitProbeKeys( + anchors []UnitAnchor, pending []int, +) ([]unitProbeKey, map[unitProbeKey]int) { + keys := make([]unitProbeKey, 0, len(pending)) + keyIdx := make(map[unitProbeKey]int, len(pending)) + for _, i := range pending { + k := unitProbeKeyOf(anchors[i]) + if _, ok := keyIdx[k]; ok { + continue + } + keyIdx[k] = len(keys) + keys = append(keys, k) + } + return keys, keyIdx +} + +// UnitBoundsFlowFactor gates the optional NearestUserBoundaries round in +// deriveProbeExtents: real user bounds are fetched only when the page packs +// at least this many probes per distinct session on average. The boundary +// fetch costs one statement over every user row of each probed session, so +// it amortizes only on dense pages — where it pays twice, by pruning the +// stop scans and by splitting probe groups at unit boundaries so run sharing +// resolves the page in one round. Exported so backend test suites can seed +// pages that provably exercise the dense flow instead of hardcoding the +// threshold. +const UnitBoundsFlowFactor = 8 + +// deriveProbeExtents runs at most two batched RunExtents calls (plus one +// optional NearestUserBoundaries call on session-dense pages, see +// UnitBoundsFlowFactor) for the distinct probe keys, returning run extents +// aligned 1:1 with keys. Every extent must cover its own anchor ordinal (the +// anchor row qualifies for its run by construction). Without the boundary +// round, probes carry the -1 / UnitOrdinalMax sentinel bounds: RunExtents' +// stop set already includes embeddable user rows, so real bounds are an +// optimization, never a correctness requirement. +// +// The two RunExtents rounds share runs between anchors: probes with the same +// (session, bounds, sidechain) group key that land in the same run have +// identical extents, so round one queries one representative per group and +// hands its extent to every group sibling the extent covers — sound because +// a rule-2 anchor is itself a run member, and a same-sidechain member inside +// [first, last] belongs to that exact run (a stop row strictly inside the +// extent would have closed the run before it reached first or last). The +// representative is the group's ordinal MEDIAN: page anchors cluster in hot +// runs, so a central anchor's run covers the most siblings (a group edge +// anchor may sit in a small neighboring run and cover nobody). Siblings in +// other runs (a page whose anchors straddle a user boundary or a sidechain +// flip) resolve in one second batch, so a page never costs more than two +// RunExtents statements. +func deriveProbeExtents( + ctx context.Context, q UnitBoundsQuerier, keys []unitProbeKey, +) ([][2]int, error) { + extentProbes, err := buildExtentProbes(ctx, q, keys) + if err != nil { + return nil, err + } + extents := make([][2]int, len(keys)) + resolved := make([]bool, len(keys)) + groups := groupExtentProbes(extentProbes) + reps := make([]int, 0, len(groups)) + for _, g := range groups { + reps = append(reps, g[len(g)/2]) + } + sort.Ints(reps) + if err := resolveExtentRound(ctx, q, extentProbes, reps, extents, resolved); err != nil { + return nil, err + } + shareGroupExtents(groups, extentProbes, extents, resolved) + var rest []int + for i := range keys { + if !resolved[i] { + rest = append(rest, i) + } + } + if len(rest) > 0 { + if err := resolveExtentRound(ctx, q, extentProbes, rest, extents, resolved); err != nil { + return nil, err + } + } + for i, k := range keys { + if extents[i][0] > k.ordinal || extents[i][1] < k.ordinal { + return nil, fmt.Errorf( + "deriving unit ranges: run extent [%d, %d] does not cover anchor %s#%d", + extents[i][0], extents[i][1], k.sessionID, k.ordinal) + } + } + return extents, nil +} + +// buildExtentProbes maps probe keys to RunExtents probes. Dense pages (at +// least UnitBoundsFlowFactor probes per distinct session) fetch the real +// exclusive user bounds with one batched NearestUserBoundaries call; sparse +// pages skip that round trip and probe with the -1 / UnitOrdinalMax +// sentinels, leaning on RunExtents' user-row stops instead. +func buildExtentProbes( + ctx context.Context, q UnitBoundsQuerier, keys []unitProbeKey, +) ([]ExtentProbe, error) { + sessions := make(map[string]struct{}, len(keys)) + for _, k := range keys { + sessions[k.sessionID] = struct{}{} + } + probes := make([]ExtentProbe, len(keys)) + for i, k := range keys { + probes[i] = ExtentProbe{ + SessionID: k.sessionID, Ordinal: k.ordinal, + Lo: -1, Hi: UnitOrdinalMax, + Sidechain: k.sidechain, + } + } + if len(keys) < UnitBoundsFlowFactor*len(sessions) { + return probes, nil + } + boundProbes := make([]UnitProbe, len(keys)) + for i, k := range keys { + boundProbes[i] = UnitProbe{SessionID: k.sessionID, Ordinal: k.ordinal} + } + bounds, err := q.NearestUserBoundaries(ctx, boundProbes) + if err != nil { + return nil, err + } + if len(bounds) != len(keys) { + return nil, fmt.Errorf( + "deriving unit ranges: NearestUserBoundaries returned %d results for %d probes", + len(bounds), len(keys)) + } + for i := range probes { + probes[i].Lo, probes[i].Hi = bounds[i].Prev, bounds[i].Next + } + return probes, nil +} + +// extentGroupKey identifies the probes that can share a run: same session, +// same exclusive bounds, same sidechain. +type extentGroupKey struct { + sessionID string + lo, hi int + sidechain bool +} + +// groupExtentProbes buckets probe indexes by extentGroupKey, each bucket +// sorted by anchor ordinal (so a bucket's median element is its central +// anchor). +func groupExtentProbes(probes []ExtentProbe) [][]int { + idx := make(map[extentGroupKey]int) + groups := make([][]int, 0, len(probes)) + for i, p := range probes { + k := extentGroupKey{ + sessionID: p.SessionID, lo: p.Lo, hi: p.Hi, sidechain: p.Sidechain, + } + gi, ok := idx[k] + if !ok { + idx[k] = len(groups) + groups = append(groups, []int{i}) + continue + } + groups[gi] = append(groups[gi], i) + } + for _, g := range groups { + sort.Slice(g, func(a, b int) bool { + return probes[g[a]].Ordinal < probes[g[b]].Ordinal + }) + } + return groups +} + +// resolveExtentRound issues one batched RunExtents call for the probes at +// idxs and records their extents. +func resolveExtentRound( + ctx context.Context, q UnitBoundsQuerier, probes []ExtentProbe, + idxs []int, extents [][2]int, resolved []bool, +) error { + batch := make([]ExtentProbe, len(idxs)) + for k, i := range idxs { + batch[k] = probes[i] + } + res, err := q.RunExtents(ctx, batch) + if err != nil { + return err + } + if len(res) != len(batch) { + return fmt.Errorf( + "deriving unit ranges: RunExtents returned %d results for %d probes", + len(res), len(batch)) + } + for k, i := range idxs { + extents[i], resolved[i] = res[k], true + } + return nil +} + +// shareGroupExtents hands each resolved group member's extent to the group +// siblings it covers (see deriveProbeExtents for why coverage implies the +// same run). +func shareGroupExtents( + groups [][]int, probes []ExtentProbe, + extents [][2]int, resolved []bool, +) { + for _, g := range groups { + for _, r := range g { + if !resolved[r] { + continue + } + ext := extents[r] + for _, i := range g { + if !resolved[i] && probes[i].Ordinal >= ext[0] && probes[i].Ordinal <= ext[1] { + extents[i], resolved[i] = ext, true + } + } + } + } +} + +// Shared backend resolvers. Every UnitBoundsQuerier implementation is the +// same backend-neutral orchestration around one batched SQL statement per +// chunk; the resolvers below own that orchestration (session/probe dedup, +// chunking, boundary resolution, alignment and invariant checks) so each +// backend supplies only its dialect's SQL builder. + +// ResolveUserBoundaries implements NearestUserBoundaries' shared +// orchestration: it dedups probe sessions in first-seen order, fetches each +// chunk of at most sessionChunk distinct sessions with one call to fetch +// (which must run ONE batched statement appending each session's embeddable +// user ordinals to its out slot, aligned 1:1 with sessions), sorts the +// ordinals, and resolves every probe's exclusive boundaries in Go. The +// result aligns 1:1 with probes, with the -1 / UnitOrdinalMax sentinels for +// missing boundaries. +func ResolveUserBoundaries( + ctx context.Context, probes []UnitProbe, sessionChunk int, + fetch func(ctx context.Context, sessions []string, out [][]int) error, +) ([]UnitBounds, error) { + out := make([]UnitBounds, len(probes)) + if len(probes) == 0 { + return out, nil + } + sessionIdx := make(map[string]int) + sessions := make([]string, 0, len(probes)) + for _, p := range probes { + if _, ok := sessionIdx[p.SessionID]; ok { + continue + } + sessionIdx[p.SessionID] = len(sessions) + sessions = append(sessions, p.SessionID) + } + ordinals := make([][]int, len(sessions)) + for start := 0; start < len(sessions); start += sessionChunk { + chunk := sessions[start:min(start+sessionChunk, len(sessions))] + if err := fetch(ctx, chunk, ordinals[start:start+len(chunk)]); err != nil { + return nil, err + } + } + for _, o := range ordinals { + sort.Ints(o) + } + for i, p := range probes { + out[i] = boundsAround(ordinals[sessionIdx[p.SessionID]], p.Ordinal) + } + return out, nil +} + +// boundsAround resolves one probe's exclusive boundaries from a session's +// sorted embeddable user ordinals: the strict MAX(< ordinal) / MIN(> ordinal) +// neighbors, with the -1 / UnitOrdinalMax sentinels when absent. +func boundsAround(userOrdinals []int, ordinal int) UnitBounds { + b := UnitBounds{Prev: -1, Next: UnitOrdinalMax} + i := sort.SearchInts(userOrdinals, ordinal) + if i > 0 { + b.Prev = userOrdinals[i-1] + } + for ; i < len(userOrdinals); i++ { + if userOrdinals[i] > ordinal { + b.Next = userOrdinals[i] + break + } + } + return b +} + +// ScanUserBoundaryRows consumes one batched boundary statement's (idx, +// ordinal) rows into out, validating each session index against the chunk — +// the shared scan half of every backend's ResolveUserBoundaries fetch. +func ScanUserBoundaryRows(rows *sql.Rows, out [][]int) error { + for rows.Next() { + var idx, ordinal int + if err := rows.Scan(&idx, &ordinal); err != nil { + return fmt.Errorf("scanning nearest user boundaries: %w", err) + } + if idx < 0 || idx >= len(out) { + return fmt.Errorf( + "nearest user boundaries: session index %d out of range [0, %d)", + idx, len(out)) + } + out[idx] = append(out[idx], ordinal) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating nearest user boundaries: %w", err) + } + return nil +} + +// ResolveRunExtents implements RunExtents' shared orchestration: duplicate +// probes share one slot, and each chunk of at most probeChunk distinct +// probes resolves with one call to lookup (which must run ONE batched +// statement filling out aligned 1:1 with its probes). The result aligns 1:1 +// with probes. +func ResolveRunExtents( + ctx context.Context, probes []ExtentProbe, probeChunk int, + lookup func(ctx context.Context, probes []ExtentProbe, out [][2]int) error, +) ([][2]int, error) { + out := make([][2]int, len(probes)) + if len(probes) == 0 { + return out, nil + } + keyIdx := make(map[ExtentProbe]int, len(probes)) + keys := make([]ExtentProbe, 0, len(probes)) + for _, p := range probes { + if _, ok := keyIdx[p]; ok { + continue + } + keyIdx[p] = len(keys) + keys = append(keys, p) + } + extents := make([][2]int, len(keys)) + for start := 0; start < len(keys); start += probeChunk { + chunk := keys[start:min(start+probeChunk, len(keys))] + if err := lookup(ctx, chunk, extents[start:start+len(chunk)]); err != nil { + return nil, err + } + } + for i, p := range probes { + out[i] = extents[keyIdx[p]] + } + return out, nil +} + +// ScanRunExtentRows consumes one batched extent statement's (idx, first, +// last) rows into out, enforcing the shared invariants — every probe index +// in range, no NULL extent side (a NULL means no same-sidechain member +// exists at the anchor, i.e. the probe was not built for a rule-2 anchor), +// and exactly one row per probe. +func ScanRunExtentRows(rows *sql.Rows, probes []ExtentProbe, out [][2]int) error { + seen := 0 + for rows.Next() { + var idx int + var first, last sql.NullInt64 + if err := rows.Scan(&idx, &first, &last); err != nil { + return fmt.Errorf("scanning run extents: %w", err) + } + if idx < 0 || idx >= len(out) { + return fmt.Errorf("run extents: probe index %d out of range [0, %d)", + idx, len(out)) + } + if !first.Valid || !last.Valid { + return fmt.Errorf( + "run extents: anchor %s#%d is not an embeddable assistant row "+ + "(probe must only be built for rule-2 anchors)", + probes[idx].SessionID, probes[idx].Ordinal) + } + out[idx] = [2]int{int(first.Int64), int(last.Int64)} + seen++ + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating run extents: %w", err) + } + if seen != len(probes) { + return fmt.Errorf("run extents: statement returned %d rows for %d probes", + seen, len(probes)) + } + return nil +} + +// SQLite seam implementation. +var _ UnitBoundsQuerier = (*DB)(nil) + +// unitSessionChunk caps sessions per NearestUserBoundaries statement so the +// VALUES CTE stays inside SQLite's bind-variable limit (see maxSQLVars): a +// session binds 2 variables (idx, session_id). +const unitSessionChunk = maxSQLVars / 2 + +// embeddableUserSQL is the SQL predicate matching an embeddable user row +// under the messages alias: user role, is_system = 0, and the SQLite dialect +// SystemPrefixSQL check, exactly as ScanEmbeddableUnits' scan predicate. +// (The assistant-side member predicate in runExtentSelectSQL skips the +// prefix check: SystemPrefixSQL constrains user rows only.) +func embeddableUserSQL(alias string) string { + return fmt.Sprintf("%[1]s.role = 'user' AND %[1]s.is_system = 0 AND %[2]s", + alias, SystemPrefixSQL(alias+".content", alias+".role")) +} + +// NearestUserBoundaries returns, per probe, the nearest embeddable user +// ordinals strictly before and after the probe ordinal (sidechain is +// irrelevant: the reducer closes runs on any embeddable user row regardless +// of its is_sidechain). Sentinels -1 / UnitOrdinalMax stand in for missing +// boundaries. Orchestration is the shared ResolveUserBoundaries; one +// statement per unitSessionChunk distinct sessions fetches each session's +// embeddable user ordinals ONCE. +func (db *DB) NearestUserBoundaries( + ctx context.Context, probes []UnitProbe, +) ([]UnitBounds, error) { + return ResolveUserBoundaries(ctx, probes, unitSessionChunk, + db.scanUserBoundaryOrdinals) +} + +// scanUserBoundaryOrdinals runs the one batched statement for a chunk of +// distinct sessions: a VALUES CTE joined against messages for every +// embeddable user ordinal of each session. out aligns 1:1 with sessions. +// Constraining the fetch to session + role only keeps it on +// idx_messages_session_role, so the statement touches each session's +// (typically sparse) user rows instead of stepping every message in an +// ordinal range. +func (db *DB) scanUserBoundaryOrdinals( + ctx context.Context, sessions []string, out [][]int, +) error { + values := make([]string, len(sessions)) + args := make([]any, 0, len(sessions)*2) + for i, sessionID := range sessions { + values[i] = "(?, ?)" + args = append(args, i, sessionID) + } + query := fmt.Sprintf(` + WITH spans(idx, session_id) AS (VALUES %s) + SELECT sp.idx, m.ordinal + FROM spans sp JOIN messages m ON m.session_id = sp.session_id + WHERE %s`, + strings.Join(values, ", "), embeddableUserSQL("m")) + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("querying nearest user boundaries: %w", err) + } + defer rows.Close() + return ScanUserBoundaryRows(rows, out) +} + +// unitExtentChunk caps extent probes per statement: a probe binds 6 +// variables (idx, session_id, o, lo, hi, sc). +const unitExtentChunk = maxSQLVars / 6 + +// RunExtents returns, per probe, the first and last member ordinals of the +// anchor's same-sidechain run: the nearest embeddable assistant rows of the +// anchor's sidechain around the anchor, bounded exclusively by (Lo, Hi) and +// by the nearest STOP row inside that interval — an embeddable user row (the +// reducer closes every run at unit boundaries) or an embeddable assistant +// row of the opposite sidechain (the flip rule; both stops only matter among +// embeddable rows). Probing with the -1 / UnitOrdinalMax sentinels therefore +// derives the full rule-2 extent on its own. The anchor row itself always +// qualifies, so a probe whose interval holds no run around its anchor was +// built for a row that is not an embeddable assistant row — an internal +// invariant violation reported as an error. +// +// Orchestration is the shared ResolveRunExtents; one statement per +// unitExtentChunk distinct probes resolves every probe with correlated point +// lookups on idx_messages_session_ordinal (nearest stop row on each side, +// then the farthest same-sidechain member inside the stop-narrowed interval) +// instead of transferring each interval's member rows to Go: an +// interval-span scan moves O(interval) rows per page across the driver +// boundary, the point lookups move exactly one result row per probe. +func (db *DB) RunExtents( + ctx context.Context, probes []ExtentProbe, +) ([][2]int, error) { + return ResolveRunExtents(ctx, probes, unitExtentChunk, + db.lookupRunExtentChunk) +} + +// runExtentSelectSQL builds the correlated point-lookup SELECT under a probes +// CTE with columns (idx, session_id, o, lo, hi, sc). Per probe and per side: +// the inner subquery seeks the nearest STOP row between the anchor and the +// interval bound — an embeddable user row (the reducer closes every run +// there) or an opposite-sidechain embeddable assistant row (the flip rule); +// ORDER BY ordinal DESC/ASC LIMIT 1 walks idx_messages_session_ordinal from +// the anchor outward and stops at the first hit. The outer subquery then +// seeks the farthest same-sidechain member inside the stop-narrowed +// interval. Folding the user boundary into the stop set is what lets +// DeriveUnitRanges probe with sentinel (Lo, Hi) bounds instead of paying a +// NearestUserBoundaries round trip first. The member predicate is role + +// is_system only: SystemPrefixSQL constrains user rows exclusively, so it is +// identically TRUE for assistant rows and deliberately omitted there. +func runExtentSelectSQL() string { + return fmt.Sprintf(` + SELECT p.idx, + (SELECT m.ordinal FROM messages m + WHERE m.session_id = p.session_id AND m.ordinal <= p.o + AND m.ordinal > COALESCE((SELECT f.ordinal FROM messages f + WHERE f.session_id = p.session_id + AND f.ordinal > p.lo AND f.ordinal < p.o + AND %[1]s + ORDER BY f.ordinal DESC LIMIT 1), p.lo) + AND m.role = 'assistant' AND m.is_system = 0 + AND m.is_sidechain = p.sc + ORDER BY m.ordinal ASC LIMIT 1), + (SELECT m.ordinal FROM messages m + WHERE m.session_id = p.session_id AND m.ordinal >= p.o + AND m.ordinal < COALESCE((SELECT f.ordinal FROM messages f + WHERE f.session_id = p.session_id + AND f.ordinal > p.o AND f.ordinal < p.hi + AND %[1]s + ORDER BY f.ordinal ASC LIMIT 1), p.hi) + AND m.role = 'assistant' AND m.is_system = 0 + AND m.is_sidechain = p.sc + ORDER BY m.ordinal DESC LIMIT 1) + FROM probes p`, runStopSQL()) +} + +// runStopSQL is the stop-row predicate under alias f, correlated on p.sc: an +// opposite-sidechain embeddable assistant row (flip) or an embeddable user +// row (unit boundary). +func runStopSQL() string { + return "((f.role = 'assistant' AND f.is_system = 0 AND f.is_sidechain <> p.sc)" + + " OR (" + embeddableUserSQL("f") + "))" +} + +// lookupRunExtentChunk runs the one batched statement for a chunk of distinct +// extent probes: a VALUES CTE with the correlated point lookups of +// runExtentSelectSQL. +func (db *DB) lookupRunExtentChunk( + ctx context.Context, probes []ExtentProbe, out [][2]int, +) error { + values := make([]string, len(probes)) + args := make([]any, 0, len(probes)*6) + for i, p := range probes { + values[i] = "(?, ?, ?, ?, ?, ?)" + args = append(args, i, p.SessionID, p.Ordinal, p.Lo, p.Hi, p.Sidechain) + } + query := "WITH probes(idx, session_id, o, lo, hi, sc) AS (VALUES " + + strings.Join(values, ", ") + ")" + runExtentSelectSQL() + + rows, err := db.getReader().QueryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("querying run extents: %w", err) + } + defer rows.Close() + return ScanRunExtentRows(rows, probes, out) +} diff --git a/internal/db/unit_range_test.go b/internal/db/unit_range_test.go new file mode 100644 index 000000000..4f0850fec --- /dev/null +++ b/internal/db/unit_range_test.go @@ -0,0 +1,684 @@ +package db + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// unitMsg builds a minimal message row for unit-range corpus seeding. +func unitMsg(sid string, ordinal int, role, content string) Message { + return Message{ + SessionID: sid, Ordinal: ordinal, Role: role, + Content: content, ContentLength: len(content), Timestamp: tsZero, + } +} + +// asSidechain marks a corpus message as is_sidechain. +func asSidechain(m Message) Message { + m.IsSidechain = true + return m +} + +// asSystem marks a corpus message as is_system. +func asSystem(m Message) Message { + m.IsSystem = true + return m +} + +// messageIsSidechain reads a message row's is_sidechain flag, the way a +// search enrichment pass carries the anchor's flag alongside the hit. +func messageIsSidechain(t *testing.T, d *DB, sessionID string, ordinal int) bool { + t.Helper() + var sidechain bool + err := d.getReader().QueryRowContext(context.Background(), + "SELECT is_sidechain FROM messages WHERE session_id = ? AND ordinal = ?", + sessionID, ordinal).Scan(&sidechain) + require.NoError(t, err) + return sidechain +} + +// unitMembers returns every member ordinal of a scanned unit: the Offsets +// walk for runs, the single ordinal for user units (Offsets is nil there). +func unitMembers(u EmbeddableUnit) []int { + if u.Kind == "user" { + return []int{u.Ordinal} + } + members := make([]int, len(u.Offsets)) + for i, o := range u.Offsets { + members[i] = o.Ordinal + } + return members +} + +// unitAnchorForMember builds the anchor a search path would construct for a +// member ordinal of a unit: role from the unit kind, sidechain from the +// message row, embeddable by definition of unit membership. +func unitAnchorForMember( + t *testing.T, d *DB, u EmbeddableUnit, ordinal int, +) UnitAnchor { + t.Helper() + role := "user" + if u.Kind == "run" { + role = "assistant" + } + return UnitAnchor{ + SessionID: u.SessionID, + Ordinal: ordinal, + Role: role, + Sidechain: messageIsSidechain(t, d, u.SessionID, ordinal), + Embeddable: true, + } +} + +// countingUnitQuerier counts seam calls and probes while delegating to a +// real backend, so tests can assert batching behavior. +type countingUnitQuerier struct { + inner UnitBoundsQuerier + boundsCalls int + boundsProbes int + extentCalls int + extentProbes int +} + +func (c *countingUnitQuerier) NearestUserBoundaries( + ctx context.Context, probes []UnitProbe, +) ([]UnitBounds, error) { + c.boundsCalls++ + c.boundsProbes += len(probes) + return c.inner.NearestUserBoundaries(ctx, probes) +} + +func (c *countingUnitQuerier) RunExtents( + ctx context.Context, probes []ExtentProbe, +) ([][2]int, error) { + c.extentCalls++ + c.extentProbes += len(probes) + return c.inner.RunExtents(ctx, probes) +} + +// noQueryUnitQuerier fails any seam call: anchors that resolve locally +// (rules 1/3, missing anchors) must never reach the backend. +type noQueryUnitQuerier struct{} + +func (noQueryUnitQuerier) NearestUserBoundaries( + context.Context, []UnitProbe, +) ([]UnitBounds, error) { + return nil, errors.New("NearestUserBoundaries must not be called") +} + +func (noQueryUnitQuerier) RunExtents( + context.Context, []ExtentProbe, +) ([][2]int, error) { + return nil, errors.New("RunExtents must not be called") +} + +func TestSubordinateSession(t *testing.T) { + tests := []struct { + name string + relationshipType string + parentSessionID string + want bool + }{ + {"Subagent", "subagent", "", true}, + {"Fork", "fork", "", true}, + {"ForkWithParent", "fork", "parent-1", true}, + {"ContinuationWithParent", "continuation", "parent-1", false}, + {"ParentLinkedEmptyRelationship", "", "parent-1", true}, + {"ParentLinkedOtherRelationship", "related", "parent-1", true}, + {"NoParentNoRelationship", "", "", false}, + {"ContinuationWithoutParent", "continuation", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, + SubordinateSession(tt.relationshipType, tt.parentSessionID)) + }) + } +} + +// seedUnitRangeCorpus seeds sessions covering every structural case the +// reducer handles: plain runs, runs at session start/end, sidechain flips +// mid-run, a sidechain user boundary, system and system-prefixed rows inside +// runs and adjacent to run boundaries, a prefix-looking assistant row (which +// stays embeddable: SystemPrefixSQL only constrains user rows), single-message +// runs, and automated/subagent/fork/continuation/parent-linked sessions. +// It returns the expected total unit count. +func seedUnitRangeCorpus(t *testing.T, d *DB) int { + t.Helper() + + insertSession(t, d, "s-plain", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-plain", 0, "user", "u0"), + unitMsg("s-plain", 1, "assistant", "a1"), + unitMsg("s-plain", 2, "assistant", "a2"), + unitMsg("s-plain", 3, "user", "u3"), + unitMsg("s-plain", 4, "assistant", "a4"), + unitMsg("s-plain", 5, "user", " not a boundary"), + unitMsg("s-plain", 6, "assistant", "a6"), + unitMsg("s-plain", 7, "user", "u7"), + ) + // Units: user[0], run[1,2], user[3], run members {4,6} -> [4,6], user[7]. + + insertSession(t, d, "s-edges", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-edges", 0, "assistant", "a0"), + unitMsg("s-edges", 1, "assistant", "a1"), + unitMsg("s-edges", 2, "user", "u2"), + unitMsg("s-edges", 3, "assistant", "a3"), + unitMsg("s-edges", 4, "assistant", "a4"), + ) + // Units: run[0,1] at session start, user[2], run[3,4] at session end. + + insertSession(t, d, "s-side", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-side", 0, "user", "u0"), + unitMsg("s-side", 1, "assistant", "a1"), + asSidechain(unitMsg("s-side", 2, "assistant", "a2")), + asSidechain(unitMsg("s-side", 3, "assistant", "a3")), + unitMsg("s-side", 4, "assistant", "a4"), + asSidechain(unitMsg("s-side", 5, "user", "u5")), + unitMsg("s-side", 6, "assistant", "a6"), + ) + // Units: user[0], run[1,1], sidechain run[2,3], run[4,4], + // sidechain user[5] (a user boundary regardless of sidechain), run[6,6]. + + insertSession(t, d, "s-system", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-system", 0, "user", "u0"), + asSystem(unitMsg("s-system", 1, "assistant", "sys adjacent to start")), + unitMsg("s-system", 2, "assistant", "a2"), + unitMsg("s-system", 3, "system", "system-role row"), + unitMsg("s-system", 4, "assistant", "a4"), + asSystem(unitMsg("s-system", 5, "assistant", "sys adjacent to end")), + unitMsg("s-system", 6, "user", "u6"), + unitMsg("s-system", 7, "assistant", " prefixed assistant stays embeddable"), + unitMsg("s-system", 8, "assistant", "a8"), + ) + // Units: user[0], run members {2,4} -> [2,4], user[6], run[7,8]. + + insertSession(t, d, "s-auto", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.IsAutomated = true + }) + insertMessages(t, d, + unitMsg("s-auto", 0, "user", "u0"), + unitMsg("s-auto", 1, "assistant", "a1"), + unitMsg("s-auto", 2, "assistant", "a2"), + ) + // Units: user[0], run[1,2]. + + insertSession(t, d, "s-subagent", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.RelationshipType = "subagent" + }) + insertMessages(t, d, + unitMsg("s-subagent", 0, "assistant", "a0"), + unitMsg("s-subagent", 1, "assistant", "a1"), + ) + // Units: run[0,1]. + + insertSession(t, d, "s-fork", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.RelationshipType = "fork" + s.ParentSessionID = Ptr("s-plain") + }) + insertMessages(t, d, unitMsg("s-fork", 0, "user", "u0")) + // Units: user[0]. + + insertSession(t, d, "s-cont", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.RelationshipType = "continuation" + s.ParentSessionID = Ptr("s-plain") + }) + insertMessages(t, d, + unitMsg("s-cont", 0, "user", "u0"), + unitMsg("s-cont", 1, "assistant", "a1"), + ) + // Units: user[0], run[1,1]. + + insertSession(t, d, "s-parent-linked", "proj", func(s *Session) { + s.EndedAt = Ptr(tsHour1) + s.ParentSessionID = Ptr("s-plain") + }) + insertMessages(t, d, unitMsg("s-parent-linked", 1, "assistant", "a1")) + // Units: run[1,1]. + + insertSession(t, d, "s-sys-flip", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-sys-flip", 0, "user", "u0"), + unitMsg("s-sys-flip", 1, "assistant", "a1"), + asSystem(asSidechain(unitMsg("s-sys-flip", 2, "assistant", "sys+sidechain, not a flip"))), + unitMsg("s-sys-flip", 3, "assistant", "a3"), + unitMsg("s-sys-flip", 4, "user", "u4"), + ) + // Units: user[0], run members {1,3} -> [1,3] (the is_system=1, + // is_sidechain=1 assistant row at 2 is invisible: it must not act as a + // flip boundary despite its opposite sidechain flag), user[4]. + + insertSession(t, d, "s-dense", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-dense", 0, "user", "u0"), + unitMsg("s-dense", 1, "assistant", "d1"), + unitMsg("s-dense", 2, "user", " not a boundary"), + unitMsg("s-dense", 3, "assistant", "d3"), + unitMsg("s-dense", 4, "assistant", "d4"), + unitMsg("s-dense", 5, "user", "u5"), + unitMsg("s-dense", 6, "assistant", "d6"), + asSystem(unitMsg("s-dense", 7, "assistant", "sys inside run")), + unitMsg("s-dense", 8, "assistant", "d8"), + unitMsg("s-dense", 9, "assistant", "d9"), + asSidechain(unitMsg("s-dense", 10, "assistant", "sc10")), + asSidechain(unitMsg("s-dense", 11, "assistant", "sc11")), + asSidechain(unitMsg("s-dense", 12, "assistant", "sc12")), + unitMsg("s-dense", 13, "assistant", "d13"), + unitMsg("s-dense", 14, "assistant", "d14"), + unitMsg("s-dense", 15, "assistant", "d15"), + unitMsg("s-dense", 16, "user", "u16"), + ) + // The dense-flow session: 12 run-member anchors in one session, so a page + // of all of them clears UnitBoundsFlowFactor. Units: user[0], run members + // {1,3,4} -> [1,4] (spanning the prefixed user row), user[5], run members + // {6,8,9} -> [6,9] (spanning the system row), sidechain run[10,12], + // run[13,15] (flip-bounded on the left, user-bounded on the right), + // user[16]. + + return 5 + 3 + 6 + 4 + 2 + 1 + 1 + 2 + 1 + 3 + 7 +} + +// TestDeriveUnitRangesReducerEquivalence is the invariant test: for every +// unit ScanEmbeddableUnits produces over the corpus (includeAutomated=true) +// and every member ordinal of that unit, DeriveUnitRanges must return exactly +// [unit.Ordinal, unit.OrdinalEnd]; user units must map to [o, o]. The units +// are checked both in one batched call over all anchors (exercising probe +// dedup across multi-run sessions) and one call per anchor (no dedup to +// exercise, each call trivially batches a single probe). +func TestDeriveUnitRangesReducerEquivalence(t *testing.T) { + d := testDB(t) + wantUnits := seedUnitRangeCorpus(t, d) + + units, _ := scanUnits(t, d, "", true) + require.Len(t, units, wantUnits, "corpus produced an unexpected unit count") + + ctx := context.Background() + var anchors []UnitAnchor + var want [][2]int + for _, u := range units { + for _, member := range unitMembers(u) { + anchors = append(anchors, unitAnchorForMember(t, d, u, member)) + want = append(want, [2]int{u.Ordinal, u.OrdinalEnd}) + if u.Kind == "user" { + assert.Equal(t, u.Ordinal, u.OrdinalEnd, + "user unit %s#%d must be a single-ordinal unit", + u.SessionID, u.Ordinal) + } + } + } + + got, err := DeriveUnitRanges(ctx, d, anchors) + require.NoError(t, err) + require.Len(t, got, len(anchors)) + for i, a := range anchors { + assert.Equal(t, want[i], got[i], + "batched derivation for anchor %s#%d", a.SessionID, a.Ordinal) + } + + for i, a := range anchors { + single, err := DeriveUnitRanges(ctx, d, []UnitAnchor{a}) + require.NoError(t, err) + require.Len(t, single, 1) + assert.Equal(t, want[i], single[0], + "per-anchor derivation for anchor %s#%d", a.SessionID, a.Ordinal) + } +} + +// TestDeriveUnitRangesReducerEquivalenceDenseFlow is the dense-flow variant +// of the invariant test: a page holding every run-member anchor of the +// structurally rich s-dense corpus session (multi-run, sidechain flip, +// prefixed user row, and system rows) clears UnitBoundsFlowFactor, so +// derivation fetches real user bounds with one NearestUserBoundaries call — +// and must still return exactly the reducer's extents, identical to the +// sparse per-anchor derivation. +func TestDeriveUnitRangesReducerEquivalenceDenseFlow(t *testing.T) { + d := testDB(t) + seedUnitRangeCorpus(t, d) + units, _ := scanUnits(t, d, "", true) + + ctx := context.Background() + var anchors []UnitAnchor + var want [][2]int + for _, u := range units { + if u.SessionID != "s-dense" || u.Kind != "run" { + continue + } + for _, member := range unitMembers(u) { + anchors = append(anchors, unitAnchorForMember(t, d, u, member)) + want = append(want, [2]int{u.Ordinal, u.OrdinalEnd}) + } + } + require.GreaterOrEqual(t, len(anchors), UnitBoundsFlowFactor, + "s-dense must supply at least UnitBoundsFlowFactor distinct run anchors "+ + "in one session; extend the corpus session if the factor grows") + + q := &countingUnitQuerier{inner: d} + got, err := DeriveUnitRanges(ctx, q, anchors) + require.NoError(t, err) + require.Len(t, got, len(anchors)) + for i, a := range anchors { + assert.Equal(t, want[i], got[i], + "dense-flow derivation for anchor %s#%d", a.SessionID, a.Ordinal) + } + assert.Equal(t, 1, q.boundsCalls, + "dense page must fetch real user bounds (dense flow)") + + // The sparse flow must agree exactly: one probe per call stays under the + // flow gate and probes with sentinel bounds. + for i, a := range anchors { + single, err := DeriveUnitRanges(ctx, d, []UnitAnchor{a}) + require.NoError(t, err) + require.Len(t, single, 1) + assert.Equal(t, want[i], single[0], + "sparse derivation for anchor %s#%d", a.SessionID, a.Ordinal) + } +} + +// TestDeriveUnitRangesLocalAnchors asserts rule-1, rule-3, and missing +// anchors resolve to [o, o] without ever touching the backend seam. +func TestDeriveUnitRangesLocalAnchors(t *testing.T) { + tests := []struct { + name string + anchor UnitAnchor + }{ + {"EmbeddableUserRule1", UnitAnchor{ + SessionID: "s", Ordinal: 3, Role: "user", Embeddable: true, + }}, + {"SystemRowInsideRun", UnitAnchor{ + SessionID: "s", Ordinal: 4, Role: "assistant", Embeddable: false, + }}, + {"ToolRoleRow", UnitAnchor{ + SessionID: "s", Ordinal: 5, Role: "tool", Embeddable: true, + }}, + {"PrefixedUserRow", UnitAnchor{ + SessionID: "s", Ordinal: 6, Role: "user", Embeddable: false, + }}, + {"MissingAnchor", UnitAnchor{ + SessionID: "s", Ordinal: 42, Missing: true, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := DeriveUnitRanges( + context.Background(), noQueryUnitQuerier{}, + []UnitAnchor{tt.anchor}, + ) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, [2]int{tt.anchor.Ordinal, tt.anchor.Ordinal}, got[0]) + }) + } +} + +// TestDeriveUnitRangesEmptyAnchors asserts an empty anchor list returns an +// empty result without backend calls. +func TestDeriveUnitRangesEmptyAnchors(t *testing.T) { + got, err := DeriveUnitRanges( + context.Background(), noQueryUnitQuerier{}, nil, + ) + require.NoError(t, err) + assert.Empty(t, got) +} + +// TestDeriveUnitRangesBatchesRunAnchors pins the DENSE flow's call counts: a +// session-dense page of anchors inside one run costs exactly one +// NearestUserBoundaries CALL (the anchor count is derived from +// UnitBoundsFlowFactor so the single-session page always clears the gate; +// every pending anchor rides one batch with duplicate (session, ordinal) +// anchors sharing one probe) and one RunExtents CALL carrying just ONE +// probe: the anchors share a (session, bounds, sidechain) group, so a single +// representative resolves the run and its extent is handed to every anchor +// it covers with no second round. +func TestDeriveUnitRangesBatchesRunAnchors(t *testing.T) { + d := testDB(t) + // Comfortably past the gate in one session, with the run extending on + // both sides of the anchored ordinals. + anchorCount := 2*UnitBoundsFlowFactor + 4 + runLen := anchorCount + 5 + insertSession(t, d, "s-batch", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + msgs := []Message{unitMsg("s-batch", 0, "user", "u0")} + for i := 1; i <= runLen; i++ { + msgs = append(msgs, unitMsg("s-batch", i, "assistant", "a")) + } + msgs = append(msgs, unitMsg("s-batch", runLen+1, "user", "u-end")) + insertMessages(t, d, msgs...) + + anchors := make([]UnitAnchor, 0, anchorCount+1) + for o := 2; o < 2+anchorCount; o++ { + anchors = append(anchors, UnitAnchor{ + SessionID: "s-batch", Ordinal: o, Role: "assistant", + Embeddable: true, + }) + } + // Duplicate of the first anchor: must reuse its probe, not add one. + anchors = append(anchors, anchors[0]) + + q := &countingUnitQuerier{inner: d} + got, err := DeriveUnitRanges(context.Background(), q, anchors) + require.NoError(t, err) + require.Len(t, got, len(anchors)) + for i := range got { + assert.Equal(t, [2]int{1, runLen}, got[i], "anchor %d", anchors[i].Ordinal) + } + + assert.Equal(t, 1, q.boundsCalls, "NearestUserBoundaries calls (dense page)") + assert.Equal(t, anchorCount, q.boundsProbes, "NearestUserBoundaries probes") + assert.Equal(t, 1, q.extentCalls, "RunExtents calls") + assert.Equal(t, 1, q.extentProbes, "RunExtents probes (one group representative)") +} + +// TestDeriveUnitRangesSecondRoundAcrossFlip seeds one user interval holding +// two runs separated by a sidechain flip and anchors both runs' main-chain +// members. The main-chain anchors share one (session, interval, sidechain) +// group but sit in different runs, so the round-one representative's extent +// cannot cover the anchors past the flip: they must resolve in exactly one +// second RunExtents round, with correct per-run extents. +func TestDeriveUnitRangesSecondRoundAcrossFlip(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s-flip2", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-flip2", 0, "user", "u0"), + unitMsg("s-flip2", 1, "assistant", "run1-a"), + unitMsg("s-flip2", 2, "assistant", "run1-b"), + asSidechain(unitMsg("s-flip2", 3, "assistant", "sidechain flip")), + unitMsg("s-flip2", 4, "assistant", "run2-a"), + unitMsg("s-flip2", 5, "assistant", "run2-b"), + unitMsg("s-flip2", 6, "user", "u6"), + ) + + anchors := []UnitAnchor{ + {SessionID: "s-flip2", Ordinal: 1, Role: "assistant", Embeddable: true}, + {SessionID: "s-flip2", Ordinal: 2, Role: "assistant", Embeddable: true}, + {SessionID: "s-flip2", Ordinal: 4, Role: "assistant", Embeddable: true}, + {SessionID: "s-flip2", Ordinal: 5, Role: "assistant", Embeddable: true}, + } + // This test pins the SPARSE flow: the page must stay under the gate so no + // NearestUserBoundaries round runs. Guard the coupling explicitly instead + // of letting a lowered UnitBoundsFlowFactor flip the flow silently. + require.Less(t, len(anchors), UnitBoundsFlowFactor, + "across-flip page must stay sparse; restructure the test if the flow factor shrinks") + q := &countingUnitQuerier{inner: d} + got, err := DeriveUnitRanges(context.Background(), q, anchors) + require.NoError(t, err) + require.Len(t, got, len(anchors)) + assert.Equal(t, [2]int{1, 2}, got[0], "run 1 anchor 1") + assert.Equal(t, [2]int{1, 2}, got[1], "run 1 anchor 2") + assert.Equal(t, [2]int{4, 5}, got[2], "run 2 anchor 4") + assert.Equal(t, [2]int{4, 5}, got[3], "run 2 anchor 5") + + assert.Equal(t, 0, q.boundsCalls, + "NearestUserBoundaries calls (sparse page probes with sentinel bounds)") + assert.Equal(t, 2, q.extentCalls, + "RunExtents calls (representative round + across-flip remainder)") + assert.Equal(t, 3, q.extentProbes, + "RunExtents probes (1 representative + 2 across the flip)") +} + +// TestDeriveUnitRangesNonMemberSpan asserts a run whose members are {5, 7} +// (system row at 6) derives [5, 7] from either member anchor, and that +// system rows at 3-4 and 8-9 sitting between the user boundaries do not +// widen the range past the first/last member. +func TestDeriveUnitRangesNonMemberSpan(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s-span", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-span", 2, "user", "u2"), + asSystem(unitMsg("s-span", 3, "assistant", "sys3")), + asSystem(unitMsg("s-span", 4, "assistant", "sys4")), + unitMsg("s-span", 5, "assistant", "member5"), + asSystem(unitMsg("s-span", 6, "assistant", "sys6")), + unitMsg("s-span", 7, "assistant", "member7"), + asSystem(unitMsg("s-span", 8, "assistant", "sys8")), + asSystem(unitMsg("s-span", 9, "assistant", "sys9")), + unitMsg("s-span", 10, "user", "u10"), + ) + + for _, ordinal := range []int{5, 7} { + got, err := DeriveUnitRanges(context.Background(), d, []UnitAnchor{{ + SessionID: "s-span", Ordinal: ordinal, Role: "assistant", + Embeddable: true, + }}) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, [2]int{5, 7}, got[0], "anchor at ordinal %d", ordinal) + } +} + +// TestNearestUserBoundariesSentinels asserts the seam returns Prev=-1 and +// Next=UnitOrdinalMax when no embeddable user row exists on that side, and +// real exclusive boundaries otherwise (ignoring system-prefixed user rows +// and the anchor's own ordinal). The empty-content user row at 5 pins the +// SQLite first-code-point guard's COALESCE path: unicode(”) is NULL, and an +// empty user row is still an embeddable boundary. +func TestNearestUserBoundariesSentinels(t *testing.T) { + d := testDB(t) + insertSession(t, d, "s-b", "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg("s-b", 0, "assistant", "a0"), + unitMsg("s-b", 1, "user", "u1"), + unitMsg("s-b", 2, "assistant", "a2"), + unitMsg("s-b", 3, "user", " prefixed, not a boundary"), + unitMsg("s-b", 4, "assistant", "a4"), + unitMsg("s-b", 5, "user", ""), + unitMsg("s-b", 6, "assistant", "a6"), + ) + + got, err := d.NearestUserBoundaries(context.Background(), []UnitProbe{ + {SessionID: "s-b", Ordinal: 0}, + {SessionID: "s-b", Ordinal: 2}, + {SessionID: "s-b", Ordinal: 4}, + {SessionID: "s-b", Ordinal: 1}, + {SessionID: "s-b", Ordinal: 6}, + }) + require.NoError(t, err) + require.Len(t, got, 5) + assert.Equal(t, UnitBounds{Prev: -1, Next: 1}, got[0], + "no user row before session start") + assert.Equal(t, UnitBounds{Prev: 1, Next: 5}, got[1], + "prefixed user row at 3 must not be a boundary; empty user row at 5 is") + assert.Equal(t, UnitBounds{Prev: 1, Next: 5}, got[2], + "empty-content user row is an embeddable boundary") + assert.Equal(t, UnitBounds{Prev: -1, Next: 5}, got[3], + "boundaries are exclusive of the probe ordinal itself") + assert.Equal(t, UnitBounds{Prev: 5, Next: UnitOrdinalMax}, got[4], + "no user row after the last assistant") +} + +// TestUnitBoundsQuerierChunkingAlignment seeds more sessions than either +// seam method batches into one statement (unitSessionChunk sessions for +// NearestUserBoundaries, unitExtentChunk probes for RunExtents), so both +// must run their per-chunk loop across multiple statements. Each session k +// gets its own ordinal base (b = 10*k), so its boundary/extent answer is +// unique to that session: {Prev: b, Next: b+3} and extent [b+1, b+2]. A +// chunk-boundary slicing bug (e.g. an off-by-one in the chunk start/end +// arithmetic) either drops a slot (surfacing as an "index out of range" or +// row-count error) or shifts results between neighboring sessions — and +// because every session's expected value differs from its neighbors', a +// shift produces a wrong value rather than a coincidentally correct one. +// Every probe's expected value is asserted individually against its own +// session's base, so a single misattributed slot fails the assertion for +// that specific session. +func TestUnitBoundsQuerierChunkingAlignment(t *testing.T) { + d := testDB(t) + ctx := context.Background() + // >1 chunk boundary crossed for both methods (unitExtentChunk is the + // smaller of the two). + const n = max(unitSessionChunk, unitExtentChunk) + 20 + + sessionID := func(k int) string { return fmt.Sprintf("s-chunk-%d", k) } + base := func(k int) int { return k * 10 } + + for k := range n { + b := base(k) + insertSession(t, d, sessionID(k), "proj", func(s *Session) { s.EndedAt = Ptr(tsHour1) }) + insertMessages(t, d, + unitMsg(sessionID(k), b, "user", "u-before"), + unitMsg(sessionID(k), b+1, "assistant", "member1"), + unitMsg(sessionID(k), b+2, "assistant", "member2"), + unitMsg(sessionID(k), b+3, "user", "u-after"), + ) + } + + boundProbes := make([]UnitProbe, n) + for k := range boundProbes { + b := base(k) + ordinal := b + 1 + if k%2 == 1 { + ordinal = b + 2 + } + boundProbes[k] = UnitProbe{SessionID: sessionID(k), Ordinal: ordinal} + } + bounds, err := d.NearestUserBoundaries(ctx, boundProbes) + require.NoError(t, err) + require.Len(t, bounds, n) + for k, got := range bounds { + b := base(k) + assert.Equal(t, UnitBounds{Prev: b, Next: b + 3}, got, + "bound probe for session %s", sessionID(k)) + } + + extentProbes := make([]ExtentProbe, n) + for k := range extentProbes { + b := base(k) + ordinal := b + 1 + if k%2 == 1 { + ordinal = b + 2 + } + extentProbes[k] = ExtentProbe{ + SessionID: sessionID(k), Ordinal: ordinal, Lo: b, Hi: b + 3, + } + } + extents, err := d.RunExtents(ctx, extentProbes) + require.NoError(t, err) + require.Len(t, extents, n) + for k, got := range extents { + b := base(k) + assert.Equal(t, [2]int{b + 1, b + 2}, got, + "extent probe for session %s", sessionID(k)) + } +} + +// TestRunExtentsAnchorRowMissingErrors asserts the seam fails fast with +// context when a probe's anchor row does not qualify (here: the session does +// not exist), instead of silently returning a zero range. +func TestRunExtentsAnchorRowMissingErrors(t *testing.T) { + d := testDB(t) + _, err := d.RunExtents(context.Background(), []ExtentProbe{{ + SessionID: "no-such-session", Ordinal: 3, + Lo: -1, Hi: UnitOrdinalMax, + }}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no-such-session") +} diff --git a/internal/db/vector.go b/internal/db/vector.go new file mode 100644 index 000000000..b1edd1c1a --- /dev/null +++ b/internal/db/vector.go @@ -0,0 +1,96 @@ +package db + +import ( + "context" + "errors" +) + +// ErrSemanticUnavailable is returned by SearchContent for modes "semantic" +// and "hybrid" when no VectorSearcher has been wired in (SetVectorSearcher +// was never called, or the concrete backend doesn't support semantic search +// at all — PostgreSQL and DuckDB always return it for these modes). +var ErrSemanticUnavailable = errors.New( + "semantic search not available: enable [vector] in config.toml and run 'agentsview embeddings build'") + +// ErrSemanticTransient is returned by SearchContent's semantic/hybrid modes +// when the wired VectorSearcher's query-time embed call itself failed (the +// embeddings endpoint is down, slow, or erroring), as distinct from +// ErrSemanticUnavailable's "not configured, or a build has never +// completed" cases: semantic search IS configured and otherwise usable, +// this particular request just failed and can be retried. It is +// deliberately not wrapped by ErrSemanticUnavailable, so +// errors.Is(err, ErrSemanticUnavailable) stays false for it and callers +// don't mistake a transient endpoint outage for "semantic search is +// disabled". +var ErrSemanticTransient = errors.New( + "semantic search embeddings endpoint is unavailable; the request can be retried") + +// VectorHit is one unit-level semantic search hit, ranked best first. +// Ordinal is the anchor ordinal: for a run document, the member message +// whose rune span contains the matched chunk's center; for a user document +// it is the message's own ordinal. OrdinalStart/OrdinalEnd span the whole +// unit (both equal Ordinal for user documents), and Subordinate carries the +// unit's sidechain/subagent classification from the vector mirror. +type VectorHit struct { + SessionID string + Ordinal int // anchor ordinal + OrdinalStart int + OrdinalEnd int + Subordinate bool + Score float32 + Snippet string +} + +// MessageRef identifies one message by its session and ordinal, the shape +// the hybrid FTS leg hands to ResolveMessageUnits. +type MessageRef struct { + SessionID string + Ordinal int +} + +// UnitRef locates the embedding unit (user document or assistant run) +// containing a message. The zero value (DocKey == "") means "no containing +// unit": the message lies outside the embeddable universe, and the hybrid +// path keeps such an FTS hit at message granularity rather than dropping it. +type UnitRef struct { + DocKey string + SessionID string + OrdinalStart int + OrdinalEnd int + Subordinate bool +} + +// VectorSearcher is the seam through which internal/db reaches the vector +// embedding index without importing internal/vector directly, which would +// create an import cycle (internal/vector depends on internal/db's schema +// helpers). The concrete implementation is internal/vector's Index, wired in +// at startup via SetVectorSearcher. +type VectorSearcher interface { + // SemanticSearch embeds query and returns up to limit unit-level hits, + // best first. + SemanticSearch(ctx context.Context, query string, limit int) ([]VectorHit, error) + // ResolveMessageUnits maps each ref to the unit containing it. The + // result is parallel to refs; a ref with no containing unit yields a + // zero UnitRef (DocKey == ""). + ResolveMessageUnits(ctx context.Context, refs []MessageRef) ([]UnitRef, error) +} + +// SetVectorSearcher wires (or, with nil, clears) the semantic search +// backend. Safe to call concurrently with SearchContent/HasSemantic. +func (db *DB) SetVectorSearcher(v VectorSearcher) { + db.vectorMu.Lock() + defer db.vectorMu.Unlock() + db.vectorSearcher = v +} + +// HasSemantic reports whether a VectorSearcher has been wired in. +func (db *DB) HasSemantic() bool { + return db.getVectorSearcher() != nil +} + +// getVectorSearcher returns the currently wired VectorSearcher, or nil. +func (db *DB) getVectorSearcher() VectorSearcher { + db.vectorMu.RLock() + defer db.vectorMu.RUnlock() + return db.vectorSearcher +} diff --git a/internal/duckdb/messages.go b/internal/duckdb/messages.go index b28ba22ca..dadaf0ea2 100644 --- a/internal/duckdb/messages.go +++ b/internal/duckdb/messages.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "slices" + "strings" "time" "go.kenn.io/agentsview/internal/db" @@ -48,6 +50,165 @@ func (s *Store) GetMessages( return msgs, nil } +// GetMessagesWindow mirrors internal/db's GetMessagesWindow: linear mode +// (optionally role-filtered) delegates to GetMessages when Roles is empty; +// Around mode merges three queries (before/anchor/after) into one ascending +// slice. The anchor query has no role predicate so the anchor row is always +// present regardless of Roles; before/after apply the role filter first, so +// Before/After count role-matching messages, not raw ordinal distance. +func (s *Store) GetMessagesWindow( + ctx context.Context, sessionID string, w db.MessageWindow, +) ([]db.Message, error) { + if w.Around != nil { + return s.getMessagesAroundAnchor(ctx, sessionID, w) + } + from := 0 + if w.From != nil { + from = *w.From + } + if len(w.Roles) == 0 { + return s.GetMessages(ctx, sessionID, from, w.Limit, w.Asc) + } + return s.getMessagesLinearRoleFiltered(ctx, sessionID, from, w.Limit, w.Asc, w.Roles) +} + +func (s *Store) getMessagesLinearRoleFiltered( + ctx context.Context, + sessionID string, from, limit int, asc bool, roles []string, +) ([]db.Message, error) { + if limit <= 0 || limit > db.MaxMessageLimit { + limit = db.DefaultMessageLimit + } + dir := "ASC" + op := ">=" + if !asc { + dir = "DESC" + op = "<=" + } + roleClause, roleArgs := duckRoleFilterClause(roles) + query := ` + SELECT id, session_id, ordinal, role, content, thinking_text, + timestamp, has_thinking, has_tool_use, content_length, + is_system, model, token_usage, context_tokens, output_tokens, + has_context_tokens, has_output_tokens, claude_message_id, + claude_request_id, source_type, source_subtype, source_uuid, + source_parent_uuid, is_sidechain, is_compact_boundary + FROM messages + WHERE session_id = ? AND ordinal ` + op + ` ?` + roleClause + ` + ORDER BY ordinal ` + dir + ` + LIMIT ?` + args := append([]any{sessionID, from}, roleArgs...) + args = append(args, limit) + + rows, err := s.queryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("querying duckdb role-filtered messages: %w", err) + } + defer rows.Close() + msgs, err := scanMessages(rows) + if err != nil { + return nil, err + } + if err := s.attachToolCalls(ctx, msgs); err != nil { + return nil, err + } + return msgs, nil +} + +func (s *Store) getMessagesAroundAnchor( + ctx context.Context, sessionID string, w db.MessageWindow, +) ([]db.Message, error) { + anchor := *w.Around + beforeLimit := max(w.Before, 0) + afterLimit := max(w.After, 0) + roleClause, roleArgs := duckRoleFilterClause(w.Roles) + + beforeQuery := ` + SELECT id, session_id, ordinal, role, content, thinking_text, + timestamp, has_thinking, has_tool_use, content_length, + is_system, model, token_usage, context_tokens, output_tokens, + has_context_tokens, has_output_tokens, claude_message_id, + claude_request_id, source_type, source_subtype, source_uuid, + source_parent_uuid, is_sidechain, is_compact_boundary + FROM messages + WHERE session_id = ? AND ordinal < ?` + roleClause + ` + ORDER BY ordinal DESC LIMIT ?` + beforeArgs := append([]any{sessionID, anchor}, roleArgs...) + beforeArgs = append(beforeArgs, beforeLimit) + before, err := s.queryMessageRows(ctx, beforeQuery, beforeArgs...) + if err != nil { + return nil, fmt.Errorf("querying duckdb before-window messages: %w", err) + } + slices.Reverse(before) + + anchorQuery := ` + SELECT id, session_id, ordinal, role, content, thinking_text, + timestamp, has_thinking, has_tool_use, content_length, + is_system, model, token_usage, context_tokens, output_tokens, + has_context_tokens, has_output_tokens, claude_message_id, + claude_request_id, source_type, source_subtype, source_uuid, + source_parent_uuid, is_sidechain, is_compact_boundary + FROM messages WHERE session_id = ? AND ordinal = ?` + anchorMsgs, err := s.queryMessageRows(ctx, anchorQuery, sessionID, anchor) + if err != nil { + return nil, fmt.Errorf("querying duckdb anchor message: %w", err) + } + + afterQuery := ` + SELECT id, session_id, ordinal, role, content, thinking_text, + timestamp, has_thinking, has_tool_use, content_length, + is_system, model, token_usage, context_tokens, output_tokens, + has_context_tokens, has_output_tokens, claude_message_id, + claude_request_id, source_type, source_subtype, source_uuid, + source_parent_uuid, is_sidechain, is_compact_boundary + FROM messages + WHERE session_id = ? AND ordinal > ?` + roleClause + ` + ORDER BY ordinal ASC LIMIT ?` + afterArgs := append([]any{sessionID, anchor}, roleArgs...) + afterArgs = append(afterArgs, afterLimit) + after, err := s.queryMessageRows(ctx, afterQuery, afterArgs...) + if err != nil { + return nil, fmt.Errorf("querying duckdb after-window messages: %w", err) + } + + msgs := make([]db.Message, 0, len(before)+len(anchorMsgs)+len(after)) + msgs = append(msgs, before...) + msgs = append(msgs, anchorMsgs...) + msgs = append(msgs, after...) + if err := s.attachToolCalls(ctx, msgs); err != nil { + return nil, err + } + return msgs, nil +} + +// queryMessageRows runs query and scans the resulting message rows without +// attaching tool calls; callers batch that across the merged window set. +func (s *Store) queryMessageRows( + ctx context.Context, query string, args ...any, +) ([]db.Message, error) { + rows, err := s.queryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanMessages(rows) +} + +// duckRoleFilterClause returns an "AND role IN (...)" clause and its bind +// args for the given roles, or ("", nil) when roles is empty. +func duckRoleFilterClause(roles []string) (string, []any) { + if len(roles) == 0 { + return "", nil + } + placeholders := make([]string, len(roles)) + args := make([]any, len(roles)) + for i, r := range roles { + placeholders[i] = "?" + args[i] = r + } + return " AND role IN (" + strings.Join(placeholders, ",") + ")", args +} + func (s *Store) GetAllMessages(ctx context.Context, sessionID string) ([]db.Message, error) { rows, err := s.queryContext(ctx, ` SELECT id, session_id, ordinal, role, content, thinking_text, diff --git a/internal/duckdb/messages_window_test.go b/internal/duckdb/messages_window_test.go new file mode 100644 index 000000000..9084e0e7c --- /dev/null +++ b/internal/duckdb/messages_window_test.go @@ -0,0 +1,181 @@ +//go:build !(windows && arm64) + +package duckdb + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +// seedDuckWindowMessages seeds a session with 12 messages (ordinals 0..11) +// with the same user/assistant/system role layout used by the SQLite +// GetMessagesWindow parity tests (internal/db/messages_window_test.go): +// +// 0 user, 1 assistant, 2 user, 3 assistant, 4 system, 5 user, +// 6 assistant, 7 user, 8 assistant, 9 system, 10 user, 11 assistant +func seedDuckWindowMessages(t *testing.T, local *db.DB, sessionID string) { + t.Helper() + s := db.Session{ + ID: sessionID, + Project: "proj", + Machine: "local", + Agent: "claude", + MessageCount: 1, + CreatedAt: "2026-01-01T00:00:00Z", + } + require.NoError(t, local.UpsertSession(s), "seedDuckWindowMessages upsertSession %s", sessionID) + roles := []string{ + "user", "assistant", "user", "assistant", "system", "user", + "assistant", "user", "assistant", "system", "user", "assistant", + } + msgs := make([]db.Message, 0, len(roles)) + for ordinal, role := range roles { + content := "msg" + msgs = append(msgs, db.Message{ + SessionID: sessionID, + Ordinal: ordinal, + Role: role, + Content: content, + ContentLength: len(content), + IsSystem: role == "system", + }) + } + require.NoError(t, local.InsertMessages(msgs), + "seedDuckWindowMessages insertMessages %s", sessionID) +} + +// newDuckWindowStore seeds the local SQLite DB via setup, pushes to a fresh +// DuckDB mirror, and returns the read-only Store. +func newDuckWindowStore(t *testing.T, setup func(local *db.DB)) *Store { + t.Helper() + ctx := context.Background() + local := newLocalDB(t) + setup(local) + syncer := newInMemoryTestSync(t, local, SyncOptions{}) + _, err := syncer.Push(ctx, true, nil) + require.NoError(t, err, "Push to DuckDB mirror") + return NewStoreFromDB(syncer.DB()) +} + +func duckOrdinalsOf(msgs []db.Message) []int { + out := make([]int, len(msgs)) + for i, m := range msgs { + out[i] = m.Ordinal + } + return out +} + +func TestDuckGetMessagesWindow_AroundMidSession(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sMid") + }) + + anchor := 6 + msgs, err := store.GetMessagesWindow(ctx, "sMid", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{4, 5, 6, 7, 8}, duckOrdinalsOf(msgs), + "unfiltered window should return anchor +/- 2 ordinals ascending") +} + +func TestDuckGetMessagesWindow_RoleFilterCountsFilteredMessages(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sRoleCount") + }) + + anchor := 6 + msgs, err := store.GetMessagesWindow(ctx, "sRoleCount", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + Roles: []string{"user", "assistant"}, + }) + require.NoError(t, err) + assert.Equal(t, []int{3, 5, 6, 7, 8}, duckOrdinalsOf(msgs), + "before/after counts should count role-filtered messages, not raw ordinals") +} + +func TestDuckGetMessagesWindow_AnchorIncludedEvenWhenRoleFiltered(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sAnchorFiltered") + }) + + anchor := 4 // role "system", excluded by the role filter + msgs, err := store.GetMessagesWindow(ctx, "sAnchorFiltered", db.MessageWindow{ + Around: &anchor, Before: 1, After: 1, + Roles: []string{"user", "assistant"}, + }) + require.NoError(t, err) + require.Equal(t, []int{3, 4, 5}, duckOrdinalsOf(msgs), + "anchor must be included even though its own role is filtered out") + assert.Equal(t, "system", msgs[1].Role) +} + +func TestDuckGetMessagesWindow_AroundOrdinalZeroHasNoBefore(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sFirst") + }) + + anchor := 0 + msgs, err := store.GetMessagesWindow(ctx, "sFirst", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{0, 1, 2}, duckOrdinalsOf(msgs), + "no before rows exist above the first ordinal") +} + +func TestDuckGetMessagesWindow_AroundLastOrdinalHasNoAfter(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sLast") + }) + + anchor := 11 + msgs, err := store.GetMessagesWindow(ctx, "sLast", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{9, 10, 11}, duckOrdinalsOf(msgs), + "no after rows exist below the last ordinal") +} + +func TestDuckGetMessagesWindow_LinearModeWithRoles(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sLinearRoles") + }) + + msgs, err := store.GetMessagesWindow(ctx, "sLinearRoles", db.MessageWindow{ + Limit: 100, Asc: true, Roles: []string{"user"}, + }) + require.NoError(t, err) + assert.Equal(t, []int{0, 2, 5, 7, 10}, duckOrdinalsOf(msgs), + "linear mode should apply the role filter like the around mode") +} + +func TestDuckGetMessagesWindow_EmptyRolesEquivalentToGetMessages(t *testing.T) { + ctx := context.Background() + store := newDuckWindowStore(t, func(local *db.DB) { + seedDuckWindowMessages(t, local, "sEquiv") + }) + + direct, err := store.GetMessages(ctx, "sEquiv", 3, 5, true) + require.NoError(t, err) + + from := 3 + windowed, err := store.GetMessagesWindow(ctx, "sEquiv", db.MessageWindow{ + From: &from, Limit: 5, Asc: true, + }) + require.NoError(t, err) + assert.Equal(t, direct, windowed, + "empty Roles should behave identically to GetMessages") +} diff --git a/internal/duckdb/search_content_units_test.go b/internal/duckdb/search_content_units_test.go new file mode 100644 index 000000000..9d7b1a2b5 --- /dev/null +++ b/internal/duckdb/search_content_units_test.go @@ -0,0 +1,527 @@ +//go:build !(windows && arm64) + +package duckdb + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +// newUnitsStore syncs the given SQLite session batch into a fresh in-memory +// DuckDB mirror and returns a read Store over it — the standard +// sync-from-SQLite seeding path for conversation-unit derivation tests. +func newUnitsStore(t *testing.T, writes []db.SessionBatchWrite) *Store { + t.Helper() + ctx := context.Background() + local := newLocalDB(t) + _, err := local.WriteSessionBatchAtomic(writes) + require.NoError(t, err) + syncer := newInMemoryTestSync(t, local, SyncOptions{}) + _, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + return NewStoreFromDB(syncer.DB()) +} + +// unitSession builds a root session for unit-derivation fixtures. +func unitSession(id string, messageCount int) db.Session { + return syncSession(id, "proj", id+" first", + "2026-05-01T10:00:00.000Z", messageCount) +} + +// unitChildSession builds a subagent child session so session lineage +// (relationship_type + parent_session_id) survives the sync path. +func unitChildSession(id, parentID string, messageCount int) db.Session { + sess := unitSession(id, messageCount) + sess.RelationshipType = "subagent" + parent := parentID + sess.ParentSessionID = &parent + return sess +} + +// unitMsg builds a message with explicit is_system and is_sidechain flags on +// top of the shared syncMessage builder, which lacks them. +func unitMsg( + sessionID string, ordinal int, role, content string, + isSystem, isSidechain bool, calls ...db.ToolCall, +) db.Message { + ts := fmt.Sprintf("2026-05-01T10:%02d:%02d.000Z", ordinal/60, ordinal%60) + m := syncMessage(sessionID, ordinal, role, content, ts, calls...) + m.IsSystem = isSystem + m.IsSidechain = isSidechain + return m +} + +// unitMatchesByOrdinal indexes a page's matches by anchor ordinal, requiring +// the ordinals to be unique. +func unitMatchesByOrdinal( + t *testing.T, page db.ContentSearchPage, +) map[int]db.ContentMatch { + t.Helper() + out := make(map[int]db.ContentMatch, len(page.Matches)) + for _, m := range page.Matches { + _, dup := out[m.Ordinal] + require.False(t, dup, "duplicate match ordinal %d", m.Ordinal) + out[m.Ordinal] = m + } + return out +} + +// TestDuckSearchContentSubstringDerivedRunRange mirrors the SQLite and PG +// tests: every substring match in one assistant run carries the run's full +// range (spanning a non-member system row), an embeddable user row and a +// system row are their own units, and ExcludeSystem changes nothing but +// which rows match. +func TestDuckSearchContentSubstringDerivedRunRange(t *testing.T) { + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-run", 6), + Messages: []db.Message{ + unitMsg("duck-unit-run", 0, "user", "the RUNHIT question", false, false), + unitMsg("duck-unit-run", 1, "assistant", "RUNHIT step one", false, false), + unitMsg("duck-unit-run", 2, "user", "sys RUNHIT note", true, false), + unitMsg("duck-unit-run", 3, "assistant", "RUNHIT step two", false, false), + unitMsg("duck-unit-run", 4, "assistant", "RUNHIT step three", false, false), + unitMsg("duck-unit-run", 5, "user", "next question", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "RUNHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 5, "matches") + byOrd := unitMatchesByOrdinal(t, got) + assert.Equal(t, [2]int{0, 0}, byOrd[0].OrdinalRange, "user row is its own unit") + assert.Equal(t, [2]int{2, 2}, byOrd[2].OrdinalRange, "system row is its own unit") + for _, o := range []int{1, 3, 4} { + m := byOrd[o] + assert.Equal(t, [2]int{1, 4}, m.OrdinalRange, "run member %d", o) + assert.Equal(t, o, m.Ordinal, "anchor ordinal %d", o) + assert.False(t, m.Subordinate, "top-level run member %d", o) + assert.False(t, m.Sidechain, "non-sidechain run member %d", o) + // The sync fixture stores relationship_type = "root"; the lineage + // fields are a passthrough of the session row on every backend. + assert.Equal(t, "root", m.Relationship, "top-level relationship %d", o) + assert.Empty(t, m.ParentSessionID, "top-level parent %d", o) + } + + // ExcludeSystem drops the system row but leaves the derived ranges of + // the surviving rows unchanged. + ex, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "RUNHIT", Mode: "substring", + Sources: []string{"messages"}, ExcludeSystem: true, + IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent ExcludeSystem") + require.Len(t, ex.Matches, 4, "ExcludeSystem matches") + exByOrd := unitMatchesByOrdinal(t, ex) + assert.NotContains(t, exByOrd, 2, "system row excluded") + assert.Equal(t, [2]int{0, 0}, exByOrd[0].OrdinalRange) + for _, o := range []int{1, 3, 4} { + assert.Equal(t, [2]int{1, 4}, exByOrd[o].OrdinalRange, + "ExcludeSystem run member %d", o) + } +} + +// TestDuckSearchContentSidechainRunSubordinate pins the sidechain rules: a +// sidechain run's members are Subordinate + Sidechain, and the sidechain +// flip bounds both the sidechain run and the following top-level run. +func TestDuckSearchContentSidechainRunSubordinate(t *testing.T) { + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-side", 4), + Messages: []db.Message{ + unitMsg("duck-unit-side", 0, "user", "the question", false, false), + unitMsg("duck-unit-side", 1, "assistant", "SIDEHIT step a", false, true), + unitMsg("duck-unit-side", 2, "assistant", "SIDEHIT step b", false, true), + unitMsg("duck-unit-side", 3, "assistant", "main MAINHIT answer", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + + ctx := context.Background() + side, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "SIDEHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent sidechain") + require.Len(t, side.Matches, 2, "sidechain matches") + for _, m := range side.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "sidechain run range") + assert.True(t, m.Subordinate, "sidechain run is subordinate") + assert.True(t, m.Sidechain, "anchor sidechain flag") + assert.Equal(t, "root", m.Relationship, + "root session lineage passthrough, no subordinate lineage") + assert.Empty(t, m.ParentSessionID, "no parent session") + } + + main, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "MAINHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent main") + require.Len(t, main.Matches, 1, "main matches") + m := main.Matches[0] + assert.Equal(t, [2]int{3, 3}, m.OrdinalRange, + "sidechain flip bounds the top-level run") + assert.False(t, m.Subordinate, "top-level run") + assert.False(t, m.Sidechain, "top-level anchor") +} + +// TestDuckSearchContentSubagentLineage pins session-level lineage on lexical +// rows: a match inside a subagent session is Subordinate with Relationship +// and ParentSessionID populated from the sessions join. +func TestDuckSearchContentSubagentLineage(t *testing.T) { + store := newUnitsStore(t, []db.SessionBatchWrite{ + { + Session: unitSession("duck-unit-parent", 1), + Messages: []db.Message{ + unitMsg("duck-unit-parent", 0, "user", "parent prompt", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: unitChildSession("duck-unit-child", "duck-unit-parent", 2), + Messages: []db.Message{ + unitMsg("duck-unit-child", 0, "user", "subagent prompt", false, false), + unitMsg("duck-unit-child", 1, "assistant", "SUBHIT answer", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "SUBHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeChildren: true, + IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 1, "matches") + m := got.Matches[0] + assert.Equal(t, [2]int{1, 1}, m.OrdinalRange, "single-member run") + assert.True(t, m.Subordinate, "subagent session is subordinate") + assert.Equal(t, "subagent", m.Relationship, "Relationship") + assert.Equal(t, "duck-unit-parent", m.ParentSessionID, "ParentSessionID") + assert.False(t, m.Sidechain, "anchor not sidechain") +} + +// TestDuckSearchContentToolDerivedRunRange pins derivation for tool_input and +// canonical tool_result rows: the anchor is the tool call's message row, so +// both locations carry the enclosing run's range while the wire Role stays +// the hard-coded "assistant". +func TestDuckSearchContentToolDerivedRunRange(t *testing.T) { + call := db.ToolCall{ + ToolName: "Bash", Category: "execution", ToolUseID: "tu1", + InputJSON: `{"command":"TOOLHIT"}`, + ResultContent: "output RESHIT data", + } + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-tool", 4), + Messages: []db.Message{ + unitMsg("duck-unit-tool", 0, "user", "the question", false, false), + unitMsg("duck-unit-tool", 1, "assistant", "running the tool", false, false, call), + unitMsg("duck-unit-tool", 2, "assistant", "continuing the answer", false, false), + unitMsg("duck-unit-tool", 3, "user", "thanks", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + + ctx := context.Background() + in, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "TOOLHIT", Mode: "substring", + Sources: []string{"tool_input"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "tool_input search") + require.Len(t, in.Matches, 1, "tool_input matches") + assert.Equal(t, "assistant", in.Matches[0].Role, "wire role stays assistant") + assert.Equal(t, 1, in.Matches[0].Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 2}, in.Matches[0].OrdinalRange, + "tool_input anchor classified from the real message row") + + res, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "RESHIT", Mode: "substring", + Sources: []string{"tool_result"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "tool_result search") + require.Len(t, res.Matches, 1, "tool_result matches") + assert.Equal(t, [2]int{1, 2}, res.Matches[0].OrdinalRange, + "canonical tool_result anchor classified from the real message row") +} + +// TestDuckSearchContentToolResultEventsDerived pins the events branch: an +// orphaned event (no message row at its ordinal) still returns its match +// (row cardinality must not change) with the [o, o] fallback and session +// lineage, while an event whose message row sits inside a run gets the run's +// range via the post-scan anchor lookup. The orphan row is seeded with a +// direct insert: the sync path only emits events attached to a synced +// message's tool call, so an event at an ordinal with no message row is not +// representable through the fixture. +func TestDuckSearchContentToolResultEventsDerived(t *testing.T) { + eventCall := db.ToolCall{ + ToolName: "Bash", Category: "execution", ToolUseID: "tu1", + InputJSON: `{"command":"x"}`, + ResultEvents: []db.ToolResultEvent{{ + ToolUseID: "tu1", Source: "agent", Status: "success", + Content: "EVHIT streamed output", + ContentLength: len("EVHIT streamed output"), + Timestamp: "2026-05-01T11:00:01.000Z", + EventIndex: 0, + }}, + } + store := newUnitsStore(t, []db.SessionBatchWrite{ + { + Session: unitSession("duck-ev-boss", 1), + Messages: []db.Message{ + unitMsg("duck-ev-boss", 0, "user", "boss prompt", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: unitChildSession("duck-ev-orph", "duck-ev-boss", 1), + Messages: []db.Message{ + unitMsg("duck-ev-orph", 0, "user", "orphan opener", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + { + Session: unitSession("duck-ev-run", 3), + Messages: []db.Message{ + unitMsg("duck-ev-run", 0, "user", "the question", false, false), + unitMsg("duck-ev-run", 1, "assistant", "running", false, false, eventCall), + unitMsg("duck-ev-run", 2, "assistant", "wrapping up", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }, + }) + // Orphan: an event at ordinal 7 with no message row behind it. + _, err := store.DB().Exec(` + INSERT INTO tool_result_events ( + session_id, tool_call_message_ordinal, call_index, + tool_use_id, source, status, content, content_length, event_index + ) VALUES (?, 7, 0, 'tux', 'agent', 'success', ?, ?, 0)`, + "duck-ev-orph", "ORPHHIT event content", len("ORPHHIT event content")) + require.NoError(t, err, "insert orphan event") + + ctx := context.Background() + orph, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "ORPHHIT", Mode: "substring", + Sources: []string{"tool_result"}, IncludeChildren: true, + IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "orphan search") + require.Len(t, orph.Matches, 1, "orphaned event row must not be dropped") + m := orph.Matches[0] + assert.Equal(t, 7, m.Ordinal, "event ordinal") + assert.Equal(t, [2]int{7, 7}, m.OrdinalRange, + "missing anchor falls back to [o, o]") + assert.False(t, m.Sidechain, "missing anchor has no sidechain flag") + assert.True(t, m.Subordinate, "session lineage still applies") + assert.Equal(t, "subagent", m.Relationship, "Relationship from sessions join") + assert.Equal(t, "duck-ev-boss", m.ParentSessionID, + "ParentSessionID from sessions join") + + ev, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "EVHIT", Mode: "substring", + Sources: []string{"tool_result"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "event search") + require.Len(t, ev.Matches, 1, "event matches") + assert.Equal(t, 1, ev.Matches[0].Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 2}, ev.Matches[0].OrdinalRange, + "event with a message row inside a run gets the run's range") +} + +// TestDuckSearchContentRegexDerivedRange spot-checks that regex mode (the +// candidate scan path) routes through the shared derivation pass. +func TestDuckSearchContentRegexDerivedRange(t *testing.T) { + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-rx", 3), + Messages: []db.Message{ + unitMsg("duck-unit-rx", 0, "user", "the question", false, false), + unitMsg("duck-unit-rx", 1, "assistant", "RXHIT alpha", false, false), + unitMsg("duck-unit-rx", 2, "assistant", "RXHIT beta", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: `RXHIT [a-z]+`, Mode: "regex", + Sources: []string{"messages"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent regex") + require.Len(t, got.Matches, 2, "regex matches") + for _, m := range got.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "derived run range") + } +} + +// TestDuckSearchContentFTSDerivedRange spot-checks that DuckDB's fts mode +// (ILIKE terms over messages) routes through the shared derivation pass. +func TestDuckSearchContentFTSDerivedRange(t *testing.T) { + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-fts", 3), + Messages: []db.Message{ + unitMsg("duck-unit-fts", 0, "user", "the question", false, false), + unitMsg("duck-unit-fts", 1, "assistant", "ftshit alpha step", false, false), + unitMsg("duck-unit-fts", 2, "assistant", "ftshit beta step", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "ftshit", Mode: "fts", + Sources: []string{"messages"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent fts") + require.Len(t, got.Matches, 2, "fts matches") + for _, m := range got.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "derived run range") + assert.False(t, m.Subordinate, "top-level run") + } +} + +// TestDuckSearchContentDenseFlowDerivedRanges exercises the DENSE derivation +// flow against DuckDB's dense-fetch SQL (scanDuckUserBoundaryOrdinals): one +// session whose runs supply at least db.UnitBoundsFlowFactor distinct run +// anchors on a single page, so the shared flow selection fetches real user +// bounds with the batched boundary statement before resolving extents. Run +// lengths are derived from the exported gate so the page stays dense if the +// factor changes. The structure packs a main run, a sidechain run, and a +// flip-bounded main run, and every match must carry its run's exact range. +func TestDuckSearchContentDenseFlowDerivedRanges(t *testing.T) { + // Three runs of runLen anchors each: 3*runLen > UnitBoundsFlowFactor. + runLen := db.UnitBoundsFlowFactor/2 + 1 + runA := [2]int{1, runLen} // main run after user 0 + side := [2]int{runLen + 2, 2*runLen + 1} // sidechain run after user runLen+1 + runC := [2]int{2*runLen + 2, 3*runLen + 1} // main run bounded left by the flip + lastUser := 3*runLen + 2 + + msgs := []db.Message{ + unitMsg("duck-unit-dense", 0, "user", "first question", false, false), + } + for o := runA[0]; o <= runA[1]; o++ { + msgs = append(msgs, unitMsg("duck-unit-dense", o, "assistant", + fmt.Sprintf("DFHIT main-a %d", o), false, false)) + } + msgs = append(msgs, unitMsg("duck-unit-dense", runLen+1, "user", + "second question", false, false)) + for o := side[0]; o <= side[1]; o++ { + msgs = append(msgs, unitMsg("duck-unit-dense", o, "assistant", + fmt.Sprintf("DFHIT side %d", o), false, true)) + } + for o := runC[0]; o <= runC[1]; o++ { + msgs = append(msgs, unitMsg("duck-unit-dense", o, "assistant", + fmt.Sprintf("DFHIT main-c %d", o), false, false)) + } + msgs = append(msgs, unitMsg("duck-unit-dense", lastUser, "user", + "done", false, false)) + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-dense", len(msgs)), + Messages: msgs, + DataVersion: 1, + ReplaceMessages: true, + }}) + + anchorCount := 3 * runLen + require.GreaterOrEqual(t, anchorCount, db.UnitBoundsFlowFactor, + "single-session page must clear the dense-flow gate") + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "DFHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeOneShot: true, + Limit: anchorCount + 10, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, anchorCount, "matches") + byOrd := unitMatchesByOrdinal(t, got) + for o := runA[0]; o <= runA[1]; o++ { + assert.Equal(t, runA, byOrd[o].OrdinalRange, "run A member %d", o) + assert.False(t, byOrd[o].Sidechain, "run A member %d flag", o) + } + for o := side[0]; o <= side[1]; o++ { + assert.Equal(t, side, byOrd[o].OrdinalRange, "sidechain member %d", o) + assert.True(t, byOrd[o].Sidechain, "sidechain member %d flag", o) + assert.True(t, byOrd[o].Subordinate, "sidechain member %d subordinate", o) + } + for o := runC[0]; o <= runC[1]; o++ { + assert.Equal(t, runC, byOrd[o].OrdinalRange, + "flip-bounded run C member %d", o) + assert.False(t, byOrd[o].Sidechain, "run C member %d flag", o) + } +} + +// TestDuckSearchContentMultiRunReducerParity is the hand-computed parity +// check against the SQLite-side embedding reducer: one session with two +// top-level runs, a sidechain run between them, and an interior system row +// that must not close the run it sits inside. +func TestDuckSearchContentMultiRunReducerParity(t *testing.T) { + store := newUnitsStore(t, []db.SessionBatchWrite{{ + Session: unitSession("duck-unit-multi", 9), + Messages: []db.Message{ + unitMsg("duck-unit-multi", 0, "user", "PARHIT q1", false, false), + unitMsg("duck-unit-multi", 1, "assistant", "PARHIT a", false, false), + unitMsg("duck-unit-multi", 2, "assistant", "PARHIT b", false, false), + unitMsg("duck-unit-multi", 3, "assistant", "PARHIT sc1", false, true), + unitMsg("duck-unit-multi", 4, "assistant", "PARHIT sc2", false, true), + unitMsg("duck-unit-multi", 5, "assistant", "PARHIT c", false, false), + unitMsg("duck-unit-multi", 6, "user", "PARHIT sys note", true, false), + unitMsg("duck-unit-multi", 7, "assistant", "PARHIT d", false, false), + unitMsg("duck-unit-multi", 8, "user", "PARHIT q2", false, false), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "PARHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeOneShot: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 9, "matches") + byOrd := unitMatchesByOrdinal(t, got) + + want := map[int][2]int{ + 0: {0, 0}, // embeddable user row: its own unit + 1: {1, 2}, // first top-level run, closed by the sidechain flip + 2: {1, 2}, + 3: {3, 4}, // sidechain run, bounded by flips on both sides + 4: {3, 4}, + 5: {5, 7}, // second top-level run, spanning the interior system row + 6: {6, 6}, // interior system row: its own unit, doesn't close the run + 7: {5, 7}, + 8: {8, 8}, // closing embeddable user row + } + for o, r := range want { + assert.Equal(t, r, byOrd[o].OrdinalRange, "ordinal %d range", o) + } + for _, o := range []int{3, 4} { + assert.True(t, byOrd[o].Subordinate, "sidechain member %d subordinate", o) + assert.True(t, byOrd[o].Sidechain, "sidechain member %d flag", o) + } + for _, o := range []int{0, 1, 2, 5, 6, 7, 8} { + assert.False(t, byOrd[o].Subordinate, "top-level row %d", o) + assert.False(t, byOrd[o].Sidechain, "top-level row %d flag", o) + } +} diff --git a/internal/duckdb/store.go b/internal/duckdb/store.go index 426ddd118..e05957ece 100644 --- a/internal/duckdb/store.go +++ b/internal/duckdb/store.go @@ -669,6 +669,11 @@ func rootSessionWhere(excludeOneShot, excludeAutomated bool) string { func (s *Store) HasFTS() bool { return true } +// HasSemantic returns false: the DuckDB store has no VectorSearcher seam +// yet, so SearchContent rejects "semantic"/"hybrid" modes up front with +// db.ErrSemanticUnavailable. +func (s *Store) HasSemantic() bool { return false } + func (s *Store) Search(ctx context.Context, f db.SearchFilter) (db.SearchPage, error) { if f.Limit <= 0 || f.Limit > db.MaxSearchLimit { f.Limit = db.DefaultSearchLimit @@ -857,6 +862,26 @@ func (s *Store) SearchContent(ctx context.Context, f db.ContentSearchFilter) (db if f.Pattern == "" { return db.ContentSearchPage{}, nil } + + // Semantic and hybrid validate Sources themselves (messages only) ahead + // of the substring/regex/fts source-set default just below, which fills + // in tool_input/tool_result that neither mode supports -- mirroring + // internal/db's SearchContent so an empty Sources field is not defaulted + // out from under ValidateSemanticFilter's empty-or-messages-only check. + if f.Mode == "semantic" || f.Mode == "hybrid" { + // Validate input the same way SQLite's semantic/hybrid paths do + // before reporting the capability gate: an invalid request (bad + // cursor, non-messages source) must return the same 400 + // SearchInputError on every backend rather than a 501 here and a + // 400 on SQLite (backend parity, see AGENTS.md). + if err := db.ValidateSemanticFilter(f); err != nil { + return db.ContentSearchPage{}, err + } + // No VectorSearcher seam on the DuckDB store yet (HasSemantic always + // false): gate after input validation. + return db.ContentSearchPage{}, db.ErrSemanticUnavailable + } + if len(f.Sources) == 0 { f.Sources = []string{"messages", "tool_input", "tool_result"} } @@ -883,6 +908,12 @@ func (s *Store) SearchContent(ctx context.Context, f db.ContentSearchFilter) (db page.Matches = matches[:f.Limit] page.NextCursor = f.Cursor + f.Limit } + // Post-truncation derivation, O(page): every lexical match gets its + // conversation-unit OrdinalRange and lineage fields via the shared + // batched pass, matching the SQLite and PG backends. + if err := s.deriveLexicalUnitsDuck(ctx, page.Matches); err != nil { + return db.ContentSearchPage{}, err + } return page, nil } diff --git a/internal/duckdb/store_contract_test.go b/internal/duckdb/store_contract_test.go index 9a3c8a27a..42ca54263 100644 --- a/internal/duckdb/store_contract_test.go +++ b/internal/duckdb/store_contract_test.go @@ -32,6 +32,61 @@ func TestDuckDBStoreContract(t *testing.T) { } } +// TestDuckDBStoreHasSemanticFalse pins that the DuckDB store reports no +// semantic search capability until it gets its own VectorSearcher seam. +func TestDuckDBStoreHasSemanticFalse(t *testing.T) { + s := &Store{} + assert.False(t, s.HasSemantic(), "DuckDB HasSemantic") +} + +// TestDuckDBSearchContentSemanticModesUnavailable pins that "semantic" and +// "hybrid" are rejected with db.ErrSemanticUnavailable before any query runs +// -- a zero-value Store (no live *sql.DB) is enough to prove that. +func TestDuckDBSearchContentSemanticModesUnavailable(t *testing.T) { + s := &Store{} + for _, mode := range []string{"semantic", "hybrid"} { + _, err := s.SearchContent(context.Background(), + db.ContentSearchFilter{Pattern: "x", Mode: mode}) + require.Error(t, err, "mode %q", mode) + assert.True(t, errors.Is(err, db.ErrSemanticUnavailable), + "mode %q: want ErrSemanticUnavailable, got %v", mode, err) + } +} + +// TestDuckDBSearchContentSemanticInvalidInputReturns400Before501 pins backend +// parity (AGENTS.md): an invalid semantic/hybrid request -- cursor pagination +// or a non-messages source -- must return the same *db.SearchInputError +// SQLite's ValidateSemanticFilter returns, not db.ErrSemanticUnavailable, even +// though DuckDB has no VectorSearcher seam and would otherwise report the +// capability gate for any request in these modes. +func TestDuckDBSearchContentSemanticInvalidInputReturns400Before501(t *testing.T) { + s := &Store{} + cases := []struct { + name string + f db.ContentSearchFilter + }{ + {"cursor rejected", db.ContentSearchFilter{Pattern: "x", Cursor: 1}}, + {"non-messages source rejected", db.ContentSearchFilter{ + Pattern: "x", Sources: []string{"tool_input"}, + }}, + } + for _, mode := range []string{"semantic", "hybrid"} { + for _, tc := range cases { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + f := tc.f + f.Mode = mode + _, err := s.SearchContent(context.Background(), f) + require.Error(t, err) + var inputErr *db.SearchInputError + assert.True(t, errors.As(err, &inputErr), + "expected *db.SearchInputError, got %T: %v", err, err) + assert.False(t, errors.Is(err, db.ErrSemanticUnavailable), + "invalid input must not be masked as ErrSemanticUnavailable") + }) + } + } +} + func TestDuckDBFindSessionIDsByPartialLiteralCaseSensitive(t *testing.T) { ctx := context.Background() local := newLocalDB(t) diff --git a/internal/duckdb/store_test.go b/internal/duckdb/store_test.go index 0187cdd10..7066f253e 100644 --- a/internal/duckdb/store_test.go +++ b/internal/duckdb/store_test.go @@ -434,6 +434,56 @@ func TestSearchContentFTSMatchesNonContiguousTerms(t *testing.T) { assert.Contains(t, got.Matches[0].Snippet, "fox") } +// TestSearchContentOrdinalRangeSelfRange pins the ordinal_range contract on +// DuckDB content search: the field is always present and derived from the +// conversation-unit rules, never a zero-valued [0, 0] at a nonzero anchor +// ordinal. This fixture's assistant anchor is a single-member run bounded by +// the user opener, so the derived range equals the self-range. Substring and +// regex modes cover the two scan paths (scanDuckContentRows and the regex +// candidate loop). +func TestSearchContentOrdinalRangeSelfRange(t *testing.T) { + ctx := context.Background() + local := newLocalDB(t) + _, err := local.WriteSessionBatchAtomic([]db.SessionBatchWrite{{ + Session: syncSession( + "duck-range", "alpha", "first", + "2026-03-22T10:00:00.000Z", 2, + ), + Messages: []db.Message{ + syncMessage("duck-range", 0, "user", + "an unrelated opener", "2026-03-22T10:00:00.000Z"), + syncMessage("duck-range", 1, "assistant", + "the rangeneedle reply", "2026-03-22T10:00:01.000Z"), + }, + DataVersion: 1, + ReplaceMessages: true, + }}) + require.NoError(t, err) + + syncer := newInMemoryTestSync(t, local, SyncOptions{}) + _, err = syncer.Push(ctx, true, nil) + require.NoError(t, err) + store := NewStoreFromDB(syncer.DB()) + + for _, mode := range []string{"substring", "regex"} { + t.Run(mode, func(t *testing.T) { + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "rangeneedle", + Mode: mode, + Sources: []string{"messages"}, + IncludeOneShot: true, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, got.Matches, 1) + m := got.Matches[0] + require.Equal(t, 1, m.Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 1}, m.OrdinalRange, + "ordinal_range must be the derived single-member run, not [0, 0]") + }) + } +} + func TestSearchContentInvalidModeReturnsInputError(t *testing.T) { ctx := context.Background() store, _ := newSyncedStore(t) diff --git a/internal/duckdb/unit_range.go b/internal/duckdb/unit_range.go new file mode 100644 index 000000000..7fc4dc312 --- /dev/null +++ b/internal/duckdb/unit_range.go @@ -0,0 +1,309 @@ +package duckdb + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "go.kenn.io/agentsview/internal/db" +) + +// DuckDB implementation of the conversation-unit seam. Orchestration — +// session/probe dedup, chunking, boundary resolution, and the alignment and +// row-count invariants — is shared with every backend via +// db.ResolveUserBoundaries, db.ResolveRunExtents, and the db.Scan*Rows +// helpers; this file supplies only the DuckDB dialect SQL and its parameter +// binding. Every statement goes through the store's queryContext wrapper so +// the quack-remote connection path keeps working. +var _ db.UnitBoundsQuerier = (*Store)(nil) + +// duckUnitSessionChunk caps sessions per NearestUserBoundaries statement, +// matching SQLite's unitSessionChunk semantics: a session binds 2 variables +// (idx, session_id). +const duckUnitSessionChunk = duckMaxSQLVars / 2 + +// duckUnitExtentChunk caps extent probes per RunExtents statement, matching +// SQLite's unitExtentChunk semantics: a probe binds 6 variables (idx, +// session_id, o, lo, hi, sc). +const duckUnitExtentChunk = duckMaxSQLVars / 6 + +// duckEmbeddableUserSQL is the DuckDB predicate matching an embeddable user +// row under the given alias: user role, is_system = FALSE, and the DuckDB +// dialect SystemPrefixSQL check — the DuckDB form of internal/db's +// embeddableUserSQL. (The assistant-side member predicate skips the prefix +// check: SystemPrefixSQL constrains user rows only.) +func duckEmbeddableUserSQL(alias string) string { + return fmt.Sprintf("%[1]s.role = 'user' AND %[1]s.is_system = FALSE AND %[2]s", + alias, db.DuckDBSystemPrefixSQL(alias+".content", alias+".role")) +} + +// NearestUserBoundaries returns, per probe, the nearest embeddable user +// ordinals strictly before and after the probe ordinal, with the -1 / +// db.UnitOrdinalMax sentinels standing in for missing boundaries — the exact +// semantics of the SQLite seam method, guaranteed by the shared +// db.ResolveUserBoundaries orchestration: one statement per +// duckUnitSessionChunk distinct sessions fetches each session's embeddable +// user ordinals ONCE. +func (s *Store) NearestUserBoundaries( + ctx context.Context, probes []db.UnitProbe, +) ([]db.UnitBounds, error) { + return db.ResolveUserBoundaries(ctx, probes, duckUnitSessionChunk, + s.scanDuckUserBoundaryOrdinals) +} + +// scanDuckUserBoundaryOrdinals runs the one batched statement for a chunk of +// distinct sessions: a VALUES CTE joined against messages for every +// embeddable user ordinal of each session. out aligns 1:1 with sessions. +func (s *Store) scanDuckUserBoundaryOrdinals( + ctx context.Context, sessions []string, out [][]int, +) error { + values := make([]string, len(sessions)) + args := make([]any, 0, len(sessions)*2) + for i, sessionID := range sessions { + values[i] = "(?, ?)" + args = append(args, i, sessionID) + } + query := fmt.Sprintf(` + WITH spans(idx, session_id) AS (VALUES %s) + SELECT sp.idx, m.ordinal + FROM spans sp JOIN messages m ON m.session_id = sp.session_id + WHERE %s`, + strings.Join(values, ", "), duckEmbeddableUserSQL("m")) + + rows, err := s.queryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("querying nearest user boundaries: %w", err) + } + defer rows.Close() + return db.ScanUserBoundaryRows(rows, out) +} + +// RunExtents returns, per probe, the first and last member ordinals of the +// anchor's same-sidechain run, bounded exclusively by (Lo, Hi) and by the +// nearest STOP row inside that interval — an embeddable user row or an +// opposite-sidechain embeddable assistant row — the exact semantics of the +// SQLite seam method, guaranteed by the shared db.ResolveRunExtents +// orchestration. Probing with the -1 / db.UnitOrdinalMax sentinels therefore +// derives the full rule-2 extent on its own. One statement per +// duckUnitExtentChunk distinct probes resolves every probe with correlated +// point lookups (nearest stop row on each side, then the farthest +// same-sidechain member inside the stop-narrowed interval), moving exactly +// one result row per probe instead of each interval's member rows. +func (s *Store) RunExtents( + ctx context.Context, probes []db.ExtentProbe, +) ([][2]int, error) { + return db.ResolveRunExtents(ctx, probes, duckUnitExtentChunk, + s.lookupDuckRunExtentChunk) +} + +// duckRunExtentSelectSQL builds the correlated point-lookup SELECT under a +// probes CTE with columns (idx, session_id, o, lo, hi, sc) — the DuckDB form +// of internal/db's runExtentSelectSQL. Per probe and per side: the inner +// subquery seeks the nearest stop row between the anchor and the interval +// bound, the outer subquery seeks the farthest same-sidechain member inside +// the stop-narrowed interval. The member predicate is role + is_system only: +// SystemPrefixSQL constrains user rows exclusively, so it is identically +// TRUE for assistant rows and deliberately omitted there. +func duckRunExtentSelectSQL() string { + stop := "((f.role = 'assistant' AND f.is_system = FALSE AND f.is_sidechain <> p.sc)" + + " OR (" + duckEmbeddableUserSQL("f") + "))" + return fmt.Sprintf(` + SELECT p.idx, + (SELECT m.ordinal FROM messages m + WHERE m.session_id = p.session_id AND m.ordinal <= p.o + AND m.ordinal > COALESCE((SELECT f.ordinal FROM messages f + WHERE f.session_id = p.session_id + AND f.ordinal > p.lo AND f.ordinal < p.o + AND %[1]s + ORDER BY f.ordinal DESC LIMIT 1), p.lo) + AND m.role = 'assistant' AND m.is_system = FALSE + AND m.is_sidechain = p.sc + ORDER BY m.ordinal ASC LIMIT 1), + (SELECT m.ordinal FROM messages m + WHERE m.session_id = p.session_id AND m.ordinal >= p.o + AND m.ordinal < COALESCE((SELECT f.ordinal FROM messages f + WHERE f.session_id = p.session_id + AND f.ordinal > p.o AND f.ordinal < p.hi + AND %[1]s + ORDER BY f.ordinal ASC LIMIT 1), p.hi) + AND m.role = 'assistant' AND m.is_system = FALSE + AND m.is_sidechain = p.sc + ORDER BY m.ordinal DESC LIMIT 1) + FROM probes p`, stop) +} + +// lookupDuckRunExtentChunk runs the one batched statement for a chunk of +// distinct extent probes: a VALUES CTE with the correlated point lookups of +// duckRunExtentSelectSQL. +func (s *Store) lookupDuckRunExtentChunk( + ctx context.Context, probes []db.ExtentProbe, out [][2]int, +) error { + values := make([]string, len(probes)) + args := make([]any, 0, len(probes)*6) + for i, p := range probes { + values[i] = "(?, ?, ?, ?, ?, ?)" + args = append(args, i, p.SessionID, p.Ordinal, p.Lo, p.Hi, p.Sidechain) + } + query := "WITH probes(idx, session_id, o, lo, hi, sc) AS (VALUES " + + strings.Join(values, ", ") + ")" + duckRunExtentSelectSQL() + + rows, err := s.queryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("querying run extents: %w", err) + } + defer rows.Close() + return db.ScanRunExtentRows(rows, probes, out) +} + +// duckAnchorMetaChunk caps (session_id, ordinal) refs per anchor-meta lookup, +// matching internal/db's enrichHitsChunk semantics (2 binds per ref). +const duckAnchorMetaChunk = duckMaxSQLVars / 2 + +// duckAnchorKey identifies one (session_id, ordinal) anchor ref. +type duckAnchorKey struct { + sessionID string + ordinal int +} + +// duckAnchorMeta is one match's anchor metadata: session lineage plus the +// anchor message row's classification columns — the DuckDB twin of +// internal/db's contentAnchorMeta. +type duckAnchorMeta struct { + relationship string + parentSessionID string + role sql.NullString + sidechain sql.NullBool + embeddable sql.NullBool + missing bool +} + +// deriveLexicalUnitsDuck is the shared post-scan pass for the DuckDB +// substring, fts, and regex modes, mirroring internal/db's +// deriveLexicalUnits: one batched anchor-meta lookup, one shared +// db.DeriveUnitRanges derivation (constant batched statement count for the +// whole page), then per-match assignment of OrdinalRange and the lineage +// fields. matches is the already truncated page, so the pass is O(page). +func (s *Store) deriveLexicalUnitsDuck( + ctx context.Context, matches []db.ContentMatch, +) error { + if len(matches) == 0 { + return nil + } + metas, err := s.fillAnchorMetaDuck(ctx, matches) + if err != nil { + return err + } + anchors := make([]db.UnitAnchor, len(matches)) + for i, m := range matches { + meta := metas[i] + anchors[i] = db.UnitAnchor{ + SessionID: m.SessionID, + Ordinal: m.Ordinal, + Role: meta.role.String, + Sidechain: meta.sidechain.Valid && meta.sidechain.Bool, + Embeddable: meta.embeddable.Valid && meta.embeddable.Bool, + Missing: meta.missing, + } + } + ranges, err := db.DeriveUnitRanges(ctx, s, anchors) + if err != nil { + return fmt.Errorf("deriving lexical units: %w", err) + } + for i := range matches { + matches[i].OrdinalRange = ranges[i] + matches[i].Relationship = metas[i].relationship + matches[i].ParentSessionID = metas[i].parentSessionID + matches[i].Sidechain = anchors[i].Sidechain + matches[i].Subordinate = anchors[i].Sidechain || + db.SubordinateSession(metas[i].relationship, metas[i].parentSessionID) + } + return nil +} + +// fillAnchorMetaDuck fetches anchor classification and session lineage for +// every page row: one batched VALUES-CTE lookup per duckAnchorMetaChunk +// distinct (session_id, ordinal) refs, never a per-row query. Refs whose +// message row does not exist (tool_result_events orphans) are marked missing +// so derivation falls back to [o, o]; their session lineage still resolves +// via the sessions join. The result aligns 1:1 with matches. +func (s *Store) fillAnchorMetaDuck( + ctx context.Context, matches []db.ContentMatch, +) ([]duckAnchorMeta, error) { + seen := make(map[duckAnchorKey]bool, len(matches)) + refs := make([]duckAnchorKey, 0, len(matches)) + for i := range matches { + key := duckAnchorKey{matches[i].SessionID, matches[i].Ordinal} + if !seen[key] { + seen[key] = true + refs = append(refs, key) + } + } + found := make(map[duckAnchorKey]duckAnchorMeta, len(refs)) + for start := 0; start < len(refs); start += duckAnchorMetaChunk { + chunk := refs[start:min(start+duckAnchorMetaChunk, len(refs))] + if err := s.lookupAnchorMetaChunkDuck(ctx, chunk, found); err != nil { + return nil, err + } + } + metas := make([]duckAnchorMeta, len(matches)) + for i := range matches { + got, ok := found[duckAnchorKey{matches[i].SessionID, matches[i].Ordinal}] + if !ok { + metas[i].missing = true + continue + } + got.missing = !got.role.Valid + metas[i] = got + } + return metas, nil +} + +// lookupAnchorMetaChunkDuck resolves one chunk of (session_id, ordinal) refs +// to session lineage plus the anchor message row's classification columns: +// role, sidechain, and the embeddable flag (is_system = FALSE AND content +// not system-prefixed, exactly the embedding reducer's predicate). messages +// is LEFT JOINed so a ref whose message row is absent still resolves +// lineage; its classification columns come back NULL. +func (s *Store) lookupAnchorMetaChunkDuck( + ctx context.Context, refs []duckAnchorKey, + out map[duckAnchorKey]duckAnchorMeta, +) error { + values := make([]string, len(refs)) + args := make([]any, 0, len(refs)*2) + for i, r := range refs { + values[i] = "(?, ?)" + args = append(args, r.sessionID, r.ordinal) + } + query := "WITH refs(session_id, ordinal) AS (VALUES " + + strings.Join(values, ", ") + ") " + + "SELECT r.session_id, r.ordinal, " + + "COALESCE(s.relationship_type, ''), COALESCE(s.parent_session_id, ''), " + + "m.role, m.is_sidechain, " + + "CASE WHEN m.is_system = FALSE AND " + + db.DuckDBSystemPrefixSQL("m.content", "m.role") + + " THEN TRUE ELSE FALSE END " + + "FROM refs r " + + "JOIN sessions s ON s.id = r.session_id " + + "LEFT JOIN messages m ON m.session_id = r.session_id AND m.ordinal = r.ordinal" + + rows, err := s.queryContext(ctx, query, args...) + if err != nil { + return fmt.Errorf("looking up match anchors: %w", err) + } + defer rows.Close() + for rows.Next() { + var key duckAnchorKey + var meta duckAnchorMeta + if err := rows.Scan(&key.sessionID, &key.ordinal, + &meta.relationship, &meta.parentSessionID, + &meta.role, &meta.sidechain, &meta.embeddable); err != nil { + return fmt.Errorf("scanning match anchor: %w", err) + } + out[key] = meta + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating match anchors: %w", err) + } + return nil +} diff --git a/internal/export/project_identity.go b/internal/export/project_identity.go index 791fcc3c0..706c3a024 100644 --- a/internal/export/project_identity.go +++ b/internal/export/project_identity.go @@ -162,12 +162,15 @@ func NormalizeRootPath(raw string) (normalized string, ok bool, err error) { return "", false, nil } cleaned := filepath.Clean(raw) - resolved, err := filepath.EvalSymlinks(cleaned) - if err != nil && !os.IsNotExist(err) { - return "", false, err - } - if err != nil { - resolved = cleaned + resolved := cleaned + if !IsAutomountNamespacePath(runtime.GOOS, cleaned) { + resolved, err = filepath.EvalSymlinks(cleaned) + if err != nil && !os.IsNotExist(err) { + return "", false, err + } + if err != nil { + resolved = cleaned + } } abs, err := filepath.Abs(resolved) if err != nil { @@ -357,7 +360,31 @@ func normalizeWindowsDriveRootPath(raw string) string { return drive + rest } +// IsAutomountNamespacePath reports whether p lies inside a macOS +// automounter namespace (/home, /net, /Network/Servers). On darwin, merely +// stat'ing such a path wakes automountd, which resolves the map through +// opendirectoryd, and negative results are not cached — an archive holding +// many /home/... roots from sessions synced off Linux machines turns every +// resolution sweep into a sustained syscall storm. Callers skip filesystem +// resolution for these paths and use the cleaned path directly, which is +// exactly what the (virtually always failing) EvalSymlinks fallback would +// have produced. goos is a parameter so the predicate is testable off-darwin. +func IsAutomountNamespacePath(goos, p string) bool { + if goos != "darwin" { + return false + } + for _, ns := range [...]string{"/home", "/net", "/Network/Servers"} { + if p == ns || strings.HasPrefix(p, ns+"/") { + return true + } + } + return false +} + func resolveStoredRootPath(cleaned string) (string, bool) { + if IsAutomountNamespacePath(runtime.GOOS, cleaned) { + return "", false + } resolved, err := filepath.EvalSymlinks(filepath.FromSlash(cleaned)) if err != nil { return "", false diff --git a/internal/export/project_identity_test.go b/internal/export/project_identity_test.go index 067bbdf0b..6d58fff87 100644 --- a/internal/export/project_identity_test.go +++ b/internal/export/project_identity_test.go @@ -320,3 +320,46 @@ func sha256Hex(s string) string { sum := sha256.Sum256([]byte(s)) return hex.EncodeToString(sum[:]) } + +func TestIsAutomountNamespacePath(t *testing.T) { + tests := []struct { + name string + goos string + path string + want bool + }{ + {"darwin home root", "darwin", "/home", true}, + {"darwin home child", "darwin", "/home/user/repo", true}, + {"darwin net child", "darwin", "/net/host/share", true}, + {"darwin network servers", "darwin", "/Network/Servers/x", true}, + {"darwin prefix collision homework", "darwin", "/homework/repo", false}, + {"darwin prefix collision netdata", "darwin", "/netdata", false}, + {"darwin regular path", "darwin", "/Users/user/repo", false}, + {"linux home is real", "linux", "/home/user/repo", false}, + {"windows never matches", "windows", "/home/user", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, IsAutomountNamespacePath(tt.goos, tt.path)) + }) + } +} + +// TestNormalizeStoredRootPathSkipsAutomountNamespace pins that on macOS a +// stored /home/... root path normalizes to its cleaned form without touching +// the filesystem: resolving it through the automounter is both futile (the +// path names a directory on another machine) and expensive (each probe wakes +// automountd/opendirectoryd, and negative results are not cached). +func TestNormalizeStoredRootPathSkipsAutomountNamespace(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("automount namespaces are a darwin-only concern") + } + got, ok := NormalizeStoredRootPath("/home/user/work/repo") + require.True(t, ok) + assert.Equal(t, "/home/user/work/repo", got) + + normalized, ok, err := NormalizeRootPath("/home/user/work/repo") + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "/home/user/work/repo", normalized) +} diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 1641ba9ae..d8cfac4d1 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -89,21 +89,25 @@ func newServer(opts ServeOptions) *mcp.Server { mcp.AddTool(s, &mcp.Tool{ Name: ToolGetMessages, - Description: "Read a slice of one session's transcript, paginated by message ordinal. Defaults " + - "return only user and assistant messages, each truncated to 2000 characters; truncated " + - "messages are flagged so you can re-fetch with a higher max_chars_per_message. Each page " + - "reports how many scanned messages the role/system filter dropped as filtered, so across a " + - "full pagination sweep, returned plus filtered messages add up to the session's " + - "message_count (which counts all stored messages, system included).", + Description: "Read a slice of one session's transcript, either by linear pagination (from/direction/" + + "limit) or a symmetric window centered on an ordinal (around, with before/after sizing each " + + "side, default 5; mutually exclusive with from/direction). Defaults return only user and " + + "assistant messages, each truncated to 2000 characters; truncated messages are flagged so you " + + "can re-fetch with a higher max_chars_per_message. Each page reports how many scanned messages " + + "the role/system filter dropped as filtered, so across a full pagination sweep, returned plus " + + "filtered messages add up to the session's message_count (which counts all stored messages, " + + "system included).", Annotations: readOnly, }, t.getMessages) mcp.AddTool(s, &mcp.Tool{ Name: ToolSearchContent, - Description: "Exact substring or regex search over raw session text, including tool inputs and results. " + - "Slower but more precise than search_sessions; use it for error messages, identifiers, or " + - "code fragments. Matches from the last 10 minutes (including the current conversation) are " + - "excluded unless include_active is set.", + Description: "Substring, regex, or semantic/hybrid embedding search over raw session text, including " + + "tool inputs and results. Slower but more precise than search_sessions; use it for error " + + "messages, identifiers, and code fragments (substring/regex), or a natural-language query when " + + "the exact wording is unknown (semantic/hybrid). Set context to include N messages of " + + "surrounding conversation with each match. Matches from the last 10 minutes (including the " + + "current conversation) are excluded unless include_active is set.", Annotations: readOnly, }, t.searchContent) diff --git a/internal/mcp/shape.go b/internal/mcp/shape.go index 8981182c7..9554712aa 100644 --- a/internal/mcp/shape.go +++ b/internal/mcp/shape.go @@ -36,6 +36,9 @@ const ( overviewMaxChars = 500 // nameMaxChars caps session display names in list/search results. nameMaxChars = 200 + // contextMessageMaxChars caps each search_content context_before/ + // context_after message. + contextMessageMaxChars = 500 ) // truncate cuts s to at most max runes on a rune boundary, returning diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index 7e9b6a753..53c3e73f0 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -311,10 +311,15 @@ func (t *toolset) sessionOverview( // --- get_messages --- type getMessagesIn struct { - SessionID string `json:"session_id" jsonschema:"The session to read."` - From *int `json:"from,omitempty" jsonschema:"Ordinal to start from (e.g. match_ordinal from search_sessions). Ordinal 0 is a valid anchor (the first message)."` - Direction string `json:"direction,omitempty" jsonschema:"asc (default, oldest first) or desc (newest first)."` - Limit int `json:"limit,omitempty" jsonschema:"Max messages scanned, default 20, max 100. System/tool messages are filtered after this limit, so a page can return fewer; use next_from to continue."` + SessionID string `json:"session_id" jsonschema:"The session to read."` + From *int `json:"from,omitempty" jsonschema:"Ordinal to start from (e.g. match_ordinal from search_sessions). Ordinal 0 is a valid anchor (the first message). Mutually exclusive with around."` + Direction string `json:"direction,omitempty" jsonschema:"asc (default, oldest first) or desc (newest first). Mutually exclusive with around."` + Limit int `json:"limit,omitempty" jsonschema:"Max messages scanned, default 20, max 100. System/tool messages are filtered after this limit, so a page can return fewer; use next_from to continue. Not used with around; before/after control the window size on that path."` + // Around switches to a symmetric window centered on an ordinal instead + // of linear pagination; it is mutually exclusive with From/Direction. + Around *int `json:"around,omitempty" jsonschema:"Center a symmetric window on this ordinal (e.g. match_ordinal from search_content or search_sessions), instead of linear pagination. Mutually exclusive with from and direction."` + Before *int `json:"before,omitempty" jsonschema:"Messages of context before the around anchor, default 5. Requires around; replaces limit on that path."` + After *int `json:"after,omitempty" jsonschema:"Messages of context after the around anchor, default 5. Requires around; replaces limit on that path."` Roles []string `json:"roles,omitempty" jsonschema:"Roles to include, e.g. tool. Default: user and assistant only. System messages are always excluded."` MaxCharsPerMessage int `json:"max_chars_per_message,omitempty" jsonschema:"Truncate each message to this many characters, default 2000, max 20000."` } @@ -339,18 +344,51 @@ type getMessagesOut struct { NextFrom *int `json:"next_from,omitempty" jsonschema:"Anchor for the next page's from parameter when more messages may remain; absent means the end. Filtering can make a page return fewer than limit messages, so keep paging until next_from is absent, not until a page comes back short."` } +// filterAndMapMessage applies the get_messages role/system contract to one +// message -- always drop system content (IsSystem or a legacy system +// content prefix), then require its role to be in roles (nil/empty roles +// falls back to user+assistant via roleAllowed) -- and shapes the survivor +// into a messageOut truncated to maxChars. ok is false when the message was +// filtered, so the caller can count it. +func filterAndMapMessage(m db.Message, roles []string, maxChars int) (messageOut, bool) { + if isSystemMessage(m) || !roleAllowed(m.Role, roles) { + return messageOut{}, false + } + content, cut := truncate(m.Content, maxChars) + mo := messageOut{ + Ordinal: m.Ordinal, + Role: m.Role, + Content: content, + Timestamp: m.Timestamp, + Model: m.Model, + HasToolUse: m.HasToolUse, + Truncated: cut, + } + if cut { + mo.FullLength = m.ContentLength + } + return mo, true +} + func (t *toolset) getMessages( ctx context.Context, _ *mcp.CallToolRequest, in getMessagesIn, ) (*mcp.CallToolResult, getMessagesOut, error) { + if in.Around != nil { + return t.getMessagesAround(ctx, in) + } // From is a *int so an explicit ordinal 0 (a valid match_ordinal) // anchors at the first message rather than being mistaken for // "omitted". A nil From lets the service default: desc to - // newest-first, asc to oldest-first. + // newest-first, asc to oldest-first. Before/After are forwarded + // unused so the service can reject them with "before/after require + // around" when set without Around. limit := clampLimit(in.Limit, defaultMessageLimit, maxMessageLimit) res, err := t.svc.Messages(ctx, in.SessionID, service.MessageFilter{ From: in.From, Direction: in.Direction, Limit: limit, + Before: in.Before, + After: in.After, }) if err != nil { return nil, getMessagesOut{}, err @@ -359,23 +397,11 @@ func (t *toolset) getMessages( in.MaxCharsPerMessage, defaultMaxCharsPerMessage, maxMaxCharsPerMessage) out := getMessagesOut{Messages: make([]messageOut, 0, len(res.Messages))} for _, m := range res.Messages { - if isSystemMessage(m) || !roleAllowed(m.Role, in.Roles) { + mo, ok := filterAndMapMessage(m, in.Roles, maxChars) + if !ok { out.Filtered++ continue } - content, cut := truncate(m.Content, maxChars) - mo := messageOut{ - Ordinal: m.Ordinal, - Role: m.Role, - Content: content, - Timestamp: m.Timestamp, - Model: m.Model, - HasToolUse: m.HasToolUse, - Truncated: cut, - } - if cut { - mo.FullLength = m.ContentLength - } out.Messages = append(out.Messages, mo) } // A full raw page means more rows may remain. Anchor next_from just past @@ -395,11 +421,61 @@ func (t *toolset) getMessages( return nil, out, nil } +// getMessagesAround handles the symmetric-window retrieval path (Around +// set). Unlike the linear path, an empty Roles must be translated to the +// MCP default (user, assistant) before it reaches the service: an empty +// service.MessageFilter.Roles means "all roles" there, not "the MCP +// default", and would leak tool/system-role dumps into the window. From and +// Direction are forwarded unused so the service can reject them as mutually +// exclusive with Around. The service's around window always includes the +// anchor row regardless of role, so it is subject to the same +// filterAndMapMessage post-filter as every other message; a suppressed +// anchor is counted in Filtered like any other dropped message. next_from +// is anchored on the last returned ordinal (there is no scan direction in a +// symmetric window), and Limit/Before/After select the window, not the +// linear path's Limit. +func (t *toolset) getMessagesAround( + ctx context.Context, in getMessagesIn, +) (*mcp.CallToolResult, getMessagesOut, error) { + roles := in.Roles + if len(roles) == 0 { + roles = []string{"user", "assistant"} + } + res, err := t.svc.Messages(ctx, in.SessionID, service.MessageFilter{ + From: in.From, + Direction: in.Direction, + Around: in.Around, + Before: in.Before, + After: in.After, + Roles: roles, + }) + if err != nil { + return nil, getMessagesOut{}, err + } + maxChars := clampLimit( + in.MaxCharsPerMessage, defaultMaxCharsPerMessage, maxMaxCharsPerMessage) + out := getMessagesOut{Messages: make([]messageOut, 0, len(res.Messages))} + for _, m := range res.Messages { + mo, ok := filterAndMapMessage(m, roles, maxChars) + if !ok { + out.Filtered++ + continue + } + out.Messages = append(out.Messages, mo) + } + if len(out.Messages) > 0 { + next := out.Messages[len(out.Messages)-1].Ordinal + 1 + out.NextFrom = &next + } + return nil, out, nil +} + // --- search_content --- type searchContentIn struct { Pattern string `json:"pattern" jsonschema:"Exact substring or regex to find across message text and tool inputs/results."` - Mode string `json:"mode,omitempty" jsonschema:"substring (default) or regex."` + Mode string `json:"mode,omitempty" jsonschema:"substring (default), regex, semantic, or hybrid."` + Scope string `json:"scope,omitempty" jsonschema:"Semantic/hybrid result scope: top, all, or subordinate (default all). Only valid with mode semantic or hybrid."` Project string `json:"project,omitempty" jsonschema:"Restrict to one project."` Agent string `json:"agent,omitempty" jsonschema:"Restrict to one agent."` DateFrom string `json:"date_from,omitempty" jsonschema:"Only sessions on or after this date (YYYY-MM-DD)."` @@ -407,17 +483,51 @@ type searchContentIn struct { Limit int `json:"limit,omitempty" jsonschema:"Max matches, default 10, max 30."` Cursor int `json:"cursor,omitempty" jsonschema:"Pagination cursor from a previous next_cursor."` IncludeActive bool `json:"include_active,omitempty" jsonschema:"Include matches from sessions active in the last 10 minutes. Default false: the conversation you are in right now is also recorded, so without this exclusion you would find yourself."` + Context int `json:"context,omitempty" jsonschema:"Messages of context before/after each match (max 10)."` +} + +// contextMessage is a truncated view of a service-level db.Message, used +// for search_content's inline context_before/context_after. +type contextMessage struct { + Ordinal int `json:"ordinal"` + Role string `json:"role"` + Content string `json:"content"` +} + +func toContextMessages(msgs []db.Message) []contextMessage { + if len(msgs) == 0 { + return nil + } + out := make([]contextMessage, 0, len(msgs)) + for _, m := range msgs { + content, _ := truncate(m.Content, contextMessageMaxChars) + out = append(out, contextMessage{ + Ordinal: m.Ordinal, Role: m.Role, Content: content, + }) + } + return out } type contentMatch struct { - SessionID string `json:"session_id"` - Project string `json:"project,omitempty"` - Agent string `json:"agent"` - Location string `json:"location" jsonschema:"Where the match occurred: one of message, tool_input, or tool_result."` - Role string `json:"role,omitempty"` - Ordinal int `json:"ordinal"` - Timestamp string `json:"timestamp"` - Snippet string `json:"snippet"` + SessionID string `json:"session_id"` + Project string `json:"project,omitempty"` + Agent string `json:"agent"` + Location string `json:"location" jsonschema:"Where the match occurred: one of message, tool_input, or tool_result."` + Role string `json:"role,omitempty"` + Ordinal int `json:"ordinal"` + Timestamp string `json:"timestamp"` + Snippet string `json:"snippet"` + Score *float64 `json:"score,omitempty" jsonschema:"Relevance score for semantic/hybrid modes; omitted for substring/regex/fts."` + OrdinalRange [2]int `json:"ordinal_range" jsonschema:"[start, end] ordinals of the conversation unit containing this match; equal to the match ordinal for single-message units."` + Subordinate bool `json:"subordinate,omitempty" jsonschema:"True when this match belongs to a subordinate unit: a sidechain run, or a subagent/fork session."` + Relationship string `json:"relationship,omitempty" jsonschema:"The matched session's relationship to its parent (for example subagent or fork), when it has one."` + ParentSessionID string `json:"parent_session_id,omitempty" jsonschema:"The parent session ID, when the matched session has one."` + Sidechain bool `json:"is_sidechain,omitempty" jsonschema:"True when the matched message itself is flagged as a sidechain message."` + // ContextBefore/ContextAfter are populated when Context > 0: the N + // messages immediately before/after this match, content truncated to + // 500 characters. + ContextBefore []contextMessage `json:"context_before,omitempty"` + ContextAfter []contextMessage `json:"context_after,omitempty"` } type searchContentOut struct { @@ -429,15 +539,24 @@ type searchContentOut struct { func (t *toolset) searchContent( ctx context.Context, _ *mcp.CallToolRequest, in searchContentIn, ) (*mcp.CallToolResult, searchContentOut, error) { + // The db layer silently ignores Scope outside semantic/hybrid, so reject + // it here with the same message the HTTP transport uses + // (internal/server/huma_routes_search.go). + if in.Scope != "" && in.Mode != "semantic" && in.Mode != "hybrid" { + return nil, searchContentOut{}, fmt.Errorf( + "scope is only supported for semantic and hybrid search modes") + } res, err := t.svc.SearchContent(ctx, service.ContentSearchRequest{ Pattern: in.Pattern, Mode: in.Mode, + Scope: in.Scope, Project: in.Project, Agent: in.Agent, DateFrom: in.DateFrom, DateTo: in.DateTo, Limit: clampLimit(in.Limit, defaultSearchLimit, maxSearchLimit), Cursor: in.Cursor, + Context: in.Context, }) if err != nil { return nil, searchContentOut{}, err @@ -467,7 +586,12 @@ func (t *toolset) searchContent( out.Matches = append(out.Matches, contentMatch{ SessionID: m.SessionID, Project: m.Project, Agent: m.Agent, Location: m.Location, Role: m.Role, Ordinal: m.Ordinal, - Timestamp: m.Timestamp, Snippet: m.Snippet, + Timestamp: m.Timestamp, Snippet: m.Snippet, Score: m.Score, + OrdinalRange: m.OrdinalRange, Subordinate: m.Subordinate, + Relationship: m.Relationship, ParentSessionID: m.ParentSessionID, + Sidechain: m.Sidechain, + ContextBefore: toContextMessages(m.ContextBefore), + ContextAfter: toContextMessages(m.ContextAfter), }) } if res.NextCursor > 0 && len(res.Matches) > 0 { diff --git a/internal/mcp/tools_test.go b/internal/mcp/tools_test.go index 2b7a5d5be..03bec51c9 100644 --- a/internal/mcp/tools_test.go +++ b/internal/mcp/tools_test.go @@ -3,6 +3,7 @@ package mcp import ( "context" "encoding/json" + "strings" "testing" "time" @@ -281,6 +282,37 @@ func TestSearchContent_SubstringMatch(t *testing.T) { assert.Equal(t, "s1", out.Matches[0].SessionID) } +// TestSearchContent_ContextRedactsSecretByDefault verifies that a secret in +// a message adjacent to a search match comes back redacted in +// context_before through the real service (not a fake), proving the MCP +// transport inherits directBackend's context redaction: MCP has no reveal +// opt-in, so this path must always come out redacted. +func TestSearchContent_ContextRedactsSecretByDefault(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = 3 + s.UserMessageCount = 2 + ended := "2024-06-15T10:00:00Z" + s.EndedAt = &ended + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("s1", 0, "my key is AKIA7QHWN2DKR4FYPLJM ok"), + dbtest.AsstMsg("s1", 1, "noted"), + dbtest.UserMsg("s1", 2, "DEADBEEF marks the match"), + })) + + _, out, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "DEADBEEF", Mode: "substring", Context: 2, + }) + require.NoError(t, err) + require.Len(t, out.Matches, 1) + require.Len(t, out.Matches[0].ContextBefore, 2) + for _, cm := range out.Matches[0].ContextBefore { + assert.NotContains(t, cm.Content, "AKIA7QHWN2DKR4FYPLJM", + "MCP has no reveal opt-in, so context must always come back redacted: %q", cm.Content) + } +} + // search_content's self-reference guard must exclude matches from sessions // that are active now, even when the matching message itself is old. A // long-running current session can match on a stale line; excluding by the @@ -443,6 +475,121 @@ func TestSearchContent_ExcludesOneShotByDefault(t *testing.T) { assert.Equal(t, "multi", out.Matches[0].SessionID) } +// search_content must surface the conversation-unit citation fields +// (ordinal_range plus lineage) copied verbatim from db.ContentMatch: every +// match in a top-level assistant run carries the run's full ordinal_range +// and none of the subordinate/lineage fields set, matching the plumbing +// pinned at the db layer in TestSearchContentSubstringDerivedRunRange. +func TestSearchContent_OrdinalRangeSpansRun(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "run1", "proj", func(s *db.Session) { + s.MessageCount = 4 + s.UserMessageCount = 2 + ended := "2024-06-15T10:00:00Z" + s.EndedAt = &ended + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("run1", 0, "the question"), + dbtest.AsstMsg("run1", 1, "RUNHIT step one"), + dbtest.AsstMsg("run1", 2, "RUNHIT step two"), + dbtest.UserMsg("run1", 3, "next question"), + })) + + _, out, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "RUNHIT", Mode: "substring", + }) + require.NoError(t, err) + require.Len(t, out.Matches, 2) + for _, m := range out.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, + "match at ordinal %d should carry the run's full range", m.Ordinal) + assert.False(t, m.Subordinate, "top-level run member") + assert.False(t, m.Sidechain, "non-sidechain run member") + assert.Empty(t, m.Relationship, "top-level relationship") + assert.Empty(t, m.ParentSessionID, "top-level parent") + } +} + +// A sidechain run's matches must round-trip Subordinate and Sidechain as +// true, with ordinal_range spanning the sidechain run rather than the +// individual anchor ordinal. +func TestSearchContent_SidechainSubordinateRoundTrip(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "side1", "proj", func(s *db.Session) { + s.MessageCount = 3 + s.UserMessageCount = 2 + ended := "2024-06-15T10:00:00Z" + s.EndedAt = &ended + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("side1", 0, "the question"), + { + SessionID: "side1", Ordinal: 1, Role: "assistant", + Content: "SIDEHIT step a", ContentLength: len("SIDEHIT step a"), + IsSidechain: true, + }, + { + SessionID: "side1", Ordinal: 2, Role: "assistant", + Content: "SIDEHIT step b", ContentLength: len("SIDEHIT step b"), + IsSidechain: true, + }, + })) + + _, out, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "SIDEHIT", Mode: "substring", + }) + require.NoError(t, err) + require.Len(t, out.Matches, 2) + for _, m := range out.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "sidechain run range") + assert.True(t, m.Subordinate, "sidechain run is subordinate") + assert.True(t, m.Sidechain, "anchor sidechain flag") + assert.Empty(t, m.Relationship, "no session lineage on a same-session sidechain") + } +} + +// A single top-level message match (its own conversation unit) must report +// ordinal_range == [o, o], and the omitempty subordinate/lineage fields must +// be entirely absent from the marshaled JSON rather than present as false/"". +func TestSearchContent_SingleMessageOrdinalRangeAndOmittedFields(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "solo1", "proj", func(s *db.Session) { + s.MessageCount = 2 + s.UserMessageCount = 2 + ended := "2024-06-15T10:00:00Z" + s.EndedAt = &ended + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("solo1", 0, "SOLOHIT alone"), + dbtest.UserMsg("solo1", 1, "an unrelated follow-up"), + })) + + _, out, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "SOLOHIT", Mode: "substring", + }) + require.NoError(t, err) + require.Len(t, out.Matches, 1) + m := out.Matches[0] + assert.Equal(t, 0, m.Ordinal) + assert.Equal(t, [2]int{0, 0}, m.OrdinalRange, "single-message unit is its own range") + assert.False(t, m.Subordinate) + assert.False(t, m.Sidechain) + assert.Empty(t, m.Relationship) + assert.Empty(t, m.ParentSessionID) + + data, err := json.Marshal(m) + require.NoError(t, err) + var raw map[string]any + require.NoError(t, json.Unmarshal(data, &raw)) + assert.Contains(t, raw, "ordinal_range", "ordinal_range is always present") + for _, key := range []string{ + "subordinate", "relationship", "parent_session_id", "is_sidechain", + } { + assert.NotContains(t, raw, key, + "zero-valued omitempty field %q must be absent from the wire shape", key) + } +} + func TestGetMessages_DescAndFromAnchor(t *testing.T) { ts, d := newTestToolset(t) dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { @@ -776,3 +923,324 @@ func TestServer_EndToEnd(t *testing.T) { require.NoError(t, ct.Close()) require.NoError(t, st.Wait()) } + +// fakeContentSearchService captures the ContentSearchRequest a tool builds +// and returns a canned result or error, so semantic-mode passthrough and +// context-mapping can be asserted without a full backend. Unused methods +// fall through to the embedded nil interface (never called by searchContent +// when IncludeActive is set, which skips the session-activity lookup). +type fakeContentSearchService struct { + service.SessionService + lastReq service.ContentSearchRequest + result *service.ContentSearchResult + err error +} + +func (f *fakeContentSearchService) SearchContent( + _ context.Context, req service.ContentSearchRequest, +) (*service.ContentSearchResult, error) { + f.lastReq = req + if f.err != nil { + return nil, f.err + } + return f.result, nil +} + +// search_content must pass Mode through to the service untouched, and map +// service.ErrSemanticUnavailable to a tool error carrying the remediation +// sentence from db.ErrSemanticUnavailable ("...run 'agentsview embeddings +// build'"), not a generic failure. +func TestSearchContent_SemanticUnavailableMapsToRemediationError(t *testing.T) { + fake := &fakeContentSearchService{err: service.ErrSemanticUnavailable} + ts := &toolset{svc: fake, now: func() time.Time { return fixedNow }} + + _, _, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "how do I configure retries", Mode: "semantic", IncludeActive: true, + }) + require.Error(t, err) + assert.ErrorIs(t, err, service.ErrSemanticUnavailable) + assert.Contains(t, err.Error(), "embeddings build") + assert.Equal(t, "semantic", fake.lastReq.Mode) +} + +// search_content must reject scope outside semantic/hybrid with the same +// message the HTTP transport uses (the db layer silently ignores Scope for +// lexical modes, so the guard lives in the transport), and must not reach +// the service at all on rejection. +func TestSearchContent_ScopeRejectedOnLexicalModes(t *testing.T) { + fake := &fakeContentSearchService{result: &service.ContentSearchResult{}} + ts := &toolset{svc: fake, now: func() time.Time { return fixedNow }} + + for _, mode := range []string{"", "substring", "regex", "fts"} { + t.Run("mode="+mode, func(t *testing.T) { + _, _, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "needle", Mode: mode, Scope: "top", IncludeActive: true, + }) + require.Error(t, err) + assert.EqualError(t, err, + "scope is only supported for semantic and hybrid search modes") + }) + } + assert.Empty(t, fake.lastReq.Pattern, + "a rejected request must not reach the service") +} + +// search_content must pass Scope through to the service untouched for +// semantic and hybrid modes; the db layer owns scope-value validation from +// there. +func TestSearchContent_ScopeForwardedForSemanticModes(t *testing.T) { + for _, mode := range []string{"semantic", "hybrid"} { + t.Run(mode, func(t *testing.T) { + fake := &fakeContentSearchService{result: &service.ContentSearchResult{}} + ts := &toolset{svc: fake, now: func() time.Time { return fixedNow }} + + _, _, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "retries", Mode: mode, Scope: "subordinate", IncludeActive: true, + }) + require.NoError(t, err) + assert.Equal(t, mode, fake.lastReq.Mode) + assert.Equal(t, "subordinate", fake.lastReq.Scope, + "scope must reach the service untouched") + }) + } +} + +// search_content's Context parameter must reach the service, and each +// match's ContextBefore/ContextAfter (full service-level db.Message) must +// map to the MCP layer's truncated contextMessage shape, along with Score. +func TestSearchContent_ContextThreading(t *testing.T) { + score := 0.83 + long := strings.Repeat("y", 600) + fake := &fakeContentSearchService{ + result: &service.ContentSearchResult{ + Matches: []db.ContentMatch{{ + SessionID: "s1", Agent: "claude", Location: "message", + Role: "user", Ordinal: 10, Timestamp: "2024-06-15T09:00:00Z", + Snippet: "hit", Score: &score, + ContextBefore: []db.Message{ + {Ordinal: 8, Role: "user", Content: "before msg"}, + {Ordinal: 9, Role: "assistant", Content: long}, + }, + ContextAfter: []db.Message{ + {Ordinal: 11, Role: "assistant", Content: "after msg"}, + }, + }}, + }, + } + ts := &toolset{svc: fake, now: func() time.Time { return fixedNow }} + + _, out, err := ts.searchContent(context.Background(), nil, searchContentIn{ + Pattern: "hit", Context: 5, IncludeActive: true, + }) + require.NoError(t, err) + assert.Equal(t, 5, fake.lastReq.Context, "context param must reach the service") + require.Len(t, out.Matches, 1) + m := out.Matches[0] + require.NotNil(t, m.Score) + assert.InDelta(t, score, *m.Score, 0.0001) + require.Len(t, m.ContextBefore, 2) + assert.Equal(t, 8, m.ContextBefore[0].Ordinal) + assert.Equal(t, "user", m.ContextBefore[0].Role) + assert.Equal(t, "before msg", m.ContextBefore[0].Content) + assert.Len(t, m.ContextBefore[1].Content, 500, "context content is truncated to 500 chars") + require.Len(t, m.ContextAfter, 1) + assert.Equal(t, 11, m.ContextAfter[0].Ordinal) + assert.Equal(t, "after msg", m.ContextAfter[0].Content) +} + +// get_messages's around/before/after form a symmetric window that is +// mutually exclusive with the linear from/direction form, and before/after +// require around. Errors come straight from the service (directBackend +// validates), so the tool error text must match its sentinels verbatim. +func TestGetMessages_AroundValidation(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = 3 + s.UserMessageCount = 2 + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("s1", 0, "m0"), + dbtest.AsstMsg("s1", 1, "m1"), + dbtest.UserMsg("s1", 2, "m2"), + })) + + anchor, from, before := 1, 0, 2 + tests := []struct { + name string + in getMessagesIn + wantErr string + }{ + { + name: "around with direction rejected", + in: getMessagesIn{SessionID: "s1", Around: &anchor, Direction: "desc"}, + wantErr: "around is mutually exclusive with from/direction", + }, + { + name: "around with from rejected", + in: getMessagesIn{SessionID: "s1", Around: &anchor, From: &from}, + wantErr: "around is mutually exclusive with from/direction", + }, + { + name: "before without around rejected", + in: getMessagesIn{SessionID: "s1", Before: &before}, + wantErr: "before/after require around", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, err := ts.getMessages(context.Background(), nil, tt.in) + require.Error(t, err) + assert.Equal(t, tt.wantErr, err.Error()) + }) + } +} + +// The around path anchors next_from on the last returned ordinal, not a +// scan-direction offset like the linear path (there is no scan direction in +// a symmetric window). +func TestGetMessages_AroundNextFromIsLastPlusOne(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = 5 + s.UserMessageCount = 3 + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("s1", 0, "m0"), + dbtest.AsstMsg("s1", 1, "m1"), + dbtest.UserMsg("s1", 2, "m2"), + dbtest.AsstMsg("s1", 3, "m3"), + dbtest.UserMsg("s1", 4, "m4"), + })) + + anchor, before, after := 2, 1, 1 + _, out, err := ts.getMessages(context.Background(), nil, getMessagesIn{ + SessionID: "s1", Around: &anchor, Before: &before, After: &after, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 3) + last := out.Messages[len(out.Messages)-1].Ordinal + require.NotNil(t, out.NextFrom) + assert.Equal(t, last+1, *out.NextFrom) + assert.Equal(t, 4, *out.NextFrom) +} + +// An empty Roles on the around path must be translated to the MCP default +// (user, assistant) before reaching the service: an empty +// service.MessageFilter.Roles means "all roles" there, which would leak +// tool-role dumps that the linear path's own default excludes. The +// translated roles reach the DB-level before/after query directly (unlike +// the linear path's post-fetch filter), so the non-anchor tool rows here +// are dropped before the MCP layer ever sees them -- Filtered stays 0; the +// anchor-bypass case is covered separately below. +func TestGetMessages_AroundDefaultRolesExcludesTool(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = 5 + s.UserMessageCount = 2 + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("s1", 0, "m0"), + {SessionID: "s1", Ordinal: 1, Role: "tool", Content: "tool dump", ContentLength: 9}, + dbtest.AsstMsg("s1", 2, "m2"), + {SessionID: "s1", Ordinal: 3, Role: "tool", Content: "tool dump 2", ContentLength: 11}, + dbtest.UserMsg("s1", 4, "m4"), + })) + + anchor := 2 + _, out, err := ts.getMessages(context.Background(), nil, getMessagesIn{ + SessionID: "s1", Around: &anchor, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 3, "ordinals 0, 2, 4 survive; the two tool rows never reach this page") + for _, m := range out.Messages { + assert.NotEqual(t, "tool", m.Role) + } + assert.Equal(t, 0, out.Filtered) +} + +// The around path always includes the anchor row server-side regardless of +// its role or system status. The MCP layer must post-filter it like any +// other message, suppressing a system anchor and counting it in Filtered +// rather than hardcoding Filtered to 0. +func TestGetMessages_AroundSuppressesSystemAnchor(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = 3 + s.UserMessageCount = 2 + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("s1", 0, "m0"), + { + SessionID: "s1", Ordinal: 1, Role: "system", + Content: "sys", IsSystem: true, ContentLength: 3, + }, + dbtest.AsstMsg("s1", 2, "m2"), + })) + + anchor, before, after := 1, 1, 1 + _, out, err := ts.getMessages(context.Background(), nil, getMessagesIn{ + SessionID: "s1", Around: &anchor, Before: &before, After: &after, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 2) + for _, m := range out.Messages { + assert.NotEqual(t, 1, m.Ordinal, "the system anchor must be suppressed") + } + assert.Equal(t, 1, out.Filtered, "the suppressed anchor is counted in Filtered") +} + +// The anchor query has no role predicate, so a tool-role anchor is returned +// by the service even under the MCP default roles (user, assistant). The +// MCP layer's post-filter must suppress it too and count it in Filtered, +// exactly like the system-anchor case above. +func TestGetMessages_AroundSuppressesToolRoleAnchor(t *testing.T) { + ts, d := newTestToolset(t) + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = 3 + s.UserMessageCount = 2 + }) + require.NoError(t, d.InsertMessages([]db.Message{ + dbtest.UserMsg("s1", 0, "m0"), + {SessionID: "s1", Ordinal: 1, Role: "tool", Content: "tool dump", ContentLength: 9}, + dbtest.AsstMsg("s1", 2, "m2"), + })) + + anchor, before, after := 1, 1, 1 + _, out, err := ts.getMessages(context.Background(), nil, getMessagesIn{ + SessionID: "s1", Around: &anchor, Before: &before, After: &after, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 2) + for _, m := range out.Messages { + assert.NotEqual(t, 1, m.Ordinal, "the tool-role anchor must be suppressed") + } + assert.Equal(t, 1, out.Filtered, "the suppressed anchor is counted in Filtered") +} + +// TestGetMessages_AroundClampsOversizedWindow verifies that an oversized +// before/after request (e.g. before=10^9) cannot bypass db.MaxMessageLimit +// through the MCP get_messages tool: directBackend.Messages clamps the +// window before it ever reaches the store, so at most db.MaxMessageLimit +// messages come back even though more than that many exist on both sides +// of the anchor. +func TestGetMessages_AroundClampsOversizedWindow(t *testing.T) { + ts, d := newTestToolset(t) + const total = db.MaxMessageLimit + 50 + dbtest.SeedSession(t, d, "s1", "proj", func(s *db.Session) { + s.MessageCount = total + s.UserMessageCount = total + }) + require.NoError(t, d.InsertMessages(dbtest.UserMessagesf("s1", total, "m%d"))) + + anchor, huge := total/2, 1_000_000_000 + _, out, err := ts.getMessages(context.Background(), nil, getMessagesIn{ + SessionID: "s1", Around: &anchor, Before: &huge, After: &huge, + Roles: []string{"user"}, + }) + require.NoError(t, err) + assert.LessOrEqual(t, len(out.Messages), db.MaxMessageLimit, + "an oversized around window must be capped at db.MaxMessageLimit") + assert.Less(t, len(out.Messages), total, + "the oversized request must actually be capped below what an "+ + "unclamped window would have returned") +} diff --git a/internal/parser/cursor_attribution.go b/internal/parser/cursor_attribution.go index 81ad66b19..a046a057e 100644 --- a/internal/parser/cursor_attribution.go +++ b/internal/parser/cursor_attribution.go @@ -165,7 +165,7 @@ func openCursorAttributionDB(path string) (*sql.DB, error) { // would be opened read-write. conn, err := sql.Open( "sqlite3", - "file:"+path+"?mode=ro&_busy_timeout=3000", + "file:"+sqliteURIPath(path)+"?mode=ro&_busy_timeout=3000", ) if err != nil { return nil, fmt.Errorf("opening cursor attribution db: %w", err) diff --git a/internal/parser/forge.go b/internal/parser/forge.go index 547d1f1b3..91aa39606 100644 --- a/internal/parser/forge.go +++ b/internal/parser/forge.go @@ -97,7 +97,8 @@ func parseForgeSession(dbPath, conversationID, machine string) (*ParsedSession, } func openForgeDB(dbPath string) (*sql.DB, error) { - dsn := dbPath + "?mode=ro&_journal_mode=WAL&_busy_timeout=3000" + dsn := "file:" + sqliteURIPath(dbPath) + + "?mode=ro&_busy_timeout=3000" db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf("opening forge db %s: %w", dbPath, err) diff --git a/internal/parser/kiro_sqlite.go b/internal/parser/kiro_sqlite.go index 828a5591b..b7d78e0f3 100644 --- a/internal/parser/kiro_sqlite.go +++ b/internal/parser/kiro_sqlite.go @@ -331,8 +331,8 @@ func (s *KiroSQLiteStore) ParseSession( } func openKiroSQLiteDB(dbPath string) (*sql.DB, error) { - dsn := dbPath + - "?mode=ro&_journal_mode=WAL&_busy_timeout=3000" + dsn := "file:" + sqliteURIPath(dbPath) + + "?mode=ro&_busy_timeout=3000" db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf( diff --git a/internal/parser/shelley.go b/internal/parser/shelley.go index f1d9efe66..d125e7bc7 100644 --- a/internal/parser/shelley.go +++ b/internal/parser/shelley.go @@ -347,7 +347,8 @@ func OpenShelleyDB(dbPath string) (*sql.DB, error) { } func openShelleyDB(dbPath string) (*sql.DB, error) { - dsn := dbPath + "?mode=ro&_journal_mode=WAL&_busy_timeout=3000" + dsn := "file:" + sqliteURIPath(dbPath) + + "?mode=ro&_busy_timeout=3000" db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf("opening shelley db %s: %w", dbPath, err) diff --git a/internal/parser/warp.go b/internal/parser/warp.go index b56a0ed97..7edd59da9 100644 --- a/internal/parser/warp.go +++ b/internal/parser/warp.go @@ -98,8 +98,8 @@ func parseWarpSession( } func openWarpDB(dbPath string) (*sql.DB, error) { - dsn := dbPath + - "?mode=ro&_journal_mode=WAL&_busy_timeout=3000" + dsn := "file:" + sqliteURIPath(dbPath) + + "?mode=ro&_busy_timeout=3000" db, err := sql.Open("sqlite3", dsn) if err != nil { return nil, fmt.Errorf( diff --git a/internal/postgres/messages.go b/internal/postgres/messages.go index 463c7967d..347cedb30 100644 --- a/internal/postgres/messages.go +++ b/internal/postgres/messages.go @@ -3,6 +3,7 @@ package postgres import ( "context" "fmt" + "slices" "strings" "time" @@ -62,6 +63,157 @@ func (s *Store) GetMessages( return msgs, nil } +const pgMessageCols = `session_id, ordinal, role, content, thinking_text, + timestamp, has_thinking, has_tool_use, + content_length, is_system, model, token_usage, + context_tokens, output_tokens, + has_context_tokens, has_output_tokens, + claude_message_id, claude_request_id, + source_type, source_subtype, source_uuid, + source_parent_uuid, is_sidechain, + is_compact_boundary` + +// GetMessagesWindow mirrors internal/db's GetMessagesWindow: linear mode +// (optionally role-filtered) delegates to GetMessages when Roles is empty; +// Around mode merges three queries (before/anchor/after) into one ascending +// slice. The anchor query has no role predicate so the anchor row is always +// present regardless of Roles; before/after apply the role filter first, so +// Before/After count role-matching messages, not raw ordinal distance. +func (s *Store) GetMessagesWindow( + ctx context.Context, sessionID string, w db.MessageWindow, +) ([]db.Message, error) { + if w.Around != nil { + return s.getMessagesAroundAnchor(ctx, sessionID, w) + } + from := 0 + if w.From != nil { + from = *w.From + } + if len(w.Roles) == 0 { + return s.GetMessages(ctx, sessionID, from, w.Limit, w.Asc) + } + return s.getMessagesLinearRoleFiltered(ctx, sessionID, from, w.Limit, w.Asc, w.Roles) +} + +func (s *Store) getMessagesLinearRoleFiltered( + ctx context.Context, + sessionID string, from, limit int, asc bool, roles []string, +) ([]db.Message, error) { + if limit <= 0 || limit > db.MaxMessageLimit { + limit = db.DefaultMessageLimit + } + dir := "ASC" + op := ">=" + if !asc { + dir = "DESC" + op = "<=" + } + roleClause, roleArgs := pgRoleFilterClause(roles, 3) + query := fmt.Sprintf(` + SELECT %s + FROM messages + WHERE session_id = $1 AND ordinal %s $2%s + ORDER BY ordinal %s + LIMIT $%d`, pgMessageCols, op, roleClause, dir, len(roleArgs)+3) + args := append([]any{sessionID, from}, roleArgs...) + args = append(args, limit) + + rows, err := s.pg.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("querying role-filtered messages: %w", err) + } + defer rows.Close() + msgs, err := scanPGMessages(rows) + if err != nil { + return nil, err + } + if err := s.attachToolCalls(ctx, msgs); err != nil { + return nil, err + } + return msgs, nil +} + +func (s *Store) getMessagesAroundAnchor( + ctx context.Context, sessionID string, w db.MessageWindow, +) ([]db.Message, error) { + anchor := *w.Around + beforeLimit := max(w.Before, 0) + afterLimit := max(w.After, 0) + roleClause, roleArgs := pgRoleFilterClause(w.Roles, 3) + + beforeQuery := fmt.Sprintf(` + SELECT %s FROM messages + WHERE session_id = $1 AND ordinal < $2%s + ORDER BY ordinal DESC LIMIT $%d`, + pgMessageCols, roleClause, len(roleArgs)+3) + beforeArgs := append([]any{sessionID, anchor}, roleArgs...) + beforeArgs = append(beforeArgs, beforeLimit) + before, err := s.queryMessageRows(ctx, beforeQuery, beforeArgs...) + if err != nil { + return nil, fmt.Errorf("querying before-window messages: %w", err) + } + slices.Reverse(before) + + anchorQuery := fmt.Sprintf(` + SELECT %s FROM messages WHERE session_id = $1 AND ordinal = $2`, + pgMessageCols) + anchorMsgs, err := s.queryMessageRows(ctx, anchorQuery, sessionID, anchor) + if err != nil { + return nil, fmt.Errorf("querying anchor message: %w", err) + } + + afterQuery := fmt.Sprintf(` + SELECT %s FROM messages + WHERE session_id = $1 AND ordinal > $2%s + ORDER BY ordinal ASC LIMIT $%d`, + pgMessageCols, roleClause, len(roleArgs)+3) + afterArgs := append([]any{sessionID, anchor}, roleArgs...) + afterArgs = append(afterArgs, afterLimit) + after, err := s.queryMessageRows(ctx, afterQuery, afterArgs...) + if err != nil { + return nil, fmt.Errorf("querying after-window messages: %w", err) + } + + msgs := make([]db.Message, 0, len(before)+len(anchorMsgs)+len(after)) + msgs = append(msgs, before...) + msgs = append(msgs, anchorMsgs...) + msgs = append(msgs, after...) + if err := s.attachToolCalls(ctx, msgs); err != nil { + return nil, err + } + return msgs, nil +} + +// queryMessageRows runs query and scans the resulting message rows without +// attaching tool calls; callers batch that across the merged window set. +func (s *Store) queryMessageRows( + ctx context.Context, query string, args ...any, +) ([]db.Message, error) { + rows, err := s.pg.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + return scanPGMessages(rows) +} + +// pgRoleFilterClause returns an "AND role IN ($n, ...)" clause and its bind +// args for the given roles, or ("", nil) when roles is empty. startAt is the +// first placeholder ordinal to use (the caller's query already consumes +// $1..$(startAt-1)). +func pgRoleFilterClause(roles []string, startAt int) (string, []any) { + if len(roles) == 0 { + return "", nil + } + placeholders := make([]string, len(roles)) + args := make([]any, len(roles)) + for i, r := range roles { + placeholders[i] = fmt.Sprintf("$%d", startAt+i) + args[i] = r + } + return " AND role IN (" + strings.Join(placeholders, ",") + ")", args +} + // GetAllMessages returns all messages for a session ordered // by ordinal. func (s *Store) GetAllMessages( @@ -143,6 +295,11 @@ func (s *Store) SearchSession( // HasFTS returns true because ILIKE search is available. func (s *Store) HasFTS() bool { return true } +// HasSemantic returns false: the PostgreSQL store has no VectorSearcher seam +// yet, so SearchContent rejects "semantic"/"hybrid" modes up front with +// db.ErrSemanticUnavailable. +func (s *Store) HasSemantic() bool { return false } + // escapeLike escapes SQL LIKE metacharacters so the bind // parameter is treated as a literal substring. func escapeLike(v string) string { diff --git a/internal/postgres/messages_window_pg_test.go b/internal/postgres/messages_window_pg_test.go new file mode 100644 index 000000000..b07f8136c --- /dev/null +++ b/internal/postgres/messages_window_pg_test.go @@ -0,0 +1,248 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "database/sql" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +const mwTestSchema = "agentsview_messages_window_test" + +func mwEnsureSchema(t *testing.T, pgURL string) *sql.DB { + t.Helper() + pg, err := Open(pgURL, mwTestSchema, true) + require.NoError(t, err, "Open") + ctx := context.Background() + _, err = pg.ExecContext(ctx, + `DROP SCHEMA IF EXISTS `+mwTestSchema+` CASCADE`, + ) + require.NoError(t, err, "drop schema") + require.NoError(t, EnsureSchema(ctx, pg, mwTestSchema), "EnsureSchema") + return pg +} + +// mwSeedWindowMessages seeds a session with 12 messages (ordinals 0..11) +// with the same user/assistant/system role layout used by the SQLite/DuckDB +// GetMessagesWindow parity tests: +// +// 0 user, 1 assistant, 2 user, 3 assistant, 4 system, 5 user, +// 6 assistant, 7 user, 8 assistant, 9 system, 10 user, 11 assistant +func mwSeedWindowMessages(t *testing.T, pg *sql.DB, sessionID string) { + t.Helper() + _, err := pg.Exec(` + INSERT INTO sessions + (id, machine, project, agent, + message_count, user_message_count) + VALUES ($1, 'test', 'proj', 'claude', 12, 8) + `, sessionID) + require.NoError(t, err, "insert session %s", sessionID) + + roles := []string{ + "user", "assistant", "user", "assistant", "system", "user", + "assistant", "user", "assistant", "system", "user", "assistant", + } + for ordinal, role := range roles { + _, err := pg.Exec(` + INSERT INTO messages + (session_id, ordinal, role, content, + content_length, is_system) + VALUES ($1, $2, $3, 'msg', 3, $4) + `, sessionID, ordinal, role, role == "system") + require.NoError(t, err, "insert message %s/%d", sessionID, ordinal) + } +} + +func mwNewStore(t *testing.T, pgURL string) *Store { + t.Helper() + store, err := NewStore(pgURL, mwTestSchema, true) + require.NoError(t, err, "NewStore") + return store +} + +func mwOrdinalsOf(msgs []db.Message) []int { + out := make([]int, len(msgs)) + for i, m := range msgs { + out[i] = m.Ordinal + } + return out +} + +// TestPGGetMessagesWindow_AroundMidSession mirrors +// TestGetMessagesWindow_AroundMidSession. +func TestPGGetMessagesWindow_AroundMidSession(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sMid") + store := mwNewStore(t, pgURL) + defer store.Close() + + anchor := 6 + msgs, err := store.GetMessagesWindow(ctx, "sMid", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{4, 5, 6, 7, 8}, mwOrdinalsOf(msgs), + "unfiltered window should return anchor +/- 2 ordinals ascending") +} + +// TestPGGetMessagesWindow_RoleFilterCountsFilteredMessages mirrors +// TestGetMessagesWindow_RoleFilterCountsFilteredMessages. +func TestPGGetMessagesWindow_RoleFilterCountsFilteredMessages(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sRoleCount") + store := mwNewStore(t, pgURL) + defer store.Close() + + anchor := 6 + msgs, err := store.GetMessagesWindow(ctx, "sRoleCount", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + Roles: []string{"user", "assistant"}, + }) + require.NoError(t, err) + assert.Equal(t, []int{3, 5, 6, 7, 8}, mwOrdinalsOf(msgs), + "before/after counts should count role-filtered messages, not raw ordinals") +} + +// TestPGGetMessagesWindow_AnchorIncludedEvenWhenRoleFiltered mirrors +// TestGetMessagesWindow_AnchorIncludedEvenWhenRoleFiltered. +func TestPGGetMessagesWindow_AnchorIncludedEvenWhenRoleFiltered(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sAnchorFiltered") + store := mwNewStore(t, pgURL) + defer store.Close() + + anchor := 4 // role "system", excluded by the role filter + msgs, err := store.GetMessagesWindow(ctx, "sAnchorFiltered", db.MessageWindow{ + Around: &anchor, Before: 1, After: 1, + Roles: []string{"user", "assistant"}, + }) + require.NoError(t, err) + require.Equal(t, []int{3, 4, 5}, mwOrdinalsOf(msgs), + "anchor must be included even though its own role is filtered out") + assert.Equal(t, "system", msgs[1].Role) +} + +// TestPGGetMessagesWindow_AroundOrdinalZeroHasNoBefore mirrors +// TestGetMessagesWindow_AroundOrdinalZeroHasNoBefore. +func TestPGGetMessagesWindow_AroundOrdinalZeroHasNoBefore(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sFirst") + store := mwNewStore(t, pgURL) + defer store.Close() + + anchor := 0 + msgs, err := store.GetMessagesWindow(ctx, "sFirst", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{0, 1, 2}, mwOrdinalsOf(msgs), + "no before rows exist above the first ordinal") +} + +// TestPGGetMessagesWindow_AroundLastOrdinalHasNoAfter mirrors +// TestGetMessagesWindow_AroundLastOrdinalHasNoAfter. +func TestPGGetMessagesWindow_AroundLastOrdinalHasNoAfter(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sLast") + store := mwNewStore(t, pgURL) + defer store.Close() + + anchor := 11 + msgs, err := store.GetMessagesWindow(ctx, "sLast", db.MessageWindow{ + Around: &anchor, Before: 2, After: 2, + }) + require.NoError(t, err) + assert.Equal(t, []int{9, 10, 11}, mwOrdinalsOf(msgs), + "no after rows exist below the last ordinal") +} + +// TestPGGetMessagesWindow_LinearModeWithRoles mirrors +// TestGetMessagesWindow_LinearModeWithRoles. +func TestPGGetMessagesWindow_LinearModeWithRoles(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sLinearRoles") + store := mwNewStore(t, pgURL) + defer store.Close() + + msgs, err := store.GetMessagesWindow(ctx, "sLinearRoles", db.MessageWindow{ + Limit: 100, Asc: true, Roles: []string{"user"}, + }) + require.NoError(t, err) + assert.Equal(t, []int{0, 2, 5, 7, 10}, mwOrdinalsOf(msgs), + "linear mode should apply the role filter like the around mode") +} + +// TestPGGetMessagesWindow_EmptyRolesEquivalentToGetMessages mirrors +// TestGetMessagesWindow_EmptyRolesEquivalentToGetMessages. +func TestPGGetMessagesWindow_EmptyRolesEquivalentToGetMessages(t *testing.T) { + pgURL := testPGURL(t) + pg := mwEnsureSchema(t, pgURL) + defer pg.Close() + defer func() { + _, _ = pg.Exec(`DROP SCHEMA IF EXISTS ` + mwTestSchema + ` CASCADE`) + }() + ctx := context.Background() + + mwSeedWindowMessages(t, pg, "sEquiv") + store := mwNewStore(t, pgURL) + defer store.Close() + + direct, err := store.GetMessages(ctx, "sEquiv", 3, 5, true) + require.NoError(t, err) + + from := 3 + windowed, err := store.GetMessagesWindow(ctx, "sEquiv", db.MessageWindow{ + From: &from, Limit: 5, Asc: true, + }) + require.NoError(t, err) + assert.Equal(t, direct, windowed, + "empty Roles should behave identically to GetMessages") +} diff --git a/internal/postgres/schema.go b/internal/postgres/schema.go index ef269907c..e984ac4a8 100644 --- a/internal/postgres/schema.go +++ b/internal/postgres/schema.go @@ -886,6 +886,10 @@ func createPartialIndexesPG(ctx context.Context, db *sql.DB) error { // SQLite partial index so legacy schemas migrate cleanly. `CREATE INDEX IF NOT EXISTS idx_tool_calls_file_path ON tool_calls(file_path) WHERE file_path IS NOT NULL`, + // idx_messages_session_role backs the dense-flow unit-range boundary + // fetch (user ordinals by session), mirroring the SQLite index. + `CREATE INDEX IF NOT EXISTS idx_messages_session_role + ON messages(session_id, role)`, } for _, ddl := range indexes { if _, err := db.ExecContext(ctx, ddl); err != nil { diff --git a/internal/postgres/search_content.go b/internal/postgres/search_content.go index f47702505..9b11f144b 100644 --- a/internal/postgres/search_content.go +++ b/internal/postgres/search_content.go @@ -29,6 +29,26 @@ func (s *Store) SearchContent( if f.Pattern == "" { return db.ContentSearchPage{}, nil } + + // Semantic and hybrid validate Sources themselves (messages only) ahead + // of the substring/regex/fts source-set default just below, which fills + // in tool_input/tool_result that neither mode supports -- mirroring + // internal/db's SearchContent so an empty Sources field is not defaulted + // out from under ValidateSemanticFilter's empty-or-messages-only check. + if f.Mode == "semantic" || f.Mode == "hybrid" { + // Validate input the same way SQLite's semantic/hybrid paths do + // before reporting the capability gate: an invalid request (bad + // cursor, non-messages source) must return the same 400 + // SearchInputError on every backend rather than a 501 here and a + // 400 on SQLite (backend parity, see AGENTS.md). + if err := db.ValidateSemanticFilter(f); err != nil { + return db.ContentSearchPage{}, err + } + // No VectorSearcher seam on the PostgreSQL store yet (HasSemantic + // always false): gate after input validation. + return db.ContentSearchPage{}, db.ErrSemanticUnavailable + } + if len(f.Sources) == 0 { f.Sources = []string{"messages", "tool_input", "tool_result"} } @@ -248,7 +268,9 @@ func pgToolResultEventsBranch( // scanPGContentMatches runs query and assembles a ContentSearchPage. The // query's final column is the full source field; makeSnippet derives the -// windowed, redacted snippet so redaction sees whole secrets. +// windowed, redacted snippet so redaction sees whole secrets. The returned +// page then gets its derived unit ranges and lineage assigned by the shared +// deriveLexicalUnitsPG pass (post-truncation, O(page)). func (s *Store) scanPGContentMatches( ctx context.Context, query string, args []any, limit, cursor int, makeSnippet func(body string) string, @@ -276,11 +298,21 @@ func (s *Store) scanPGContentMatches( if err := rows.Err(); err != nil { return db.ContentSearchPage{}, err } + // Close the cursor before deriving units (exhausting Next already + // auto-closed it; this keeps the release explicit): deriveLexicalUnitsPG + // issues new queries, which must never wait on a connection this cursor + // would otherwise still pin. + if err := rows.Close(); err != nil { + return db.ContentSearchPage{}, fmt.Errorf("closing pg content matches: %w", err) + } page := db.ContentSearchPage{Matches: out} if len(out) > limit { page.Matches = out[:limit] page.NextCursor = cursor + limit } + if err := s.deriveLexicalUnitsPG(ctx, page.Matches); err != nil { + return db.ContentSearchPage{}, err + } return page, nil } @@ -331,11 +363,21 @@ func (s *Store) searchContentRegexPG( if err := rows.Err(); err != nil { return db.ContentSearchPage{}, err } + // Close the candidate cursor before deriving units: the loop breaks out + // with rows still open once Limit+1 matches are collected, and + // deriveLexicalUnitsPG issues new queries that could otherwise block on a + // constrained connection pool while this cursor pins a connection. + if err := rows.Close(); err != nil { + return db.ContentSearchPage{}, fmt.Errorf("closing pg regex candidates: %w", err) + } page := db.ContentSearchPage{Matches: out} if len(out) > f.Limit { page.Matches = out[:f.Limit] page.NextCursor = f.Cursor + f.Limit } + if err := s.deriveLexicalUnitsPG(ctx, page.Matches); err != nil { + return db.ContentSearchPage{}, err + } return page, nil } diff --git a/internal/postgres/search_content_units_pgtest_test.go b/internal/postgres/search_content_units_pgtest_test.go new file mode 100644 index 000000000..e0b64664b --- /dev/null +++ b/internal/postgres/search_content_units_pgtest_test.go @@ -0,0 +1,455 @@ +//go:build pgtest + +package postgres + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" +) + +// insertCSUnitMessage inserts a message with explicit is_system and +// is_sidechain flags for conversation-unit derivation tests. +func insertCSUnitMessage( + t *testing.T, store *Store, + sessionID string, ordinal int, role, content string, + isSystem, isSidechain bool, +) { + t.Helper() + ts := fmt.Sprintf("2026-05-01T10:00:%02dZ", ordinal) + _, err := store.DB().Exec(` + INSERT INTO messages + (session_id, ordinal, role, content, timestamp, + content_length, is_system, is_sidechain) + VALUES ($1, $2, $3, $4, $5::timestamptz, $6, $7, $8) + ON CONFLICT DO NOTHING`, + sessionID, ordinal, role, content, ts, len(content), + isSystem, isSidechain, + ) + require.NoError(t, err, "insert message ord=%d", ordinal) +} + +// csMatchesByOrdinal indexes a page's matches by anchor ordinal, requiring +// the ordinals to be unique. +func csMatchesByOrdinal( + t *testing.T, page db.ContentSearchPage, +) map[int]db.ContentMatch { + t.Helper() + out := make(map[int]db.ContentMatch, len(page.Matches)) + for _, m := range page.Matches { + _, dup := out[m.Ordinal] + require.False(t, dup, "duplicate match ordinal %d", m.Ordinal) + out[m.Ordinal] = m + } + return out +} + +// TestPGSearchContentSubstringDerivedRunRange mirrors the SQLite test: every +// substring match in one assistant run carries the run's full range (spanning +// a non-member system row), an embeddable user row and a system row are their +// own units, and ExcludeSystem changes nothing but which rows match. +func TestPGSearchContentSubstringDerivedRunRange(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-run", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSUnitMessage(t, store, "cs-unit-run", 0, "user", + "the RUNHIT question", false, false) + insertCSUnitMessage(t, store, "cs-unit-run", 1, "assistant", + "RUNHIT step one", false, false) + insertCSUnitMessage(t, store, "cs-unit-run", 2, "user", + "sys RUNHIT note", true, false) + insertCSUnitMessage(t, store, "cs-unit-run", 3, "assistant", + "RUNHIT step two", false, false) + insertCSUnitMessage(t, store, "cs-unit-run", 4, "assistant", + "RUNHIT step three", false, false) + insertCSUnitMessage(t, store, "cs-unit-run", 5, "user", + "next question", false, false) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "RUNHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 5, "matches") + byOrd := csMatchesByOrdinal(t, got) + assert.Equal(t, [2]int{0, 0}, byOrd[0].OrdinalRange, "user row is its own unit") + assert.Equal(t, [2]int{2, 2}, byOrd[2].OrdinalRange, "system row is its own unit") + for _, o := range []int{1, 3, 4} { + m := byOrd[o] + assert.Equal(t, [2]int{1, 4}, m.OrdinalRange, "run member %d", o) + assert.Equal(t, o, m.Ordinal, "anchor ordinal %d", o) + assert.False(t, m.Subordinate, "top-level run member %d", o) + assert.False(t, m.Sidechain, "non-sidechain run member %d", o) + assert.Empty(t, m.Relationship, "top-level relationship %d", o) + assert.Empty(t, m.ParentSessionID, "top-level parent %d", o) + } + + // ExcludeSystem drops the system row but leaves the derived ranges of + // the surviving rows unchanged. + ex, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "RUNHIT", Mode: "substring", + Sources: []string{"messages"}, ExcludeSystem: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent ExcludeSystem") + require.Len(t, ex.Matches, 4, "ExcludeSystem matches") + exByOrd := csMatchesByOrdinal(t, ex) + assert.NotContains(t, exByOrd, 2, "system row excluded") + assert.Equal(t, [2]int{0, 0}, exByOrd[0].OrdinalRange) + for _, o := range []int{1, 3, 4} { + assert.Equal(t, [2]int{1, 4}, exByOrd[o].OrdinalRange, + "ExcludeSystem run member %d", o) + } +} + +// TestPGSearchContentSidechainRunSubordinate pins the sidechain rules on PG: +// a sidechain run's members are Subordinate + Sidechain, and the sidechain +// flip bounds both the sidechain run and the following top-level run. +func TestPGSearchContentSidechainRunSubordinate(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-side", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSUnitMessage(t, store, "cs-unit-side", 0, "user", + "the question", false, false) + insertCSUnitMessage(t, store, "cs-unit-side", 1, "assistant", + "SIDEHIT step a", false, true) + insertCSUnitMessage(t, store, "cs-unit-side", 2, "assistant", + "SIDEHIT step b", false, true) + insertCSUnitMessage(t, store, "cs-unit-side", 3, "assistant", + "main MAINHIT answer", false, false) + + ctx := context.Background() + side, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "SIDEHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent sidechain") + require.Len(t, side.Matches, 2, "sidechain matches") + for _, m := range side.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "sidechain run range") + assert.True(t, m.Subordinate, "sidechain run is subordinate") + assert.True(t, m.Sidechain, "anchor sidechain flag") + assert.Empty(t, m.Relationship, "no session lineage") + } + + main, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "MAINHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent main") + require.Len(t, main.Matches, 1, "main matches") + m := main.Matches[0] + assert.Equal(t, [2]int{3, 3}, m.OrdinalRange, + "sidechain flip bounds the top-level run") + assert.False(t, m.Subordinate, "top-level run") + assert.False(t, m.Sidechain, "top-level anchor") +} + +// TestPGSearchContentSubagentLineage pins session-level lineage on lexical +// rows: a match inside a subagent session is Subordinate with Relationship +// and ParentSessionID populated from the sessions join. +func TestPGSearchContentSubagentLineage(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-parent", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSChildSession(t, store, "cs-unit-child", "proj", "claude", + "cs-unit-parent", "2026-05-01T10:05:00Z", "2026-05-01T10:25:00Z") + insertCSUnitMessage(t, store, "cs-unit-child", 0, "user", + "subagent prompt", false, false) + insertCSUnitMessage(t, store, "cs-unit-child", 1, "assistant", + "SUBHIT answer", false, false) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "SUBHIT", Mode: "substring", + Sources: []string{"messages"}, IncludeChildren: true, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 1, "matches") + m := got.Matches[0] + assert.Equal(t, [2]int{1, 1}, m.OrdinalRange, "single-member run") + assert.True(t, m.Subordinate, "subagent session is subordinate") + assert.Equal(t, "subagent", m.Relationship, "Relationship") + assert.Equal(t, "cs-unit-parent", m.ParentSessionID, "ParentSessionID") + assert.False(t, m.Sidechain, "anchor not sidechain") +} + +// TestPGSearchContentToolDerivedRunRange pins derivation for tool_input and +// canonical tool_result rows: the anchor is the tool call's message row, so +// both locations carry the enclosing run's range while the wire Role stays +// the hard-coded "assistant". +func TestPGSearchContentToolDerivedRunRange(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-tool", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSUnitMessage(t, store, "cs-unit-tool", 0, "user", + "the question", false, false) + insertCSUnitMessage(t, store, "cs-unit-tool", 1, "assistant", + "running the tool", false, false) + insertCSUnitMessage(t, store, "cs-unit-tool", 2, "assistant", + "continuing the answer", false, false) + insertCSUnitMessage(t, store, "cs-unit-tool", 3, "user", + "thanks", false, false) + insertCSToolCall(t, store, "cs-unit-tool", 1, 0, + "Bash", "tu1", `{"command":"TOOLHIT"}`, "output RESHIT data") + + ctx := context.Background() + in, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "TOOLHIT", Mode: "substring", + Sources: []string{"tool_input"}, Limit: 50, + }) + require.NoError(t, err, "tool_input search") + require.Len(t, in.Matches, 1, "tool_input matches") + assert.Equal(t, "assistant", in.Matches[0].Role, "wire role stays assistant") + assert.Equal(t, 1, in.Matches[0].Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 2}, in.Matches[0].OrdinalRange, + "tool_input anchor classified from the real message row") + + res, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "RESHIT", Mode: "substring", + Sources: []string{"tool_result"}, Limit: 50, + }) + require.NoError(t, err, "tool_result search") + require.Len(t, res.Matches, 1, "tool_result matches") + assert.Equal(t, [2]int{1, 2}, res.Matches[0].OrdinalRange, + "canonical tool_result anchor classified from the real message row") +} + +// TestPGSearchContentToolResultEventsDerived pins the events branch: an +// orphaned event (no message row at its ordinal) still returns its match +// (row cardinality must not change) with the [o, o] fallback and session +// lineage, while an event whose message row sits inside a run gets the run's +// range via the post-scan anchor lookup. +func TestPGSearchContentToolResultEventsDerived(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-ev-boss", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSChildSession(t, store, "cs-ev-orph", "proj", "claude", + "cs-ev-boss", "2026-05-01T10:05:00Z", "2026-05-01T10:25:00Z") + // Orphan: no message row at ordinal 7. + insertCSToolResultEvent(t, store, "cs-ev-orph", 7, 0, 0, + "tux", "ORPHHIT event content") + + insertCSSession(t, store, "cs-ev-run", "proj", "claude", + "2026-05-01T11:00:00Z", "2026-05-01T11:30:00Z") + insertCSUnitMessage(t, store, "cs-ev-run", 0, "user", + "the question", false, false) + insertCSUnitMessage(t, store, "cs-ev-run", 1, "assistant", + "running", false, false) + insertCSUnitMessage(t, store, "cs-ev-run", 2, "assistant", + "wrapping up", false, false) + insertCSToolCall(t, store, "cs-ev-run", 1, 0, + "Bash", "tu1", `{"command":"x"}`, "") + insertCSToolResultEvent(t, store, "cs-ev-run", 1, 0, 0, + "tu1", "EVHIT streamed output") + + ctx := context.Background() + orph, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "ORPHHIT", Mode: "substring", + Sources: []string{"tool_result"}, IncludeChildren: true, Limit: 50, + }) + require.NoError(t, err, "orphan search") + require.Len(t, orph.Matches, 1, "orphaned event row must not be dropped") + m := orph.Matches[0] + assert.Equal(t, 7, m.Ordinal, "event ordinal") + assert.Equal(t, [2]int{7, 7}, m.OrdinalRange, + "missing anchor falls back to [o, o]") + assert.False(t, m.Sidechain, "missing anchor has no sidechain flag") + assert.True(t, m.Subordinate, "session lineage still applies") + assert.Equal(t, "subagent", m.Relationship, "Relationship from sessions join") + assert.Equal(t, "cs-ev-boss", m.ParentSessionID, + "ParentSessionID from sessions join") + + ev, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "EVHIT", Mode: "substring", + Sources: []string{"tool_result"}, Limit: 50, + }) + require.NoError(t, err, "event search") + require.Len(t, ev.Matches, 1, "event matches") + assert.Equal(t, 1, ev.Matches[0].Ordinal, "anchor ordinal") + assert.Equal(t, [2]int{1, 2}, ev.Matches[0].OrdinalRange, + "event with a message row inside a run gets the run's range") +} + +// TestPGSearchContentRegexDerivedRange spot-checks that regex mode routes +// through the shared derivation pass. +func TestPGSearchContentRegexDerivedRange(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-rx", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSUnitMessage(t, store, "cs-unit-rx", 0, "user", + "the question", false, false) + insertCSUnitMessage(t, store, "cs-unit-rx", 1, "assistant", + "RXHIT alpha", false, false) + insertCSUnitMessage(t, store, "cs-unit-rx", 2, "assistant", + "RXHIT beta", false, false) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: `RXHIT [a-z]+`, Mode: "regex", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent regex") + require.Len(t, got.Matches, 2, "regex matches") + for _, m := range got.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "derived run range") + } +} + +// TestPGSearchContentFTSDerivedRange spot-checks that PG's fts fallback +// (ILIKE terms over messages) routes through the shared derivation pass. +func TestPGSearchContentFTSDerivedRange(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-fts", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSUnitMessage(t, store, "cs-unit-fts", 0, "user", + "the question", false, false) + insertCSUnitMessage(t, store, "cs-unit-fts", 1, "assistant", + "ftshit alpha step", false, false) + insertCSUnitMessage(t, store, "cs-unit-fts", 2, "assistant", + "ftshit beta step", false, false) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "ftshit", Mode: "fts", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent fts") + require.Len(t, got.Matches, 2, "fts matches") + for _, m := range got.Matches { + assert.Equal(t, [2]int{1, 2}, m.OrdinalRange, "derived run range") + assert.False(t, m.Subordinate, "top-level run") + } +} + +// TestPGSearchContentDenseFlowDerivedRanges exercises the DENSE derivation +// flow against PG's dense-fetch SQL (scanPGUserBoundaryOrdinals): one session +// whose runs supply at least db.UnitBoundsFlowFactor distinct run anchors on +// a single page, so the shared flow selection fetches real user bounds with +// PG's batched boundary statement before resolving extents. Run lengths are +// derived from the exported gate so the page stays dense if the factor +// changes. The structure packs a main run, a sidechain run, and a +// flip-bounded main run, and every match must carry its run's exact range. +func TestPGSearchContentDenseFlowDerivedRanges(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-dense", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + + // Three runs of runLen anchors each: 3*runLen > UnitBoundsFlowFactor. + runLen := db.UnitBoundsFlowFactor/2 + 1 + runA := [2]int{1, runLen} // main run after user 0 + side := [2]int{runLen + 2, 2*runLen + 1} // sidechain run after user runLen+1 + runC := [2]int{2*runLen + 2, 3*runLen + 1} // main run bounded left by the flip + lastUser := 3*runLen + 2 + + insertCSUnitMessage(t, store, "cs-unit-dense", 0, "user", + "first question", false, false) + for o := runA[0]; o <= runA[1]; o++ { + insertCSUnitMessage(t, store, "cs-unit-dense", o, "assistant", + fmt.Sprintf("DFHIT main-a %d", o), false, false) + } + insertCSUnitMessage(t, store, "cs-unit-dense", runLen+1, "user", + "second question", false, false) + for o := side[0]; o <= side[1]; o++ { + insertCSUnitMessage(t, store, "cs-unit-dense", o, "assistant", + fmt.Sprintf("DFHIT side %d", o), false, true) + } + for o := runC[0]; o <= runC[1]; o++ { + insertCSUnitMessage(t, store, "cs-unit-dense", o, "assistant", + fmt.Sprintf("DFHIT main-c %d", o), false, false) + } + insertCSUnitMessage(t, store, "cs-unit-dense", lastUser, "user", + "done", false, false) + + anchorCount := 3 * runLen + require.GreaterOrEqual(t, anchorCount, db.UnitBoundsFlowFactor, + "single-session page must clear the dense-flow gate") + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "DFHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: anchorCount + 10, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, anchorCount, "matches") + byOrd := csMatchesByOrdinal(t, got) + for o := runA[0]; o <= runA[1]; o++ { + assert.Equal(t, runA, byOrd[o].OrdinalRange, "run A member %d", o) + assert.False(t, byOrd[o].Sidechain, "run A member %d flag", o) + } + for o := side[0]; o <= side[1]; o++ { + assert.Equal(t, side, byOrd[o].OrdinalRange, "sidechain member %d", o) + assert.True(t, byOrd[o].Sidechain, "sidechain member %d flag", o) + assert.True(t, byOrd[o].Subordinate, "sidechain member %d subordinate", o) + } + for o := runC[0]; o <= runC[1]; o++ { + assert.Equal(t, runC, byOrd[o].OrdinalRange, + "flip-bounded run C member %d", o) + assert.False(t, byOrd[o].Sidechain, "run C member %d flag", o) + } +} + +// TestPGSearchContentMultiRunReducerParity is the hand-computed parity check +// against the SQLite-side embedding reducer: one session with two top-level +// runs, a sidechain run between them, and an interior system row that must +// not close the run it sits inside. +func TestPGSearchContentMultiRunReducerParity(t *testing.T) { + store := setupContentSearch(t) + insertCSSession(t, store, "cs-unit-multi", "proj", "claude", + "2026-05-01T10:00:00Z", "2026-05-01T10:30:00Z") + insertCSUnitMessage(t, store, "cs-unit-multi", 0, "user", + "PARHIT q1", false, false) + insertCSUnitMessage(t, store, "cs-unit-multi", 1, "assistant", + "PARHIT a", false, false) + insertCSUnitMessage(t, store, "cs-unit-multi", 2, "assistant", + "PARHIT b", false, false) + insertCSUnitMessage(t, store, "cs-unit-multi", 3, "assistant", + "PARHIT sc1", false, true) + insertCSUnitMessage(t, store, "cs-unit-multi", 4, "assistant", + "PARHIT sc2", false, true) + insertCSUnitMessage(t, store, "cs-unit-multi", 5, "assistant", + "PARHIT c", false, false) + insertCSUnitMessage(t, store, "cs-unit-multi", 6, "user", + "PARHIT sys note", true, false) + insertCSUnitMessage(t, store, "cs-unit-multi", 7, "assistant", + "PARHIT d", false, false) + insertCSUnitMessage(t, store, "cs-unit-multi", 8, "user", + "PARHIT q2", false, false) + + ctx := context.Background() + got, err := store.SearchContent(ctx, db.ContentSearchFilter{ + Pattern: "PARHIT", Mode: "substring", + Sources: []string{"messages"}, Limit: 50, + }) + require.NoError(t, err, "SearchContent") + require.Len(t, got.Matches, 9, "matches") + byOrd := csMatchesByOrdinal(t, got) + + want := map[int][2]int{ + 0: {0, 0}, // embeddable user row: its own unit + 1: {1, 2}, // first top-level run, closed by the sidechain flip + 2: {1, 2}, + 3: {3, 4}, // sidechain run, bounded by flips on both sides + 4: {3, 4}, + 5: {5, 7}, // second top-level run, spanning the interior system row + 6: {6, 6}, // interior system row: its own unit, doesn't close the run + 7: {5, 7}, + 8: {8, 8}, // closing embeddable user row + } + for o, r := range want { + assert.Equal(t, r, byOrd[o].OrdinalRange, "ordinal %d range", o) + } + for _, o := range []int{3, 4} { + assert.True(t, byOrd[o].Subordinate, "sidechain member %d subordinate", o) + assert.True(t, byOrd[o].Sidechain, "sidechain member %d flag", o) + } + for _, o := range []int{0, 1, 2, 5, 6, 7, 8} { + assert.False(t, byOrd[o].Subordinate, "top-level row %d", o) + assert.False(t, byOrd[o].Sidechain, "top-level row %d flag", o) + } +} diff --git a/internal/postgres/store_unit_test.go b/internal/postgres/store_unit_test.go index 7ec3d69f8..4383a035e 100644 --- a/internal/postgres/store_unit_test.go +++ b/internal/postgres/store_unit_test.go @@ -18,6 +18,61 @@ import ( "go.kenn.io/agentsview/internal/db" ) +// TestStoreHasSemanticFalse pins that the PostgreSQL store reports no +// semantic search capability until it gets its own VectorSearcher seam. +func TestStoreHasSemanticFalse(t *testing.T) { + s := &Store{} + assert.False(t, s.HasSemantic(), "PostgreSQL HasSemantic") +} + +// TestStoreSearchContentSemanticModesUnavailable pins that "semantic" and +// "hybrid" are rejected with db.ErrSemanticUnavailable before any query runs +// -- a zero-value Store (no live *sql.DB) is enough to prove that. +func TestStoreSearchContentSemanticModesUnavailable(t *testing.T) { + s := &Store{} + for _, mode := range []string{"semantic", "hybrid"} { + _, err := s.SearchContent(context.Background(), + db.ContentSearchFilter{Pattern: "x", Mode: mode}) + require.Error(t, err, "mode %q", mode) + assert.True(t, errors.Is(err, db.ErrSemanticUnavailable), + "mode %q: want ErrSemanticUnavailable, got %v", mode, err) + } +} + +// TestStoreSearchContentSemanticInvalidInputReturns400Before501 pins backend +// parity (AGENTS.md): an invalid semantic/hybrid request -- cursor pagination +// or a non-messages source -- must return the same *db.SearchInputError +// SQLite's ValidateSemanticFilter returns, not db.ErrSemanticUnavailable, even +// though PostgreSQL has no VectorSearcher seam and would otherwise report the +// capability gate for any request in these modes. +func TestStoreSearchContentSemanticInvalidInputReturns400Before501(t *testing.T) { + s := &Store{} + cases := []struct { + name string + f db.ContentSearchFilter + }{ + {"cursor rejected", db.ContentSearchFilter{Pattern: "x", Cursor: 1}}, + {"non-messages source rejected", db.ContentSearchFilter{ + Pattern: "x", Sources: []string{"tool_input"}, + }}, + } + for _, mode := range []string{"semantic", "hybrid"} { + for _, tc := range cases { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + f := tc.f + f.Mode = mode + _, err := s.SearchContent(context.Background(), f) + require.Error(t, err) + var inputErr *db.SearchInputError + assert.True(t, errors.As(err, &inputErr), + "expected *db.SearchInputError, got %T: %v", err, err) + assert.False(t, errors.Is(err, db.ErrSemanticUnavailable), + "invalid input must not be masked as ErrSemanticUnavailable") + }) + } + } +} + // TestStripFTSQuotes pins the de-quoting behavior the PostgreSQL Search path // relies on. The canonical implementation lives in the db package and is // shared with the SQLite and HTTP paths so the backends stay in parity. diff --git a/internal/postgres/unit_range.go b/internal/postgres/unit_range.go new file mode 100644 index 000000000..9aa6a904d --- /dev/null +++ b/internal/postgres/unit_range.go @@ -0,0 +1,309 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" + "strings" + + "go.kenn.io/agentsview/internal/db" +) + +// PostgreSQL implementation of the conversation-unit seam. Orchestration — +// session/probe dedup, chunking, boundary resolution, and the alignment and +// row-count invariants — is shared with every backend via +// db.ResolveUserBoundaries, db.ResolveRunExtents, and the db.Scan*Rows +// helpers; this file supplies only the PG dialect SQL and its parameter +// binding. +var _ db.UnitBoundsQuerier = (*Store)(nil) + +// pgUnitSessionChunk caps sessions per NearestUserBoundaries statement, +// matching SQLite's unitSessionChunk semantics: a session binds 2 variables +// (idx, session_id). +const pgUnitSessionChunk = maxPGVars / 2 + +// pgUnitExtentChunk caps extent probes per RunExtents statement, matching +// SQLite's unitExtentChunk semantics: a probe binds 6 variables (idx, +// session_id, o, lo, hi, sc). +const pgUnitExtentChunk = maxPGVars / 6 + +// pgEmbeddableUserSQL is the PostgreSQL predicate matching an embeddable +// user row under the given alias: user role, is_system = FALSE, and the +// PG dialect SystemPrefixSQL check — the PG form of internal/db's +// embeddableUserSQL. (The assistant-side member predicate skips the prefix +// check: SystemPrefixSQL constrains user rows only.) +func pgEmbeddableUserSQL(alias string) string { + return fmt.Sprintf("%[1]s.role = 'user' AND %[1]s.is_system = FALSE AND %[2]s", + alias, db.PostgresSystemPrefixSQL(alias+".content", alias+".role")) +} + +// NearestUserBoundaries returns, per probe, the nearest embeddable user +// ordinals strictly before and after the probe ordinal, with the -1 / +// db.UnitOrdinalMax sentinels standing in for missing boundaries — the exact +// semantics of the SQLite seam method, guaranteed by the shared +// db.ResolveUserBoundaries orchestration: one statement per +// pgUnitSessionChunk distinct sessions fetches each session's embeddable +// user ordinals ONCE. +func (s *Store) NearestUserBoundaries( + ctx context.Context, probes []db.UnitProbe, +) ([]db.UnitBounds, error) { + return db.ResolveUserBoundaries(ctx, probes, pgUnitSessionChunk, + s.scanPGUserBoundaryOrdinals) +} + +// scanPGUserBoundaryOrdinals runs the one batched statement for a chunk of +// distinct sessions: a VALUES CTE joined against messages for every +// embeddable user ordinal of each session. out aligns 1:1 with sessions. +func (s *Store) scanPGUserBoundaryOrdinals( + ctx context.Context, sessions []string, out [][]int, +) error { + pb := ¶mBuilder{} + values := make([]string, len(sessions)) + for i, sessionID := range sessions { + values[i] = fmt.Sprintf("(%s::int, %s::text)", pb.add(i), pb.add(sessionID)) + } + query := fmt.Sprintf(` + WITH spans(idx, session_id) AS (VALUES %s) + SELECT sp.idx, m.ordinal + FROM spans sp JOIN messages m ON m.session_id = sp.session_id + WHERE %s`, + strings.Join(values, ", "), pgEmbeddableUserSQL("m")) + + rows, err := s.pg.QueryContext(ctx, query, pb.args...) + if err != nil { + return fmt.Errorf("querying nearest user boundaries: %w", err) + } + defer rows.Close() + return db.ScanUserBoundaryRows(rows, out) +} + +// RunExtents returns, per probe, the first and last member ordinals of the +// anchor's same-sidechain run, bounded exclusively by (Lo, Hi) and by the +// nearest STOP row inside that interval — an embeddable user row or an +// opposite-sidechain embeddable assistant row — the exact semantics of the +// SQLite seam method, guaranteed by the shared db.ResolveRunExtents +// orchestration. Probing with the -1 / db.UnitOrdinalMax sentinels therefore +// derives the full rule-2 extent on its own. One statement per +// pgUnitExtentChunk distinct probes resolves every probe with correlated +// point lookups (nearest stop row on each side, then the farthest +// same-sidechain member inside the stop-narrowed interval), moving exactly +// one result row per probe instead of each interval's member rows. +func (s *Store) RunExtents( + ctx context.Context, probes []db.ExtentProbe, +) ([][2]int, error) { + return db.ResolveRunExtents(ctx, probes, pgUnitExtentChunk, + s.lookupPGRunExtentChunk) +} + +// pgRunExtentSelectSQL builds the correlated point-lookup SELECT under a +// probes CTE with columns (idx, session_id, o, lo, hi, sc) — the PG form of +// internal/db's runExtentSelectSQL. Per probe and per side: the inner +// subquery seeks the nearest stop row between the anchor and the interval +// bound, the outer subquery seeks the farthest same-sidechain member inside +// the stop-narrowed interval. The member predicate is role + is_system only: +// SystemPrefixSQL constrains user rows exclusively, so it is identically +// TRUE for assistant rows and deliberately omitted there. +func pgRunExtentSelectSQL() string { + stop := "((f.role = 'assistant' AND f.is_system = FALSE AND f.is_sidechain <> p.sc)" + + " OR (" + pgEmbeddableUserSQL("f") + "))" + return fmt.Sprintf(` + SELECT p.idx, + (SELECT m.ordinal FROM messages m + WHERE m.session_id = p.session_id AND m.ordinal <= p.o + AND m.ordinal > COALESCE((SELECT f.ordinal FROM messages f + WHERE f.session_id = p.session_id + AND f.ordinal > p.lo AND f.ordinal < p.o + AND %[1]s + ORDER BY f.ordinal DESC LIMIT 1), p.lo) + AND m.role = 'assistant' AND m.is_system = FALSE + AND m.is_sidechain = p.sc + ORDER BY m.ordinal ASC LIMIT 1), + (SELECT m.ordinal FROM messages m + WHERE m.session_id = p.session_id AND m.ordinal >= p.o + AND m.ordinal < COALESCE((SELECT f.ordinal FROM messages f + WHERE f.session_id = p.session_id + AND f.ordinal > p.o AND f.ordinal < p.hi + AND %[1]s + ORDER BY f.ordinal ASC LIMIT 1), p.hi) + AND m.role = 'assistant' AND m.is_system = FALSE + AND m.is_sidechain = p.sc + ORDER BY m.ordinal DESC LIMIT 1) + FROM probes p`, stop) +} + +// lookupPGRunExtentChunk runs the one batched statement for a chunk of +// distinct extent probes: a VALUES CTE with the correlated point lookups of +// pgRunExtentSelectSQL. +func (s *Store) lookupPGRunExtentChunk( + ctx context.Context, probes []db.ExtentProbe, out [][2]int, +) error { + pb := ¶mBuilder{} + values := make([]string, len(probes)) + for i, p := range probes { + values[i] = fmt.Sprintf( + "(%s::int, %s::text, %s::int, %s::int, %s::int, %s::boolean)", + pb.add(i), pb.add(p.SessionID), pb.add(p.Ordinal), + pb.add(p.Lo), pb.add(p.Hi), pb.add(p.Sidechain)) + } + query := "WITH probes(idx, session_id, o, lo, hi, sc) AS (VALUES " + + strings.Join(values, ", ") + ")" + pgRunExtentSelectSQL() + + rows, err := s.pg.QueryContext(ctx, query, pb.args...) + if err != nil { + return fmt.Errorf("querying run extents: %w", err) + } + defer rows.Close() + return db.ScanRunExtentRows(rows, probes, out) +} + +// pgAnchorMetaChunk caps (session_id, ordinal) refs per anchor-meta lookup, +// matching internal/db's enrichHitsChunk semantics (2 binds per ref). +const pgAnchorMetaChunk = maxPGVars / 2 + +// pgAnchorKey identifies one (session_id, ordinal) anchor ref. +type pgAnchorKey struct { + sessionID string + ordinal int +} + +// pgAnchorMeta is one match's anchor metadata: session lineage plus the +// anchor message row's classification columns — the PG twin of internal/db's +// contentAnchorMeta. +type pgAnchorMeta struct { + relationship string + parentSessionID string + role sql.NullString + sidechain sql.NullBool + embeddable sql.NullBool + missing bool +} + +// deriveLexicalUnitsPG is the shared post-scan pass for the PG substring, +// regex, and fts-fallback modes, mirroring internal/db's deriveLexicalUnits: +// one batched anchor-meta lookup, one shared db.DeriveUnitRanges derivation +// (constant batched statement count for the whole page), then per-match +// assignment of OrdinalRange and the lineage fields. matches is the already +// truncated page, so the pass is O(page). +func (s *Store) deriveLexicalUnitsPG( + ctx context.Context, matches []db.ContentMatch, +) error { + if len(matches) == 0 { + return nil + } + metas, err := s.fillAnchorMetaPG(ctx, matches) + if err != nil { + return err + } + anchors := make([]db.UnitAnchor, len(matches)) + for i, m := range matches { + meta := metas[i] + anchors[i] = db.UnitAnchor{ + SessionID: m.SessionID, + Ordinal: m.Ordinal, + Role: meta.role.String, + Sidechain: meta.sidechain.Valid && meta.sidechain.Bool, + Embeddable: meta.embeddable.Valid && meta.embeddable.Bool, + Missing: meta.missing, + } + } + ranges, err := db.DeriveUnitRanges(ctx, s, anchors) + if err != nil { + return fmt.Errorf("deriving lexical units: %w", err) + } + for i := range matches { + matches[i].OrdinalRange = ranges[i] + matches[i].Relationship = metas[i].relationship + matches[i].ParentSessionID = metas[i].parentSessionID + matches[i].Sidechain = anchors[i].Sidechain + matches[i].Subordinate = anchors[i].Sidechain || + db.SubordinateSession(metas[i].relationship, metas[i].parentSessionID) + } + return nil +} + +// fillAnchorMetaPG fetches anchor classification and session lineage for +// every page row: one batched VALUES-CTE lookup per pgAnchorMetaChunk +// distinct (session_id, ordinal) refs, never a per-row query. Refs whose +// message row does not exist (tool_result_events orphans) are marked missing +// so derivation falls back to [o, o]; their session lineage still resolves +// via the sessions join. The result aligns 1:1 with matches. +func (s *Store) fillAnchorMetaPG( + ctx context.Context, matches []db.ContentMatch, +) ([]pgAnchorMeta, error) { + seen := make(map[pgAnchorKey]bool, len(matches)) + refs := make([]pgAnchorKey, 0, len(matches)) + for i := range matches { + key := pgAnchorKey{matches[i].SessionID, matches[i].Ordinal} + if !seen[key] { + seen[key] = true + refs = append(refs, key) + } + } + found := make(map[pgAnchorKey]pgAnchorMeta, len(refs)) + for start := 0; start < len(refs); start += pgAnchorMetaChunk { + chunk := refs[start:min(start+pgAnchorMetaChunk, len(refs))] + if err := s.lookupAnchorMetaChunkPG(ctx, chunk, found); err != nil { + return nil, err + } + } + metas := make([]pgAnchorMeta, len(matches)) + for i := range matches { + got, ok := found[pgAnchorKey{matches[i].SessionID, matches[i].Ordinal}] + if !ok { + metas[i].missing = true + continue + } + got.missing = !got.role.Valid + metas[i] = got + } + return metas, nil +} + +// lookupAnchorMetaChunkPG resolves one chunk of (session_id, ordinal) refs to +// session lineage plus the anchor message row's classification columns: +// role, sidechain, and the embeddable flag (is_system = FALSE AND content not +// system-prefixed, exactly the embedding reducer's predicate). messages is +// LEFT JOINed so a ref whose message row is absent still resolves lineage; +// its classification columns come back NULL. +func (s *Store) lookupAnchorMetaChunkPG( + ctx context.Context, refs []pgAnchorKey, + out map[pgAnchorKey]pgAnchorMeta, +) error { + pb := ¶mBuilder{} + values := make([]string, len(refs)) + for i, r := range refs { + values[i] = fmt.Sprintf("(%s::text, %s::int)", + pb.add(r.sessionID), pb.add(r.ordinal)) + } + query := "WITH refs(session_id, ordinal) AS (VALUES " + + strings.Join(values, ", ") + ") " + + "SELECT r.session_id, r.ordinal, " + + "COALESCE(s.relationship_type,''), COALESCE(s.parent_session_id,''), " + + "m.role, m.is_sidechain, " + + "CASE WHEN m.is_system = FALSE AND " + + db.PostgresSystemPrefixSQL("m.content", "m.role") + + " THEN TRUE ELSE FALSE END " + + "FROM refs r " + + "JOIN sessions s ON s.id = r.session_id " + + "LEFT JOIN messages m ON m.session_id = r.session_id AND m.ordinal = r.ordinal" + + rows, err := s.pg.QueryContext(ctx, query, pb.args...) + if err != nil { + return fmt.Errorf("looking up match anchors: %w", err) + } + defer rows.Close() + for rows.Next() { + var key pgAnchorKey + var meta pgAnchorMeta + if err := rows.Scan(&key.sessionID, &key.ordinal, + &meta.relationship, &meta.parentSessionID, + &meta.role, &meta.sidechain, &meta.embeddable); err != nil { + return fmt.Errorf("scanning match anchor: %w", err) + } + out[key] = meta + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating match anchors: %w", err) + } + return nil +} diff --git a/internal/server/huma_route_groups.go b/internal/server/huma_route_groups.go index a87c0b76d..1f9c99657 100644 --- a/internal/server/huma_route_groups.go +++ b/internal/server/huma_route_groups.go @@ -26,6 +26,7 @@ func (s *Server) registerTypedAPIRoutes() { s.registerPinRoutes() s.registerImportRoutes() s.registerAssetRoutes() + s.registerEmbeddingsRoutes() } type routeGroup struct { diff --git a/internal/server/huma_routes_embeddings.go b/internal/server/huma_routes_embeddings.go new file mode 100644 index 000000000..578658584 --- /dev/null +++ b/internal/server/huma_routes_embeddings.go @@ -0,0 +1,154 @@ +package server + +import ( + "context" + "errors" + "net/http" + + "go.kenn.io/agentsview/internal/vector" +) + +// EmbeddingsManager is the subset of *vector.Manager's API the embeddings +// build lifecycle routes need. Declaring it here (rather than depending on +// *vector.Manager directly) lets tests substitute a fake. TryBuild is +// intentionally excluded: it is the scheduler's synchronous entry point +// (Task 16), not part of the HTTP surface. +type EmbeddingsManager interface { + StartBuild(req vector.BuildRequest) error + Status() vector.BuildStatus + Generations(ctx context.Context) ([]vector.GenerationInfo, error) + Activate(ctx context.Context, id int64, force bool) error + Retire(ctx context.Context, id int64, force bool) error +} + +// WithEmbeddingsManager wires the implementation behind the embeddings build +// lifecycle routes (/api/v1/embeddings/...). The routes are registered even +// when no manager is present so OpenAPI and generated clients expose the full +// API surface; handlers return 501 until vector serving is configured. +func WithEmbeddingsManager(m EmbeddingsManager) Option { + return func(s *Server) { s.embeddingsManager = m } +} + +func (s *Server) registerEmbeddingsRoutes() { + group := newRouteGroup(s.api, "/api/v1/embeddings", "Embeddings") + + post(s, group, "/build", "Start an embeddings build", s.humaEmbeddingsBuild) + get(s, group, "/status", "Embeddings build status", s.humaEmbeddingsStatus) + get(s, group, "/generations", "List embedding generations", s.humaEmbeddingsGenerations) + post(s, group, "/generations/{id}/activate", "Activate an embedding generation", + s.humaEmbeddingsActivate) + post(s, group, "/generations/{id}/retire", "Retire an embedding generation", + s.humaEmbeddingsRetire) +} + +type embeddingsBuildInput struct { + Body vector.BuildRequest +} + +type embeddingsBuildResponse struct { + Started bool `json:"started"` +} + +type embeddingsBuildOutput struct { + Status int `status:"202"` + Body embeddingsBuildResponse +} + +type embeddingsGenerationsResponse struct { + Generations []vector.GenerationInfo `json:"generations"` +} + +type embeddingsGenerationActionRequest struct { + Force bool `json:"force,omitempty"` +} + +type embeddingsGenerationActionInput struct { + ID int64 `path:"id" required:"true" doc:"Generation ordinal ID"` + Body embeddingsGenerationActionRequest +} + +func (s *Server) humaEmbeddingsBuild( + _ context.Context, in *embeddingsBuildInput, +) (*embeddingsBuildOutput, error) { + if s.embeddingsManager == nil { + return nil, apiError(http.StatusNotImplemented, "embeddings manager not available") + } + if err := s.embeddingsManager.StartBuild(in.Body); err != nil { + if errors.Is(err, vector.ErrBuildRunning) { + return nil, apiError(http.StatusConflict, err.Error()) + } + if errors.Is(err, vector.ErrUnknownServer) { + return nil, apiError(http.StatusBadRequest, err.Error()) + } + return nil, internalError("start embeddings build", err) + } + return &embeddingsBuildOutput{ + Status: http.StatusAccepted, + Body: embeddingsBuildResponse{Started: true}, + }, nil +} + +func (s *Server) humaEmbeddingsStatus( + _ context.Context, _ *emptyInput, +) (*jsonOutput[vector.BuildStatus], error) { + if s.embeddingsManager == nil { + return nil, apiError(http.StatusNotImplemented, "embeddings manager not available") + } + return &jsonOutput[vector.BuildStatus]{Body: s.embeddingsManager.Status()}, nil +} + +func (s *Server) humaEmbeddingsGenerations( + ctx context.Context, _ *emptyInput, +) (*jsonOutput[embeddingsGenerationsResponse], error) { + if s.embeddingsManager == nil { + return nil, apiError(http.StatusNotImplemented, "embeddings manager not available") + } + gens, err := s.embeddingsManager.Generations(ctx) + if err != nil { + return nil, internalError("list embedding generations", err) + } + if gens == nil { + gens = []vector.GenerationInfo{} + } + return &jsonOutput[embeddingsGenerationsResponse]{ + Body: embeddingsGenerationsResponse{Generations: gens}, + }, nil +} + +func (s *Server) humaEmbeddingsActivate( + ctx context.Context, in *embeddingsGenerationActionInput, +) (*noContentOutput, error) { + if s.embeddingsManager == nil { + return nil, apiError(http.StatusNotImplemented, "embeddings manager not available") + } + if err := s.embeddingsManager.Activate(ctx, in.ID, in.Body.Force); err != nil { + return nil, embeddingsActionError(err) + } + return &noContentOutput{Status: http.StatusNoContent}, nil +} + +func (s *Server) humaEmbeddingsRetire( + ctx context.Context, in *embeddingsGenerationActionInput, +) (*noContentOutput, error) { + if s.embeddingsManager == nil { + return nil, apiError(http.StatusNotImplemented, "embeddings manager not available") + } + if err := s.embeddingsManager.Retire(ctx, in.ID, in.Body.Force); err != nil { + return nil, embeddingsActionError(err) + } + return &noContentOutput{Status: http.StatusNoContent}, nil +} + +// embeddingsActionError maps Activate/Retire's sentinels — ErrBuildRunning +// and ErrGenerationRefused to 409 Conflict, ErrGenerationNotFound to 404 Not +// Found — with the underlying message, and anything else to a generic +// internal error. +func embeddingsActionError(err error) error { + if errors.Is(err, vector.ErrBuildRunning) || errors.Is(err, vector.ErrGenerationRefused) { + return apiError(http.StatusConflict, err.Error()) + } + if errors.Is(err, vector.ErrGenerationNotFound) { + return apiError(http.StatusNotFound, err.Error()) + } + return internalError("embeddings generation action", err) +} diff --git a/internal/server/huma_routes_embeddings_test.go b/internal/server/huma_routes_embeddings_test.go new file mode 100644 index 000000000..523bc6d6e --- /dev/null +++ b/internal/server/huma_routes_embeddings_test.go @@ -0,0 +1,295 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/vector" +) + +// activateCall records one Activate or Retire invocation on +// fakeEmbeddingsManager, for assertions on what the route handlers passed +// through. +type activateCall struct { + id int64 + force bool +} + +// fakeEmbeddingsManager is a test double for EmbeddingsManager: each method's +// return value is scripted via the corresponding field, and calls are +// recorded for assertions. Safe for concurrent use since huma may invoke +// handlers from more than one goroutine. +type fakeEmbeddingsManager struct { + mu sync.Mutex + + startBuildErr error + startBuildCalls []vector.BuildRequest + + status vector.BuildStatus + + generations []vector.GenerationInfo + generationsErr error + + activateErr error + activateCalls []activateCall + + retireErr error + retireCalls []activateCall +} + +func (f *fakeEmbeddingsManager) StartBuild(req vector.BuildRequest) error { + f.mu.Lock() + defer f.mu.Unlock() + f.startBuildCalls = append(f.startBuildCalls, req) + return f.startBuildErr +} + +func (f *fakeEmbeddingsManager) Status() vector.BuildStatus { + f.mu.Lock() + defer f.mu.Unlock() + return f.status +} + +func (f *fakeEmbeddingsManager) Generations(_ context.Context) ([]vector.GenerationInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.generations, f.generationsErr +} + +func (f *fakeEmbeddingsManager) Activate(_ context.Context, id int64, force bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.activateCalls = append(f.activateCalls, activateCall{id: id, force: force}) + return f.activateErr +} + +func (f *fakeEmbeddingsManager) Retire(_ context.Context, id int64, force bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.retireCalls = append(f.retireCalls, activateCall{id: id, force: force}) + return f.retireErr +} + +// newEmbeddingsTestServer builds a full Server (via testServer, so the SPA +// fallback and route registration match production) with m wired in as the +// embeddings manager. A nil m leaves the routes registered but unavailable. +func newEmbeddingsTestServer(t *testing.T, m EmbeddingsManager) *Server { + t.Helper() + var opts []Option + if m != nil { + opts = append(opts, WithEmbeddingsManager(m)) + } + return testServer(t, 0, opts...) +} + +func TestEmbeddingsRoutesRegisteredWhenManagerNil(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/v1/embeddings/status", nil) + + withoutManager := newEmbeddingsTestServer(t, nil) + _, patternWithout := withoutManager.mux.Handler(req) + assert.NotEqual(t, "/", patternWithout) + + w := serveGet(t, withoutManager, "/api/v1/embeddings/status") + assertRecorderStatus(t, w, http.StatusNotImplemented) + + withManager := newEmbeddingsTestServer(t, &fakeEmbeddingsManager{}) + _, patternWith := withManager.mux.Handler(req) + assert.NotEqual(t, "/", patternWith) +} + +func TestOpenAPIDocumentsEmbeddingsRoutesWithoutManager(t *testing.T) { + spec := readOpenAPISpec(t, testServer(t, 0).Handler()) + + for _, tt := range []struct { + method string + path string + }{ + {method: "post", path: "/api/v1/embeddings/build"}, + {method: "get", path: "/api/v1/embeddings/status"}, + {method: "get", path: "/api/v1/embeddings/generations"}, + {method: "post", path: "/api/v1/embeddings/generations/{id}/activate"}, + {method: "post", path: "/api/v1/embeddings/generations/{id}/retire"}, + } { + requireOpenAPIOperation(t, spec, tt.method, tt.path) + } +} + +func TestEmbeddingsBuildReturnsAcceptedAndStartsBuild(t *testing.T) { + fake := &fakeEmbeddingsManager{} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/build", + vector.BuildRequest{FullRebuild: true}) + assertRecorderStatus(t, w, http.StatusAccepted) + + var body struct { + Started bool `json:"started"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + assert.True(t, body.Started) + + require.Len(t, fake.startBuildCalls, 1) + assert.True(t, fake.startBuildCalls[0].FullRebuild) +} + +func TestEmbeddingsBuildReturnsConflictWhenAlreadyRunning(t *testing.T) { + fake := &fakeEmbeddingsManager{startBuildErr: vector.ErrBuildRunning} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/build", vector.BuildRequest{}) + assertRecorderStatus(t, w, http.StatusConflict) + assert.Contains(t, w.Body.String(), "already running") +} + +func TestEmbeddingsBuildUnknownServerReturnsBadRequest(t *testing.T) { + fake := &fakeEmbeddingsManager{startBuildErr: fmt.Errorf( + "resolve encoder: %w", vector.ErrUnknownServer)} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/build", + vector.BuildRequest{Using: "nope"}) + assertRecorderStatus(t, w, http.StatusBadRequest) + assert.Contains(t, w.Body.String(), "unknown embeddings server", + "the response must carry the manager's actionable message, not a generic 500") +} + +func TestEmbeddingsStatusReturnsCurrentStatus(t *testing.T) { + fake := &fakeEmbeddingsManager{status: vector.BuildStatus{ + Running: true, Phase: "embedding", Done: 3, Total: 10, + }} + s := newEmbeddingsTestServer(t, fake) + + w := serveGet(t, s, "/api/v1/embeddings/status") + assertRecorderStatus(t, w, http.StatusOK) + + var status vector.BuildStatus + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &status)) + assert.Equal(t, fake.status, status) +} + +func TestEmbeddingsGenerationsReturnsWrappedList(t *testing.T) { + fake := &fakeEmbeddingsManager{generations: []vector.GenerationInfo{ + {ID: 1, State: "active", Model: "m", Dimension: 3, Fingerprint: "fp1", Embedded: 5}, + }} + s := newEmbeddingsTestServer(t, fake) + + w := serveGet(t, s, "/api/v1/embeddings/generations") + assertRecorderStatus(t, w, http.StatusOK) + + var body struct { + Generations []vector.GenerationInfo `json:"generations"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + require.Len(t, body.Generations, 1) + assert.Equal(t, fake.generations[0], body.Generations[0]) +} + +func TestEmbeddingsGenerationsPropagatesError(t *testing.T) { + fake := &fakeEmbeddingsManager{generationsErr: errors.New("boom")} + s := newEmbeddingsTestServer(t, fake) + + w := serveGet(t, s, "/api/v1/embeddings/generations") + assertRecorderStatus(t, w, http.StatusInternalServerError) +} + +func TestEmbeddingsActivateReturnsNoContentAndPassesForce(t *testing.T) { + fake := &fakeEmbeddingsManager{} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/5/activate", + map[string]bool{"force": true}) + assertRecorderStatus(t, w, http.StatusNoContent) + + require.Len(t, fake.activateCalls, 1) + assert.Equal(t, activateCall{id: 5, force: true}, fake.activateCalls[0]) +} + +func TestEmbeddingsActivateRefusalReturnsConflict(t *testing.T) { + fake := &fakeEmbeddingsManager{ + activateErr: fmt.Errorf("%w: generation 5 still has 2 messages needing embedding; use --force", + vector.ErrGenerationRefused), + } + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/5/activate", + map[string]bool{"force": false}) + assertRecorderStatus(t, w, http.StatusConflict) + assert.Contains(t, w.Body.String(), "still has 2 messages needing embedding") +} + +func TestEmbeddingsActivateBuildRunningReturnsConflict(t *testing.T) { + fake := &fakeEmbeddingsManager{activateErr: vector.ErrBuildRunning} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/5/activate", + map[string]bool{"force": false}) + assertRecorderStatus(t, w, http.StatusConflict) +} + +func TestEmbeddingsActivateUnknownGenerationReturnsNotFound(t *testing.T) { + fake := &fakeEmbeddingsManager{ + activateErr: fmt.Errorf("generation %d: %w", 999, vector.ErrGenerationNotFound), + } + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/999/activate", + map[string]bool{"force": false}) + assertRecorderStatus(t, w, http.StatusNotFound) + assert.Contains(t, w.Body.String(), "999") +} + +func TestEmbeddingsActivateOtherErrorReturnsInternalError(t *testing.T) { + fake := &fakeEmbeddingsManager{activateErr: errors.New("boom")} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/5/activate", + map[string]bool{"force": false}) + assertRecorderStatus(t, w, http.StatusInternalServerError) +} + +func TestEmbeddingsRetireReturnsNoContentAndPassesForce(t *testing.T) { + fake := &fakeEmbeddingsManager{} + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/7/retire", + map[string]bool{"force": true}) + assertRecorderStatus(t, w, http.StatusNoContent) + + require.Len(t, fake.retireCalls, 1) + assert.Equal(t, activateCall{id: 7, force: true}, fake.retireCalls[0]) +} + +func TestEmbeddingsRetireRefusalReturnsConflict(t *testing.T) { + fake := &fakeEmbeddingsManager{ + retireErr: fmt.Errorf("%w: generation 7 is active; use --force to retire it", + vector.ErrGenerationRefused), + } + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/7/retire", + map[string]bool{"force": false}) + assertRecorderStatus(t, w, http.StatusConflict) + assert.True(t, strings.Contains(w.Body.String(), "is active")) +} + +func TestEmbeddingsRetireUnknownGenerationReturnsNotFound(t *testing.T) { + fake := &fakeEmbeddingsManager{ + retireErr: fmt.Errorf("generation %d: %w", 999, vector.ErrGenerationNotFound), + } + s := newEmbeddingsTestServer(t, fake) + + w := serveJSON(t, s.mux, http.MethodPost, "/api/v1/embeddings/generations/999/retire", + map[string]bool{"force": false}) + assertRecorderStatus(t, w, http.StatusNotFound) + assert.Contains(t, w.Body.String(), "999") +} diff --git a/internal/server/huma_routes_search.go b/internal/server/huma_routes_search.go index 7b87d9bf7..2b0107e62 100644 --- a/internal/server/huma_routes_search.go +++ b/internal/server/huma_routes_search.go @@ -21,6 +21,8 @@ type searchSort string type contentSearchMode string +type contentSearchScope string + type searchInput struct { Query string `query:"q" required:"true" doc:"Search query"` Project string `query:"project" doc:"Filter by project"` @@ -30,25 +32,28 @@ type searchInput struct { } type contentSearchInput struct { - Pattern string `query:"pattern" required:"true" doc:"Pattern to search for"` - Mode contentSearchMode `query:"mode" enum:"substring,regex,fts" doc:"Search mode"` - In string `query:"in" doc:"Comma-separated content sources"` - ExcludeSystem bool `query:"exclude_system" doc:"Exclude system messages"` - Reveal bool `query:"reveal" doc:"Return unredacted secret matches for localhost callers"` - Project string `query:"project" doc:"Filter by project"` - ExcludeProject string `query:"exclude_project" doc:"Exclude a project"` - Machine string `query:"machine" doc:"Filter by machine"` - GitBranch string `query:"git_branch" doc:"Filter by git branch; opaque (project, branch) tokens from the /branches endpoint"` - Agent string `query:"agent" doc:"Filter by agent"` - Date string `query:"date" format:"date" doc:"Filter to a single YYYY-MM-DD date"` - DateFrom string `query:"date_from" format:"date" doc:"Filter start date"` - DateTo string `query:"date_to" format:"date" doc:"Filter end date"` - ActiveSince string `query:"active_since" format:"date-time" doc:"Filter sessions active since this RFC3339 timestamp"` - IncludeChildren bool `query:"include_children" doc:"Include child sessions"` - IncludeAutomated bool `query:"include_automated" doc:"Include automated sessions"` - IncludeOneShot bool `query:"include_one_shot" doc:"Include one-shot sessions"` - Limit int `query:"limit" minimum:"0" doc:"Maximum number of results"` - Cursor int `query:"cursor" minimum:"0" doc:"Pagination cursor"` + Pattern string `query:"pattern" required:"true" doc:"Pattern to search for"` + Mode contentSearchMode `query:"mode" enum:"substring,regex,fts,semantic,hybrid" doc:"Search mode"` + Scope contentSearchScope `query:"scope" enum:"top,all,subordinate" doc:"Semantic/hybrid result scope: top, all, or subordinate (default all)"` + SearchIntent string `header:"X-AgentsView-Search-Intent" doc:"Required for semantic/hybrid GET searches"` + In string `query:"in" doc:"Comma-separated content sources"` + ExcludeSystem bool `query:"exclude_system" doc:"Exclude system messages"` + Reveal bool `query:"reveal" doc:"Return unredacted secret matches for localhost callers"` + Project string `query:"project" doc:"Filter by project"` + ExcludeProject string `query:"exclude_project" doc:"Exclude a project"` + Machine string `query:"machine" doc:"Filter by machine"` + GitBranch string `query:"git_branch" doc:"Filter by git branch; opaque (project, branch) tokens from the /branches endpoint"` + Agent string `query:"agent" doc:"Filter by agent"` + Date string `query:"date" format:"date" doc:"Filter to a single YYYY-MM-DD date"` + DateFrom string `query:"date_from" format:"date" doc:"Filter start date"` + DateTo string `query:"date_to" format:"date" doc:"Filter end date"` + ActiveSince string `query:"active_since" format:"date-time" doc:"Filter sessions active since this RFC3339 timestamp"` + IncludeChildren bool `query:"include_children" doc:"Include child sessions"` + IncludeAutomated bool `query:"include_automated" doc:"Include automated sessions"` + IncludeOneShot bool `query:"include_one_shot" doc:"Include one-shot sessions"` + Limit int `query:"limit" minimum:"0" doc:"Maximum number of results"` + Cursor int `query:"cursor" minimum:"0" doc:"Pagination cursor"` + Context int `query:"context" doc:"Include N messages of context before and after each match (max 10)"` } func (s *Server) humaSearch( @@ -93,6 +98,15 @@ func (s *Server) humaSearchContent( if in.Reveal && !isLocalhostContext(ctx) { return nil, apiError(http.StatusForbidden, "reveal is only permitted from localhost") } + if requiresSemanticSearchIntent(in.Mode) && + in.SearchIntent != service.SemanticSearchIntentValue { + return nil, apiError(http.StatusForbidden, + "semantic and hybrid search require "+service.SemanticSearchIntentHeader) + } + if in.Scope != "" && !requiresSemanticSearchIntent(in.Mode) { + return nil, apiError(http.StatusBadRequest, + "scope is only supported for semantic and hybrid search modes") + } var sources []string if in.In != "" { sources = strings.Split(in.In, ",") @@ -118,8 +132,10 @@ func (s *Server) humaSearchContent( IncludeChildren: in.IncludeChildren, IncludeAutomated: in.IncludeAutomated, IncludeOneShot: in.IncludeOneShot, + Scope: string(in.Scope), Limit: in.Limit, Cursor: in.Cursor, + Context: in.Context, }) if err != nil { if handled := handleHumaContextError(err); handled != nil { @@ -128,6 +144,16 @@ func (s *Server) humaSearchContent( if handled := handleHumaReadOnly(err); handled != nil { return nil, handled } + if errors.Is(err, db.ErrSemanticTransient) { + // The embeddings endpoint itself is unreachable at query + // time; semantic search is configured and otherwise ready, + // so this must read as "temporarily unavailable, retry" (503) + // rather than 501's "not implemented / disabled". + return nil, apiError(http.StatusServiceUnavailable, err.Error()) + } + if errors.Is(err, db.ErrSemanticUnavailable) { + return nil, apiError(http.StatusNotImplemented, err.Error()) + } var inputErr *db.SearchInputError if errors.As(err, &inputErr) { return nil, apiError(http.StatusBadRequest, err.Error()) @@ -139,3 +165,7 @@ func (s *Server) humaSearchContent( } return &jsonOutput[*service.ContentSearchResult]{Body: res}, nil } + +func requiresSemanticSearchIntent(mode contentSearchMode) bool { + return mode == "semantic" || mode == "hybrid" +} diff --git a/internal/server/huma_routes_sessions.go b/internal/server/huma_routes_sessions.go index 8ce00ad65..459f47d13 100644 --- a/internal/server/huma_routes_sessions.go +++ b/internal/server/huma_routes_sessions.go @@ -90,6 +90,10 @@ type messageListInput struct { Limit int `query:"limit" minimum:"0" doc:"Maximum number of messages"` Direction messageDirection `query:"direction" enum:"asc,desc" doc:"Message ordering direction"` From optionalIntParam `query:"from" minimum:"0" doc:"Starting message ordinal"` + Around optionalIntParam `query:"around" minimum:"0" doc:"Center a symmetric window on this ordinal (mutually exclusive with from/direction)"` + Before optionalIntParam `query:"before" minimum:"0" doc:"Messages before the around anchor (default 5)"` + After optionalIntParam `query:"after" minimum:"0" doc:"Messages after the around anchor (default 5)"` + Roles string `query:"roles" doc:"Comma-separated roles to include, e.g. user,assistant"` } type searchSessionInput struct { @@ -278,13 +282,44 @@ func (s *Server) humaGetMessages( if in.From.IsSet { filter.From = &in.From.Value } + if in.Around.IsSet { + filter.Around = &in.Around.Value + } + if in.Before.IsSet { + filter.Before = &in.Before.Value + } + if in.After.IsSet { + filter.After = &in.After.Value + } + if in.Roles != "" { + filter.Roles = splitTrimmedNonEmpty(in.Roles) + } list, err := s.sessions.Messages(ctx, in.ID, filter) if err != nil { + if errors.Is(err, service.ErrAroundMutuallyExclusive) || + errors.Is(err, service.ErrBeforeAfterRequireAround) { + return nil, apiError(http.StatusBadRequest, err.Error()) + } return nil, serverError(err) } return &jsonOutput[*service.MessageList]{Body: list}, nil } +// splitTrimmedNonEmpty splits s on commas, trims surrounding whitespace from +// each part, and drops empty parts. This matches the CLI's `session search +// --in` convention (cmd/agentsview/session_search.go) so a trailing or +// doubled comma (e.g. "user,") narrows the filter by one intended value +// instead of silently adding a spurious "" element that matches nothing. +func splitTrimmedNonEmpty(s string) []string { + var out []string + for part := range strings.SplitSeq(s, ",") { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out +} + func (s *Server) humaToolCalls( ctx context.Context, in *idPathInput, @@ -631,7 +666,7 @@ func (s *Server) humaDeleteSession( type batchDeleteInput struct { Body struct { - SessionIDs []string `json:"session_ids" required:"true" doc:"Session IDs to soft-delete"` + SessionIDs []string `json:"session_ids" required:"true" nullable:"false" doc:"Session IDs to soft-delete"` } } diff --git a/internal/server/search_scope_test.go b/internal/server/search_scope_test.go new file mode 100644 index 000000000..c6eaae82e --- /dev/null +++ b/internal/server/search_scope_test.go @@ -0,0 +1,181 @@ +package server_test + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.kenn.io/agentsview/internal/db" + "go.kenn.io/agentsview/internal/service" +) + +// fakeHitsVectorSearcher returns canned semantic hits. ResolveMessageUnits +// resolves against the units implied by the hits, mirroring the db-layer +// fake, so hybrid requests do not error. +type fakeHitsVectorSearcher struct{ hits []db.VectorHit } + +func (f fakeHitsVectorSearcher) SemanticSearch( + _ context.Context, _ string, _ int, +) ([]db.VectorHit, error) { + return f.hits, nil +} + +func (f fakeHitsVectorSearcher) ResolveMessageUnits( + _ context.Context, refs []db.MessageRef, +) ([]db.UnitRef, error) { + return make([]db.UnitRef, len(refs)), nil +} + +// TestSearchContentScopeInvalidValueRejected pins the enum gate on the +// scope query param: an out-of-enum value is rejected up front (Huma 422 +// remapped to 400), matching how mode is validated. +func TestSearchContentScopeInvalidValueRejected(t *testing.T) { + te := setup(t) + te.db.SetVectorSearcher(fakeTransientVectorSearcher{}) + + w := te.wrappedRequest(http.MethodGet, + "/api/v1/search/content?pattern=fox&mode=semantic&scope=bogus", + withHeader("X-AgentsView-Search-Intent", "semantic")) + assertStatus(t, w, http.StatusBadRequest) + assert.Contains(t, w.Body.String(), "scope") +} + +// TestSearchContentScopeRequiresSemanticOrHybridMode pins that scope is +// only meaningful for semantic/hybrid: setting it on any other mode is +// rejected rather than silently ignored. +func TestSearchContentScopeRequiresSemanticOrHybridMode(t *testing.T) { + te := setup(t) + + for _, q := range []string{ + "pattern=fox&scope=top", + "pattern=fox&mode=substring&scope=top", + "pattern=fox&mode=regex&scope=all", + "pattern=fox&mode=fts&scope=subordinate", + } { + w := te.get(t, "/api/v1/search/content?"+q) + assertStatus(t, w, http.StatusBadRequest) + assert.Contains(t, w.Body.String(), "semantic", + "error should point at the semantic/hybrid-only restriction") + } +} + +// TestSearchContentScopeFiltersSemanticResults exercises the scope param +// end to end: scope=top drops the subordinate unit, scope=subordinate +// keeps only it, and the default returns both even though include_children +// is not set (precedence over the sidebar-child exclusion). +func TestSearchContentScopeFiltersSemanticResults(t *testing.T) { + te := setup(t) + te.seedSession(t, "top-sess", "proj", 2) + te.seedMessages(t, "top-sess", 1, func(_ int, m *db.Message) { + m.Content = "zebra at top level" + }) + te.seedSession(t, "sub-sess", "proj", 2, func(s *db.Session) { + s.ParentSessionID = new("top-sess") + s.RelationshipType = "subagent" + }) + te.seedMessages(t, "sub-sess", 1, func(_ int, m *db.Message) { + m.Content = "zebra inside the subagent" + }) + te.db.SetVectorSearcher(fakeHitsVectorSearcher{hits: []db.VectorHit{ + {SessionID: "sub-sess", Ordinal: 0, Subordinate: true, Score: 0.9, + Snippet: "zebra inside the subagent"}, + {SessionID: "top-sess", Ordinal: 0, Score: 0.5, + Snippet: "zebra at top level"}, + }}) + + search := func(t *testing.T, scope string) []string { + t.Helper() + path := "/api/v1/search/content?pattern=zebra&mode=semantic" + if scope != "" { + path += "&scope=" + scope + } + w := te.wrappedRequest(http.MethodGet, path, + withHeader("X-AgentsView-Search-Intent", "semantic")) + assertStatus(t, w, http.StatusOK) + res := decode[service.ContentSearchResult](t, w) + ids := make([]string, 0, len(res.Matches)) + for _, m := range res.Matches { + ids = append(ids, m.SessionID) + } + return ids + } + + def := search(t, "") + require.Contains(t, def, "sub-sess", + "default scope must return the subordinate unit despite include_children being unset") + assert.Contains(t, def, "top-sess") + + assert.Equal(t, []string{"top-sess"}, search(t, "top")) + assert.Equal(t, []string{"sub-sess"}, search(t, "subordinate")) + assert.ElementsMatch(t, []string{"top-sess", "sub-sess"}, search(t, "all")) +} + +// TestSearchContentSemanticResponseCarriesUnitRangeAndLineage pins the HTTP +// wire shape for run-grouped semantic hits: ordinal stays the anchor while +// ordinal_range, subordinate, and the lineage keys ride along. ordinal_range +// is always present, so the fixture's top-level single-message hit at +// ordinal 0 still serializes "ordinal_range":[0,0] even though its other +// zero-valued unit/lineage fields are omitted via omitempty. +func TestSearchContentSemanticResponseCarriesUnitRangeAndLineage(t *testing.T) { + te := setup(t) + te.seedSession(t, "top-sess", "proj", 2) + te.seedMessages(t, "top-sess", 1, func(_ int, m *db.Message) { + m.Content = "zebra at top level" + }) + te.seedSession(t, "sub-sess", "proj", 3, func(s *db.Session) { + s.ParentSessionID = new("top-sess") + s.RelationshipType = "subagent" + }) + te.seedMessages(t, "sub-sess", 3, func(i int, m *db.Message) { + if i > 0 { + m.Role = "assistant" + m.IsSidechain = true + m.Content = "zebra step inside the subagent" + } + }) + te.db.SetVectorSearcher(fakeHitsVectorSearcher{hits: []db.VectorHit{ + {SessionID: "sub-sess", Ordinal: 1, OrdinalStart: 1, OrdinalEnd: 2, + Subordinate: true, Score: 0.9, Snippet: "zebra step inside the subagent"}, + {SessionID: "top-sess", Ordinal: 0, Score: 0.5, + Snippet: "zebra at top level"}, + }}) + + w := te.wrappedRequest(http.MethodGet, + "/api/v1/search/content?pattern=zebra&mode=semantic", + withHeader("X-AgentsView-Search-Intent", "semantic")) + assertStatus(t, w, http.StatusOK) + + var res struct { + Matches []map[string]any `json:"matches"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &res)) + require.Len(t, res.Matches, 2) + byID := map[string]map[string]any{} + for _, m := range res.Matches { + byID[m["session_id"].(string)] = m + } + + sub, ok := byID["sub-sess"] + require.True(t, ok, "subordinate run hit present") + assert.EqualValues(t, 1, sub["ordinal"], "ordinal stays the anchor") + assert.Equal(t, []any{float64(1), float64(2)}, sub["ordinal_range"]) + assert.Equal(t, true, sub["subordinate"]) + assert.Equal(t, "subagent", sub["relationship"]) + assert.Equal(t, "top-sess", sub["parent_session_id"]) + assert.Equal(t, true, sub["is_sidechain"]) + + top, ok := byID["top-sess"] + require.True(t, ok, "top-level hit present") + assert.Equal(t, []any{float64(0), float64(0)}, top["ordinal_range"], + "ordinal_range is always present, even for a zero-valued single-message hit") + for _, key := range []string{ + "subordinate", "relationship", "parent_session_id", "is_sidechain", + } { + assert.NotContains(t, top, key, + "zero-valued lineage keys must be omitted from the wire") + } +} diff --git a/internal/server/server.go b/internal/server/server.go index b09ffb6a3..4fdf1482a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -94,6 +94,11 @@ type Server struct { // /debug/pprof/ so a running daemon can be profiled. Off by // default; enabled by the hidden serve --pprof flag. pprofEnabled bool + + // embeddingsManager, when set, backs the /api/v1/embeddings/... + // build lifecycle routes. Nil (the default) leaves those routes + // unregistered, e.g. when semantic search is not configured. + embeddingsManager EmbeddingsManager } // New creates a new Server. @@ -965,7 +970,7 @@ func corsMiddleware( ) w.Header().Set( "Access-Control-Allow-Headers", - "Content-Type, Authorization", + "Content-Type, Authorization, "+service.SemanticSearchIntentHeader, ) if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) @@ -1002,7 +1007,7 @@ func corsMiddleware( ) w.Header().Set( "Access-Control-Allow-Headers", - "Content-Type, Authorization", + "Content-Type, Authorization, "+service.SemanticSearchIntentHeader, ) if r.Method == http.MethodOptions { if !safeForReads { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 9e97704c7..dc629fdaa 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -733,7 +733,13 @@ func TestOpenAPIEndpointDocumentsEnumsAndRequestBodies(t *testing.T) { path: "/api/v1/search/content", method: "get", name: "mode", - want: []string{"substring", "regex", "fts"}, + want: []string{"substring", "regex", "fts", "semantic", "hybrid"}, + }, + { + path: "/api/v1/search/content", + method: "get", + name: "scope", + want: []string{"top", "all", "subordinate"}, }, { path: "/api/v1/sessions/{id}/md", @@ -821,6 +827,109 @@ func TestOpenAPIEndpointDocumentsEnumsAndRequestBodies(t *testing.T) { assert.Equal(t, []string{"auto", "custom", "clipboard"}, mode.Enum) } +func TestSearchContentSemanticGETRequiresIntentHeader(t *testing.T) { + te := setup(t) + te.db.SetVectorSearcher(fakeTransientVectorSearcher{}) + + w := te.wrappedRequest(http.MethodGet, "/api/v1/search/content?pattern=fox&mode=semantic", + withOrigin("http://evil-site.com")) + assertStatus(t, w, http.StatusForbidden) + assert.Contains(t, w.Body.String(), "X-AgentsView-Search-Intent") +} + +func TestSearchContentSemanticGETWithIntentHeaderReachesSearcher(t *testing.T) { + te := setup(t) + te.db.SetVectorSearcher(fakeTransientVectorSearcher{}) + + w := te.get(t, "/api/v1/search/content?pattern=fox&mode=semantic") + assertStatus(t, w, http.StatusForbidden) + + w = te.wrappedRequest(http.MethodGet, "/api/v1/search/content?pattern=fox&mode=semantic", + withHeader("X-AgentsView-Search-Intent", "semantic")) + assertStatus(t, w, http.StatusServiceUnavailable) +} + +// TestSearchContentSemanticModeUnavailable pins the end-to-end capability +// gate: a test server has no VectorSearcher wired in (db.HasSemantic is +// false), so a semantic or hybrid content search must respond 501 rather +// than 500 or a silently-empty page. +func TestSearchContentSemanticModeUnavailable(t *testing.T) { + te := setup(t) + + for _, mode := range []string{"semantic", "hybrid"} { + w := te.wrappedRequest(http.MethodGet, "/api/v1/search/content?pattern=fox&mode="+mode, + withHeader("X-AgentsView-Search-Intent", "semantic")) + assertStatus(t, w, http.StatusNotImplemented) + } +} + +// fakeTransientVectorSearcher implements db.VectorSearcher, always failing +// with an error wrapping db.ErrSemanticTransient — standing in for what +// cmd/agentsview's searcherAdapter returns when the embeddings endpoint +// itself is unreachable at query time (translateSearchError wraps a +// vector.QueryEncodeError this way). +type fakeTransientVectorSearcher struct{} + +func (fakeTransientVectorSearcher) SemanticSearch( + _ context.Context, _ string, _ int, +) ([]db.VectorHit, error) { + return nil, fmt.Errorf("%w: dial tcp: connection refused", db.ErrSemanticTransient) +} + +func (fakeTransientVectorSearcher) ResolveMessageUnits( + _ context.Context, refs []db.MessageRef, +) ([]db.UnitRef, error) { + return make([]db.UnitRef, len(refs)), nil +} + +// TestSearchContentSemanticQueryEncodeFailureReturns503 covers the +// query-time embeddings-endpoint-down case: it must map to 503 (the +// feature is configured and the request can be retried), not 501 (which +// would read as "semantic search is disabled") or a bare 500. +func TestSearchContentSemanticQueryEncodeFailureReturns503(t *testing.T) { + te := setup(t) + te.db.SetVectorSearcher(fakeTransientVectorSearcher{}) + + w := te.wrappedRequest(http.MethodGet, "/api/v1/search/content?pattern=fox&mode=semantic", + withHeader("X-AgentsView-Search-Intent", "semantic")) + assertStatus(t, w, http.StatusServiceUnavailable) +} + +// TestOpenAPIEndpointDocumentsBatchDeleteSessionIDsAsNonNullableArray guards +// the schema huma emits for the batch-delete request body: session_ids must +// serialize as a plain, non-nullable string array (schema type "array"), not +// the OpenAPI 3.1 nullable union ["array", "null"] huma's DefaultArrayNullable +// otherwise applies to every slice field. Losing that keeps the generated +// TypeScript client's session_ids typed as Array rather than +// loosening to any[] | null. +func TestOpenAPIEndpointDocumentsBatchDeleteSessionIDsAsNonNullableArray(t *testing.T) { + te := setup(t) + + w := te.get(t, "/api/openapi.json") + require.Equal(t, http.StatusOK, w.Code, "body: %s", w.Body.String()) + + var spec struct { + Components struct { + Schemas map[string]struct { + Required []string `json:"required"` + Properties map[string]struct { + Type json.RawMessage `json:"type"` + } `json:"properties"` + } `json:"schemas"` + } `json:"components"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &spec)) + + schema, ok := spec.Components.Schemas["BatchDeleteInputBody"] + require.True(t, ok, "spec missing BatchDeleteInputBody schema") + assert.Contains(t, schema.Required, "session_ids") + + prop, ok := schema.Properties["session_ids"] + require.True(t, ok, "session_ids property missing from BatchDeleteInputBody schema") + assert.JSONEq(t, `"array"`, string(prop.Type), + `session_ids must be a non-nullable array, not the nullable ["array","null"] union`) +} + func TestOpenAPIEndpointDocumentsQualitySignalResponses(t *testing.T) { te := setup(t) @@ -1808,6 +1917,24 @@ func TestGetMessages_DescWithFrom(t *testing.T) { } } +// TestGetMessages_RolesTrimsSpacesAndDropsTrailingEmpty covers a regression +// where "roles=user, assistant," (a space after the comma, plus a trailing +// comma) silently narrowed the filter: an untrimmed " assistant" role never +// matches any stored row's plain "assistant" value, so assistant messages +// were dropped even though the caller asked for both roles. +func TestGetMessages_RolesTrimsSpacesAndDropsTrailingEmpty(t *testing.T) { + te := setup(t) + te.seedSession(t, "s1", "my-app", 4) + te.seedMessages(t, "s1", 4) + + w := te.get(t, "/api/v1/sessions/s1/messages?roles=user,%20assistant,") + assertStatus(t, w, http.StatusOK) + + resp := decode[messageListResponse](t, w) + require.Len(t, resp.Messages, 4, + "a space after the comma or a trailing empty element must not narrow the role filter") +} + func TestGetMessages_Pagination(t *testing.T) { te := setup(t) te.seedSession(t, "s1", "my-app", 20) diff --git a/internal/service/direct.go b/internal/service/direct.go index 8a9c39fda..a09024c0d 100644 --- a/internal/service/direct.go +++ b/internal/service/direct.go @@ -229,9 +229,19 @@ func hideStaleSecretCount(s *db.Session, activeVersions []string) { s.SecretLeakCount = 0 } +// defaultAroundSpan is the number of messages returned on each side of the +// anchor when Around is set but Before/After are omitted. +const defaultAroundSpan = 5 + func (b *directBackend) Messages( ctx context.Context, id string, f MessageFilter, ) (*MessageList, error) { + if f.Around != nil && (f.From != nil || f.Direction != "") { + return nil, ErrAroundMutuallyExclusive + } + if (f.Before != nil || f.After != nil) && f.Around == nil { + return nil, ErrBeforeAfterRequireAround + } switch f.Direction { case "", "asc", "desc": default: @@ -248,21 +258,90 @@ func (b *directBackend) Messages( if limit > db.MaxMessageLimit { limit = db.MaxMessageLimit } - // An omitted From means "newest" in descending mode and 0 in - // ascending mode. An explicit 0 is a real ordinal and must be - // honored in both directions. - var from int - switch { - case f.From != nil: - from = *f.From - case !asc: - from = math.MaxInt32 + + w := db.MessageWindow{Limit: limit, Asc: asc, Roles: f.Roles} + if f.Around != nil { + w.Around = f.Around + w.Before = defaultAroundSpan + if f.Before != nil { + w.Before = *f.Before + } + w.After = defaultAroundSpan + if f.After != nil { + w.After = *f.After + } + w.Before, w.After = clampAroundSpan(w.Before, w.After) + } else { + // An omitted From means "newest" in descending mode and 0 in + // ascending mode. An explicit 0 is a real ordinal and must be + // honored in both directions. + var from int + switch { + case f.From != nil: + from = *f.From + case !asc: + from = math.MaxInt32 + } + w.From = &from } - msgs, err := b.db.GetMessages(ctx, id, from, limit, asc) + + msgs, err := b.db.GetMessagesWindow(ctx, id, w) if err != nil { return nil, err } - return &MessageList{Messages: msgs, Count: len(msgs)}, nil + list := &MessageList{Messages: msgs, Count: len(msgs)} + if len(msgs) > 0 { + first := msgs[0].Ordinal + last := msgs[len(msgs)-1].Ordinal + list.FirstOrdinal = &first + list.LastOrdinal = &last + } + return list, nil +} + +// clampAroundSpan bounds a requested Before/After pair for an around window +// so the resulting response (before + after + 1 anchor row) can never exceed +// db.MaxMessageLimit, the same cap the linear path silently clamps Limit to +// (see the limit clamp a few lines up in Messages). Negative values +// (reachable via a direct SessionService caller or a negative CLI/API flag) +// are floored to 0 first, matching the floor the db layer already applies +// via max(w.Before, 0). +// +// Before and After share one budget rather than each independently capping +// to the max (which would still let the combined window reach ~2x the max), +// so an oversized request on one side (e.g. before=10^9) must not be able to +// starve a modest request on the other side down toward zero. This is a +// two-sided water-fill: a side asking for at most its fair (even) share of +// the budget gets exactly what it asked for, and the other side absorbs +// whatever budget remains; only when both sides exceed their fair share +// does the budget get split evenly between them. +func clampAroundSpan(before, after int) (int, int) { + if before < 0 { + before = 0 + } + if after < 0 { + after = 0 + } + const budget = db.MaxMessageLimit - 1 // reserve the anchor row + // Compare each side against budget individually rather than summing + // before+after: before and after are untrusted ints (reachable via a + // direct SessionService caller or the API), and before+after can + // overflow and wrap negative when either side is near math.MaxInt, + // which would slip an unbounded window past this cap. + if before <= budget && after <= budget-before { + return before, after + } + fairShare := budget / 2 + switch { + case before <= fairShare: + after = budget - before + case after <= fairShare: + before = budget - after + default: + before = fairShare + after = budget - fairShare + } + return before, after } func (b *directBackend) ToolCalls( @@ -632,9 +711,17 @@ func (b *directBackend) UsagePairwiseComparison( return &out, nil } +// maxContentSearchContext is the largest --context value SearchContent +// accepts. Larger requests are rejected rather than silently clamped, so a +// caller who asks for more context than the store will give is told, rather +// than getting a page that quietly carries less than requested. +const maxContentSearchContext = 10 + // SearchContent maps the transport-neutral request to a // db.ContentSearchFilter, calls the store, and redacts secret-shaped -// spans from each snippet unless Reveal is set. +// spans from each snippet unless Reveal is set. When Context > 0, each +// match is additionally enriched with ContextBefore/ContextAfter (see +// enrichContentContext). func (b *directBackend) SearchContent( ctx context.Context, req ContentSearchRequest, ) (*ContentSearchResult, error) { @@ -647,6 +734,15 @@ func (b *directBackend) SearchContent( } req.Sources = []string{"messages"} } + // Context < 0 (reachable via `--context -5` or a direct SessionService + // caller) is treated as "off" rather than rejected; only exceeding the + // max is a hard error. + if req.Context < 0 { + req.Context = 0 + } + if req.Context > maxContentSearchContext { + return nil, &db.SearchInputError{Msg: "context: maximum is 10"} + } page, err := b.db.SearchContent(ctx, db.ContentSearchFilter{ Pattern: req.Pattern, Mode: req.Mode, @@ -664,6 +760,7 @@ func (b *directBackend) SearchContent( IncludeChildren: req.IncludeChildren, IncludeAutomated: req.IncludeAutomated, IncludeOneShot: req.IncludeOneShot, + Scope: req.Scope, // The store builds snippets from the full source field and redacts // secrets (including ones straddling the snippet window) unless reveal // is set. Redacting the pre-truncated snippet here would miss those. @@ -674,12 +771,102 @@ func (b *directBackend) SearchContent( if err != nil { return nil, err } + if req.Context > 0 { + if err := b.enrichContentContext( + ctx, page.Matches, req.Context, req.Reveal, + ); err != nil { + return nil, err + } + } return &ContentSearchResult{ Matches: page.Matches, NextCursor: page.NextCursor, }, nil } +// enrichContentContext populates ContextBefore/ContextAfter on each match +// with a non-negative Ordinal, fetching n messages of context on each side +// of the match's ordinal and splitting the returned (ascending, anchor +// included) window on the anchor -- the anchor row is dropped from both +// slices since it duplicates the match already in the page. Matches with a +// negative ordinal are left unenriched: no current search path produces +// one, but a defensive skip here means a future one (e.g. a name-only +// match with no message row) degrades gracefully instead of panicking on +// the *w.Around dereference or fetching the wrong window. +// +// Unlike the match's own Snippet, context messages are full db.Message +// values pulled straight from GetMessagesWindow with no redaction applied +// by the store -- they were never part of the search hit, so +// ContentSearchFilter.RevealSecrets (which only governs snippet redaction) +// never touches them. When reveal is false, every context message is +// redacted here via redactMessageSecrets before being attached to the +// match, so context_before/context_after never leak a secret from an +// adjacent message regardless of transport (HTTP, CLI, MCP all share this +// path). When reveal is true the raw messages are attached unchanged, same +// as the snippet path. +func (b *directBackend) enrichContentContext( + ctx context.Context, matches []db.ContentMatch, n int, reveal bool, +) error { + for i := range matches { + m := &matches[i] + if m.Ordinal < 0 { + continue + } + anchor := m.Ordinal + msgs, err := b.db.GetMessagesWindow(ctx, m.SessionID, db.MessageWindow{ + Around: &anchor, Before: n, After: n, + }) + if err != nil { + return fmt.Errorf("content search context: %w", err) + } + for _, msg := range msgs { + if !reveal { + msg = redactMessageSecrets(msg) + } + switch { + case msg.Ordinal < anchor: + m.ContextBefore = append(m.ContextBefore, msg) + case msg.Ordinal > anchor: + m.ContextAfter = append(m.ContextAfter, msg) + } + } + } + return nil +} + +// redactMessageSecrets returns a copy of m with every secret-shaped span +// masked in its user-visible text fields: message content, thinking text, +// and each tool call's input/output payloads (InputJSON, ResultContent, and +// every ResultEvent's Content). It mirrors the masking the search-snippet +// path already applies via secrets.RedactWindow, but a context message has +// no known match offset to window around, so the whole-string secrets.Redact +// scan is used instead. m.ToolResults is not touched: it is a transient +// parse-time field (json:"-") that GetMessagesWindow never populates, so it +// never reaches a transport response. +func redactMessageSecrets(m db.Message) db.Message { + m.Content = secrets.Redact(m.Content) + m.ThinkingText = secrets.Redact(m.ThinkingText) + if len(m.ToolCalls) == 0 { + return m + } + toolCalls := make([]db.ToolCall, len(m.ToolCalls)) + for i, tc := range m.ToolCalls { + tc.InputJSON = secrets.Redact(tc.InputJSON) + tc.ResultContent = secrets.Redact(tc.ResultContent) + if len(tc.ResultEvents) > 0 { + events := make([]db.ToolResultEvent, len(tc.ResultEvents)) + for j, ev := range tc.ResultEvents { + ev.Content = secrets.Redact(ev.Content) + events[j] = ev + } + tc.ResultEvents = events + } + toolCalls[i] = tc + } + m.ToolCalls = toolCalls + return m +} + const secretSourceChanged = "source changed; cannot reveal" func (b *directBackend) ListSecrets( diff --git a/internal/service/direct_test.go b/internal/service/direct_test.go index dc98d1d10..cdddebeb8 100644 --- a/internal/service/direct_test.go +++ b/internal/service/direct_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "path/filepath" "strings" @@ -1138,6 +1139,323 @@ func TestDirectBackend_Messages_DescExplicitZeroFrom(t *testing.T) { assert.Equal(t, 0, list.Messages[0].Ordinal) } +// TestDirectBackend_Messages_AroundMutuallyExclusiveWithFrom verifies that +// combining Around with an explicit From is rejected: the two retrieval +// modes (symmetric window vs. linear pagination) cannot both be requested. +func TestDirectBackend_Messages_AroundMutuallyExclusiveWithFrom(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, 5, "m%d")...) + + around, from := 2, 1 + _, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + From: &from, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "around is mutually exclusive with from/direction") +} + +// TestDirectBackend_Messages_AroundMutuallyExclusiveWithDirection is the +// Direction half of the same guard: an explicit non-default Direction +// alongside Around must also be rejected. +func TestDirectBackend_Messages_AroundMutuallyExclusiveWithDirection(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, 5, "m%d")...) + + around := 2 + _, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + Direction: "desc", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "around is mutually exclusive with from/direction") +} + +// TestDirectBackend_Messages_BeforeAfterRequireAround verifies that Before +// or After without Around is rejected rather than silently ignored. +func TestDirectBackend_Messages_BeforeAfterRequireAround(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, 5, "m%d")...) + + before := 2 + _, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Before: &before, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "before/after require around") +} + +// TestDirectBackend_Messages_AroundDefaultsBeforeAfter verifies that Around +// with Before/After both omitted defaults to 5 messages on each side. +func TestDirectBackend_Messages_AroundDefaultsBeforeAfter(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + // 12 messages, ordinals 0..11. + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, 12, "m%d")...) + + around := 6 + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + }) + require.NoError(t, err) + require.NotNil(t, list) + require.Equal(t, 11, list.Count, "default before=5/after=5 around ordinal 6 "+ + "spans ordinals 1..11 (only 5 exist after 6)") + assert.Equal(t, 1, list.Messages[0].Ordinal) + assert.Equal(t, 11, list.Messages[len(list.Messages)-1].Ordinal) +} + +// TestDirectBackend_Messages_AroundWithRoles verifies that Roles reaches +// GetMessagesWindow: only messages matching a role in Roles are returned, +// with the anchor always included. +func TestDirectBackend_Messages_AroundWithRoles(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + dbtest.SeedMessages(t, env.db, + dbtest.UserMsg(sid, 0, "u0"), + dbtest.AsstMsg(sid, 1, "a1"), + dbtest.UserMsg(sid, 2, "u2"), + dbtest.AsstMsg(sid, 3, "a3"), + dbtest.UserMsg(sid, 4, "u4"), + ) + + around := 2 + before, after := 1, 1 + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + Before: &before, + After: &after, + Roles: []string{"user"}, + }) + require.NoError(t, err) + require.NotNil(t, list) + require.Equal(t, 3, list.Count) + for _, m := range list.Messages { + assert.Equal(t, "user", m.Role) + } + assert.Equal(t, []int{0, 2, 4}, []int{ + list.Messages[0].Ordinal, list.Messages[1].Ordinal, list.Messages[2].Ordinal, + }) +} + +// TestDirectBackend_Messages_ResponseWindowBounds verifies that MessageList +// reports FirstOrdinal/LastOrdinal from the returned window (non-empty +// case) and leaves them nil when the result is empty. +func TestDirectBackend_Messages_ResponseWindowBounds(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, 5, "m%d")...) + + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Limit: 10, + }) + require.NoError(t, err) + require.NotNil(t, list) + require.NotNil(t, list.FirstOrdinal) + require.NotNil(t, list.LastOrdinal) + assert.Equal(t, 0, *list.FirstOrdinal) + assert.Equal(t, 4, *list.LastOrdinal) + + emptyList, err := svc.Messages(context.Background(), "no-such-session", + service.MessageFilter{Limit: 10}) + require.NoError(t, err) + require.NotNil(t, emptyList) + assert.Equal(t, 0, emptyList.Count) + assert.Nil(t, emptyList.FirstOrdinal) + assert.Nil(t, emptyList.LastOrdinal) +} + +// TestDirectBackend_Messages_AroundOmittedBeforeAfterNoOtherFlags mirrors +// the CLI's zero-flag `--around N` invocation: only Around is set (no +// Before/After/From/Direction), which must succeed using the default +// before/after window rather than tripping the mutual-exclusion guard. +func TestDirectBackend_Messages_AroundOmittedBeforeAfterNoOtherFlags(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, 12, "m%d")...) + + around := 5 + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + }) + require.NoError(t, err) + require.NotNil(t, list) + assert.Equal(t, 11, list.Count) +} + +// capturingWindowStore is a minimal db.Store fake that records the +// db.MessageWindow passed to GetMessagesWindow so tests can assert on what +// directBackend.Messages forwards to the store without needing a real +// dataset. Every other db.Store method comes from the embedded nil +// interface and would panic if a test path reached it. +type capturingWindowStore struct { + db.Store + captured db.MessageWindow +} + +func (f *capturingWindowStore) GetMessagesWindow( + _ context.Context, _ string, w db.MessageWindow, +) ([]db.Message, error) { + f.captured = w + return nil, nil +} + +// TestDirectBackend_Messages_AroundClampsOversizedBefore verifies that an +// arbitrarily large --before value (e.g. before=10^9) cannot bypass +// db.MaxMessageLimit: the window forwarded to the store is capped so +// before+after+1 never exceeds the max, matching the silent-clamp +// convention the linear path already applies to Limit. +func TestDirectBackend_Messages_AroundClampsOversizedBefore(t *testing.T) { + t.Parallel() + store := &capturingWindowStore{} + svc := service.NewReadOnlyBackend(store) + + around, huge := 100, 1_000_000_000 + _, err := svc.Messages(context.Background(), "sid", service.MessageFilter{ + Around: &around, + Before: &huge, + }) + require.NoError(t, err) + total := store.captured.Before + store.captured.After + 1 + assert.LessOrEqual(t, total, db.MaxMessageLimit, + "oversized before must be clamped so the window never exceeds MaxMessageLimit") + assert.Positive(t, store.captured.After, + "the other side of the window must not be starved to zero") +} + +// TestDirectBackend_Messages_AroundClampsOversizedAfter is the After half of +// the same guard. +func TestDirectBackend_Messages_AroundClampsOversizedAfter(t *testing.T) { + t.Parallel() + store := &capturingWindowStore{} + svc := service.NewReadOnlyBackend(store) + + around, huge := 100, 1_000_000_000 + _, err := svc.Messages(context.Background(), "sid", service.MessageFilter{ + Around: &around, + After: &huge, + }) + require.NoError(t, err) + total := store.captured.Before + store.captured.After + 1 + assert.LessOrEqual(t, total, db.MaxMessageLimit, + "oversized after must be clamped so the window never exceeds MaxMessageLimit") + assert.Positive(t, store.captured.Before, + "the other side of the window must not be starved to zero") +} + +// TestDirectBackend_Messages_AroundClampsCombinedOversizedWindow verifies +// that Before and After sharing one budget are scaled down proportionally +// (not independently capped to MaxMessageLimit each, which would still let +// the combined window reach ~2x the max) when both are oversized. +func TestDirectBackend_Messages_AroundClampsCombinedOversizedWindow(t *testing.T) { + t.Parallel() + store := &capturingWindowStore{} + svc := service.NewReadOnlyBackend(store) + + around, hugeBefore, hugeAfter := 100, 1_000_000_000, 1_000_000_000 + _, err := svc.Messages(context.Background(), "sid", service.MessageFilter{ + Around: &around, + Before: &hugeBefore, + After: &hugeAfter, + }) + require.NoError(t, err) + total := store.captured.Before + store.captured.After + 1 + assert.LessOrEqual(t, total, db.MaxMessageLimit) + // Equal requests should split the shared budget evenly. + assert.InDelta(t, store.captured.Before, store.captured.After, 1) +} + +// TestDirectBackend_Messages_AroundClampsMaxIntOverflow guards against a +// regression where before+after overflows and wraps negative once either +// side approaches math.MaxInt, which would slip the sum past the budget +// check and forward an effectively unbounded window. Both sides, and the +// combination, must still be clamped to the shared budget. +func TestDirectBackend_Messages_AroundClampsMaxIntOverflow(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + before int + after int + }{ + {"before overflow", math.MaxInt, 1}, + {"after overflow", 1, math.MaxInt}, + {"both overflow", math.MaxInt, math.MaxInt}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + store := &capturingWindowStore{} + svc := service.NewReadOnlyBackend(store) + + around := 100 + _, err := svc.Messages(context.Background(), "sid", service.MessageFilter{ + Around: &around, + Before: &tc.before, + After: &tc.after, + }) + require.NoError(t, err) + + // Check each side against the bound individually, and with + // require (not assert), before ever adding them together: + // summing two still-untrusted huge values here would hit the + // exact same overflow-wraps-negative trap this test exists to + // catch, silently passing a LessOrEqual check against a + // wrapped-negative "total" regardless of whether the fix + // under test is applied. + require.LessOrEqual(t, store.captured.Before, db.MaxMessageLimit, + "clamped before must never exceed MaxMessageLimit on its own") + require.LessOrEqual(t, store.captured.After, db.MaxMessageLimit, + "clamped after must never exceed MaxMessageLimit on its own") + assert.GreaterOrEqual(t, store.captured.Before, 0, + "clamped before must never be negative") + assert.GreaterOrEqual(t, store.captured.After, 0, + "clamped after must never be negative") + total := store.captured.Before + store.captured.After + 1 + assert.LessOrEqual(t, total, db.MaxMessageLimit, + "an overflow-inducing before/after must still be capped to MaxMessageLimit") + }) + } +} + +// TestDirectBackend_Messages_AroundClampsOversizedWindowEndToEnd is an +// end-to-end regression check with a real SQLite-backed session: an +// oversized --before/--after request must return at most +// db.MaxMessageLimit messages even though more than that many exist on +// both sides of the anchor. +func TestDirectBackend_Messages_AroundClampsOversizedWindowEndToEnd(t *testing.T) { + t.Parallel() + svc, env := newDirectTestSvc(t) + sid := env.InsertSession(t) + const total = db.MaxMessageLimit + 50 + dbtest.SeedMessages(t, env.db, dbtest.UserMessagesf(sid, total, "m%d")...) + + around, huge := total/2, 1_000_000_000 + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + Before: &huge, + After: &huge, + }) + require.NoError(t, err) + require.NotNil(t, list) + assert.LessOrEqual(t, list.Count, db.MaxMessageLimit, + "the returned window must not exceed MaxMessageLimit even though "+ + "more than that many messages exist on both sides of the anchor") + assert.Less(t, list.Count, total, + "the oversized request must actually be capped below what an "+ + "unclamped window would have returned") +} + // vsCopilotChatTraceLine builds one Visual Studio Copilot trace JSONL line // carrying a single user-prompt chat span for the given conversation. func vsCopilotChatTraceLine(conversationID, spanID, prompt string) string { diff --git a/internal/service/http.go b/internal/service/http.go index 92749379a..e5bae2509 100644 --- a/internal/service/http.go +++ b/internal/service/http.go @@ -26,11 +26,38 @@ import ( // explicitly below. var errHTTPNotFound = errors.New("http: not found") -// errHTTPNotImplemented is returned by getJSON for 501 responses so -// callers can map a capability-absent daemon (e.g. search with no FTS -// index) to a typed sentinel instead of string-matching the status. +// errHTTPNotImplemented is returned (wrapped in *errNotImplementedBody) by +// getJSON for 501 responses so callers can map a capability-absent daemon +// (e.g. search with no FTS index) to a typed sentinel instead of +// string-matching the status. var errHTTPNotImplemented = errors.New("http: not implemented") +// errNotImplementedBody wraps errHTTPNotImplemented with the 501 response's +// error message, so callers that need cause-specific detail — e.g. +// SearchContent's "index is building: N% complete" or "index is stale ... +// --full-rebuild" remediation — can recover it instead of seeing only the +// bare sentinel. errors.Is(err, errHTTPNotImplemented) still holds for every +// caller that only cares about the status. +type errNotImplementedBody struct { + message string +} + +func (e *errNotImplementedBody) Error() string { return errHTTPNotImplemented.Error() } +func (e *errNotImplementedBody) Unwrap() error { return errHTTPNotImplemented } + +// notImplementedMessage extracts the {"error": "..."} message huma's error +// responses carry, falling back to the raw (trimmed) body when it isn't in +// that shape. +func notImplementedMessage(body []byte) string { + var apiErr struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &apiErr) == nil && apiErr.Error != "" { + return apiErr.Error + } + return strings.TrimSpace(string(body)) +} + type httpBackend struct { baseURL string client *http.Client @@ -173,6 +200,18 @@ func (b *httpBackend) Messages( if f.Direction != "" { q.Set("direction", f.Direction) } + if f.Around != nil { + q.Set("around", strconv.Itoa(*f.Around)) + } + if f.Before != nil { + q.Set("before", strconv.Itoa(*f.Before)) + } + if f.After != nil { + q.Set("after", strconv.Itoa(*f.After)) + } + if len(f.Roles) > 0 { + q.Set("roles", strings.Join(f.Roles, ",")) + } path := "/api/v1/sessions/" + url.PathEscape(id) + "/messages?" + q.Encode() var out MessageList @@ -395,6 +434,7 @@ func (b *httpBackend) SearchContent( "date_from": req.DateFrom, "date_to": req.DateTo, "active_since": req.ActiveSince, + "scope": req.Scope, } { if v != "" { q.Set(k, v) @@ -415,13 +455,47 @@ func (b *httpBackend) SearchContent( if req.Cursor > 0 { q.Set("cursor", strconv.Itoa(req.Cursor)) } + if req.Context > 0 { + q.Set("context", strconv.Itoa(req.Context)) + } var out ContentSearchResult - if err := b.getJSON(ctx, "/api/v1/search/content?"+q.Encode(), &out); err != nil { + var opts []func(*http.Request) + if req.Mode == "semantic" || req.Mode == "hybrid" { + opts = append(opts, func(r *http.Request) { + r.Header.Set(SemanticSearchIntentHeader, SemanticSearchIntentValue) + }) + } + if err := b.getJSON(ctx, "/api/v1/search/content?"+q.Encode(), &out, opts...); err != nil { + var notImpl *errNotImplementedBody + if errors.As(err, ¬Impl) { + return nil, wrapSemanticUnavailable(notImpl.message) + } return nil, err } return &out, nil } +// wrapSemanticUnavailable turns a search/content 501 response's error +// message into an error wrapping ErrSemanticUnavailable, preserving +// whatever cause-specific remediation text the server attached (e.g. "index +// is building: N% complete" or "... run 'agentsview embeddings build +// --full-rebuild'") instead of discarding it for the bare sentinel. +// errors.Is(result, ErrSemanticUnavailable) always holds. When message is +// empty or is exactly the sentinel's own text (no extra cause), the bare +// sentinel is returned rather than duplicating it. +func wrapSemanticUnavailable(message string) error { + sentinel := ErrSemanticUnavailable.Error() + if message == "" || message == sentinel { + return ErrSemanticUnavailable + } + if cause, ok := strings.CutPrefix(message, sentinel); ok { + return fmt.Errorf("%w%s", ErrSemanticUnavailable, cause) + } + // An unexpected body shape (e.g. a differently worded 501); still wrap + // the sentinel so errors.Is holds, and keep the server's text. + return fmt.Errorf("%w: %s", ErrSemanticUnavailable, message) +} + func (b *httpBackend) UsageSummary( ctx context.Context, req UsageRequest, ) (*UsageSummaryResult, error) { @@ -700,7 +774,7 @@ func (b *httpBackend) addAuth(req *http.Request) { } func (b *httpBackend) getJSON( - ctx context.Context, path string, out any, + ctx context.Context, path string, out any, opts ...func(*http.Request), ) error { req, err := http.NewRequestWithContext( ctx, http.MethodGet, b.baseURL+path, nil, @@ -709,6 +783,9 @@ func (b *httpBackend) getJSON( return err } b.addAuth(req) + for _, opt := range opts { + opt(req) + } resp, err := b.client.Do(req) if err != nil { return err @@ -718,7 +795,8 @@ func (b *httpBackend) getJSON( return errHTTPNotFound } if resp.StatusCode == http.StatusNotImplemented { - return errHTTPNotImplemented + body, _ := io.ReadAll(resp.Body) + return &errNotImplementedBody{message: notImplementedMessage(body)} } if resp.StatusCode != http.StatusOK { msg, _ := io.ReadAll(resp.Body) diff --git a/internal/service/http_test.go b/internal/service/http_test.go index 6b66d13cb..198cebdb1 100644 --- a/internal/service/http_test.go +++ b/internal/service/http_test.go @@ -310,6 +310,71 @@ func TestHTTPBackend_Messages_DescDirection(t *testing.T) { "desc iteration should return highest ordinal first") } +func TestHTTPBackend_Messages_AroundRoundtrip(t *testing.T) { + t.Parallel() + env := newHTTPBackendEnv(t) + const sid = "msg-around" + dbtest.SeedSessionWithMessages(t, env.DB, sid, "p1", + dbtest.UserMessagesf(sid, 12, "m%d"), dbtest.WithMessageCount(12)) + + svc := env.Backend("", false) + around := 6 + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + }) + require.NoError(t, err) + require.NotNil(t, list) + require.Equal(t, 11, list.Count, + "default before=5/after=5 around ordinal 6 spans ordinals 1..11") + assert.Equal(t, 1, list.Messages[0].Ordinal) + assert.Equal(t, 11, list.Messages[len(list.Messages)-1].Ordinal) + require.NotNil(t, list.FirstOrdinal) + require.NotNil(t, list.LastOrdinal) + assert.Equal(t, 1, *list.FirstOrdinal) + assert.Equal(t, 11, *list.LastOrdinal) +} + +func TestHTTPBackend_Messages_RolesRoundtrip(t *testing.T) { + t.Parallel() + env := newHTTPBackendEnv(t) + const sid = "msg-roles" + dbtest.SeedSessionWithMessages(t, env.DB, sid, "p1", []db.Message{ + dbtest.UserMsg(sid, 0, "u0"), + dbtest.AsstMsg(sid, 1, "a1"), + dbtest.UserMsg(sid, 2, "u2"), + }, dbtest.WithMessageCount(3)) + + svc := env.Backend("", false) + zero := 0 + list, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + From: &zero, + Limit: 100, + Roles: []string{"user"}, + }) + require.NoError(t, err) + require.NotNil(t, list) + require.Equal(t, 2, list.Count) + for _, m := range list.Messages { + assert.Equal(t, "user", m.Role) + } +} + +func TestHTTPBackend_Messages_AroundValidationErrorSurfaces(t *testing.T) { + t.Parallel() + env := newHTTPBackendEnv(t) + const sid = "msg-validation" + dbtest.SeedSessionWithMessages(t, env.DB, sid, "p1", + dbtest.UserMessagesf(sid, 3, "m%d"), dbtest.WithMessageCount(3)) + + svc := env.Backend("", false) + around, from := 1, 0 + _, err := svc.Messages(context.Background(), sid, service.MessageFilter{ + Around: &around, + From: &from, + }) + require.Error(t, err) +} + func TestHTTPBackend_ToolCalls_Empty(t *testing.T) { t.Parallel() env := newHTTPBackendEnv(t) @@ -430,6 +495,26 @@ func TestHTTPSearchContent(t *testing.T) { assert.Equal(t, "s1", res.Matches[0].SessionID) } +func TestHTTPSearchContentSemanticSetsIntentHeader(t *testing.T) { + t.Parallel() + var gotIntent string + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + gotIntent = r.Header.Get("X-AgentsView-Search-Intent") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"matches":[],"next_cursor":0}`)) + })) + defer srv.Close() + be := service.NewHTTPBackend(srv.URL, "", true) + + _, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "needle", Mode: "semantic", Limit: 50, + }) + + require.NoError(t, err) + assert.Equal(t, "semantic", gotIntent) +} + func TestHTTPSearchContent_RealServer(t *testing.T) { t.Parallel() env := newHTTPBackendEnv(t) @@ -450,6 +535,51 @@ func TestHTTPSearchContent_RealServer(t *testing.T) { assert.Equal(t, "message", res.Matches[0].Location) } +// TestHTTPSearchContent_501PreservesCauseDetail asserts that a 501 response +// carrying cause-specific remediation text (the shape searcherAdapter's +// translateSearchError produces for a still-building index) survives the +// daemon round-trip instead of being collapsed to the bare +// ErrSemanticUnavailable sentinel message. +func TestHTTPSearchContent_501PreservesCauseDetail(t *testing.T) { + t.Parallel() + body := service.ErrSemanticUnavailable.Error() + ": index is building: 40% complete" + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotImplemented) + _, _ = w.Write([]byte(`{"error":"` + body + `"}`)) + })) + defer srv.Close() + + be := service.NewHTTPBackend(srv.URL, "", true) + _, err := be.SearchContent(context.Background(), service.ContentSearchRequest{Pattern: "needle"}) + require.Error(t, err) + assert.ErrorIs(t, err, service.ErrSemanticUnavailable) + assert.Contains(t, err.Error(), "index is building: 40% complete") +} + +// TestHTTPSearchContent_501IdenticalToSentinelDoesNotDuplicate asserts that +// when the 501 body's message is exactly the sentinel's own text (no extra +// cause — e.g. ErrNoActiveGeneration's case), the client returns the bare +// sentinel rather than a message with the sentinel text repeated twice. +func TestHTTPSearchContent_501IdenticalToSentinelDoesNotDuplicate(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotImplemented) + _, _ = w.Write([]byte(`{"error":"` + service.ErrSemanticUnavailable.Error() + `"}`)) + })) + defer srv.Close() + + be := service.NewHTTPBackend(srv.URL, "", true) + _, err := be.SearchContent(context.Background(), service.ContentSearchRequest{Pattern: "needle"}) + require.Error(t, err) + assert.ErrorIs(t, err, service.ErrSemanticUnavailable) + assert.Equal(t, service.ErrSemanticUnavailable.Error(), err.Error(), + "the sentinel text must not be duplicated when the body carries no extra cause") +} + func TestNewHTTPBackend_TrimsTrailingSlash(t *testing.T) { t.Parallel() env := newHTTPBackendEnv(t) diff --git a/internal/service/search_content_test.go b/internal/service/search_content_test.go index b635a496d..0e3ed9f2b 100644 --- a/internal/service/search_content_test.go +++ b/internal/service/search_content_test.go @@ -2,6 +2,8 @@ package service_test import ( "context" + "fmt" + "slices" "strings" "testing" @@ -65,3 +67,226 @@ func TestDirectSearchContentFTSSourceGuard(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "messages only") } + +// fakeContentStore is a minimal db.Store fake for context-enrichment tests: +// only SearchContent and GetMessagesWindow are implemented; every other +// Store method comes from the embedded nil interface and would panic if a +// test path reached it (none of these tests exercise anything else). +type fakeContentStore struct { + db.Store + page db.ContentSearchPage + windows map[string][]db.Message // keyed by contextWindowKey +} + +func contextWindowKey(sessionID string, anchor int) string { + return fmt.Sprintf("%s:%d", sessionID, anchor) +} + +func (f *fakeContentStore) SearchContent( + context.Context, db.ContentSearchFilter, +) (db.ContentSearchPage, error) { + return f.page, nil +} + +func (f *fakeContentStore) GetMessagesWindow( + _ context.Context, sessionID string, w db.MessageWindow, +) ([]db.Message, error) { + return f.windows[contextWindowKey(sessionID, *w.Around)], nil +} + +// contextWindowFixture builds the before/anchor/after messages +// GetMessagesWindow(Around) would return for an anchor ordinal, ascending. +func contextWindowFixture(sessionID string, anchor int) []db.Message { + return []db.Message{ + {SessionID: sessionID, Ordinal: anchor - 2, Role: "user", Content: "before2"}, + {SessionID: sessionID, Ordinal: anchor - 1, Role: "assistant", Content: "before1"}, + {SessionID: sessionID, Ordinal: anchor, Role: "user", Content: "anchor"}, + {SessionID: sessionID, Ordinal: anchor + 1, Role: "assistant", Content: "after1"}, + {SessionID: sessionID, Ordinal: anchor + 2, Role: "user", Content: "after2"}, + } +} + +func TestDirectSearchContentContextEnrichment(t *testing.T) { + t.Parallel() + const sess = "s1" + matches := []db.ContentMatch{ + {SessionID: sess, Ordinal: 5, Snippet: "match one"}, + {SessionID: sess, Ordinal: 20, Snippet: "match two"}, + } + store := &fakeContentStore{ + page: db.ContentSearchPage{Matches: matches}, + windows: map[string][]db.Message{ + contextWindowKey(sess, 5): contextWindowFixture(sess, 5), + contextWindowKey(sess, 20): contextWindowFixture(sess, 20), + }, + } + be := service.NewReadOnlyBackend(store) + + res, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", Context: 2, + }) + require.NoError(t, err) + require.Len(t, res.Matches, 2) + for _, m := range res.Matches { + require.Len(t, m.ContextBefore, 2) + assert.Equal(t, "before2", m.ContextBefore[0].Content) + assert.Equal(t, "before1", m.ContextBefore[1].Content) + require.Len(t, m.ContextAfter, 2) + assert.Equal(t, "after1", m.ContextAfter[0].Content) + assert.Equal(t, "after2", m.ContextAfter[1].Content) + combined := slices.Concat(m.ContextBefore, m.ContextAfter) + for _, cm := range combined { + assert.NotEqual(t, m.Ordinal, cm.Ordinal, "anchor row must be excluded") + } + } +} + +func TestDirectSearchContentContextZeroLeavesNil(t *testing.T) { + t.Parallel() + store := &fakeContentStore{ + page: db.ContentSearchPage{ + Matches: []db.ContentMatch{{SessionID: "s1", Ordinal: 5}}, + }, + } + be := service.NewReadOnlyBackend(store) + + res, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", + }) + require.NoError(t, err) + require.Len(t, res.Matches, 1) + assert.Nil(t, res.Matches[0].ContextBefore) + assert.Nil(t, res.Matches[0].ContextAfter) +} + +func TestDirectSearchContentContextRejectsOverMax(t *testing.T) { + t.Parallel() + be := service.NewReadOnlyBackend(&fakeContentStore{}) + + _, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", Context: 11, + }) + require.Error(t, err) + assert.Equal(t, "context: maximum is 10", err.Error()) +} + +// contextWindowFixtureWithSecret is contextWindowFixture but with an AWS +// access key planted in the message immediately before the anchor, so tests +// can assert that context enrichment redacts (or reveals) it independently +// of the match's own Snippet redaction. +func contextWindowFixtureWithSecret(sessionID string, anchor int) []db.Message { + msgs := contextWindowFixture(sessionID, anchor) + msgs[1].Content = "my key is AKIA7QHWN2DKR4FYPLJM ok" + return msgs +} + +// TestDirectSearchContentContextRedactsSecretsByDefault verifies that a +// secret-shaped span in a context message (not the match itself) is +// redacted in ContextBefore/ContextAfter when the request does not reveal, +// and left intact when it does. This must hold regardless of transport +// (HTTP, CLI, MCP) since the redaction happens once in +// directBackend.enrichContentContext. +func TestDirectSearchContentContextRedactsSecretsByDefault(t *testing.T) { + t.Parallel() + const sess = "s1" + newStore := func() *fakeContentStore { + return &fakeContentStore{ + page: db.ContentSearchPage{ + Matches: []db.ContentMatch{{SessionID: sess, Ordinal: 5, Snippet: "match one"}}, + }, + windows: map[string][]db.Message{ + contextWindowKey(sess, 5): contextWindowFixtureWithSecret(sess, 5), + }, + } + } + + redacted := service.NewReadOnlyBackend(newStore()) + res, err := redacted.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", Context: 2, + }) + require.NoError(t, err) + require.Len(t, res.Matches, 1) + require.Len(t, res.Matches[0].ContextBefore, 2) + assert.False(t, + strings.Contains(res.Matches[0].ContextBefore[1].Content, "AKIA7QHWN2DKR4FYPLJM"), + "default (Reveal=false) must redact a secret in a context message: %q", + res.Matches[0].ContextBefore[1].Content) + + revealed := service.NewReadOnlyBackend(newStore()) + rev, err := revealed.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", Context: 2, Reveal: true, + }) + require.NoError(t, err) + require.Len(t, rev.Matches, 1) + require.Len(t, rev.Matches[0].ContextBefore, 2) + assert.True(t, + strings.Contains(rev.Matches[0].ContextBefore[1].Content, "AKIA7QHWN2DKR4FYPLJM"), + "Reveal=true must leave a context message's secret intact: %q", + rev.Matches[0].ContextBefore[1].Content) +} + +// TestDirectSearchContentContextRedactsToolPayloads verifies that a secret +// carried in a context message's tool call payloads (input_json, +// result_content, and a result event's content) is also redacted by +// default, not just the message's own Content field. +func TestDirectSearchContentContextRedactsToolPayloads(t *testing.T) { + t.Parallel() + const sess = "s1" + secret := "AKIA7QHWN2DKR4FYPLJM" + store := &fakeContentStore{ + page: db.ContentSearchPage{ + Matches: []db.ContentMatch{{SessionID: sess, Ordinal: 5, Snippet: "match one"}}, + }, + windows: map[string][]db.Message{ + contextWindowKey(sess, 5): { + { + SessionID: sess, Ordinal: 4, Role: "assistant", + Content: "calling a tool", + ToolCalls: []db.ToolCall{{ + ToolName: "bash", + InputJSON: fmt.Sprintf(`{"cmd":"export KEY=%s"}`, secret), + ResultContent: "key is " + secret, + ResultEvents: []db.ToolResultEvent{ + {Source: "stdout", Content: "leaked: " + secret}, + }, + }}, + }, + {SessionID: sess, Ordinal: 5, Role: "user", Content: "anchor"}, + {SessionID: sess, Ordinal: 6, Role: "assistant", Content: "after"}, + }, + }, + } + be := service.NewReadOnlyBackend(store) + + res, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", Context: 2, + }) + require.NoError(t, err) + require.Len(t, res.Matches, 1) + require.Len(t, res.Matches[0].ContextBefore, 1) + tc := res.Matches[0].ContextBefore[0].ToolCalls + require.Len(t, tc, 1) + assert.NotContains(t, tc[0].InputJSON, secret, "tool input_json must be redacted") + assert.NotContains(t, tc[0].ResultContent, secret, "tool result_content must be redacted") + require.Len(t, tc[0].ResultEvents, 1) + assert.NotContains(t, tc[0].ResultEvents[0].Content, secret, + "tool result event content must be redacted") +} + +func TestDirectSearchContentContextSkipsNegativeOrdinal(t *testing.T) { + t.Parallel() + store := &fakeContentStore{ + page: db.ContentSearchPage{ + Matches: []db.ContentMatch{{SessionID: "s1", Ordinal: -1}}, + }, + } + be := service.NewReadOnlyBackend(store) + + res, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "match", Context: 2, + }) + require.NoError(t, err) + require.Len(t, res.Matches, 1) + assert.Nil(t, res.Matches[0].ContextBefore) + assert.Nil(t, res.Matches[0].ContextAfter) +} diff --git a/internal/service/search_test.go b/internal/service/search_test.go index f929ea0e9..4bcd3c5a0 100644 --- a/internal/service/search_test.go +++ b/internal/service/search_test.go @@ -155,3 +155,39 @@ func TestHTTPBackend_Search_Unavailable(t *testing.T) { assert.True(t, errors.Is(err, service.ErrSearchUnavailable), "501 should map to ErrSearchUnavailable, got %v", err) } + +// The direct backend has no VectorSearcher wired into the test DB, so +// semantic content search must surface db.ErrSemanticUnavailable unwrapped +// (errors.Is must still see through it -- no extra wrapping in between). +func TestDirectBackend_SearchContent_SemanticUnavailable(t *testing.T) { + t.Parallel() + d := dbtest.OpenTestDB(t) + be := service.NewDirectBackend(d, nil) + + _, err := be.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "fox", Mode: "semantic", + }) + require.Error(t, err) + assert.True(t, errors.Is(err, service.ErrSemanticUnavailable), + "expected ErrSemanticUnavailable, got %v", err) +} + +// A daemon without a VectorSearcher responds 501 to a semantic content +// search; the HTTP backend maps that to the shared ErrSemanticUnavailable +// sentinel, mirroring the ErrSearchUnavailable mapping above. +func TestHTTPBackend_SearchContent_SemanticUnavailable(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + })) + t.Cleanup(srv.Close) + svc := service.NewHTTPBackend(srv.URL, "", true) + + _, err := svc.SearchContent(context.Background(), service.ContentSearchRequest{ + Pattern: "fox", Mode: "semantic", + }) + require.Error(t, err) + assert.True(t, errors.Is(err, service.ErrSemanticUnavailable), + "501 should map to ErrSemanticUnavailable, got %v", err) +} diff --git a/internal/service/service.go b/internal/service/service.go index 3340f5f94..9f2dfc46b 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -18,6 +18,34 @@ import ( // regardless of transport (the REST handler maps it back to HTTP 501). var ErrSearchUnavailable = errors.New("search not available") +// ErrAroundMutuallyExclusive is returned by Messages when Around is combined +// with From or a non-default Direction: the two retrieval modes (symmetric +// window vs. linear pagination) cannot both be requested. The HTTP handler +// maps it to a 400 response. +var ErrAroundMutuallyExclusive = errors.New( + "around is mutually exclusive with from/direction", +) + +// ErrBeforeAfterRequireAround is returned by Messages when Before or After +// is set without Around. The HTTP handler maps it to a 400 response. +var ErrBeforeAfterRequireAround = errors.New("before/after require around") + +// ErrSemanticUnavailable is returned by SearchContent for modes +// "semantic"/"hybrid" when the backing store has no VectorSearcher wired in. +// It is the same sentinel as db.ErrSemanticUnavailable so direct callers can +// errors.Is it without transport-specific handling; the HTTP backend maps a +// 501 response back to it for daemon-backed callers. +var ErrSemanticUnavailable = db.ErrSemanticUnavailable + +const ( + // SemanticSearchIntentHeader is required on HTTP GET semantic/hybrid + // content searches. It forces browser callers to use an explicit fetch with + // a non-simple header, preventing blind no-CORS cross-origin GETs from + // spending embeddings quota through the local daemon. + SemanticSearchIntentHeader = "X-AgentsView-Search-Intent" + SemanticSearchIntentValue = "semantic" +) + // SessionService is the canonical per-session operation interface. // Two implementations: directBackend (wraps *db.DB) and httpBackend // (proxies to a running daemon). @@ -110,10 +138,13 @@ type SessionSearchResult struct { // ContentSearchRequest is the transport-neutral content-search input. type ContentSearchRequest struct { Pattern string `json:"pattern"` - Mode string `json:"mode,omitempty"` // substring|regex|fts + Mode string `json:"mode,omitempty"` // substring|regex|fts|semantic|hybrid Sources []string `json:"sources,omitempty"` ExcludeSystem bool `json:"exclude_system,omitempty"` Reveal bool `json:"reveal,omitempty"` + // Context requests N messages of inline context before and after each + // match (0 = off, max 10). See directBackend.SearchContent. + Context int `json:"context,omitempty"` Project, ExcludeProject, Machine, Agent string Date, DateFrom, DateTo, ActiveSince string @@ -121,6 +152,11 @@ type ContentSearchRequest struct { // GitBranch is a branchListSep-joined list of opaque (project, branch) tokens (EncodeBranchFilterToken). GitBranch string + // Scope governs semantic/hybrid unit visibility ("top", "all", or + // "subordinate"; "" means "all") and supersedes IncludeChildren in + // those modes. See db.ContentSearchFilter.Scope. + Scope string `json:"scope,omitempty"` + Limit int `json:"limit,omitempty"` Cursor int `json:"cursor,omitempty"` } @@ -238,16 +274,29 @@ type ListFilter struct { // From is a pointer so callers can distinguish "omitted" from "0". An // omitted From in descending mode means "start from the newest message"; // an explicit 0 means "start at ordinal 0". +// +// Around/Before/After select a symmetric window centered on an ordinal +// instead of linear pagination; they are mutually exclusive with +// From/Direction (see directBackend.Messages). Roles filters the result to +// the given roles (empty = all roles) in either mode. type MessageFilter struct { - From *int `json:"from,omitempty"` - Limit int `json:"limit,omitempty"` - Direction string `json:"direction,omitempty"` // "asc" (default) or "desc" + From *int `json:"from,omitempty"` + Limit int `json:"limit,omitempty"` + Direction string `json:"direction,omitempty"` // "asc" (default) or "desc" + Around *int `json:"around,omitempty"` + Before *int `json:"before,omitempty"` // default 5 when Around set + After *int `json:"after,omitempty"` // default 5 when Around set + Roles []string `json:"roles,omitempty"` } -// MessageList mirrors {messages, count}. +// MessageList mirrors {messages, count}. FirstOrdinal/LastOrdinal report the +// returned window's bounds (nil when Messages is empty) so callers can page +// on with from = last_ordinal + 1. type MessageList struct { - Messages []db.Message `json:"messages"` - Count int `json:"count"` + Messages []db.Message `json:"messages"` + Count int `json:"count"` + FirstOrdinal *int `json:"first_ordinal,omitempty"` + LastOrdinal *int `json:"last_ordinal,omitempty"` } // ToolCall mirrors a flattened tool call with its enclosing message's diff --git a/internal/skills/skills.go b/internal/skills/skills.go new file mode 100644 index 000000000..f2e928506 --- /dev/null +++ b/internal/skills/skills.go @@ -0,0 +1,177 @@ +// Package skills renders the AgentsView skill files that teach coding +// agents (Claude Code, Codex, and similar harnesses) how to search the +// AgentsView archive for prior session history. Each harness has its own +// discovery convention (~/.claude/skills, ~/.agents/skills), but shares +// one template body with a harness-specific delegation instruction. +package skills + +import ( + "bytes" + "crypto/sha256" + "embed" + "encoding/hex" + "fmt" + "path/filepath" + "regexp" + "strings" + "text/template" +) + +//go:embed templates/finding-history.md.tmpl +var templatesFS embed.FS + +// Harness identifies a skill discovery convention. +type Harness string + +const ( + HarnessClaude Harness = "claude" // ~/.claude/skills + HarnessAgents Harness = "agents" // ~/.agents/skills (Codex et al.) +) + +// AllHarnesses returns every harness a skill can be rendered for. +func AllHarnesses() []Harness { + return []Harness{HarnessClaude, HarnessAgents} +} + +// skillName is the directory and frontmatter name for the only skill this +// package currently renders. +const skillName = "agentsview-finding-history" + +// delegatePhrases supplies the harness-specific instruction that replaces +// {{.Delegate}} in the template: whether the harness can dispatch a search +// subagent or must run the bounded probes itself. +var delegatePhrases = map[Harness]string{ + HarnessClaude: "Dispatch a search subagent (e.g. the Task/Agent tool)", + HarnessAgents: "Delegate to a search subagent if your harness supports one; " + + "otherwise run the bounded probes yourself in order", +} + +// skillsSubdir is the harness-specific path segment under the install base, +// e.g. ".claude/skills" or ".agents/skills". +var skillsSubdir = map[Harness]string{ + HarnessClaude: filepath.Join(".claude", "skills"), + HarnessAgents: filepath.Join(".agents", "skills"), +} + +// headerFormat is the second line of every rendered file: a YAML comment +// inserted just inside the frontmatter fence, so the file still begins with +// "---" and frontmatter parsers (which require the fence as the first bytes) +// keep discovering the skill. version is recorded for humans; hash is +// authoritative for staleness and tamper detection. +const headerFormat = "# generated-by: agentsview %s hash:%s — do not edit; " + + "re-run `agentsview skills install`" + +// headerPattern extracts the hash recorded in a generated-by header line. +// It must match headerFormat exactly so parsing round-trips. +var headerPattern = regexp.MustCompile( + "^# generated-by: agentsview \\S+ hash:([0-9a-f]{64}) — do not edit; " + + "re-run `agentsview skills install`$", +) + +// frontmatterFence opens every skill file; the template body starts with it +// and Render re-emits it above the generated-by header. +const frontmatterFence = "---\n" + +var tmpl = template.Must(template.ParseFS(templatesFS, "templates/finding-history.md.tmpl")) + +// templateData is the data passed to the finding-history template. +type templateData struct { + Delegate string +} + +// Rendered is one skill file ready to install. +type Rendered struct { + Name string // "agentsview-finding-history" + Content string // full file: frontmatter fence, generated-by header, rest + Hash string // sha256 hex of Content minus the header line +} + +// Render produces the skill for a harness. version is the CLI version +// string, recorded in the header for humans (hash is authoritative). The +// generated-by header is inserted as line two, inside the frontmatter +// fence, so the rendered file still begins with "---". +func Render(h Harness, version string) (Rendered, error) { + delegate, ok := delegatePhrases[h] + if !ok { + return Rendered{}, fmt.Errorf("skills: unknown harness %q", h) + } + + var body bytes.Buffer + data := templateData{Delegate: delegate} + if err := tmpl.ExecuteTemplate(&body, "finding-history.md.tmpl", data); err != nil { + return Rendered{}, fmt.Errorf("skills: render %s template: %w", h, err) + } + if !strings.HasPrefix(body.String(), frontmatterFence) { + return Rendered{}, fmt.Errorf( + "skills: %s template must start with a %q frontmatter fence", h, "---") + } + + hash := bodyHash(body.String()) + header := fmt.Sprintf(headerFormat, version, hash) + content := frontmatterFence + header + "\n" + + strings.TrimPrefix(body.String(), frontmatterFence) + + return Rendered{ + Name: skillName, + Content: content, + Hash: hash, + }, nil +} + +// TargetDir returns the directory the skill installs into for a harness: +// //agentsview-finding-history. base is the +// home dir for user-level installs or the project root for --project. +func TargetDir(h Harness, base string) string { + return filepath.Join(base, skillsSubdir[h], skillName) +} + +// InstalledState classifies an existing file against a fresh render. +type InstalledState int + +const ( + StateMissing InstalledState = iota // no file at the target path + StateCurrent // content == fresh render + StateStale // unmodified generated file, but older render + StateModified // content no longer matches its recorded hash + StateForeign // no generated-by header +) + +// Classify compares an existing file's content against a fresh render. +// existing is the file's current content, or nil if no file exists at the +// target path. It never mutates fresh or existing. +func Classify(existing []byte, fresh Rendered) InstalledState { + if existing == nil { + return StateMissing + } + + content := string(existing) + if !strings.HasPrefix(content, frontmatterFence) { + return StateForeign + } + headerLine, rest, hasRest := strings.Cut( + strings.TrimPrefix(content, frontmatterFence), "\n") + if !hasRest { + rest = "" + } + + match := headerPattern.FindStringSubmatch(headerLine) + if match == nil { + return StateForeign + } + recordedHash := match[1] + + if recordedHash != bodyHash(frontmatterFence+rest) { + return StateModified + } + if recordedHash == fresh.Hash { + return StateCurrent + } + return StateStale +} + +// bodyHash returns the sha256 hex digest of a rendered file's body, i.e. +// its content minus the generated-by header line. +func bodyHash(body string) string { + sum := sha256.Sum256([]byte(body)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/skills/skills_test.go b/internal/skills/skills_test.go new file mode 100644 index 000000000..057d667b4 --- /dev/null +++ b/internal/skills/skills_test.go @@ -0,0 +1,144 @@ +package skills + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// frontmatterField extracts a "key: value" field from a YAML frontmatter +// block. It's a minimal stand-in for a full YAML parse, sufficient to prove +// the rendered frontmatter is well-formed and names the skill. +func frontmatterField(t *testing.T, frontmatter, key string) string { + t.Helper() + for line := range strings.SplitSeq(frontmatter, "\n") { + field, value, ok := strings.Cut(line, ":") + if ok && strings.TrimSpace(field) == key { + return strings.TrimSpace(value) + } + } + t.Fatalf("frontmatter field %q not found in:\n%s", key, frontmatter) + return "" +} + +func TestAllHarnesses(t *testing.T) { + assert.Equal(t, []Harness{HarnessClaude, HarnessAgents}, AllHarnesses()) +} + +func TestRenderProducesParseableFrontmatterAndDelegatePhrase(t *testing.T) { + tests := []struct { + harness Harness + delegate string + }{ + {HarnessClaude, "Dispatch a search subagent (e.g. the Task/Agent tool)"}, + { + HarnessAgents, + "Delegate to a search subagent if your harness supports one; " + + "otherwise run the bounded probes yourself in order", + }, + } + + for _, tt := range tests { + t.Run(string(tt.harness), func(t *testing.T) { + rendered, err := Render(tt.harness, "1.2.3") + require.NoError(t, err) + + assert.Equal(t, "agentsview-finding-history", rendered.Name) + + // Frontmatter parsers require the fence as the first bytes, so + // the generated-by header must sit on line two, inside the fence. + lines := strings.SplitN(rendered.Content, "\n", 3) + require.Len(t, lines, 3, "content must have a fence line, header line, and body") + require.Equal(t, "---", lines[0], "content must start with the frontmatter fence") + headerLine := lines[1] + + // The header hash must equal the hash of the content minus the + // header line (the pure template render). + match := headerPattern.FindStringSubmatch(headerLine) + require.NotNil(t, match, "second line must match the generated-by format: %q", headerLine) + assert.Equal(t, rendered.Hash, match[1]) + sum := sha256.Sum256([]byte("---\n" + lines[2])) + assert.Equal(t, hex.EncodeToString(sum[:]), rendered.Hash) + assert.Contains(t, headerLine, "1.2.3") + + // The frontmatter must be well-formed YAML naming the skill; the + // header is a YAML comment frontmatterField skips over. + parts := strings.SplitN(rendered.Content, "---", 3) + require.Len(t, parts, 3, "content must have exactly two frontmatter fences") + assert.Equal(t, "agentsview-finding-history", frontmatterField(t, parts[1], "name")) + assert.NotEmpty(t, frontmatterField(t, parts[1], "description")) + + assert.Contains(t, rendered.Content, tt.delegate) + }) + } +} + +func TestRenderUnknownHarness(t *testing.T) { + _, err := Render(Harness("bogus"), "1.2.3") + require.Error(t, err) +} + +func TestTargetDir(t *testing.T) { + tests := []struct { + harness Harness + base string + want string + }{ + {HarnessClaude, filepath.Join("home", "user"), + filepath.Join("home", "user", ".claude", "skills", "agentsview-finding-history")}, + {HarnessAgents, filepath.Join("home", "user"), + filepath.Join("home", "user", ".agents", "skills", "agentsview-finding-history")}, + {HarnessClaude, "repo", + filepath.Join("repo", ".claude", "skills", "agentsview-finding-history")}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/%s", tt.harness, tt.base), func(t *testing.T) { + assert.Equal(t, tt.want, TargetDir(tt.harness, tt.base)) + }) + } +} + +func TestClassify(t *testing.T) { + fresh, err := Render(HarnessClaude, "1.2.3") + require.NoError(t, err) + + oldBody := "---\nname: agentsview-finding-history\n---\n\nAn earlier revision of the skill body.\n" + oldHash := bodyHash(oldBody) + oldContent := "---\n" + fmt.Sprintf(headerFormat, "1.0.0", oldHash) + "\n" + + strings.TrimPrefix(oldBody, "---\n") + + tamperedContent := fresh.Content + "\nan uninvited edit\n" + + // The pre-release rendered shape put the header above the fence; those + // files no longer classify as generated. + header, rest, _ := strings.Cut(strings.TrimPrefix(fresh.Content, "---\n"), "\n") + headerFirstContent := header + "\n---\n" + rest + + tests := []struct { + name string + existing []byte + want InstalledState + }{ + {"missing file", nil, StateMissing}, + {"current install", []byte(fresh.Content), StateCurrent}, + {"stale but unmodified install", []byte(oldContent), StateStale}, + {"modified install", []byte(tamperedContent), StateModified}, + {"foreign file with no header", []byte("# Just a regular file\n\nsome text\n"), StateForeign}, + {"fenced file without a generated-by header", []byte("---\nname: x\n---\nbody\n"), StateForeign}, + {"header above the fence", []byte(headerFirstContent), StateForeign}, + {"empty file", []byte(""), StateForeign}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, Classify(tt.existing, fresh)) + }) + } +} diff --git a/internal/skills/templates/finding-history.md.tmpl b/internal/skills/templates/finding-history.md.tmpl new file mode 100644 index 000000000..aececc4e9 --- /dev/null +++ b/internal/skills/templates/finding-history.md.tmpl @@ -0,0 +1,106 @@ +--- +name: agentsview-finding-history +description: Use when asked why a decision was made, how something was done before, or to recover prior instructions, examples, or conversations from recorded agent history — searches the AgentsView archive for evidence. +--- + +# Finding AgentsView History + +Use AgentsView as an evidence source for past agent behavior: find relevant +recorded sessions, inspect the surrounding conversation, and extract +decisions, instructions, or patterns. A snippet is a lead, not an answer. + +## Core Workflow + +{{.Delegate}}. The coordinator decides what evidence it wants, delegates +the archive crawl, then synthesizes and verifies. + +1. Translate the ask into a target behavior and desired evidence: likely + projects, agents, time windows, vocabulary. +2. Concept questions ("why did we…", "how did we handle…") start with + hybrid search; exact identifiers (paths, error strings, tool names) + start with plain or FTS search: + + ```bash + agentsview session search "" --hybrid --context 2 --json --limit 8 + agentsview session search "" --fts --json --limit 8 + ``` + +3. Narrow by recency when the ask implies it: `--since 14d`, `--since 3m` + (h/d/w/m/y units; `m` is months, not minutes). Widen or drop `--since` + if results are thin. +4. Triage from the inline `context_before`/`context_after` in the results. + Every hit, in every mode, carries an `ordinal_range` conversation-unit + span (`[start, end]`; a run of assistant messages when end > start) with + `ordinal` as the anchor message; center the window on the anchor. + Deep-dive only the strongest candidates: + + ```bash + agentsview session messages --around --before 8 --after 8 --role user,assistant --json + ``` + +5. Give the search a mechanical budget: 4-6 probes, top 2-4 sessions, one + window per session, then stop and report. +6. Answer from evidence, citing session IDs and ordinals. Cite the session + plus the hit's `ordinal_range` with the `@ordinal` anchor (e.g. + `#12-40 @19`), in every mode — not just the single anchor ordinal. +7. A hit marked `subordinate` (sidechain or subagent/fork content) is + supporting evidence only: corroborate it from its parent session + (`parent_session_id`) before treating it as a decision or instruction. + +## Mode Fallbacks + +- "semantic search not available" (HTTP 501): embeddings are not set up on + this archive. Fall back to FTS probes — several short queries with + synonyms beat one long phrase: + + ```bash + agentsview session search "" --fts --json --limit 8 + ``` + +- "temporarily unavailable" (HTTP 503): the embeddings endpoint is down. + Retry once; if it persists, say so and continue with FTS — do not + silently downgrade, the user should know their embeddings are broken. + +## Reconstructing a Decision + +1. Find the earliest substantive mention: hybrid query for the decision's + subject with `--scope top` (delegated subagent sessions echo their + parent's instructions and bury the conversation where the decision was + made), starting `--since 3m` and widening (6m, 1y, none) until the + origin appears. +2. Walk forward from the origin with message windows. User messages carry + intent and constraints; assistant messages carry rationale, options + considered, and tradeoffs. +3. Watch for durable artifacts referenced in the conversation (spec or + plan documents, ADRs, PR descriptions) and read those files if they + still exist. +4. Produce a decision record: what was decided, when, by whom, the stated + reasons, alternatives that were rejected, and citations + (`session-id` + ordinals) for each claim. + +## Rationalization Table + +| Rationalization | Reality | +| --- | --- | +| "One good snippet is enough." | Snippets are leads. Inspect a window before extracting a pattern. | +| "A longer query is more semantic." | Hybrid works best with a focused phrase; FTS works best with 2-3 word probes. | +| "Tool output matches are just as good." | Start in messages for concepts; add tool sources only for artifacts, commands, or errors. | +| "This current session mentions it, so it counts." | Down-rank active-session echoes; historical evidence needs older sessions. | +| "A subordinate hit settles it." | Sidechain/subagent hits restate delegated work; confirm against the parent session before citing. | +| "Keep searching until certain." | Return a bounded, evidence-backed pass and list follow-ups. | + +## Output Shape + +```markdown +## Searches +- `query` (mode) -> why it was useful or not + +## Strong Matches +- `` (`project`, `agent`, ordinals N-M): finding and evidence + +## Synthesis +- Decision/pattern grounded in the matches, with citations + +## Gaps / Follow-ups +- What was not found and the next narrower probe +``` diff --git a/internal/sync/engine.go b/internal/sync/engine.go index 9ec9926b7..af2ae8a67 100644 --- a/internal/sync/engine.go +++ b/internal/sync/engine.go @@ -6662,7 +6662,13 @@ func (e *Engine) cachedProjectIdentity(machine, rootPath string) projectIdentity return cached } identity := projectIdentityCacheEntry{rootPath: rootPath} - if e.idPrefix == "" && e.pathRewriter == nil { + // Only probe the local filesystem for sessions recorded on this + // machine: another machine's cwd (e.g. /home/... from a synced Linux + // host) is meaningless here, and on macOS merely stat'ing such paths + // wakes the /home automounter — with tens of thousands of remote + // sessions and a one-minute cache TTL that becomes a sustained + // automountd/opendirectoryd CPU storm. + if e.idPrefix == "" && e.pathRewriter == nil && machine == e.machine { if gitRoot, remotes := discoverLocalGitIdentity(rootPath); gitRoot != "" { identity.rootPath = gitRoot if name, raw, ok := export.SelectRemote(remotes); ok { @@ -6726,6 +6732,12 @@ func discoverLocalGitIdentity(cwd string) (string, map[string]string) { if !safeLocalAbsolutePath(cwd) { return "", nil } + // Skip macOS automounter namespaces: probing them wakes + // automountd/opendirectoryd for paths that virtually never exist + // locally (see export.IsAutomountNamespacePath). + if export.IsAutomountNamespacePath(runtime.GOOS, filepath.Clean(cwd)) { + return "", nil + } resolved, err := filepath.EvalSymlinks(filepath.Clean(cwd)) if err != nil { return "", nil diff --git a/internal/sync/engine_test.go b/internal/sync/engine_test.go index 486d76e49..9d3c49bd1 100644 --- a/internal/sync/engine_test.go +++ b/internal/sync/engine_test.go @@ -1091,6 +1091,46 @@ func TestProjectIdentityObservationCachesLocalGitDiscovery(t *testing.T) { assert.Equal(t, "github.com/acme/cache", observations[0].NormalizedRemote) } +// TestProjectIdentityObservationSkipsDiscoveryForRemoteMachine pins that +// local git discovery never probes the filesystem for a session recorded on +// another machine: the cwd names a path on that machine, so resolving it +// locally is wrong (and on macOS, stat'ing a remote /home/... cwd wakes the +// automounter — see cachedProjectIdentity). The cwd here is a real local git +// repo, so if discovery ran anyway the observation would carry its remote. +func TestProjectIdentityObservationSkipsDiscoveryForRemoteMachine(t *testing.T) { + database := openTestDB(t) + root := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(root, ".git"), 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(root, ".git", "config"), + []byte("[remote \"origin\"]\n\turl = https://github.com/acme/remote.git\n"), + 0o644, + )) + cwd := filepath.Join(root, "subdir") + require.NoError(t, os.Mkdir(cwd, 0o755)) + e := NewEngine(database, EngineConfig{Machine: "laptop"}) + ctx := context.Background() + + require.NoError(t, e.writeProjectIdentityObservation(ctx, db.Session{ + ID: "identity-remote-machine", + Project: "remote-proj", + Machine: "remote-linux", + Agent: "codex", + Cwd: cwd, + StartedAt: strPtr(time.Now().UTC().Format(time.RFC3339Nano)), + })) + + observations, err := database.ListProjectIdentityObservations( + ctx, []string{"remote-proj"}, + ) + require.NoError(t, err) + require.Len(t, observations, 1) + assert.Empty(t, observations[0].GitRemote, + "foreign-machine cwd must not be probed for a local git identity") + assert.Equal(t, cwd, observations[0].RootPath, + "root path must stay the raw cwd, not a locally resolved git root") +} + func TestProjectIdentitySafeLocalAbsolutePathHandlesWindowsDriveRootsByOS(t *testing.T) { wantWindowsDriveLocal := runtime.GOOS == "windows" assert.Equal(t, wantWindowsDriveLocal, safeLocalAbsolutePath(`C:\repo`)) diff --git a/internal/timeutil/timeutil.go b/internal/timeutil/timeutil.go index 3a0ef923d..77fdbdca7 100644 --- a/internal/timeutil/timeutil.go +++ b/internal/timeutil/timeutil.go @@ -1,7 +1,9 @@ package timeutil import ( + "fmt" "os" + "strconv" "time" ) @@ -40,6 +42,46 @@ func IsValidTimestamp(s string) bool { return err == nil } +// ParseSince resolves a --since value against now: relative forms +// "Nh", "Nd", "Nw", "Nm" (months), "Ny", or an absolute YYYY-MM-DD +// (that date's midnight in now's location). N is a positive integer. +// Months/years use now.AddDate(-y, -m, 0) for calendar-aware arithmetic; +// hours/days/weeks use now.Add(-d). +func ParseSince(now time.Time, s string) (time.Time, error) { + if IsValidDate(s) { + return time.ParseInLocation("2006-01-02", s, now.Location()) + } + if len(s) < 2 { + return time.Time{}, sinceFormatError(s) + } + unit := s[len(s)-1] + n, err := strconv.Atoi(s[:len(s)-1]) + if err != nil || n <= 0 { + return time.Time{}, sinceFormatError(s) + } + switch unit { + case 'h': + return now.Add(-time.Duration(n) * time.Hour), nil + case 'd': + return now.Add(-time.Duration(n) * 24 * time.Hour), nil + case 'w': + return now.Add(-time.Duration(n) * 7 * 24 * time.Hour), nil + case 'm': + return now.AddDate(0, -n, 0), nil + case 'y': + return now.AddDate(-n, 0, 0), nil + default: + return time.Time{}, sinceFormatError(s) + } +} + +// sinceFormatError names the accepted --since forms in the error message so +// callers can react without consulting docs. +func sinceFormatError(s string) error { + return fmt.Errorf( + "invalid --since %q: use Nh, Nd, Nw, Nm, Ny, or YYYY-MM-DD", s) +} + func BestEffortLocalTimezone() string { return bestEffortLocalTimezone( os.Getenv("TZ"), diff --git a/internal/timeutil/timeutil_test.go b/internal/timeutil/timeutil_test.go index 97fbeacb0..69fc627d9 100644 --- a/internal/timeutil/timeutil_test.go +++ b/internal/timeutil/timeutil_test.go @@ -104,6 +104,89 @@ func TestIsValidTimestamp(t *testing.T) { } } +func TestParseSince(t *testing.T) { + now := time.Date(2026, 3, 15, 10, 0, 0, 0, time.UTC) + + tests := []struct { + name string + in string + want time.Time + }{ + {"hours", "3h", now.Add(-3 * time.Hour)}, + {"days", "14d", now.Add(-14 * 24 * time.Hour)}, + {"weeks", "2w", now.Add(-14 * 24 * time.Hour)}, + {"months", "3m", now.AddDate(0, -3, 0)}, + {"years", "1y", now.AddDate(-1, 0, 0)}, + {"single digit unit", "1d", now.Add(-24 * time.Hour)}, + { + "absolute date is midnight in now's location", + "2026-01-01", + time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseSince(now, tt.in) + require.NoError(t, err) + assert.True(t, tt.want.Equal(got), + "ParseSince(%q) = %v, want %v", tt.in, got, tt.want) + }) + } +} + +// TestParseSince_MonthArithmeticCrossesYearBoundary verifies month/year units +// use calendar-aware AddDate rather than a fixed-duration approximation, so +// "2m" from mid-January lands in the previous November/year rather than a +// rough 60-day subtraction. +func TestParseSince_MonthArithmeticCrossesYearBoundary(t *testing.T) { + now := time.Date(2026, 1, 15, 8, 30, 0, 0, time.UTC) + got, err := ParseSince(now, "2m") + require.NoError(t, err) + want := time.Date(2025, 11, 15, 8, 30, 0, 0, time.UTC) + assert.True(t, want.Equal(got), "got %v, want %v", got, want) +} + +// TestParseSince_AbsoluteDateUsesNowsLocation verifies the YYYY-MM-DD form +// resolves to that date's midnight in now's location rather than always UTC. +func TestParseSince_AbsoluteDateUsesNowsLocation(t *testing.T) { + ny, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + now := time.Date(2026, 3, 15, 10, 0, 0, 0, ny) + + got, err := ParseSince(now, "2026-01-01") + require.NoError(t, err) + want := time.Date(2026, 1, 1, 0, 0, 0, 0, ny) + assert.True(t, want.Equal(got), "got %v, want %v", got, want) + assert.Equal(t, ny.String(), got.Location().String()) +} + +func TestParseSince_RejectsInvalidForms(t *testing.T) { + now := time.Date(2026, 3, 15, 10, 0, 0, 0, time.UTC) + + tests := []struct { + name string + in string + }{ + {"empty string", ""}, + {"unknown unit", "3x"}, + {"unit before number", "m3"}, + {"negative number", "-3d"}, + {"zero is not positive", "0d"}, + {"unit only", "d"}, + {"double unit", "3dd"}, + {"decimal number", "3.5d"}, + {"trailing space", "3d "}, + {"leading space", " 3d"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseSince(now, tt.in) + require.Error(t, err) + assert.Contains(t, err.Error(), tt.in) + }) + } +} + func TestBestEffortLocalTimezone(t *testing.T) { la, err := time.LoadLocation("America/Los_Angeles") require.NoError(t, err) diff --git a/internal/vector/build.go b/internal/vector/build.go new file mode 100644 index 000000000..409e395ca --- /dev/null +++ b/internal/vector/build.go @@ -0,0 +1,467 @@ +package vector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "sync" + "sync/atomic" + "time" + + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// progressInterval bounds how often BuildOptions.Progress is invoked during +// an embedding pass. A final call always fires once Fill returns, +// regardless of this interval, so callers see a Done total that matches +// the completed (or aborted) run. Tests lower it to 0 for determinism. +var progressInterval = 2 * time.Second + +// chunksTable is the kit-managed table mapping each embedded chunk to its +// vec0 row, scoped by generation ordinal and doc_key. +const chunksTable = vectorsPrefix + "_chunks" + +// BuildOptions configures one Build pass. +type BuildOptions struct { + // FullRebuild forces every document to be re-embedded under the target + // generation's fingerprint, even if it is already the active one. + FullRebuild bool + // Backstop forces a full mirror reconciliation scan (ignoring the + // refresh watermark) without forcing a re-embed. + Backstop bool + // IncludeAutomated controls whether automated sessions' units are + // scanned into the mirror at all (see UnitSource.ScanEmbeddableUnits). + // It is part of the mirror's identity: Build compares it against the + // scope the mirror was last refreshed under (vector_meta) and forces a + // full reconciliation scan on any change, so now-out-of-scope rows (and + // their vectors) are removed and newly-in-scope sessions older than the + // refresh watermark are picked up. It does not force a re-embed of + // documents that stay in scope. + IncludeAutomated bool + // BatchSize is the encode batch size (config batch_size). + BatchSize int + // Concurrency is the number of documents encoded in parallel (config + // concurrency). Values <= 0 encode sequentially. + Concurrency int + // Progress, if non-nil, is called at most ~every 2s with incremental + // embedding progress, plus once more after the run completes. + Progress func(BuildProgress) +} + +// BuildProgress reports incremental embedding progress during Build. Done +// and Total are both counted in chunks (not documents): kit's Fill hands the +// wrapped encoder each document's chunks, sometimes split across several +// sub-batches when BatchSize is smaller than a document's chunk count, so +// there is no seam to count completed documents from the encoder wrapper +// alone. Counting chunks on both sides keeps the percentage bounded at 100% +// regardless of how many chunks a message splits into. +type BuildProgress struct { + Phase string // "scanning" | "embedding" + Done int64 // chunks encoded so far + Total int64 // pending chunks at start (approximate denominator) +} + +// BuildResult summarizes one Build call. +type BuildResult struct { + Fingerprint string + Activated bool // building generation auto-activated on completion + Refresh RefreshStats + Fill kitvec.FillStats +} + +// Build runs one embedding pass against gen (the desired vector space, from +// config: Model, Dimensions, and the fingerprinted Params — max_input_chars, +// doc_unit_scheme, and chunk_overlap_chars; see vectorGeneration in +// cmd/agentsview/embeddings.go). It +// refreshes the vector_messages mirror, resolves which generation to fill +// (top-up the active one, start a new building generation, or reset and +// refill the active one for FullRebuild), fills pending documents, and +// auto-activates a building generation once it fully covers the mirror. +func (ix *Index) Build( + ctx context.Context, src UnitSource, enc kitvec.EncodeFunc, + gen kitvec.Generation, o BuildOptions, +) (BuildResult, error) { + if err := ix.requireWritable(); err != nil { + return BuildResult{}, err + } + + firstEver, err := ix.noWatermarkYet(ctx) + if err != nil { + return BuildResult{}, err + } + storedScope, hasScope, err := ix.storedIncludeAutomatedScope(ctx) + if err != nil { + return BuildResult{}, err + } + // A mirror refreshed before the scope key existed (a refresh watermark + // is stamped, but scope_include_automated is not) predates this scope + // feature entirely. Treat that the same as a genuine scope change: force + // one full reconciliation now so any automated rows that were never + // meant to be in scope (or, if the config default is true, newly + // in-scope automated sessions older than the watermark) get resolved, + // then setIncludeAutomatedScope below stamps the key so every later + // build compares against a real stored scope again. + scopeChanged := !hasScope || storedScope != o.IncludeAutomated + full := o.FullRebuild || o.Backstop || firstEver || scopeChanged + refreshStats, err := ix.Refresh(ctx, src, full, o.IncludeAutomated) + if err != nil { + return BuildResult{}, err + } + if err := ix.setIncludeAutomatedScope(ctx, o.IncludeAutomated); err != nil { + return BuildResult{}, err + } + + fp := gen.Fingerprint() + target, wasBuilding, err := ix.resolveBuildTarget(ctx, gen, fp, o.FullRebuild) + if err != nil { + return BuildResult{}, err + } + + total, err := ix.countPending(ctx, target) + if err != nil { + return BuildResult{}, err + } + + wrapped, finish := ix.wrapProgress(enc, total, o.Progress) + fillStats, fillErr := kitvec.Fill[string, string](ctx, ix.store, target, wrapped, kitvec.FillOptions[string]{ + Split: ix.split, + Batch: kitvec.BatchOptions{BatchSize: o.BatchSize, Concurrency: 1}, + Concurrency: o.Concurrency, + OnEncodeError: skipPermanentEncodeError, + }) + finish() + result := BuildResult{Fingerprint: target, Refresh: refreshStats, Fill: fillStats} + if fillErr != nil { + return result, fillErr + } + + activated, err := ix.maybeActivate(ctx, target, wasBuilding) + if err != nil { + return result, err + } + result.Activated = activated + return result, nil +} + +// skipPermanentEncodeError implements kitvec.FillOptions.OnEncodeError: a +// document the embeddings endpoint permanently rejects for input-specific +// reasons (e.g. a token-window overflow, whitespace-only content some servers +// refuse, or a content-policy rejection) is skipped — kit stamps it for the +// generation with no vectors so it stops being pending — instead of aborting +// the whole fill. Without this, one poison document would wedge every future +// build at the same doc_key-ordered scan position: later documents would never +// embed, a first build would never reach Missing==0, and auto-activation would +// never fire. +// +// Every other failure (5xx, network, timeout, 429 rate-limiting, auth, route, +// model, media-type, or other config/API failures) still aborts the fill, since +// the next scheduled build should retry the document rather than silently +// giving up on it. +func skipPermanentEncodeError(doc string, err error) bool { + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) || !statusErr.Permanent() { + return false + } + log.Printf("vector build: skipping document %s: permanently rejected by embeddings endpoint: %v", + doc, err) + return true +} + +// noWatermarkYet reports whether Refresh has never advanced the stored +// refresh watermark, i.e. this would be the mirror's first scan. +func (ix *Index) noWatermarkYet(ctx context.Context) (bool, error) { + watermark, err := ix.refreshWatermark(ctx) + if err != nil { + return false, err + } + return watermark == "", nil +} + +// resolveBuildTarget decides which generation fingerprint Build should fill +// and whether it is a newly (or still) building generation that should be +// auto-activated once it fully covers the mirror. See the package's build +// brief for the exact decision table. +// +// FullRebuild resets the target generation whenever it already exists — +// active, building, or retired — not only when it happens to be the active +// one: a fingerprint that already exists as a retired (or still building) +// generation carries stamps and vectors from its earlier life, and without +// a reset EnsureGeneration would reuse them, letting Fill skip every +// document and silently reactivate stale embeddings instead of performing +// the requested full rebuild. +func (ix *Index) resolveBuildTarget( + ctx context.Context, gen kitvec.Generation, fp string, fullRebuild bool, +) (target string, wasBuilding bool, err error) { + active, hasActive, err := ix.ActiveFingerprint(ctx) + if err != nil { + return "", false, err + } + + if hasActive && active == fp { + if fullRebuild { + if err := ix.resetGeneration(ctx, fp); err != nil { + return "", false, err + } + } + // fp is already the active generation, so anything else still in + // state building was abandoned by an earlier failed first build + // (the config since reverted back to this active fingerprint) and + // would otherwise stay building forever: this is the only path + // through resolveBuildTarget that reaches an active fp without + // going through EnsureGeneration, which is where the abandoned-gen + // retirement normally happens. + if err := ix.retireAbandonedBuildingGenerations(ctx, fp); err != nil { + return "", false, err + } + return fp, false, nil + } + + existed, err := ix.generationExists(ctx, fp) + if err != nil { + return "", false, err + } + + target, err = ix.EnsureGeneration(ctx, gen, sqlitevec.StateBuilding) + if err != nil { + return "", false, err + } + if err := ix.retireAbandonedBuildingGenerations(ctx, target); err != nil { + return "", false, err + } + if fullRebuild && existed { + if err := ix.resetGeneration(ctx, target); err != nil { + return "", false, err + } + } + return target, true, nil +} + +// retireAbandonedBuildingGenerations transitions every generation still in +// state building other than keep to retired. A generation is abandoned +// when the embedding config (model, dimensions, or params) changes mid +// first-build: resolveBuildTarget starts a fresh building generation under +// the new fingerprint, but the old one never got the chance to activate or +// retire itself and would otherwise stay in state building forever — +// fingerprintByState's ORDER BY ordinal LIMIT 1 could then report the +// abandoned generation's stale coverage as BuildingError's percent instead +// of the generation actually being built. +// +// This only changes state; kit's store has no API to drop a generation's +// vec0 table, chunk map, or stamps, so an abandoned generation's rows stay +// on disk (bloating vectors.db) until an operator rebuilds vectors.db from +// scratch or a future kit API adds reclamation. +func (ix *Index) retireAbandonedBuildingGenerations(ctx context.Context, keep string) error { + if _, err := ix.db.ExecContext(ctx, + `UPDATE `+generationsTable+` SET state = ? WHERE state = ? AND gen_key != ?`, + string(sqlitevec.StateRetired), string(sqlitevec.StateBuilding), keep, + ); err != nil { + return fmt.Errorf("retire abandoned building generations: %w", err) + } + return nil +} + +// generationExists reports whether a generation with fingerprint fp has +// already been registered, in any state. +func (ix *Index) generationExists(ctx context.Context, fp string) (bool, error) { + var ordinal int64 + err := ix.db.QueryRowContext(ctx, + `SELECT ordinal FROM `+generationsTable+` WHERE gen_key = ?`, fp, + ).Scan(&ordinal) + if err == sql.ErrNoRows { + return false, nil + } + if err != nil { + return false, fmt.Errorf("check generation exists for fingerprint %s: %w", fp, err) + } + return true, nil +} + +// countPending returns the total number of chunks the documents not yet +// stamped at their current content_hash (for fp's generation) would produce +// under the index's split configuration — the denominator BuildProgress.Total +// reports. It counts chunks rather than documents so the denominator stays +// in the same unit as BuildProgress.Done (chunks encoded so far); see +// BuildProgress's doc comment for why a per-document count isn't reachable +// from the encoder wrapper. It applies the same s.revision = d.content_hash +// predicate generationCoverageQuery's Missing column uses, so a document +// whose content changed since it was last stamped (a stale revision) counts +// as pending rather than complete — kit's Fill treats it as pending re-embed +// for the same reason. +func (ix *Index) countPending(ctx context.Context, fp string) (int64, error) { + ordinal, err := ix.ordinalForFingerprint(ctx, fp) + if err != nil { + return 0, err + } + rows, err := ix.db.QueryContext(ctx, ` +SELECT content FROM vector_messages d WHERE NOT EXISTS ( + SELECT 1 FROM `+stampsTable+` s WHERE s.ordinal = ? AND s.doc_key = d.doc_key + AND s.revision = d.content_hash)`, + ordinal, + ) + if err != nil { + return 0, fmt.Errorf("count pending documents: %w", err) + } + defer rows.Close() + + var total int64 + for rows.Next() { + var content string + if err := rows.Scan(&content); err != nil { + return 0, fmt.Errorf("scanning pending document content: %w", err) + } + total += int64(len(kitvec.Split(content, ix.split))) + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("iterating pending documents: %w", err) + } + return total, nil +} + +// ordinalForFingerprint looks up a generation's generations-table ordinal +// from its fingerprint (kit's store uses the fingerprint as gen_key). +func (ix *Index) ordinalForFingerprint(ctx context.Context, fp string) (int64, error) { + var ordinal int64 + if err := ix.db.QueryRowContext(ctx, + `SELECT ordinal FROM `+generationsTable+` WHERE gen_key = ?`, fp, + ).Scan(&ordinal); err != nil { + return 0, fmt.Errorf("lookup generation ordinal for fingerprint %s: %w", fp, err) + } + return ordinal, nil +} + +// resetGeneration clears fp's generation of all embedded state (its vec0 +// vectors, chunk map, and stamps) in a single transaction, so a subsequent +// Fill call re-embeds every document from scratch. It leaves the +// generation row itself (and its state) untouched. +func (ix *Index) resetGeneration(ctx context.Context, fp string) error { + tx, err := ix.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin reset generation: %w", err) + } + defer func() { _ = tx.Rollback() }() + + var ordinal int64 + if err := tx.QueryRowContext(ctx, + `SELECT ordinal FROM `+generationsTable+` WHERE gen_key = ?`, fp, + ).Scan(&ordinal); err != nil { + return fmt.Errorf("lookup generation ordinal for fingerprint %s: %w", fp, err) + } + + vecTable := fmt.Sprintf("%s_v%d", vectorsPrefix, ordinal) + if _, err := tx.ExecContext(ctx, `DELETE FROM `+vecTable); err != nil { + return fmt.Errorf("clearing vec0 table: %w", err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM `+chunksTable+` WHERE ordinal = ?`, ordinal); err != nil { + return fmt.Errorf("clearing chunk map: %w", err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM `+stampsTable+` WHERE ordinal = ?`, ordinal); err != nil { + return fmt.Errorf("clearing stamps: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit reset generation: %w", err) + } + return nil +} + +// maybeActivate activates target (retiring the previous active generation) +// when it was a building generation whose fill just brought its coverage +// of the mirror to zero Missing documents. It is a no-op, returning false, +// for the active-generation top-up and full-rebuild-in-place cases, which +// never pass wasBuilding=true. +func (ix *Index) maybeActivate(ctx context.Context, target string, wasBuilding bool) (bool, error) { + if !wasBuilding { + return false, nil + } + ordinal, err := ix.ordinalForFingerprint(ctx, target) + if err != nil { + return false, err + } + info, err := ix.GenerationByID(ctx, ordinal) + if err != nil { + return false, err + } + if info.Missing != 0 { + return false, nil + } + if err := ix.activateGeneration(ctx, target); err != nil { + return false, err + } + return true, nil +} + +// activateGeneration retires whichever generation is currently active +// (other than target, a no-op when there is none) and activates target, +// in one transaction. +func (ix *Index) activateGeneration(ctx context.Context, target string) error { + tx, err := ix.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin activate generation: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if _, err := tx.ExecContext(ctx, + `UPDATE `+generationsTable+` SET state = ? WHERE state = ? AND gen_key != ?`, + string(sqlitevec.StateRetired), string(sqlitevec.StateActive), target, + ); err != nil { + return fmt.Errorf("retire old active generation: %w", err) + } + if _, err := tx.ExecContext(ctx, + `UPDATE `+generationsTable+` SET state = ? WHERE gen_key = ?`, + string(sqlitevec.StateActive), target, + ); err != nil { + return fmt.Errorf("activate generation: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit activate generation: %w", err) + } + return nil +} + +// wrapProgress wraps enc so every successful encode call atomically adds +// its chunk count to a running total and, when onProgress is non-nil, +// reports it at most once per progressInterval. The returned finish func +// always reports the final count once, regardless of the interval, so the +// caller sees a Done total matching the completed (or aborted) run. +func (ix *Index) wrapProgress( + enc kitvec.EncodeFunc, total int64, onProgress func(BuildProgress), +) (kitvec.EncodeFunc, func()) { + if onProgress == nil { + return enc, func() {} + } + + var ( + done atomic.Int64 + mu sync.Mutex + last time.Time + ) + report := func(force bool) { + mu.Lock() + if !force && time.Since(last) < progressInterval { + mu.Unlock() + return + } + last = time.Now() + mu.Unlock() + onProgress(BuildProgress{ + Phase: "embedding", + Done: done.Load(), + Total: total, + }) + } + + wrapped := func(ctx context.Context, texts []string) ([][]float32, error) { + vectors, err := enc(ctx, texts) + if err != nil { + return nil, err + } + done.Add(int64(len(texts))) + report(false) + return vectors, nil + } + return wrapped, func() { report(true) } +} diff --git a/internal/vector/build_test.go b/internal/vector/build_test.go new file mode 100644 index 000000000..d971bb03a --- /dev/null +++ b/internal/vector/build_test.go @@ -0,0 +1,779 @@ +package vector + +import ( + "context" + "fmt" + "net/http" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// fakeBuildEncoder returns a deterministic 3-dimensional encoder that never +// fails, for tests that only care about fill/activation bookkeeping rather +// than the vectors themselves. +func fakeBuildEncoder() kitvec.EncodeFunc { + return func(_ context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } +} + +// twoDocSource returns a fakeUnitSource with two distinct user documents +// in one session, the small corpus most build tests share. +func twoDocSource() *fakeUnitSource { + return &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "u1", 0, "hello"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "u2", 1, "world"), + endedAt: "2024-01-01T00:00:01Z", + }, + }} +} + +func fakeGeneration(model string) kitvec.Generation { + return kitvec.Generation{Model: model, Dimensions: 3} +} + +func TestBuildFirstBuildEmbedsAllAndActivates(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + assert.True(t, result.Activated) + assert.Equal(t, gen.Fingerprint(), result.Fingerprint) + assert.Equal(t, 2, result.Fill.Documents) + + active, ok, err := ix.ActiveFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, gen.Fingerprint(), active) +} + +func TestBuildSecondBuildNoChangesFillsZero(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + assert.Equal(t, 0, result.Fill.Documents) + assert.False(t, result.Activated, "already active, no re-activation") +} + +func TestBuildContentChangeReembedsExactlyOne(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + // A real edit bumps the session's ended_at, so give the changed row a + // newer timestamp than the watermark the first build advanced to, or + // the fake source's incremental scan (mimicking the real one) would + // never resurface it. + src.rows[0].unit.Content = "changed" + src.rows[0].endedAt = "2024-01-02T00:00:00Z" + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + assert.Equal(t, 1, result.Fill.Documents, "only the changed document is re-embedded") + assert.False(t, result.Activated, "target was already active") +} + +func TestBuildModelChangeBuildsSecondGenerationAndRetiresOld(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen1 := fakeGeneration("model-a") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), gen1, BuildOptions{}) + require.NoError(t, err) + + gen2 := fakeGeneration("model-b") + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen2, BuildOptions{}) + require.NoError(t, err) + assert.True(t, result.Activated) + assert.Equal(t, gen2.Fingerprint(), result.Fingerprint) + assert.Equal(t, 2, result.Fill.Documents) + + active, ok, err := ix.ActiveFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, gen2.Fingerprint(), active) + + gens, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, gens, 2) + var oldState string + for _, g := range gens { + if g.Fingerprint == gen1.Fingerprint() { + oldState = g.State + } + } + assert.Equal(t, string(sqlitevec.StateRetired), oldState, "old active generation is retired") +} + +func TestBuildFullRebuildSameFingerprintReembedsEverything(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{FullRebuild: true}) + require.NoError(t, err) + assert.Equal(t, 2, result.Fill.Documents, "full rebuild re-embeds every document") + assert.False(t, result.Activated, "already-active generation stays active without reactivation") + + active, ok, err := ix.ActiveFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, gen.Fingerprint(), active) +} + +// TestBuildFullRebuildRetiredGenerationReembeds covers the resolveBuildTarget +// gap where a FullRebuild request targets a fingerprint that already exists +// as a retired generation (from an earlier model switch): without resetting +// it, EnsureGeneration would reuse its old stamps and Fill would find +// nothing pending, silently reactivating stale embeddings instead of +// performing the requested rebuild. +func TestBuildFullRebuildRetiredGenerationReembeds(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + genA := fakeGeneration("model-a") + genB := fakeGeneration("model-b") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), genA, BuildOptions{}) + require.NoError(t, err) + _, err = ix.Build(ctx, src, fakeBuildEncoder(), genB, BuildOptions{}) + require.NoError(t, err, "genA is now retired, genB active") + + var encodeCalls int + countingEncoder := func(_ context.Context, texts []string) ([][]float32, error) { + encodeCalls++ + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } + + result, err := ix.Build(ctx, src, countingEncoder, genA, BuildOptions{FullRebuild: true}) + require.NoError(t, err) + assert.Equal(t, 2, result.Fill.Documents, + "full rebuild on a retired generation must re-embed every document") + assert.Positive(t, encodeCalls, "encoder must actually be invoked, not skipped") +} + +// TestBuildScopeChangeToIncludeAutomatedForcesFullRefreshAndEmbedsOlderDoc +// covers the interplay between a widening include-automated scope change and +// the refresh watermark: the automated doc is older than the first build's +// watermark (set from the human doc's later ended_at, since the automated +// doc never entered the scan at all under the default scope). Without +// scope-change detection forcing a full (since="") rescan, the second +// build's incremental scan would stay restricted to the stored watermark and +// permanently miss the now-in-scope but chronologically older document. +func TestBuildScopeChangeToIncludeAutomatedForcesFullRefreshAndEmbedsOlderDoc(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + gen := fakeGeneration("fake-model") + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "human", 0, "hello"), + endedAt: "2024-01-02T00:00:00Z", + }, + { + unit: userDoc("s2", "auto", 0, "roborev output"), + endedAt: "2024-01-01T00:00:00Z", // older than the human doc + automated: true, + }, + }} + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + assert.Equal(t, 1, result.Fill.Documents, "the automated doc is excluded by the default scope") + assert.ElementsMatch(t, []string{"u:s1:human"}, mirrorDocKeys(t, ix)) + + result, err = ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{IncludeAutomated: true}) + require.NoError(t, err) + assert.Equal(t, 1, result.Fill.Documents, "only the newly in-scope automated doc is embedded") + assert.ElementsMatch(t, []string{"u:s1:human", "u:s2:auto"}, mirrorDocKeys(t, ix), + "the older automated doc must be picked up despite predating the stored refresh watermark") +} + +// TestBuildScopeChangeToExcludeAutomatedRemovesOutOfScopeMirrorRow covers the +// narrowing direction: reverting to the default scope after building with +// IncludeAutomated: true must reconcile the automated document's mirror row +// (and its vectors) away, without touching the still-in-scope human +// document's existing embedding. +func TestBuildScopeChangeToExcludeAutomatedRemovesOutOfScopeMirrorRow(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + gen := fakeGeneration("fake-model") + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "human", 0, "hello"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s2", "auto", 0, "roborev output"), + endedAt: "2024-01-02T00:00:00Z", + automated: true, + }, + }} + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{IncludeAutomated: true}) + require.NoError(t, err) + assert.Equal(t, 2, result.Fill.Documents) + assert.ElementsMatch(t, []string{"u:s1:human", "u:s2:auto"}, mirrorDocKeys(t, ix)) + + result, err = ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{IncludeAutomated: false}) + require.NoError(t, err) + assert.Equal(t, 0, result.Fill.Documents, + "no embedding work: the human doc was already embedded and stays so") + assert.Equal(t, 1, result.Refresh.Deleted, + "the now-out-of-scope automated row must be reconciled away") + assert.ElementsMatch(t, []string{"u:s1:human"}, mirrorDocKeys(t, ix)) + + _, ok := readMirrorRow(t, ix, "u:s2:auto") + assert.False(t, ok, "the out-of-scope mirror row must be removed") +} + +// TestBuildLegacyMirrorMissingScopeKeyForcesFullRefresh covers a mirror built +// before the include-automated scope feature existed: a refresh watermark is +// already stamped (this is not a first-ever build), but +// scope_include_automated was never written since setIncludeAutomatedScope +// did not exist yet. Without treating that missing key as a scope change, +// Build would run an incremental (since=watermark) scan forever and never +// pick up a document older than the stored watermark, nor would it ever +// reconcile away now-out-of-scope automated rows a legacy mirror might carry. +func TestBuildLegacyMirrorMissingScopeKeyForcesFullRefresh(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + gen := fakeGeneration("fake-model") + + // Simulate the pre-scope-feature mirror state directly: a stamped + // watermark with no scope_include_automated row in vector_meta. + require.NoError(t, ix.setRefreshWatermark(ctx, "2024-06-01T00:00:00Z")) + _, hasScope, err := ix.storedIncludeAutomatedScope(ctx) + require.NoError(t, err) + require.False(t, hasScope, "test setup must not pre-seed a scope key") + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "human", 0, "hello"), + endedAt: "2024-01-01T00:00:00Z", // older than the stored watermark + }, + }} + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + assert.Empty(t, src.gotSince, + "a legacy mirror with no stored scope key must force a full (since=\"\") rescan") + assert.Equal(t, 1, result.Fill.Documents, + "the pre-watermark document must be picked up despite predating the stored refresh watermark") + assert.ElementsMatch(t, []string{"u:s1:human"}, mirrorDocKeys(t, ix)) + + storedScope, hasScope, err := ix.storedIncludeAutomatedScope(ctx) + require.NoError(t, err) + assert.True(t, hasScope, + "Build must stamp the scope key so later builds compare against a real stored value") + assert.False(t, storedScope) +} + +// TestCountPendingIncludesRevisionChangedDocs covers countPending's +// BuildProgress.Total denominator: a document whose mirror content_hash +// changed since it was last stamped must still count as pending, matching +// the s.revision = d.content_hash predicate generationCoverageQuery's +// Missing column uses, or Total under-reports outstanding work. +func TestCountPendingIncludesRevisionChangedDocs(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + fp := gen.Fingerprint() + total, err := ix.countPending(ctx, fp) + require.NoError(t, err) + assert.Zero(t, total, "fully embedded generation has nothing pending") + + // Simulate content changing without a mirror refresh reconciling the + // stamp: the stamp's revision no longer matches content_hash. + _, err = ix.db.ExecContext(ctx, + `UPDATE vector_messages SET content_hash = 'changed-hash' WHERE doc_key = 'u:s1:u1'`) + require.NoError(t, err) + + total, err = ix.countPending(ctx, fp) + require.NoError(t, err) + assert.EqualValues(t, 1, total, "content-changed doc must count as pending, not complete") +} + +// TestCountPendingSumsChunksAcrossMultiChunkDocuments covers the units bug +// where BuildProgress.Total counted pending documents while Done counted +// encoded chunks: a message that splits into several chunks would drive the +// reported percentage past 100%. countPending must sum chunks, matching +// Done's unit, not count the document once. +func TestCountPendingSumsChunksAcrossMultiChunkDocuments(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + longContent := strings.Repeat("word ", 2000) // far past the 4000-rune split threshold + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "u1", 0, "short"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "u2", 1, longContent), + endedAt: "2024-01-01T00:00:01Z", + }, + }} + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + fp := gen.Fingerprint() + + longChunks := len(kitvec.Split(longContent, ix.split)) + require.Greater(t, longChunks, 1, + "content must actually split into multiple chunks for this test to be meaningful") + + // Simulate the long document changing without a mirror refresh + // reconciling the stamp, so it counts as pending again (same technique + // as TestCountPendingIncludesRevisionChangedDocs). + _, err = ix.db.ExecContext(ctx, + `UPDATE vector_messages SET content_hash = 'changed-hash' WHERE doc_key = 'u:s1:u2'`) + require.NoError(t, err) + + total, err := ix.countPending(ctx, fp) + require.NoError(t, err) + assert.EqualValues(t, longChunks, total, + "Total must sum the pending document's chunks, not count it as one document") +} + +func TestBuildProgressReceivesFinalDoneEqualToTotalChunks(t *testing.T) { + previous := progressInterval + progressInterval = 0 + t.Cleanup(func() { progressInterval = previous }) + + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + var calls []BuildProgress + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{ + Progress: func(p BuildProgress) { calls = append(calls, p) }, + }) + require.NoError(t, err) + require.NotEmpty(t, calls) + last := calls[len(calls)-1] + assert.EqualValues(t, result.Fill.Chunks, last.Done) + assert.EqualValues(t, result.Fill.Chunks, last.Total, + "Total must also be in chunks so Done/Total settles at exactly 100%") + assert.Equal(t, "embedding", last.Phase) +} + +// TestBuildProgressNeverExceedsTotalWithMultiChunkMessage covers the +// regression this units fix addresses directly: a message long enough to +// split into several chunks must never push a progress call's Done past its +// Total (which would render as a percentage over 100%). +func TestBuildProgressNeverExceedsTotalWithMultiChunkMessage(t *testing.T) { + previous := progressInterval + progressInterval = 0 + t.Cleanup(func() { progressInterval = previous }) + + ix := openTestIndex(t) + ctx := context.Background() + longContent := strings.Repeat("word ", 2000) + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "u1", 0, "short"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "u2", 1, longContent), + endedAt: "2024-01-01T00:00:01Z", + }, + }} + gen := fakeGeneration("fake-model") + require.Greater(t, len(kitvec.Split(longContent, ix.split)), 1, + "content must actually split into multiple chunks for this test to be meaningful") + + var calls []BuildProgress + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{ + Progress: func(p BuildProgress) { calls = append(calls, p) }, + }) + require.NoError(t, err) + require.NotEmpty(t, calls) + for _, p := range calls { + assert.LessOrEqualf(t, p.Done, p.Total, "progress must never exceed 100%%: %+v", p) + } + last := calls[len(calls)-1] + assert.EqualValues(t, result.Fill.Chunks, last.Done) + assert.EqualValues(t, result.Fill.Chunks, last.Total) +} + +func TestBuildEncoderErrorAbortsAndRetryResumesWithoutReembedding(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "", 0, "one"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "", 1, "bad"), + endedAt: "2024-01-01T00:00:01Z", + }, + { + unit: userDoc("s1", "", 2, "three"), + endedAt: "2024-01-01T00:00:02Z", + }, + }} + gen := fakeGeneration("fake-model") + + failOnBad := func(_ context.Context, texts []string) ([][]float32, error) { + if slices.Contains(texts, "bad") { + return nil, fmt.Errorf("encoder rejected input") + } + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } + + _, err := ix.Build(ctx, src, failOnBad, gen, BuildOptions{}) + require.Error(t, err) + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps`, + ).Scan(&stampCount)) + assert.Equal(t, 1, stampCount, "only the document before the failing one was stamped") + + result, err := ix.Build(ctx, src, fakeBuildEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + assert.Equal(t, 2, result.Fill.Documents, "retry embeds only the two remaining documents") + assert.True(t, result.Activated) +} + +// TestBuildSkipsPermanentlyRejectedDocumentAndContinues is the fix-1 +// regression test: a single document the endpoint permanently rejects +// (400) must not wedge the whole build. kit stamps it without vectors so +// the scan moves past it, later documents still embed, and the generation +// still auto-activates once every document — including the skipped one — +// is stamped. +func TestBuildSkipsPermanentlyRejectedDocumentAndContinues(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "", 0, "one"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "", 1, "poison"), + endedAt: "2024-01-01T00:00:01Z", + }, + { + unit: userDoc("s1", "", 2, "three"), + endedAt: "2024-01-01T00:00:02Z", + }, + }} + gen := fakeGeneration("fake-model") + + var calls int + rejectPoison := func(_ context.Context, texts []string) ([][]float32, error) { + calls++ + if slices.Contains(texts, "poison") { + return nil, &HTTPStatusError{Status: http.StatusBadRequest, Body: "token window overflow"} + } + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } + + result, err := ix.Build(ctx, src, rejectPoison, gen, BuildOptions{}) + require.NoError(t, err, "a permanently-rejected document must not abort the whole build") + assert.Equal(t, 2, result.Fill.Documents, "the two good documents still embed") + assert.Equal(t, 1, result.Fill.Skipped, "the poison document is counted as skipped") + assert.True(t, result.Activated, + "coverage is complete (every document stamped) once the poison doc is stamped-skipped") + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps`, + ).Scan(&stampCount)) + assert.Equal(t, 3, stampCount, "the skipped document is still stamped, just without vectors") + + // kit only re-embeds a skipped document once its content_hash changes; + // a later build over unchanged content must not retry the encoder for + // it at all. + callsBefore := calls + result2, err := ix.Build(ctx, src, rejectPoison, gen, BuildOptions{}) + require.NoError(t, err) + assert.Equal(t, 0, result2.Fill.Documents) + assert.Equal(t, 0, result2.Fill.Skipped) + assert.Equal(t, callsBefore, calls, + "unchanged content must not re-invoke the encoder for the already-skipped document") +} + +// TestBuildConfig404EncodeErrorAbortsAndLeavesDocumentsPending covers a +// config/API failure that applies to every input, such as a wrong embeddings +// route. It must abort rather than stamp-skipping the whole corpus and +// auto-activating an empty generation. +func TestBuildConfig404EncodeErrorAbortsAndLeavesDocumentsPending(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + notFound := func(_ context.Context, _ []string) ([][]float32, error) { + return nil, &HTTPStatusError{Status: http.StatusNotFound, Body: "not found"} + } + + result, err := ix.Build(ctx, src, notFound, gen, BuildOptions{}) + require.Error(t, err) + assert.Equal(t, 0, result.Fill.Documents) + assert.Equal(t, 0, result.Fill.Skipped, + "route/config failures must not stamp-skip every document") + + active, ok, err := ix.ActiveFingerprint(ctx) + require.NoError(t, err) + assert.False(t, ok, "an empty failed generation must not activate") + assert.Empty(t, active) + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps`, + ).Scan(&stampCount)) + assert.Equal(t, 0, stampCount, "failed route/config errors leave documents retryable") + + pending, err := ix.countPending(ctx, gen.Fingerprint()) + require.NoError(t, err) + assert.EqualValues(t, 2, pending, + "later builds must still see the same documents as pending") +} + +func TestBuildSchema400EncodeErrorAbortsAndLeavesDocumentsPending(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + + badSchema := func(_ context.Context, _ []string) ([][]float32, error) { + return nil, &HTTPStatusError{Status: http.StatusBadRequest, Body: "invalid input type"} + } + + result, err := ix.Build(ctx, src, badSchema, gen, BuildOptions{}) + require.Error(t, err) + assert.Equal(t, 0, result.Fill.Documents) + assert.Equal(t, 0, result.Fill.Skipped) + + pending, err := ix.countPending(ctx, gen.Fingerprint()) + require.NoError(t, err) + assert.EqualValues(t, 2, pending) +} + +// TestBuild5xxEncodeErrorStillAbortsFill guards the other side of the OnEncodeError +// wiring: a transient (5xx) failure must still abort the whole fill rather +// than being skipped, since it is likely to succeed on a later retry and +// permanently giving up on the document would lose it from the index. +func TestBuild5xxEncodeErrorStillAbortsFill(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "", 0, "one"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "", 1, "bad"), + endedAt: "2024-01-01T00:00:01Z", + }, + }} + gen := fakeGeneration("fake-model") + + fail500 := func(_ context.Context, texts []string) ([][]float32, error) { + if slices.Contains(texts, "bad") { + return nil, &HTTPStatusError{Status: http.StatusInternalServerError, Body: "boom"} + } + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } + + _, err := ix.Build(ctx, src, fail500, gen, BuildOptions{}) + require.Error(t, err, "a transient (5xx) encode error must still abort the fill") + var statusErr *HTTPStatusError + require.ErrorAs(t, err, &statusErr) + assert.Equal(t, http.StatusInternalServerError, statusErr.Status) +} + +// TestResolveBuildTargetRetiresOtherBuildingGeneration is the fix-3 +// regression test: a generation left in state building by an abandoned +// (config changed mid-build) first build must be retired once a new +// building generation is established, so fingerprintByState's ORDER BY +// ordinal LIMIT 1 lookup (used for both BuildingFingerprint and the +// building-percent shown to a caller with no active generation) resolves +// to the generation actually being built, not the abandoned one. +func TestResolveBuildTargetRetiresOtherBuildingGeneration(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + genA := fakeGeneration("model-a") + fpA, err := ix.EnsureGeneration(ctx, genA, sqlitevec.StateBuilding) + require.NoError(t, err) + + genB := fakeGeneration("model-b") + target, wasBuilding, err := ix.resolveBuildTarget(ctx, genB, genB.Fingerprint(), false) + require.NoError(t, err) + assert.True(t, wasBuilding) + assert.Equal(t, genB.Fingerprint(), target) + + gens, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, gens, 2) + for _, g := range gens { + switch g.Fingerprint { + case fpA: + assert.Equal(t, string(sqlitevec.StateRetired), g.State, + "the abandoned generation must be retired, not left building forever") + case target: + assert.Equal(t, string(sqlitevec.StateBuilding), g.State) + } + } + + building, ok, err := ix.BuildingFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, target, building, + "the building-generation lookup must resolve to B, not the abandoned A") +} + +// TestBuildRetiresAbandonedBuildingGenerationEndToEnd drives the same +// scenario through the public Build entry point: an interrupted first +// build (genA registered as building but never filled, standing in for a +// crashed process) followed by a full Build under a different config +// (genB) must retire genA rather than leave two generations in state +// building. +func TestBuildRetiresAbandonedBuildingGenerationEndToEnd(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + + genA := fakeGeneration("model-a") + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + fpA, err := ix.EnsureGeneration(ctx, genA, sqlitevec.StateBuilding) + require.NoError(t, err) + + genB := fakeGeneration("model-b") + result, err := ix.Build(ctx, src, fakeBuildEncoder(), genB, BuildOptions{}) + require.NoError(t, err) + assert.True(t, result.Activated) + + gens, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, gens, 2) + var stateA, stateB string + for _, g := range gens { + switch g.Fingerprint { + case fpA: + stateA = g.State + case genB.Fingerprint(): + stateB = g.State + } + } + assert.Equal(t, string(sqlitevec.StateRetired), stateA, "abandoned generation A must be retired") + assert.Equal(t, string(sqlitevec.StateActive), stateB) + + _, ok, err := ix.BuildingFingerprint(ctx) + require.NoError(t, err) + assert.False(t, ok, "no generation should remain in state building") +} + +// TestBuildActiveFingerprintEarlyReturnRetiresAbandonedBuildingGeneration +// covers a gap in resolveBuildTarget's active-fingerprint early return: when +// the requested generation is already active, the target-resolution path +// never reaches EnsureGeneration, which is the only other place that retires +// abandoned building generations. Without also retiring on this path, a +// first build of some other fingerprint that registered as building and +// then failed (a crashed process, or config that got reverted back to the +// still-active fingerprint before the failed build could be retried) stays +// in state building forever once every subsequent build targets the active +// generation again. +func TestBuildActiveFingerprintEarlyReturnRetiresAbandonedBuildingGeneration(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + genA := fakeGeneration("model-a") + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), genA, BuildOptions{}) + require.NoError(t, err, "genA is now active") + + // Simulate an abandoned first build of a different config: registered as + // building, but never filled or retired (standing in for a crashed + // process, or a config change that was reverted before the failed build + // could be cleaned up). + genB := fakeGeneration("model-b") + fpB, err := ix.EnsureGeneration(ctx, genB, sqlitevec.StateBuilding) + require.NoError(t, err) + + // Build again with genA, the still-active fingerprint: this must take + // the active-fingerprint early-return path in resolveBuildTarget, not + // the EnsureGeneration path. + result, err := ix.Build(ctx, src, fakeBuildEncoder(), genA, BuildOptions{}) + require.NoError(t, err) + assert.False(t, result.Activated, "already-active generation stays active without reactivation") + + gens, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, gens, 2) + for _, g := range gens { + if g.Fingerprint == fpB { + assert.Equal(t, string(sqlitevec.StateRetired), g.State, + "the abandoned building generation must be retired once a build "+ + "resolves back to the active fingerprint") + } + } + + _, ok, err := ix.BuildingFingerprint(ctx) + require.NoError(t, err) + assert.False(t, ok, "no generation should remain in state building") +} diff --git a/internal/vector/chunk_test.go b/internal/vector/chunk_test.go new file mode 100644 index 000000000..83f8d3ab7 --- /dev/null +++ b/internal/vector/chunk_test.go @@ -0,0 +1,175 @@ +package vector + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// forceIndexVarLimit pins ix's underlying vectors.db connection pool to a +// single connection and lowers its SQLITE_LIMIT_VARIABLE_NUMBER, mirroring +// internal/db's forceReaderVarLimit: some SQLite builds compile against the +// older documented 999-variable limit rather than the modern default +// (32766), and a test seeding a key count above maxSQLVars only genuinely +// regression-guards chunkKeys's chunking when the connection's real limit is +// low enough for that chunking to matter — otherwise a single unchunked IN +// (...) query would still succeed under the modern default and the test +// would pass even with chunking deleted. The driver-specific limit call +// lives in setConnVarLimit (varlimit_cgo_test.go / varlimit_modernc_test.go), +// matching the per-platform driver split in driver_cgo.go / driver_modernc.go. +func forceIndexVarLimit(t *testing.T, ix *Index, limit int) { + t.Helper() + ix.db.SetMaxOpenConns(1) + ix.db.SetMaxIdleConns(1) + conn, err := ix.db.Conn(context.Background()) + require.NoError(t, err) + defer func() { require.NoError(t, conn.Close()) }() + setConnVarLimit(t, conn, limit) +} + +// requireIndexVarLimitConstrained probes ix's connection with an +// over-the-limit IN (...) query, failing the test if it does not error -- +// proof the lowered limit from forceIndexVarLimit is actually live, so a +// setup bug cannot silently mask the regression the caller checks next. +func requireIndexVarLimitConstrained(t *testing.T, ix *Index) { + t.Helper() + ctx := context.Background() + overLimitPh, overLimitArgs := inPlaceholders(make([]string, 1001)) + _, probeErr := ix.db.QueryContext(ctx, "SELECT 1 WHERE '' IN "+overLimitPh, overLimitArgs...) + require.Error(t, probeErr, "index variable limit was not constrained") +} + +// TestChunkKeysSplitsAtMaxSQLVars asserts chunkKeys never hands fn more than +// maxSQLVars keys at a time, that every key is visited exactly once, and +// that a non-multiple-of-maxSQLVars input yields a shorter final chunk +// rather than an empty trailing one. +func TestChunkKeysSplitsAtMaxSQLVars(t *testing.T) { + total := maxSQLVars*2 + 137 + keys := make([]string, total) + for i := range keys { + keys[i] = fmt.Sprintf("key-%d", i) + } + + var chunkSizes []int + seen := make(map[string]int, total) + err := chunkKeys(keys, func(chunk []string) error { + chunkSizes = append(chunkSizes, len(chunk)) + for _, k := range chunk { + seen[k]++ + } + return nil + }) + require.NoError(t, err) + + require.Len(t, chunkSizes, 3) + assert.Equal(t, []int{maxSQLVars, maxSQLVars, 137}, chunkSizes) + assert.Len(t, seen, total, "every key must be visited") + for _, k := range keys { + assert.Equal(t, 1, seen[k], "key %s must be visited exactly once", k) + } +} + +// TestChunkKeysEmptyInputInvokesNothing asserts an empty key slice never +// calls fn. +func TestChunkKeysEmptyInputInvokesNothing(t *testing.T) { + calls := 0 + err := chunkKeys(nil, func([]string) error { + calls++ + return nil + }) + require.NoError(t, err) + assert.Zero(t, calls) +} + +// seedVectorMessages bulk-inserts n vector_messages rows with distinct +// doc_key/session_id/ordinal/content/content_hash values, inside one +// transaction so a large n (well past SQLite's 999-bind-variable limit) +// stays fast. +func seedVectorMessages(t *testing.T, ix *Index, n int) []string { + t.Helper() + ctx := context.Background() + tx, err := ix.db.BeginTx(ctx, nil) + require.NoError(t, err) + + keys := make([]string, n) + for i := range n { + key := fmt.Sprintf("d%d", i) + keys[i] = key + _, err := tx.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?)`, + key, fmt.Sprintf("s%d", i), i, i, fmt.Sprintf("content %d", i), fmt.Sprintf("h%d", i)) + require.NoError(t, err) + } + require.NoError(t, tx.Commit()) + return keys +} + +// TestLookupMirrorDocsOverMaxSQLVars asserts lookupMirrorDocs resolves every +// doc_key when the requested key count exceeds SQLite's 999-bind-variable +// limit (and this package's maxSQLVars chunk size), which a deep semantic +// overfetch (limit * over-fetch factor, in the low thousands) can trigger in +// a single Search call. +func TestLookupMirrorDocsOverMaxSQLVars(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + forceIndexVarLimit(t, ix, 999) + requireIndexVarLimitConstrained(t, ix) + + n := maxSQLVars*3 + 42 + keys := seedVectorMessages(t, ix, n) + + docs, err := ix.lookupMirrorDocs(ctx, keys) + require.NoError(t, err) + + require.Len(t, docs, n) + for i, key := range keys { + doc, ok := docs[key] + require.True(t, ok, "doc_key %s missing from result", key) + assert.Equal(t, fmt.Sprintf("s%d", i), doc.sessionID) + assert.Equal(t, i, doc.ordinal) + assert.Equal(t, fmt.Sprintf("content %d", i), doc.content) + } +} + +// TestLookupMirrorDocsMissingKeyOmittedNotZeroValued asserts a doc_key with +// no matching row is simply absent from the result map even when it is +// mixed into a chunk of thousands of keys that do resolve. +func TestLookupMirrorDocsMissingKeyOmittedNotZeroValued(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + keys := seedVectorMessages(t, ix, maxSQLVars+10) + keys = append(keys, "does-not-exist") + + docs, err := ix.lookupMirrorDocs(ctx, keys) + require.NoError(t, err) + + _, ok := docs["does-not-exist"] + assert.False(t, ok) + assert.Len(t, docs, maxSQLVars+10) +} + +// TestCurrentOrdinalsOverMaxSQLVars asserts currentOrdinals resolves every +// key's ordinal when the key count exceeds SQLite's 999-bind-variable limit, +// which a pathological refresh with a large same-scan eviction batch could +// trigger. +func TestCurrentOrdinalsOverMaxSQLVars(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + forceIndexVarLimit(t, ix, 999) + requireIndexVarLimitConstrained(t, ix) + + n := maxSQLVars*3 + 42 + keys := seedVectorMessages(t, ix, n) + + ordinals, err := ix.currentOrdinals(ctx, keys) + require.NoError(t, err) + + require.Len(t, ordinals, n) + for i, key := range keys { + assert.Equal(t, i, ordinals[key]) + } +} diff --git a/internal/vector/driver_cgo.go b/internal/vector/driver_cgo.go new file mode 100644 index 000000000..82f508e26 --- /dev/null +++ b/internal/vector/driver_cgo.go @@ -0,0 +1,46 @@ +//go:build !windows && cgo + +package vector + +import ( + "net/url" + + _ "github.com/mattn/go-sqlite3" +) + +// vectorDriverName selects the database/sql driver vectors.db opens with. +// On Unix with cgo it is mattn/go-sqlite3 — the same driver as the main +// archive — with the cgo sqlite-vec extension kit's sqlitevec.Register +// loads. On Windows the cgo sqlite-vec bindings do not build, so +// driver_modernc.go substitutes the pure-Go modernc driver instead. +const vectorDriverName = "sqlite3" + +// vectorDSN builds the sqlite3 DSN for vectors.db. The rw path mirrors +// internal/db.Open's pragmas (WAL, busy timeout, NORMAL synchronous); the +// ro path opens with mode=ro and no immutable hint, since vectors.db can be +// concurrently rewritten by another agentsview process, and carries its own +// busy timeout so a reader waits out a concurrent writer's lock instead of +// failing immediately with SQLITE_BUSY. +// +// Both branches emit a file: URI. mattn/go-sqlite3 forwards the `_`-prefixed +// pragma params either way, but it only honors mode=ro when the DSN carries +// the file: scheme — a bare path silently opens read-write, so the ro +// contract depends on the prefix. +// +// The path component is percent-encoded (slashes kept intact): SQLite +// percent-decodes URI paths and splits params at `?`, so a raw path +// containing `%`, `?`, or `#` would be misparsed — e.g. a literal "%41" in a +// directory name would silently open a different file. +func vectorDSN(path string, readOnly bool) string { + params := url.Values{} + if readOnly { + params.Set("mode", "ro") + params.Set("_busy_timeout", "5000") + } else { + params.Set("_journal_mode", "WAL") + params.Set("_busy_timeout", "5000") + params.Set("_synchronous", "NORMAL") + } + escaped := (&url.URL{Path: path}).EscapedPath() + return "file:" + escaped + "?" + params.Encode() +} diff --git a/internal/vector/driver_modernc.go b/internal/vector/driver_modernc.go new file mode 100644 index 000000000..8451a1c50 --- /dev/null +++ b/internal/vector/driver_modernc.go @@ -0,0 +1,41 @@ +//go:build windows || !cgo + +package vector + +import ( + "net/url" + + _ "modernc.org/sqlite" +) + +// vectorDriverName selects the database/sql driver vectors.db opens with. +// The cgo sqlite-vec bindings do not build on Windows, so kit's sqlitevec +// registers the pure-Go modernc.org/sqlite/vec extension there (via +// sqlite3_auto_extension at package init) and expects databases opened with +// modernc's "sqlite" driver; sqlitevec.Register is a no-op in this build. +// The main archive keeps its own driver — only vectors.db switches. +const vectorDriverName = "sqlite" + +// vectorDSN builds the modernc "sqlite" DSN for vectors.db, matching the +// pragmas driver_cgo.go sets through mattn's `_`-prefixed params: WAL, busy +// timeout, and NORMAL synchronous on the rw path; mode=ro plus a busy +// timeout on the ro path so a reader waits out a concurrent writer's lock +// instead of failing immediately with SQLITE_BUSY. modernc expresses +// connection pragmas as repeated `_pragma=name(value)` params on a file: +// URI. +// +// The path component is percent-encoded (slashes kept intact) for the same +// reason as driver_cgo.go: SQLite percent-decodes URI paths and splits +// params at `?`, so a raw `%`, `?`, or `#` in the path would be misparsed. +func vectorDSN(path string, readOnly bool) string { + params := url.Values{} + params.Add("_pragma", "busy_timeout(5000)") + if readOnly { + params.Set("mode", "ro") + } else { + params.Add("_pragma", "journal_mode(WAL)") + params.Add("_pragma", "synchronous(NORMAL)") + } + escaped := (&url.URL{Path: path}).EscapedPath() + return "file:" + escaped + "?" + params.Encode() +} diff --git a/internal/vector/encoder.go b/internal/vector/encoder.go new file mode 100644 index 000000000..3d2c3e1f5 --- /dev/null +++ b/internal/vector/encoder.go @@ -0,0 +1,425 @@ +// Package vector wires agentsview into kit's vector package for semantic +// search: SQLite-backed vector storage and OpenAI-compatible embeddings. +package vector + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "strconv" + "strings" + "sync/atomic" + "time" + + kitvec "go.kenn.io/kit/vector" +) + +// EncoderConfig configures an OpenAI-compatible embeddings HTTP client. +type EncoderConfig struct { + // Endpoint is the base URL including "/v1"; "/embeddings" is appended. + Endpoint string + // APIKey is sent as a Bearer token when non-empty. Empty means + // anonymous, unauthenticated requests. + APIKey string + // Model is the embeddings model name sent in the request body. + Model string + // Dimension is the length every returned vector must have. + Dimension int + // Timeout bounds each individual HTTP request. + Timeout time.Duration + // MaxRetries is the maximum total attempts on 429/5xx/network errors + // (4xx fails fast); values <= 0 mean one attempt. + MaxRetries int + // InputSuffix is appended verbatim to every input text before it is + // sent, for models that expect a terminator the serving layer does not + // add (e.g. "<|endoftext|>" for Qwen3-Embedding under llama.cpp). + // Empty means inputs are sent unmodified. + InputSuffix string +} + +const ( + backoffBase = 250 * time.Millisecond + backoffMax = 5 * time.Second + // retryAfterCap bounds how long a Retry-After header can push a single + // wait, so a misbehaving or hostile endpoint cannot stall a build for + // arbitrarily long. + retryAfterCap = 60 * time.Second +) + +// HTTPStatusError reports a non-200 response from the embeddings endpoint. +// It carries the status code so callers can distinguish retryable failures +// (429, 5xx) from non-retryable ones, and, for known input-specific 4xx +// rejections, skip-stamp one poison document without aborting a whole build. +// Route/model/schema/config failures are not input-specific and must abort so +// later builds can retry the corpus after the configuration is fixed. When the +// response is a 429 and carried a parseable Retry-After header, RetryAfter +// holds the delay the server asked for (clamped to retryAfterCap) so retry +// backoff can honor it instead of guessing. +type HTTPStatusError struct { + // Status is the HTTP status code the embeddings endpoint returned. + Status int + // Body is a trimmed snippet (up to 512 bytes) of the response body. + Body string + // RetryAfter is the parsed Retry-After delay from a 429 response, or + // nil when the response carried none or it could not be parsed. + RetryAfter *time.Duration +} + +func (e *HTTPStatusError) Error() string { + return fmt.Sprintf("[vector.embeddings] status %d: %s", e.Status, e.Body) +} + +// Permanent reports whether the embeddings endpoint's response indicates a +// rejection of this specific input that will never succeed on retry. It is +// intentionally conservative: generic 4xx statuses often mean a bad route, +// model, media type, or credentials, and skip-stamping those would silently +// mark an entire corpus embedded-with-no-vectors. +func (e *HTTPStatusError) Permanent() bool { + switch e.Status { + case http.StatusBadRequest, + http.StatusRequestEntityTooLarge, + http.StatusUnprocessableEntity: + return hasDocumentSpecificEmbeddingError(e.Body) + } + return false +} + +// hasDocumentSpecificEmbeddingError reports whether an embeddings error body +// describes a rejection of the input document itself: an input/token/context +// length overflow, or a content-policy refusal. Bare keywords are not enough +// — "invalid token" is an auth failure and "unsupported content type" is a +// media-type failure, and skip-stamping those would silently mark the whole +// corpus embedded-with-no-vectors — so a size word must pair with an input +// word, and "content" must pair with "policy". +func hasDocumentSpecificEmbeddingError(body string) bool { + body = strings.ToLower(body) + if strings.Contains(body, "content") && strings.Contains(body, "policy") { + return true + } + overLimit := strings.Contains(body, "too long") || + strings.Contains(body, "too large") || + strings.Contains(body, "too many") || + strings.Contains(body, "length") || + strings.Contains(body, "limit") || + strings.Contains(body, "maximum") || + strings.Contains(body, "exceed") || + strings.Contains(body, "overflow") + if !overLimit { + return false + } + return strings.Contains(body, "token") || + strings.Contains(body, "context") || + strings.Contains(body, "input") || + strings.Contains(body, "text") +} + +// embeddingsRequestBody is the OpenAI-compatible embeddings request. +type embeddingsRequestBody struct { + Model string `json:"model"` + Input []string `json:"input"` + // EncodingFormat asks for "base64" responses (raw little-endian float32 + // bytes), roughly 4x smaller than the default JSON float arrays — the + // difference dominates round-trip time on slow links. Empty omits the + // field for servers that reject it. + EncodingFormat string `json:"encoding_format,omitempty"` +} + +// embeddingsResponseBody is the OpenAI-compatible embeddings response. +type embeddingsResponseBody struct { + Data []struct { + Index int `json:"index"` + Embedding embeddingVector `json:"embedding"` + } `json:"data"` +} + +// embeddingVector decodes an OpenAI-compatible embedding that arrives either +// as a JSON float array (the default) or as a base64 string of little-endian +// float32 bytes (encoding_format "base64"). Accepting both means the encoder +// keeps working against servers that silently ignore encoding_format. +type embeddingVector []float32 + +func (v *embeddingVector) UnmarshalJSON(b []byte) error { + if len(b) > 0 && b[0] == '"' { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + raw, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return fmt.Errorf("decode base64 embedding: %w", err) + } + if len(raw)%4 != 0 { + return fmt.Errorf("base64 embedding is %d bytes, not a multiple of 4", len(raw)) + } + out := make([]float32, len(raw)/4) + for i := range out { + out[i] = math.Float32frombits(binary.LittleEndian.Uint32(raw[4*i:])) + } + *v = out + return nil + } + var floats []float32 + if err := json.Unmarshal(b, &floats); err != nil { + return err + } + *v = floats + return nil +} + +// encoderClient carries the HTTP client, resolved URL, and the runtime +// float-fallback state shared by every call the EncodeFunc makes. +type encoderClient struct { + client *http.Client + url string + cfg EncoderConfig + // floatMode flips to true (for the encoder's lifetime) when the server + // rejects the encoding_format field, so every later request goes back + // to plain JSON float arrays instead of failing the same way again. + floatMode atomic.Bool +} + +// NewEncoder returns a kitvec.EncodeFunc that POSTs to an OpenAI-compatible +// embeddings endpoint. Each invocation of the returned func makes exactly +// one HTTP call; batching and concurrency are the caller's responsibility +// via kitvec.EncodeBatched. +// +// Requests ask for base64-encoded embeddings (encoding_format "base64", +// ~4x smaller than JSON float arrays); responses in either format are +// accepted, and a server that rejects the field outright downgrades this +// encoder to plain float requests for its lifetime. +func NewEncoder(cfg EncoderConfig) kitvec.EncodeFunc { + ec := &encoderClient{ + client: &http.Client{Timeout: cfg.Timeout}, + url: strings.TrimRight(cfg.Endpoint, "/") + "/embeddings", + cfg: cfg, + } + return ec.encode +} + +// marshalRequest builds the request body, applying the configured input +// suffix and, unless the encoder has downgraded to float mode, asking for +// base64 embeddings. +func (ec *encoderClient) marshalRequest(texts []string) ([]byte, error) { + inputs := texts + if ec.cfg.InputSuffix != "" { + inputs = make([]string, len(texts)) + for i, t := range texts { + inputs[i] = t + ec.cfg.InputSuffix + } + } + body := embeddingsRequestBody{Model: ec.cfg.Model, Input: inputs} + if !ec.floatMode.Load() { + body.EncodingFormat = "base64" + } + reqBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("[vector.embeddings] marshal request: %w", err) + } + return reqBody, nil +} + +// encode performs the retrying HTTP call and validates the response shape. +func (ec *encoderClient) encode(ctx context.Context, texts []string) ([][]float32, error) { + usedBase64 := !ec.floatMode.Load() + reqBody, err := ec.marshalRequest(texts) + if err != nil { + return nil, err + } + + attempts := ec.cfg.MaxRetries + if attempts <= 0 { + attempts = 1 + } + + var lastErr error + for attempt := 1; attempt <= attempts; attempt++ { + vectors, retryable, err := ec.attemptEncode(ctx, reqBody, texts) + if err == nil { + return vectors, nil + } + if usedBase64 && isEncodingFormatRejection(err) { + // The server rejected the encoding_format field itself (not this + // input). Downgrade to float mode permanently and redo the call; + // without this, every request would fail identically and the + // build would abort on a transport nicety. + ec.floatMode.Store(true) + return ec.encode(ctx, texts) + } + lastErr = err + if !retryable || attempt == attempts { + return nil, lastErr + } + if err := sleepBackoff(ctx, attempt, err); err != nil { + return nil, err + } + } + return nil, lastErr +} + +// isEncodingFormatRejection reports whether err is a client-error response +// that names the encoding_format field, i.e. a server refusing the base64 +// request format rather than the input. The match is deliberately narrow: +// generic 4xx bodies must keep flowing through the normal retry/permanence +// classification. +func isEncodingFormatRejection(err error) bool { + var statusErr *HTTPStatusError + if !errors.As(err, &statusErr) { + return false + } + if statusErr.Status < 400 || statusErr.Status >= 500 { + return false + } + return strings.Contains(strings.ToLower(statusErr.Body), "encoding_format") +} + +// attemptEncode makes a single HTTP request and decodes the response. The +// retryable return value indicates whether the error is worth retrying +// (429, 5xx, or a transport-level failure). +func (ec *encoderClient) attemptEncode( + ctx context.Context, reqBody []byte, texts []string, +) ([][]float32, bool, error) { + client, url, cfg := ec.client, ec.url, ec.cfg + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(reqBody)) + if err != nil { + return nil, false, fmt.Errorf("[vector.embeddings] build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if cfg.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+cfg.APIKey) + } + + resp, err := client.Do(req) + if err != nil { + return nil, true, fmt.Errorf("[vector.embeddings] request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + statusErr := &HTTPStatusError{Status: resp.StatusCode, Body: strings.TrimSpace(string(body))} + if resp.StatusCode == http.StatusTooManyRequests { + if d, ok := parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()); ok { + statusErr.RetryAfter = &d + } + } + retryable := resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 + return nil, retryable, statusErr + } + + var decoded embeddingsResponseBody + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + // A decode failure almost always means the connection died + // mid-stream (truncated body), not that the endpoint sent a + // deliberately malformed response; treat it as transient so the + // caller retries rather than giving up immediately. + return nil, true, fmt.Errorf("[vector.embeddings] decode response: %w", err) + } + + vectors, err := reorderAndValidate(decoded, texts, cfg.Dimension) + if err != nil { + return nil, false, err + } + return vectors, false, nil +} + +// reorderAndValidate reorders the decoded embeddings by their reported +// index and validates counts and dimensions against the request. +func reorderAndValidate( + decoded embeddingsResponseBody, texts []string, dimension int, +) ([][]float32, error) { + if len(decoded.Data) != len(texts) { + return nil, fmt.Errorf( + "[vector.embeddings] count mismatch: got %d embeddings, want %d", + len(decoded.Data), len(texts)) + } + + out := make([][]float32, len(texts)) + seen := make([]bool, len(texts)) + for _, d := range decoded.Data { + if d.Index < 0 || d.Index >= len(texts) { + return nil, fmt.Errorf( + "[vector.embeddings] index %d out of range for %d texts", d.Index, len(texts)) + } + if len(d.Embedding) != dimension { + return nil, fmt.Errorf( + "[vector.embeddings] dimension mismatch at index %d: got %d, want %d", + d.Index, len(d.Embedding), dimension) + } + out[d.Index] = []float32(d.Embedding) + seen[d.Index] = true + } + for i, ok := range seen { + if !ok { + return nil, fmt.Errorf("[vector.embeddings] missing embedding for index %d", i) + } + } + return out, nil +} + +// sleepBackoff waits before the next attempt — honoring a 429 response's +// Retry-After delay when lastErr carries one, falling back to capped +// exponential backoff otherwise — returning ctx.Err() promptly if ctx is +// cancelled during the wait. +func sleepBackoff(ctx context.Context, attempt int, lastErr error) error { + delay := backoffDelay(attempt, lastErr) + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// backoffDelay picks the wait before the next retry: a 429's parsed +// Retry-After delay when present, otherwise capped exponential backoff +// from attempt. +func backoffDelay(attempt int, lastErr error) time.Duration { + var statusErr *HTTPStatusError + if errors.As(lastErr, &statusErr) && statusErr.RetryAfter != nil { + return *statusErr.RetryAfter + } + delay := backoffBase << (attempt - 1) + if delay > backoffMax || delay <= 0 { + delay = backoffMax + } + return delay +} + +// parseRetryAfter parses an HTTP Retry-After header value in either the +// delta-seconds or HTTP-date form (RFC 9110 §10.2.3), relative to now, +// clamped to [0, retryAfterCap]. It reports ok=false for an empty or +// unparseable header. A delta-seconds value of 0 (or an HTTP-date already +// in the past) means "retry immediately", reported as a zero duration. +func parseRetryAfter(header string, now time.Time) (time.Duration, bool) { + header = strings.TrimSpace(header) + if header == "" { + return 0, false + } + if seconds, err := strconv.Atoi(header); err == nil { + return clampRetryAfter(time.Duration(seconds) * time.Second), true + } + if when, err := http.ParseTime(header); err == nil { + return clampRetryAfter(when.Sub(now)), true + } + return 0, false +} + +// clampRetryAfter bounds d to [0, retryAfterCap]. +func clampRetryAfter(d time.Duration) time.Duration { + if d < 0 { + return 0 + } + if d > retryAfterCap { + return retryAfterCap + } + return d +} diff --git a/internal/vector/encoder_test.go b/internal/vector/encoder_test.go new file mode 100644 index 000000000..22ed574c4 --- /dev/null +++ b/internal/vector/encoder_test.go @@ -0,0 +1,625 @@ +package vector + +import ( + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "io" + "math" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// embeddingsRequest mirrors the OpenAI-compatible request body the encoder +// sends, for use in test assertions. +type embeddingsRequest struct { + Model string `json:"model"` + Input []string `json:"input"` + EncodingFormat string `json:"encoding_format"` +} + +// embeddingDatum mirrors one element of the OpenAI-compatible response. +type embeddingDatum struct { + Index int `json:"index"` + Embedding []float32 `json:"embedding"` +} + +type embeddingsResponse struct { + Data []embeddingDatum `json:"data"` +} + +func writeJSON(t *testing.T, w http.ResponseWriter, status int, v any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + require.NoError(t, json.NewEncoder(w).Encode(v)) +} + +func TestEncoderHappyPath(t *testing.T) { + var gotPath string + var gotAuth string + var gotReq embeddingsRequest + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &gotReq)) + + // Return data out of order to verify reordering by index. + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{ + {Index: 1, Embedding: []float32{4, 5, 6}}, + {Index: 0, Embedding: []float32{1, 2, 3}}, + }, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + APIKey: "secret-key", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + out, err := enc(context.Background(), []string{"hello", "world"}) + require.NoError(t, err) + + assert.Equal(t, "/v1/embeddings", gotPath) + assert.Equal(t, "Bearer secret-key", gotAuth) + assert.Equal(t, "test-model", gotReq.Model) + assert.Equal(t, []string{"hello", "world"}, gotReq.Input) + assert.Equal(t, "base64", gotReq.EncodingFormat, + "requests ask for the compact base64 wire format") + + require.Len(t, out, 2) + assert.Equal(t, []float32{1, 2, 3}, out[0]) + assert.Equal(t, []float32{4, 5, 6}, out[1]) +} + +// base64Embedding encodes floats the way encoding_format "base64" responses +// carry them: raw little-endian float32 bytes, base64-encoded. +func base64Embedding(floats []float32) string { + raw := make([]byte, 4*len(floats)) + for i, f := range floats { + binary.LittleEndian.PutUint32(raw[4*i:], math.Float32bits(f)) + } + return base64.StdEncoding.EncodeToString(raw) +} + +// TestEncoderDecodesBase64Embeddings asserts a server honoring +// encoding_format "base64" round-trips to the same float vectors, including +// reordering by index. +func TestEncoderDecodesBase64Embeddings(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, map[string]any{ + "data": []map[string]any{ + {"index": 1, "embedding": base64Embedding([]float32{4, 5, 6})}, + {"index": 0, "embedding": base64Embedding([]float32{1, 2, 3})}, + }, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + }) + + out, err := enc(context.Background(), []string{"hello", "world"}) + require.NoError(t, err) + require.Len(t, out, 2) + assert.Equal(t, []float32{1, 2, 3}, out[0]) + assert.Equal(t, []float32{4, 5, 6}, out[1]) +} + +// TestEncoderBase64WrongByteCountFailsDimensionCheck asserts a base64 +// payload whose float count disagrees with the configured dimension is +// rejected rather than silently stored. +func TestEncoderBase64WrongByteCountFailsDimensionCheck(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, map[string]any{ + "data": []map[string]any{ + {"index": 0, "embedding": base64Embedding([]float32{1, 2})}, + }, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.ErrorContains(t, err, "dimension mismatch") +} + +// TestEncoderFallsBackToFloatsWhenBase64Rejected asserts that a server which +// 400s the encoding_format field triggers a permanent downgrade: the request +// is redone without the field immediately, and later calls never ask for +// base64 again. +func TestEncoderFallsBackToFloatsWhenBase64Rejected(t *testing.T) { + var base64Requests, floatRequests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + var req embeddingsRequest + require.NoError(t, json.Unmarshal(body, &req)) + + if req.EncodingFormat != "" { + base64Requests.Add(1) + writeJSON(t, w, http.StatusBadRequest, map[string]any{ + "error": "unknown field: encoding_format", + }) + return + } + floatRequests.Add(1) + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2, 3}}}, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + }) + + out, err := enc(context.Background(), []string{"hello"}) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, []float32{1, 2, 3}, out[0]) + + _, err = enc(context.Background(), []string{"again"}) + require.NoError(t, err) + + assert.Equal(t, int32(1), base64Requests.Load(), + "the rejection downgrades the encoder for its lifetime") + assert.Equal(t, int32(2), floatRequests.Load()) +} + +// TestEncoderInputSuffixAppended asserts a configured InputSuffix is appended +// to every input in the request body, while the returned vectors still map +// back to the original (unsuffixed) texts by index. +func TestEncoderInputSuffixAppended(t *testing.T) { + var gotReq embeddingsRequest + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &gotReq)) + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{ + {Index: 0, Embedding: []float32{1, 2, 3}}, + {Index: 1, Embedding: []float32{4, 5, 6}}, + }, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 1, + InputSuffix: "<|endoftext|>", + }) + + out, err := enc(context.Background(), []string{"hello", "world"}) + require.NoError(t, err) + + assert.Equal(t, []string{"hello<|endoftext|>", "world<|endoftext|>"}, gotReq.Input) + require.Len(t, out, 2) + assert.Equal(t, []float32{1, 2, 3}, out[0]) + assert.Equal(t, []float32{4, 5, 6}, out[1]) +} + +func TestEncoderAnonymousNoAuthHeader(t *testing.T) { + var gotAuth string + var authSet bool + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + _, authSet = r.Header["Authorization"] + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2, 3}}}, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.NoError(t, err) + assert.False(t, authSet) + assert.Empty(t, gotAuth) +} + +func TestEncoderDimensionMismatch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2}}}, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "got") + assert.Contains(t, err.Error(), "want") + assert.Contains(t, err.Error(), "[vector.embeddings] dimension") +} + +func TestEncoderCountMismatch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2, 3}}}, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + _, err := enc(context.Background(), []string{"hello", "world"}) + require.Error(t, err) +} + +func TestEncoderRetries429ThenSucceeds(t *testing.T) { + var attempts atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := attempts.Add(1) + if n == 1 { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte("rate limited")) + return + } + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2, 3}}}, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + out, err := enc(context.Background(), []string{"hello"}) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, int32(2), attempts.Load()) +} + +func TestEncoder500ExhaustsRetries(t *testing.T) { + var attempts atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("server error")) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.Error(t, err) + assert.Equal(t, int32(3), attempts.Load()) +} + +func TestEncoder400FailsWithoutRetry(t *testing.T) { + var attempts atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("bad request: invalid input")) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.Error(t, err) + assert.Equal(t, int32(1), attempts.Load()) + assert.Contains(t, err.Error(), "400") +} + +func TestEncoderContextCancellationAbortsBackoffPromptly(t *testing.T) { + var attempts atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte("rate limited")) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", + Model: "test-model", + Dimension: 3, + Timeout: 5 * time.Second, + MaxRetries: 10, + }) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(50 * time.Millisecond) + cancel() + }() + + start := time.Now() + _, err := enc(ctx, []string{"hello"}) + elapsed := time.Since(start) + + require.Error(t, err) + assert.Less(t, elapsed, 2*time.Second, "backoff should abort promptly on context cancellation") +} + +// TestEncoder400ReturnsPermanentHTTPStatusError covers fix 1a: a non-200 +// response must come back as a *HTTPStatusError carrying the status code, +// so callers (kit's FillOptions.OnEncodeError) can distinguish a permanent +// rejection from a transient one instead of string-matching the message. +func TestEncoder400ReturnsPermanentHTTPStatusError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("token window overflow")) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", Model: "test-model", Dimension: 3, + Timeout: 5 * time.Second, MaxRetries: 1, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.Error(t, err) + var statusErr *HTTPStatusError + require.ErrorAs(t, err, &statusErr) + assert.Equal(t, http.StatusBadRequest, statusErr.Status) + assert.True(t, statusErr.Permanent(), "a 400 is a permanent rejection") +} + +// TestEncoder429ReturnsNonPermanentHTTPStatusError guards the except-429 +// carve-out: rate-limiting is a 4xx status but must not be treated as a +// permanent content rejection, or a poison-document skip would wrongly +// swallow a document that would have succeeded once the rate limit +// cleared. +func TestEncoder429ReturnsNonPermanentHTTPStatusError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte("rate limited")) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", Model: "test-model", Dimension: 3, + Timeout: 5 * time.Second, MaxRetries: 1, + }) + + _, err := enc(context.Background(), []string{"hello"}) + require.Error(t, err) + var statusErr *HTTPStatusError + require.ErrorAs(t, err, &statusErr) + assert.Equal(t, http.StatusTooManyRequests, statusErr.Status) + assert.False(t, statusErr.Permanent(), "429 is transient rate-limiting, not a content rejection") +} + +// TestHTTPStatusErrorPermanentClassification pins the skip-vs-abort +// classification: only input-specific rejections (400/413/422 whose body +// describes an input-size overflow or a content-policy refusal) may +// skip-stamp a poison document. Everything else — auth, routing, model, +// media-type, rate-limit, server errors, or an allowlisted status with a +// nonspecific body — must abort the build, or a config mistake would +// silently skip-stamp an entire corpus as embedded-with-no-vectors. +func TestHTTPStatusErrorPermanentClassification(t *testing.T) { + cases := []struct { + status int + body string + permanent bool + }{ + {http.StatusBadRequest, "token window overflow", true}, + {http.StatusBadRequest, "maximum context length is 8192 tokens", true}, + {http.StatusRequestEntityTooLarge, "input too large", true}, + {http.StatusUnprocessableEntity, "content policy violation", true}, + {http.StatusBadRequest, "", false}, + {http.StatusBadRequest, "no route", false}, + {http.StatusBadRequest, "invalid token", false}, + {http.StatusBadRequest, "invalid model", false}, + {http.StatusUnprocessableEntity, "unsupported content type", false}, + {http.StatusNotFound, "model not found", false}, + {http.StatusUnauthorized, "invalid token", false}, + {http.StatusForbidden, "forbidden", false}, + {http.StatusTooManyRequests, "input rate limited", false}, + {http.StatusInternalServerError, "token error", false}, + {http.StatusBadGateway, "", false}, + } + for _, tc := range cases { + err := &HTTPStatusError{Status: tc.status, Body: tc.body} + assert.Equalf(t, tc.permanent, err.Permanent(), + "status %d body %q: Permanent() classification", tc.status, tc.body) + } +} + +// TestEncoderDecodeErrorIsRetried covers fix 2: a decoding failure almost +// always means the connection died mid-stream, not that the endpoint sent +// a deliberately malformed response, so it must be retried rather than +// failing the whole encode on the first garbled response. +func TestEncoderDecodeErrorIsRetried(t *testing.T) { + var attempts atomic.Int32 + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := attempts.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if n == 1 { + _, _ = w.Write([]byte("{not valid json")) + return + } + require.NoError(t, json.NewEncoder(w).Encode(embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2, 3}}}, + })) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", Model: "test-model", Dimension: 3, + Timeout: 5 * time.Second, MaxRetries: 3, + }) + + out, err := enc(context.Background(), []string{"hello"}) + require.NoError(t, err, "a truncated/garbled body must be retried, not fail outright") + require.Len(t, out, 1) + assert.Equal(t, int32(2), attempts.Load()) +} + +// --- fix 5: Retry-After --- + +func TestParseRetryAfterDeltaSeconds(t *testing.T) { + d, ok := parseRetryAfter("30", time.Now()) + require.True(t, ok) + assert.Equal(t, 30*time.Second, d) +} + +func TestParseRetryAfterHTTPDate(t *testing.T) { + now := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + future := now.Add(45 * time.Second) + d, ok := parseRetryAfter(future.Format(http.TimeFormat), now) + require.True(t, ok) + assert.InDelta(t, float64(45*time.Second), float64(d), float64(time.Second)) +} + +func TestParseRetryAfterZeroMeansImmediate(t *testing.T) { + d, ok := parseRetryAfter("0", time.Now()) + require.True(t, ok) + assert.Zero(t, d) +} + +func TestParseRetryAfterAbsentOrUnparseableReturnsNotOK(t *testing.T) { + _, ok := parseRetryAfter("", time.Now()) + assert.False(t, ok, "empty header") + + _, ok = parseRetryAfter("not-a-value", time.Now()) + assert.False(t, ok, "unparseable header") +} + +func TestParseRetryAfterCappedAtSixtySeconds(t *testing.T) { + d, ok := parseRetryAfter("3600", time.Now()) + require.True(t, ok) + assert.Equal(t, 60*time.Second, d, "a huge Retry-After must be capped rather than honored verbatim") +} + +func TestParseRetryAfterPastHTTPDateClampsToZero(t *testing.T) { + now := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + past := now.Add(-time.Hour) + d, ok := parseRetryAfter(past.Format(http.TimeFormat), now) + require.True(t, ok) + assert.Zero(t, d, "an already-past Retry-After date means retry immediately") +} + +func TestBackoffDelayHonorsRetryAfterOn429(t *testing.T) { + d := 3 * time.Second + err := &HTTPStatusError{Status: http.StatusTooManyRequests, RetryAfter: &d} + assert.Equal(t, 3*time.Second, backoffDelay(1, err)) +} + +func TestBackoffDelayFallsBackToExponentialWithoutRetryAfter(t *testing.T) { + err := &HTTPStatusError{Status: http.StatusTooManyRequests} + assert.Equal(t, backoffBase, backoffDelay(1, err)) + assert.Equal(t, 2*backoffBase, backoffDelay(2, err)) +} + +func TestBackoffDelayFallsBackForNonStatusErrors(t *testing.T) { + assert.Equal(t, backoffBase, backoffDelay(1, errors.New("network error"))) +} + +// TestEncoderHonorsRetryAfterHeaderOn429 drives the full retry path end to +// end: a 429 response carrying Retry-After must make the encoder wait at +// least that long (not the default 250ms backoff) before its next attempt. +func TestEncoderHonorsRetryAfterHeaderOn429(t *testing.T) { + var attempts atomic.Int32 + var firstAt, secondAt time.Time + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := attempts.Add(1) + if n == 1 { + firstAt = time.Now() + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + secondAt = time.Now() + writeJSON(t, w, http.StatusOK, embeddingsResponse{ + Data: []embeddingDatum{{Index: 0, Embedding: []float32{1, 2, 3}}}, + }) + })) + defer srv.Close() + + enc := NewEncoder(EncoderConfig{ + Endpoint: srv.URL + "/v1", Model: "test-model", Dimension: 3, + Timeout: 5 * time.Second, MaxRetries: 2, + }) + + out, err := enc(context.Background(), []string{"hello"}) + require.NoError(t, err) + require.Len(t, out, 1) + assert.GreaterOrEqual(t, secondAt.Sub(firstAt), 900*time.Millisecond, + "the encoder must wait out the server's Retry-After: 1 rather than the default ~250ms backoff") +} diff --git a/internal/vector/index.go b/internal/vector/index.go new file mode 100644 index 000000000..63c72ebb8 --- /dev/null +++ b/internal/vector/index.go @@ -0,0 +1,490 @@ +package vector + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sync" + + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// registerVecOnce guards the process-wide sqlite-vec extension registration +// that sqlitevec.Register performs; calling it more than once would attempt +// to register the same SQL functions twice. +var registerVecOnce sync.Once + +// vectorsPrefix names the vec0/bookkeeping tables kit's sqlitevec store +// manages for our documents table (message_vectors_generations, +// message_vectors_chunks, message_vectors_stamps, message_vectors_v). +const vectorsPrefix = "message_vectors" + +// vectorSchema binds kit's sqlitevec store to our vector_messages mirror +// table. +var vectorSchema = sqlitevec.Schema{ + DocsTable: "vector_messages", + IDColumn: "doc_key", + ContentColumn: "content", + EmbedGenColumn: "embed_gen", + RevisionColumn: "content_hash", + VectorsPrefix: vectorsPrefix, +} + +// generationsTable is the kit-managed table holding one row per embedding +// generation (ordinal, gen_key, fingerprint, dimension, state). +const generationsTable = vectorsPrefix + "_generations" + +// stampsTable is the kit-managed table recording which documents are +// embedded under which generation ordinal. +const stampsTable = vectorsPrefix + "_stamps" + +// mirrorDDL creates agentsview's mirror of embeddable message content plus +// a small key/value table for metadata kit's store does not track (the +// display model name for a generation). ordinal is the unit's first +// (start) ordinal; ordinal_end is its last member's ordinal, equal to +// ordinal for single-message (user) documents. subordinate and offsets +// default to the "no run grouping yet" shape so a row inserted without +// them (see mirror.go) is still valid. +const mirrorDDL = ` +CREATE TABLE IF NOT EXISTS vector_messages ( + doc_key TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + source_uuid TEXT NOT NULL DEFAULT '', + ordinal INTEGER NOT NULL, + ordinal_end INTEGER NOT NULL, + subordinate INTEGER NOT NULL DEFAULT 0, + offsets TEXT NOT NULL DEFAULT '[]', + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + embed_gen TEXT +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_vector_messages_session_ordinal + ON vector_messages(session_id, ordinal); +CREATE TABLE IF NOT EXISTS vector_meta ( + key TEXT PRIMARY KEY, value TEXT NOT NULL +); +` + +// mirrorSchemaVersion is the current vectors.db mirror schema generation, +// stamped into vector_meta under mirrorSchemaVersionKey. It covers both the +// mirror's DDL shape (mirrorDDL's column set) AND its document-identity +// scheme (what one vector_messages row means); bump it whenever either +// changes in a way old rows cannot simply be read as-is. Open resets +// vectors.db on the write path, and flags ErrMirrorVersionMismatch on the +// read path, whenever the stamped value differs (or is absent while any +// mirror state already exists). +// +// History: "2" added the ordinal_end/subordinate/offsets columns but still +// held one row per message; "3" switched document identity to run-grouped +// units (one row per user message or per run of contiguous assistant +// messages) with no DDL change. +const mirrorSchemaVersion = "3" + +// mirrorSchemaVersionKey is the vector_meta key holding mirrorSchemaVersion. +const mirrorSchemaVersionKey = "mirror_schema_version" + +// ErrMirrorVersionMismatch reports a vectors.db written by a different +// mirror schema version than this binary expects. Write-path Open calls +// reset the mirror instead of returning this error (see prepareMirrorSchema); +// read-path Open calls (CLI reads, direct-install search) succeed regardless, +// but every subsequent Search call on that Index fails with this sentinel +// until a build recreates the mirror. +var ErrMirrorVersionMismatch = errors.New( + "vector index was built by an incompatible version: run `agentsview embeddings build`") + +// Index wraps vectors.db: agentsview's mirror of embeddable message content +// plus kit's sqlitevec store, which owns the generation and vec0 tables +// derived from vectorsPrefix. +type Index struct { + db *sql.DB + store *sqlitevec.Store[string, string] + split kitvec.SplitOptions + readOnly bool + + // versionMismatch records a read-path Open's version-gate finding: the + // mirror was written by a different mirrorSchemaVersion. Write-path + // Opens always resolve the mismatch themselves (reset and restamp), so + // this is only ever true on a read-only Index. Search checks it before + // touching any table. + versionMismatch bool +} + +// GenerationInfo describes one embedding generation and its coverage of the +// current vector_messages mirror, for CLI and status display. +type GenerationInfo struct { + ID int64 `json:"id"` // generations-table ordinal, CLI-facing + State string `json:"state"` + Model string `json:"model"` + Dimension int `json:"dimension"` + Fingerprint string `json:"fingerprint"` + Embedded int64 `json:"embedded"` // stamped docs + Missing int64 `json:"missing"` // mirror docs not stamped +} + +// ChunkOverlap derives the SplitOptions.Overlap rune count from +// maxInputChars: 15% of the chunk size, so consecutive chunks share enough +// context for the anchor/window logic to bridge a run split mid-message. +// Open and vectorGeneration (cmd/agentsview/embeddings.go) both call this so +// the split behavior and its fingerprint can never drift apart. +func ChunkOverlap(maxInputChars int) int { + return maxInputChars * 15 / 100 +} + +// Open opens (creating when rw) vectors.db and the kit sqlitevec store atop +// it. maxInputChars bounds the rune length of a single embedding request via +// SplitOptions{MaxRunes: maxInputChars, Overlap: ChunkOverlap(maxInputChars)}. +// +// When readOnly is true the file must already exist; Open never creates or +// migrates schema in that mode, matching internal/db.OpenReadOnly's +// cold-CLI-read contract. +func Open(ctx context.Context, path string, readOnly bool, maxInputChars int) (*Index, error) { + registerVecOnce.Do(sqlitevec.Register) + + if readOnly { + if _, err := os.Stat(path); err != nil { + return nil, fmt.Errorf("opening read-only vectors.db: %w", err) + } + } else if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, fmt.Errorf("creating vectors.db directory: %w", err) + } + + db, err := sql.Open(vectorDriverName, vectorDSN(path, readOnly)) + if err != nil { + return nil, fmt.Errorf("opening vectors.db: %w", err) + } + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, fmt.Errorf("opening vectors.db: %w", err) + } + + versionMismatch, err := prepareMirrorSchema(ctx, db, readOnly) + if err != nil { + db.Close() + return nil, err + } + + store, err := sqlitevec.New[string, string](ctx, db, vectorSchema) + if err != nil { + db.Close() + return nil, fmt.Errorf("opening vector store: %w", err) + } + + overlap := ChunkOverlap(maxInputChars) + return &Index{ + db: db, + store: store, + split: kitvec.SplitOptions{MaxRunes: maxInputChars, Overlap: overlap}, + readOnly: readOnly, + versionMismatch: versionMismatch, + }, nil +} + +// prepareMirrorSchema checks vectors.db's stamped mirror_schema_version +// against mirrorSchemaVersion and, on the write path, brings the schema +// current. On a mismatch — including the version key being absent while any +// mirror-state table already exists — the write path drops every such table +// and recreates the current schema from scratch: vectors.db is disposable by +// design (unlike sessions.db, which is never reset this way), so a clean +// rebuild is simpler and safer than an in-place column migration. On the +// read path a mismatch is reported back as mismatch=true without touching +// any table, leaving stale rows exactly as they are; the caller (Search) +// then fails closed with ErrMirrorVersionMismatch instead of risking a +// misread of old-shaped rows. +func prepareMirrorSchema(ctx context.Context, db *sql.DB, readOnly bool) (mismatch bool, err error) { + mismatch, tables, err := mirrorVersionMismatch(ctx, db) + if err != nil { + return false, err + } + if readOnly { + return mismatch, nil + } + + if mismatch { + if err := dropMirrorTables(ctx, db, tables); err != nil { + return false, err + } + } + if _, err := db.ExecContext(ctx, mirrorDDL); err != nil { + return false, fmt.Errorf("creating vectors.db schema: %w", err) + } + if err := stampMirrorSchemaVersion(ctx, db); err != nil { + return false, err + } + return false, nil +} + +// mirrorStateTableNames lists the sqlite_master table names considered part +// of the versioned mirror: the mirror's own tables, plus any kit-owned +// message_vectors* table (the generations and stamps bookkeeping tables and +// one vec0 table per embedding generation, including retired or abandoned +// ones left behind by a prior build). +func mirrorStateTableNames(ctx context.Context, db *sql.DB) ([]string, error) { + rows, err := db.QueryContext(ctx, ` +SELECT name FROM sqlite_master + WHERE type = 'table' + AND (name IN ('vector_messages', 'vector_meta') OR name LIKE ?)`, + vectorsPrefix+"%") + if err != nil { + return nil, fmt.Errorf("listing mirror tables: %w", err) + } + defer rows.Close() + + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("scanning mirror table name: %w", err) + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("listing mirror tables: %w", err) + } + return names, nil +} + +// mirrorVersionMismatch reports whether vectors.db's stamped +// mirror_schema_version differs from mirrorSchemaVersion. An empty/fresh +// database (no mirror-state table at all) is never a mismatch — there is +// nothing yet to be incompatible with. Otherwise, a version key that is +// absent, or holds a different value, while any mirror-state table already +// exists is a mismatch: that table predates versioning entirely, or was +// stamped by a different scheme. tables is always returned so a write-path +// caller can drop exactly the set that was inspected. +func mirrorVersionMismatch(ctx context.Context, db *sql.DB) (mismatch bool, tables []string, err error) { + tables, err = mirrorStateTableNames(ctx, db) + if err != nil { + return false, nil, err + } + if len(tables) == 0 { + return false, tables, nil + } + if !slices.Contains(tables, "vector_meta") { + return true, tables, nil + } + + var stamped string + err = db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, mirrorSchemaVersionKey, + ).Scan(&stamped) + if err == sql.ErrNoRows { + return true, tables, nil + } + if err != nil { + return false, nil, fmt.Errorf("reading mirror schema version: %w", err) + } + return stamped != mirrorSchemaVersion, tables, nil +} + +// dropMirrorTables drops every named table, resetting vectors.db's mirror +// state ahead of a fresh mirrorDDL + stampMirrorSchemaVersion. Table names +// come from sqlite_master (mirrorStateTableNames), not caller input, so +// building the DROP statement by concatenation is safe; SQL does not allow +// binding identifiers as query parameters. +func dropMirrorTables(ctx context.Context, db *sql.DB, tables []string) error { + for _, name := range tables { + if _, err := db.ExecContext(ctx, `DROP TABLE IF EXISTS "`+name+`"`); err != nil { + return fmt.Errorf("dropping stale mirror table %s: %w", name, err) + } + } + return nil +} + +// stampMirrorSchemaVersion records mirrorSchemaVersion into vector_meta, +// overwriting any prior value. +func stampMirrorSchemaVersion(ctx context.Context, db *sql.DB) error { + if _, err := db.ExecContext(ctx, ` +INSERT INTO vector_meta (key, value) VALUES (?, ?) +ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + mirrorSchemaVersionKey, mirrorSchemaVersion, + ); err != nil { + return fmt.Errorf("stamping mirror schema version: %w", err) + } + return nil +} + +// Close closes the underlying vectors.db connection. +func (ix *Index) Close() error { + return ix.db.Close() +} + +// requireWritable rejects generation-mutating calls on an Index opened +// read-only, matching internal/db's read-only guard pattern. +func (ix *Index) requireWritable() error { + if ix.readOnly { + return fmt.Errorf("vectors.db is opened read-only") + } + return nil +} + +// EnsureGeneration registers gen (a model + dimension configuration) with +// kit's store under its own fingerprint as the gen_key, creating its vec0 +// table on first use, and records the model's display name in vector_meta +// so Generations can show it (kit's store persists only the fingerprint). +// Calling it again for the same fingerprint updates only the state. +func (ix *Index) EnsureGeneration( + ctx context.Context, gen kitvec.Generation, state sqlitevec.State, +) (string, error) { + if err := ix.requireWritable(); err != nil { + return "", err + } + fingerprint := gen.Fingerprint() + if err := ix.store.EnsureGeneration(ctx, fingerprint, gen, state); err != nil { + return "", fmt.Errorf("ensure generation: %w", err) + } + if _, err := ix.db.ExecContext(ctx, ` +INSERT INTO vector_meta (key, value) VALUES (?, ?) +ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + "gen_model:"+fingerprint, gen.Model); err != nil { + return "", fmt.Errorf("record generation model: %w", err) + } + return fingerprint, nil +} + +// SetStateByID transitions the generation identified by its generations-table +// ordinal (the CLI-facing ID) to state. +func (ix *Index) SetStateByID(ctx context.Context, id int64, state sqlitevec.State) error { + if err := ix.requireWritable(); err != nil { + return err + } + res, err := ix.db.ExecContext(ctx, + `UPDATE `+generationsTable+` SET state = ? WHERE ordinal = ?`, string(state), id) + if err != nil { + return fmt.Errorf("set generation state: %w", err) + } + if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("set generation state rows: %w", err) + } else if n == 0 { + return fmt.Errorf("generation %d: %w", id, ErrGenerationNotFound) + } + return nil +} + +// ActiveFingerprint returns the fingerprint of the generation currently in +// the active state, if any. +func (ix *Index) ActiveFingerprint(ctx context.Context) (string, bool, error) { + return ix.fingerprintByState(ctx, string(sqlitevec.StateActive)) +} + +// BuildingFingerprint returns the fingerprint of the generation currently in +// the building state, if any. +func (ix *Index) BuildingFingerprint(ctx context.Context) (string, bool, error) { + return ix.fingerprintByState(ctx, string(sqlitevec.StateBuilding)) +} + +func (ix *Index) fingerprintByState(ctx context.Context, state string) (string, bool, error) { + var fingerprint string + err := ix.db.QueryRowContext(ctx, + `SELECT gen_key FROM `+generationsTable+` WHERE state = ? ORDER BY ordinal LIMIT 1`, + state).Scan(&fingerprint) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("lookup %s generation: %w", state, err) + } + return fingerprint, true, nil +} + +// generationCoverageQuery is the exact join agentsview uses to report each +// generation's coverage of the current vector_messages mirror: Embedded +// counts stamps for that generation ordinal whose revision still matches +// the mirror row's current content_hash, Missing counts mirror documents +// with no such matching-revision stamp. A stamp whose revision no longer +// matches (the mirror row's content changed since it was embedded) counts +// as Missing rather than Embedded, since kit's store treats it as pending +// re-embed. +const generationCoverageQuery = ` +SELECT g.ordinal, g.gen_key, g.fingerprint, g.dimension, g.state, + (SELECT COUNT(*) FROM ` + stampsTable + ` s WHERE s.ordinal = g.ordinal + AND EXISTS (SELECT 1 FROM vector_messages d + WHERE s.doc_key = d.doc_key AND s.revision = d.content_hash)), + (SELECT COUNT(*) FROM vector_messages d WHERE NOT EXISTS + (SELECT 1 FROM ` + stampsTable + ` s + WHERE s.ordinal = g.ordinal AND s.doc_key = d.doc_key AND s.revision = d.content_hash)) +FROM ` + generationsTable + ` g` + +// Generations returns every generation with its coverage counts against the +// current vector_messages mirror, ordered by ordinal. Like Search and +// StaleActive it fails closed with ErrMirrorVersionMismatch on a read-only +// Index over a mismatched mirror, rather than reporting coverage counts +// computed over stale-shape rows. +func (ix *Index) Generations(ctx context.Context) ([]GenerationInfo, error) { + if ix.versionMismatch { + return nil, ErrMirrorVersionMismatch + } + rows, err := ix.db.QueryContext(ctx, generationCoverageQuery+` ORDER BY g.ordinal`) + if err != nil { + return nil, fmt.Errorf("list generations: %w", err) + } + defer rows.Close() + + var infos []GenerationInfo + for rows.Next() { + info, err := ix.scanGenerationInfo(ctx, rows) + if err != nil { + return nil, err + } + infos = append(infos, info) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list generations: %w", err) + } + return infos, nil +} + +// ErrGenerationNotFound is returned by GenerationByID (and propagated by +// Manager.Activate/Retire) when id does not match any row in the +// generations table. Match it with errors.Is; the wrapping error's message +// still carries the specific id for logs and direct display. +var ErrGenerationNotFound = errors.New("generation not found") + +// GenerationByID returns the single generation identified by its +// generations-table ordinal. +func (ix *Index) GenerationByID(ctx context.Context, id int64) (GenerationInfo, error) { + row := ix.db.QueryRowContext(ctx, generationCoverageQuery+` WHERE g.ordinal = ?`, id) + info, err := ix.scanGenerationInfo(ctx, row) + if err == sql.ErrNoRows { + return GenerationInfo{}, fmt.Errorf("generation %d: %w", id, ErrGenerationNotFound) + } + if err != nil { + return GenerationInfo{}, err + } + return info, nil +} + +// genInfoScanner is the subset of *sql.Row / *sql.Rows Scan needs, letting +// scanGenerationInfo serve both Generations (rows) and GenerationByID (row). +type genInfoScanner interface { + Scan(dest ...any) error +} + +func (ix *Index) scanGenerationInfo(ctx context.Context, src genInfoScanner) (GenerationInfo, error) { + var ( + info GenerationInfo + genKey string + ) + if err := src.Scan( + &info.ID, &genKey, &info.Fingerprint, &info.Dimension, &info.State, + &info.Embedded, &info.Missing, + ); err != nil { + if err == sql.ErrNoRows { + return GenerationInfo{}, err + } + return GenerationInfo{}, fmt.Errorf("scan generation: %w", err) + } + + var model sql.NullString + err := ix.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, "gen_model:"+info.Fingerprint).Scan(&model) + if err != nil && err != sql.ErrNoRows { + return GenerationInfo{}, fmt.Errorf("lookup generation model: %w", err) + } + info.Model = model.String + return info, nil +} diff --git a/internal/vector/index_test.go b/internal/vector/index_test.go new file mode 100644 index 000000000..e1c6b4378 --- /dev/null +++ b/internal/vector/index_test.go @@ -0,0 +1,572 @@ +package vector + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +func TestOpenCreatesSchema(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + var name string + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT name FROM sqlite_master WHERE type='table' AND name='vector_messages'`).Scan(&name)) + require.Equal(t, "vector_messages", name) + + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT name FROM sqlite_master WHERE type='table' AND name='message_vectors_generations'`).Scan(&name)) + require.Equal(t, "message_vectors_generations", name) +} + +func TestOpenReadOnlyOnMissingFileFails(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "missing-vectors.db") + + _, err := Open(ctx, path, true, 4000) + require.Error(t, err) +} + +// TestOpenReadOnlyRefusesWrites pins the read-only contract: a mattn DSN +// only honors mode=ro with a file: URI prefix, so a bare-path DSN silently +// handed out writable handles. A read-only Open on a valid vectors.db must +// refuse writes at the SQLite level. +func TestOpenReadOnlyRefusesWrites(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + rw, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + require.NoError(t, rw.Close()) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err) + defer ro.Close() + + _, err = ro.db.ExecContext(ctx, + `INSERT INTO vector_meta (key, value) VALUES ('probe', 'x')`) + require.Error(t, err, "a read-only vectors.db handle must refuse writes") + assert.Contains(t, err.Error(), "readonly", + "the refusal must be SQLite's readonly-database error, got: %v", err) +} + +// TestOpenPathWithSpecialCharacters pins vectorDSN's path escaping: SQLite +// percent-decodes file: URI paths and splits params at `?`, so a directory +// name containing a space and a literal %-hex sequence ("%41") would, raw, +// be decoded to a different path ("weArd dir") and fail to open. Both the +// writable and read-only branches must escape the path, and read-only must +// still refuse writes. +func TestOpenPathWithSpecialCharacters(t *testing.T) { + ctx := context.Background() + dir := filepath.Join(t.TempDir(), "we%41rd dir") + require.NoError(t, os.MkdirAll(dir, 0o755)) + path := filepath.Join(dir, "vectors.db") + + rw, err := Open(ctx, path, false, 4000) + require.NoError(t, err, "writable Open must succeed on a path with %% and space") + _, err = rw.db.ExecContext(ctx, + `INSERT INTO vector_meta (key, value) VALUES ('probe', 'x')`) + require.NoError(t, err) + require.NoError(t, rw.Close()) + + _, err = os.Stat(path) + require.NoError(t, err, "the database file must exist at the literal path, not a decoded one") + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err, "read-only Open must succeed on a path with %% and space") + defer ro.Close() + + var value string + require.NoError(t, ro.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = 'probe'`).Scan(&value)) + assert.Equal(t, "x", value) + + _, err = ro.db.ExecContext(ctx, + `INSERT INTO vector_meta (key, value) VALUES ('probe2', 'y')`) + require.Error(t, err, "a read-only vectors.db handle must refuse writes") + assert.Contains(t, err.Error(), "readonly", + "the refusal must be SQLite's readonly-database error, got: %v", err) +} + +func TestOpenSplitOptionsUse15PercentOverlap(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + assert.Equal(t, 4000, ix.split.MaxRunes) + assert.Equal(t, 600, ix.split.Overlap, "15%% of 4000 is 600") + assert.Equal(t, ChunkOverlap(4000), ix.split.Overlap, + "Open must derive Overlap from the shared ChunkOverlap helper") +} + +func TestChunkOverlapIs15Percent(t *testing.T) { + assert.Equal(t, 600, ChunkOverlap(4000)) + assert.Equal(t, 150, ChunkOverlap(1000)) + assert.Equal(t, 0, ChunkOverlap(0)) +} + +func TestEnsureGenerationLifecycle(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateBuilding) + require.NoError(t, err) + require.NotEmpty(t, fingerprint) + + infos, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, infos, 1) + require.Equal(t, "building", infos[0].State) + require.Equal(t, "fake-model", infos[0].Model) + require.Equal(t, 3, infos[0].Dimension) + require.Equal(t, fingerprint, infos[0].Fingerprint) + require.NotZero(t, infos[0].ID) + + building, ok, err := ix.BuildingFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, fingerprint, building) + + _, ok, err = ix.ActiveFingerprint(ctx) + require.NoError(t, err) + require.False(t, ok) + + require.NoError(t, ix.SetStateByID(ctx, infos[0].ID, sqlitevec.StateActive)) + + active, ok, err := ix.ActiveFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, fingerprint, active) + + _, ok, err = ix.BuildingFingerprint(ctx) + require.NoError(t, err) + require.False(t, ok) + + info, err := ix.GenerationByID(ctx, infos[0].ID) + require.NoError(t, err) + require.Equal(t, "active", info.State) +} + +// TestGenerationByIDUnknownIDReturnsSentinel guards the HTTP layer's 404 +// mapping: an id with no matching row must return an error matching +// ErrGenerationNotFound via errors.Is, not just any error, so callers can +// distinguish "not found" from other failures. +func TestGenerationByIDUnknownIDReturnsSentinel(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + _, err = ix.GenerationByID(ctx, 999) + require.Error(t, err) + require.ErrorIs(t, err, ErrGenerationNotFound) + require.Contains(t, err.Error(), "999") +} + +func TestGenerationCoverageCounts(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateBuilding) + require.NoError(t, err) + + _, err = ix.db.ExecContext(ctx, + `INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) + VALUES (?, ?, ?, ?, ?, ?)`, + "d1", "s1", 0, 0, "hello world", "h1") + require.NoError(t, err) + _, err = ix.db.ExecContext(ctx, + `INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) + VALUES (?, ?, ?, ?, ?, ?)`, + "d2", "s1", 1, 1, "goodbye world", "h2") + require.NoError(t, err) + + err = ix.store.SaveVectors(ctx, fingerprint, "d1", "h1", []kitvec.ChunkVector{ + {ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}, + }) + require.NoError(t, err) + + infos, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, infos, 1) + require.EqualValues(t, 1, infos[0].Embedded) + require.EqualValues(t, 1, infos[0].Missing) +} + +// TestGenerationCoverageStaleRevisionCountsAsMissing asserts that a stamp +// whose revision no longer matches the mirror row's current content_hash +// (the content changed since it was embedded) counts as Missing, not +// Embedded, since kit's store treats such a document as pending re-embed. +func TestGenerationCoverageStaleRevisionCountsAsMissing(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateBuilding) + require.NoError(t, err) + + _, err = ix.db.ExecContext(ctx, + `INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) + VALUES (?, ?, ?, ?, ?, ?)`, + "d1", "s1", 0, 0, "hello world", "h1") + require.NoError(t, err) + + err = ix.store.SaveVectors(ctx, fingerprint, "d1", "h1", []kitvec.ChunkVector{ + {ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}, + }) + require.NoError(t, err) + + infos, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, infos, 1) + require.EqualValues(t, 1, infos[0].Embedded) + require.EqualValues(t, 0, infos[0].Missing) + + _, err = ix.db.ExecContext(ctx, + `UPDATE vector_messages SET content_hash = 'changed' WHERE doc_key = ?`, "d1") + require.NoError(t, err) + + infos, err = ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, infos, 1) + require.EqualValues(t, 0, infos[0].Embedded, "stale stamp revision no longer counts as embedded") + require.EqualValues(t, 1, infos[0].Missing, "stale stamp revision counts as missing") +} + +// v1MirrorDDL is the pre-versioning mirror schema (no ordinal_end/ +// subordinate/offsets columns, no mirror_schema_version key), used to +// simulate a vectors.db left behind by an older agentsview build. +const v1MirrorDDL = ` +CREATE TABLE vector_messages ( + doc_key TEXT PRIMARY KEY, + session_id TEXT NOT NULL, + source_uuid TEXT NOT NULL DEFAULT '', + ordinal INTEGER NOT NULL, + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + embed_gen TEXT +); +CREATE UNIQUE INDEX idx_vector_messages_session_ordinal + ON vector_messages(session_id, ordinal); +CREATE TABLE vector_meta ( + key TEXT PRIMARY KEY, value TEXT NOT NULL +); +` + +// seedV1Mirror writes path as a vectors.db with the pre-versioning v1 mirror +// schema plus the state a real build would have left behind: a mirror row, +// fake kit generations/chunks/stamps tables (sharing kit's table names but +// not its real column sets, standing in for whatever kit's store last +// wrote — every real build created all three, and a read-only Open must not +// need to CREATE any of them), a stray abandoned per-generation table, and +// meta keys with no mirror_schema_version — exactly what a writable Open +// must detect as a mismatch (absent key, mirror state present) and reset. +func seedV1Mirror(t *testing.T, path string) { + t.Helper() + ctx := context.Background() + raw, err := sql.Open("sqlite3", path) + require.NoError(t, err) + defer raw.Close() + + _, err = raw.ExecContext(ctx, v1MirrorDDL) + require.NoError(t, err) + _, err = raw.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, ordinal, content, content_hash) +VALUES (?, ?, ?, ?, ?)`, + "d1", "s1", 0, "hello world", "h1") + require.NoError(t, err) + _, err = raw.ExecContext(ctx, ` +INSERT INTO vector_meta (key, value) VALUES (?, ?), (?, ?)`, + refreshWatermarkKey, "2024-01-01T00:00:00Z", + scopeIncludeAutomatedKey, "true") + require.NoError(t, err) + _, err = raw.ExecContext(ctx, `CREATE TABLE `+generationsTable+` (ordinal INTEGER)`) + require.NoError(t, err) + // Kit's other bookkeeping tables and indexes, by their real names so a + // read-only Open's CREATE ... IF NOT EXISTS statements all no-op. + _, err = raw.ExecContext(ctx, ` +CREATE TABLE `+chunksTable+` (ordinal INTEGER, doc_key, vec_rowid INTEGER); +CREATE INDEX `+vectorsPrefix+`_chunks_by_vector ON `+chunksTable+` (ordinal, vec_rowid); +CREATE INDEX `+vectorsPrefix+`_chunks_by_doc ON `+chunksTable+` (doc_key, ordinal, vec_rowid); +CREATE TABLE `+stampsTable+` (ordinal INTEGER, doc_key, revision); +CREATE INDEX `+vectorsPrefix+`_stamps_by_doc_revision ON `+stampsTable+` (doc_key, revision);`) + require.NoError(t, err) + _, err = raw.ExecContext(ctx, `CREATE TABLE message_vectors_gen7 (id INTEGER)`) + require.NoError(t, err) +} + +// seedV2Mirror writes path as a vectors.db stamped mirror_schema_version +// "2": the window between the v2 columns landing and the run-grouped +// document-identity change, when rows were still one-per-message. It lays +// down a full current-DDL file (mirror tables, kit generation/chunk tables, +// version stamp) via a writable Open, then rewrites the stamp to "2" and +// inserts a per-message row — the DDL shape matches the current schema +// exactly, so only the version stamp can tell the two apart, the case the +// "3" bump exists for. Seeding the kit tables too matters for read-only +// Opens: sqlitevec.New must not need any CREATE TABLE on a mode=ro handle. +func seedV2Mirror(t *testing.T, path string) { + t.Helper() + ctx := context.Background() + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + require.NoError(t, ix.Close()) + + raw, err := sql.Open("sqlite3", path) + require.NoError(t, err) + defer raw.Close() + + _, err = raw.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?)`, + "s1:0", "s1", 0, 0, "a per-message row", "h1") + require.NoError(t, err) + _, err = raw.ExecContext(ctx, ` +UPDATE vector_meta SET value = '2' WHERE key = ?`, mirrorSchemaVersionKey) + require.NoError(t, err) +} + +func TestMirrorSchemaVersionFreshDBStampsVersionNothingDropped(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + var version string + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, mirrorSchemaVersionKey, + ).Scan(&version)) + assert.Equal(t, mirrorSchemaVersion, version) + + var metaCount int + require.NoError(t, ix.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vector_meta`).Scan(&metaCount)) + assert.Equal(t, 1, metaCount, "a fresh DB has nothing to drop, only the stamped version key") +} + +func TestMirrorSchemaVersionCurrentVersionUntouchedOnReopen(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + _, err = ix.db.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?)`, + "d1", "s1", 0, 0, "hello world", "h1") + require.NoError(t, err) + require.NoError(t, ix.setRefreshWatermark(ctx, "2024-01-01T00:00:00Z")) + require.NoError(t, ix.Close()) + + ix2, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix2.Close() + + var content string + require.NoError(t, ix2.db.QueryRowContext(ctx, + `SELECT content FROM vector_messages WHERE doc_key = ?`, "d1").Scan(&content)) + assert.Equal(t, "hello world", content, "current-version data must survive a reopen untouched") + + watermark, err := ix2.refreshWatermark(ctx) + require.NoError(t, err) + assert.Equal(t, "2024-01-01T00:00:00Z", watermark) +} + +// TestMirrorSchemaVersionMismatchResetsWritePath covers the full write-path +// reset: a v1-shaped mirror plus stray kit tables (including a fake +// generations table and an abandoned per-generation table) must be dropped +// and recreated with the v2 columns and defaults, and vector_meta must be +// cleared except for the freshly stamped version key. +func TestMirrorSchemaVersionMismatchResetsWritePath(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + seedV1Mirror(t, path) + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + var rowCount int + require.NoError(t, ix.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vector_messages`).Scan(&rowCount)) + assert.Zero(t, rowCount, "v1 mirror rows must not survive a version reset") + + _, err = ix.db.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?)`, + "d1", "s1", 0, 0, "hello", "h1") + require.NoError(t, err) + var subordinate int + var offsets string + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT subordinate, offsets FROM vector_messages WHERE doc_key = ?`, "d1", + ).Scan(&subordinate, &offsets)) + assert.Zero(t, subordinate, "subordinate defaults to 0") + assert.Equal(t, "[]", offsets, "offsets defaults to an empty JSON array") + + _, err = ix.db.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?)`, + "d2", "s1", 0, 0, "duplicate slot", "h2") + assert.Error(t, err, "the unique (session_id, ordinal) index must be retained") + + var metaCount int + require.NoError(t, ix.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vector_meta`).Scan(&metaCount)) + assert.Equal(t, 1, metaCount, "vector_meta is cleared of the old watermark/scope keys") + var version string + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, mirrorSchemaVersionKey, + ).Scan(&version)) + assert.Equal(t, mirrorSchemaVersion, version) + + err = ix.db.QueryRowContext(ctx, + `SELECT name FROM sqlite_master WHERE name = 'message_vectors_gen7'`).Scan(new(string)) + assert.ErrorIs(t, err, sql.ErrNoRows, "the stray abandoned per-generation table must be dropped") + + var genCount int + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM `+generationsTable).Scan(&genCount)) + assert.Zero(t, genCount, "kit must have recreated its generations table fresh, not kept the fake row") +} + +// TestMirrorSchemaVersionV2StampResetsWritePath covers the document-identity +// half of the version gate: a vectors.db whose DDL already matches the +// current shape but whose rows predate run grouping (stamped "2", one row +// per message) must still be reset on writable open — the stamp, not the +// column set, is what marks the rows incompatible. +func TestMirrorSchemaVersionV2StampResetsWritePath(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + seedV2Mirror(t, path) + + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + defer ix.Close() + + var rowCount int + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM vector_messages`).Scan(&rowCount)) + assert.Zero(t, rowCount, "per-message v2 rows must not survive a version reset") + + var version string + require.NoError(t, ix.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, mirrorSchemaVersionKey, + ).Scan(&version)) + assert.Equal(t, mirrorSchemaVersion, version) +} + +// TestMirrorSchemaVersionV2StampReadOnlyReturnsSentinel covers the read path +// for the same document-identity mismatch: a read-only Open against a +// "2"-stamped vectors.db must succeed, but both Search and StaleActive must +// fail closed with ErrMirrorVersionMismatch — StaleActive runs before Search +// in the real serving path, so without its gate a caller would query the +// generation tables of a mirror shaped by a different identity scheme and +// never reach the sentinel. +func TestMirrorSchemaVersionV2StampReadOnlyReturnsSentinel(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + seedV2Mirror(t, path) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err, "read-only Open must succeed even against a v2-stamped mirror") + defer ro.Close() + + _, err = ro.Search(ctx, fakeSearchEncoder(), "alpha", 10) + assert.ErrorIs(t, err, ErrMirrorVersionMismatch) + + _, err = ro.StaleActive(ctx, "any-fingerprint") + assert.ErrorIs(t, err, ErrMirrorVersionMismatch, + "StaleActive must apply the same version gate Search does") +} + +// TestMirrorSchemaVersionReadOnlyMismatchSearchReturnsSentinel covers the +// read path: Open against a version-mismatched vectors.db must still +// succeed (a read-only CLI process cannot reset the file), but Search must +// fail closed with ErrMirrorVersionMismatch before touching any table, +// rather than misreading v1-shaped rows or falling through to +// ErrNoActiveGeneration. +func TestMirrorSchemaVersionReadOnlyMismatchSearchReturnsSentinel(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + seedV1Mirror(t, path) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err, "read-only Open must succeed even against a mismatched mirror") + defer ro.Close() + + _, err = ro.Search(ctx, fakeSearchEncoder(), "alpha", 10) + require.Error(t, err) + assert.ErrorIs(t, err, ErrMirrorVersionMismatch) + assert.NotErrorIs(t, err, ErrNoActiveGeneration, + "a version mismatch must not be reported as an empty index") +} + +// TestMirrorSchemaVersionReadOnlyCurrentVersionSearchUnaffected is a +// regression guard: a read-only Open against an up-to-date mirror must not +// be flagged as a mismatch, so Search proceeds to its normal +// ErrNoActiveGeneration/BuildingError/hit-returning behavior. +func TestMirrorSchemaVersionReadOnlyCurrentVersionSearchUnaffected(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + + rw, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + require.NoError(t, rw.Close()) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err) + defer ro.Close() + + _, err = ro.Search(ctx, fakeSearchEncoder(), "alpha", 10) + assert.ErrorIs(t, err, ErrNoActiveGeneration, + "a current-version empty mirror must fall through to the normal empty-index error") +} + +// TestGenerationsReadOnlyMismatchReturnsSentinel closes the version-gate gap +// on the generation-listing read path: a read-only Index over a mismatched +// mirror must refuse Generations with ErrMirrorVersionMismatch (the same +// rebuild-required sentinel Search and StaleActive return) instead of +// reporting coverage counts computed over stale-shape rows. +func TestGenerationsReadOnlyMismatchReturnsSentinel(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + seedV2Mirror(t, path) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err, "read-only Open must succeed even against a v2-stamped mirror") + defer ro.Close() + + _, err = ro.Generations(ctx) + assert.ErrorIs(t, err, ErrMirrorVersionMismatch, + "Generations must apply the same version gate Search does") +} diff --git a/internal/vector/kit_smoke_test.go b/internal/vector/kit_smoke_test.go new file mode 100644 index 000000000..3d3cd35c2 --- /dev/null +++ b/internal/vector/kit_smoke_test.go @@ -0,0 +1,51 @@ +package vector + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +func TestKitSqlitevecRoundTrip(t *testing.T) { + sqlitevec.Register() + db, err := sql.Open(vectorDriverName, vectorDSN(filepath.Join(t.TempDir(), "v.db"), false)) + require.NoError(t, err) + defer db.Close() + ctx := context.Background() + _, err = db.ExecContext(ctx, `CREATE TABLE docs ( + doc_key TEXT PRIMARY KEY, content TEXT NOT NULL, + content_hash TEXT NOT NULL, embed_gen TEXT)`) + require.NoError(t, err) + store, err := sqlitevec.New[string, string](ctx, db, sqlitevec.Schema{ + DocsTable: "docs", IDColumn: "doc_key", ContentColumn: "content", + EmbedGenColumn: "embed_gen", RevisionColumn: "content_hash", + VectorsPrefix: "docs_vectors", + }) + require.NoError(t, err) + gen := kitvec.Generation{Model: "fake", Dimensions: 3} + fp := gen.Fingerprint() + require.NoError(t, store.EnsureGeneration(ctx, fp, gen, sqlitevec.StateActive)) + _, err = db.ExecContext(ctx, + `INSERT INTO docs VALUES ('d1', 'hello world', 'h1', NULL)`) + require.NoError(t, err) + enc := func(_ context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } + stats, err := kitvec.Fill[string, string](ctx, store, fp, enc, + kitvec.FillOptions[string]{}) + require.NoError(t, err) + require.Equal(t, 1, stats.Documents) + hits, err := store.QueryGeneration(ctx, fp, kitvec.Vector{1, 0, 0}, 5) + require.NoError(t, err) + require.Len(t, hits, 1) + require.Equal(t, "d1", hits[0].Doc) +} diff --git a/internal/vector/manager.go b/internal/vector/manager.go new file mode 100644 index 000000000..a9e0a00f4 --- /dev/null +++ b/internal/vector/manager.go @@ -0,0 +1,357 @@ +package vector + +import ( + "context" + "errors" + "fmt" + "sync" + + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// ErrBuildRunning is returned by StartBuild when a build is already in +// flight, and by Activate/Retire when they are called while one is running. +var ErrBuildRunning = errors.New("an embeddings build is already running") + +// ErrGenerationRefused indicates Activate or Retire declined to change a +// generation's state without --force (incomplete coverage, or retiring the +// active generation). Match it with errors.Is; the error's own message +// carries the specific, user-facing reason rather than a fixed string. +var ErrGenerationRefused = errors.New("generation state change refused") + +// refusalError carries a specific, literal refusal message while still +// satisfying errors.Is(err, ErrGenerationRefused), so callers get an +// unprefixed message (e.g. for direct display) alongside a stable sentinel +// for status-code mapping. +type refusalError struct { + msg string +} + +func refusedf(format string, args ...any) error { + return &refusalError{msg: fmt.Sprintf(format, args...)} +} + +func (e *refusalError) Error() string { return e.msg } + +func (e *refusalError) Is(target error) bool { return target == ErrGenerationRefused } + +// ErrUnknownServer indicates a build request named an embeddings server that +// is not defined in the manager's encoder set — caller input, not a manager +// fault. Match it with errors.Is; the error's own message carries the +// offending name. +var ErrUnknownServer = errors.New("unknown embeddings server") + +type unknownServerError struct { + name string +} + +func (e *unknownServerError) Error() string { + return fmt.Sprintf("no embeddings server named %q", e.name) +} + +func (e *unknownServerError) Is(target error) bool { return target == ErrUnknownServer } + +// Manager serializes embedding builds over one Index: only one Build call +// may run at a time, whether triggered via StartBuild (async, for the HTTP +// API) or TryBuild (sync, for a periodic scheduler). Activate and Retire +// are likewise serialized against each other and against build starts, so +// their check-then-act refusal invariants (Missing coverage, the +// active-generation check) hold under concurrent calls. +type Manager struct { + ix *Index + src UnitSource + encoders EncoderSet + gen kitvec.Generation + + // opMu serializes lifecycle operations: build starts (begin) and the + // whole of Activate/Retire. It is never held across a running build — + // begin releases it once running is set — so StartBuild stays + // non-blocking while a build is in flight. + opMu sync.Mutex + + // mu guards running and status; held only for short field updates so + // Status() stays responsive during a build. + mu sync.Mutex + running bool + status BuildStatus +} + +// BuildRequest is the caller-controlled subset of BuildOptions the manager +// exposes; encode settings and Progress are the manager's own concerns. +type BuildRequest struct { + FullRebuild bool `json:"full_rebuild,omitempty"` + Backstop bool `json:"backstop,omitempty"` + // IncludeAutomated is the resolved include-automated scope for this + // build (caller-resolved from config and, for the CLI's one-off + // --include-automated flag, its override). See BuildOptions.IncludeAutomated. + IncludeAutomated bool `json:"include_automated,omitempty"` + // Using names the embeddings server (an EncoderSet entry) this build + // encodes against; empty selects the set's default. Server choice is + // transport only — every server encodes the same model, so it never + // affects the generation fingerprint. + Using string `json:"using,omitempty"` +} + +// BuildStatus reports the manager's current build state, for polling +// clients (CLI and HTTP API). +type BuildStatus struct { + Running bool `json:"running"` + Phase string `json:"phase,omitempty"` + Done int64 `json:"done"` + Total int64 `json:"total"` + LastError string `json:"last_error,omitempty"` + LastResult *BuildResult `json:"last_result,omitempty"` +} + +// EncodeSettings groups the encode-shape knobs of one embeddings server, +// resolved from config ([vector.embeddings.servers.] batch_size and +// concurrency). +type EncodeSettings struct { + // BatchSize is the number of inputs sent per HTTP call. + BatchSize int + // Concurrency is the number of documents encoded in parallel. + Concurrency int +} + +// ManagedEncoder pairs one embeddings server's encoder with the encode +// settings tuned for that server. +type ManagedEncoder struct { + Encode kitvec.EncodeFunc + Settings EncodeSettings +} + +// EncoderSet is the named embeddings servers a Manager can build with. All +// entries encode the same model — the embedding-space identity is global +// config — so a build may use any of them interchangeably; Default names +// the entry used when a BuildRequest doesn't select one. +type EncoderSet struct { + Default string + ByName map[string]ManagedEncoder +} + +// NewManager creates a Manager that builds gen's embedding space over ix, +// scanning src and encoding with one of encoders' entries per build (the +// default, or BuildRequest.Using). Each encoder is wrapped so a panic +// inside it surfaces as an encode error rather than crashing the process +// (see recoveringEncoder). +func NewManager( + ix *Index, src UnitSource, encoders EncoderSet, gen kitvec.Generation, +) *Manager { + wrapped := EncoderSet{Default: encoders.Default, ByName: make(map[string]ManagedEncoder, len(encoders.ByName))} + for name, me := range encoders.ByName { + me.Encode = recoveringEncoder(me.Encode) + wrapped.ByName[name] = me + } + return &Manager{ix: ix, src: src, encoders: wrapped, gen: gen} +} + +// resolveEncoder picks the encoder a build request encodes with: the named +// entry when Using is set, the set's default otherwise. It fails when the +// name is unknown so a mistyped --using errors before a build starts. +func (m *Manager) resolveEncoder(req BuildRequest) (ManagedEncoder, error) { + name := req.Using + if name == "" { + name = m.encoders.Default + } + me, ok := m.encoders.ByName[name] + if !ok { + return ManagedEncoder{}, &unknownServerError{name: name} + } + return me, nil +} + +// recoveringEncoder converts a panic in enc (a caller-supplied network +// client) into an ordinary encode error. This must wrap the encoder itself +// rather than rely on runBuild's recover: kit's EncodeBatched invokes +// encoders on its own worker goroutines, where a recover on the manager's +// build goroutine cannot reach. +func recoveringEncoder(enc kitvec.EncodeFunc) kitvec.EncodeFunc { + return func(ctx context.Context, texts []string) (vectors [][]float32, err error) { + defer func() { + if r := recover(); r != nil { + vectors = nil + err = fmt.Errorf("encoder panicked: %v", r) + } + }() + return enc(ctx, texts) + } +} + +// StartBuild launches a Build in a background goroutine, returning +// ErrBuildRunning if one is already in flight rather than queuing behind it. +// The goroutine runs against context.Background() so it outlives the HTTP +// request that triggered it. +func (m *Manager) StartBuild(req BuildRequest) error { + me, err := m.resolveEncoder(req) + if err != nil { + return err + } + if err := m.begin(); err != nil { + return err + } + go func() { + result, err := m.runBuild(context.Background(), req, me) + m.finish(result, err) + }() + return nil +} + +// TryBuild runs one Build synchronously, for a periodic scheduler that +// should drop a scheduled run rather than queue it: it returns (false, nil) +// without starting anything if a build is already running. +func (m *Manager) TryBuild(ctx context.Context, req BuildRequest) (bool, error) { + me, err := m.resolveEncoder(req) + if err != nil { + return false, err + } + if err := m.begin(); err != nil { + return false, nil + } + result, err := m.runBuild(ctx, req, me) + m.finish(result, err) + return true, err +} + +// Status returns a snapshot of the manager's current build state. +func (m *Manager) Status() BuildStatus { + m.mu.Lock() + defer m.mu.Unlock() + status := m.status + if status.LastResult != nil { + result := *status.LastResult + status.LastResult = &result + } + return status +} + +// Generations delegates to the underlying Index, listing every generation +// with its coverage of the current mirror. +func (m *Manager) Generations(ctx context.Context) ([]GenerationInfo, error) { + return m.ix.Generations(ctx) +} + +// Activate transitions the generation identified by id to active, retiring +// whichever generation was previously active (in one transaction, via the +// same activateGeneration primitive Build's auto-activation uses, so two +// generations can never end up active simultaneously). Without force, it +// refuses when id's generation has documents still needing embedding +// (Missing > 0) or while a build is running. Serialized against Retire, +// other Activate calls, and build starts via opMu. +func (m *Manager) Activate(ctx context.Context, id int64, force bool) error { + m.opMu.Lock() + defer m.opMu.Unlock() + if m.isRunning() { + return ErrBuildRunning + } + + target, err := m.ix.GenerationByID(ctx, id) + if err != nil { + return err + } + if !force && target.Missing > 0 { + return refusedf("generation %d still has %d documents needing embedding; use --force", + id, target.Missing) + } + return m.ix.activateGeneration(ctx, target.Fingerprint) +} + +// Retire transitions the generation identified by id to retired. Without +// force, it refuses when id is the active generation or while a build is +// running. Serialized against Activate, other Retire calls, and build +// starts via opMu. +func (m *Manager) Retire(ctx context.Context, id int64, force bool) error { + m.opMu.Lock() + defer m.opMu.Unlock() + if m.isRunning() { + return ErrBuildRunning + } + + target, err := m.ix.GenerationByID(ctx, id) + if err != nil { + return err + } + if !force && target.State == string(sqlitevec.StateActive) { + return refusedf("generation %d is active; use --force to retire it", id) + } + return m.ix.SetStateByID(ctx, id, sqlitevec.StateRetired) +} + +// begin transitions the manager into the running state, resetting the +// progress fields of status for the new run. It returns ErrBuildRunning +// without changing anything if a build is already in flight. Taking opMu +// first serializes build starts behind any in-flight Activate/Retire. +func (m *Manager) begin() error { + m.opMu.Lock() + defer m.opMu.Unlock() + m.mu.Lock() + defer m.mu.Unlock() + if m.running { + return ErrBuildRunning + } + m.running = true + m.status.Running = true + m.status.Phase = "" + m.status.Done = 0 + m.status.Total = 0 + return nil +} + +// runBuild performs the actual Index.Build call, wiring the manager's +// reportProgress method in as the BuildOptions.Progress callback so Status +// reflects incremental progress while the build is in flight. It converts +// a panic (e.g. from the caller-supplied encoder, a network client) into an +// error so StartBuild's detached goroutine can never crash the process and +// TryBuild's caller sees a failure rather than a propagating panic; either +// way finish records it in LastError and clears the running state. +func (m *Manager) runBuild( + ctx context.Context, req BuildRequest, me ManagedEncoder, +) (result BuildResult, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("build panicked: %v", r) + } + }() + return m.ix.Build(ctx, m.src, me.Encode, m.gen, BuildOptions{ + FullRebuild: req.FullRebuild, + Backstop: req.Backstop, + IncludeAutomated: req.IncludeAutomated, + BatchSize: me.Settings.BatchSize, + Concurrency: me.Settings.Concurrency, + Progress: m.reportProgress, + }) +} + +// reportProgress updates status's progress fields under the manager's lock; +// it is passed as BuildOptions.Progress and so runs on the build goroutine. +func (m *Manager) reportProgress(p BuildProgress) { + m.mu.Lock() + defer m.mu.Unlock() + m.status.Phase = p.Phase + m.status.Done = p.Done + m.status.Total = p.Total +} + +// finish records a completed build's outcome and clears the running state. +// A successful build sets LastResult and clears any previous LastError; a +// failed build sets LastError and leaves the last successful LastResult (if +// any) untouched. +func (m *Manager) finish(result BuildResult, err error) { + m.mu.Lock() + defer m.mu.Unlock() + m.running = false + m.status.Running = false + if err != nil { + m.status.LastError = err.Error() + return + } + m.status.LastError = "" + r := result + m.status.LastResult = &r +} + +func (m *Manager) isRunning() bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.running +} diff --git a/internal/vector/manager_test.go b/internal/vector/manager_test.go new file mode 100644 index 000000000..84a2957b8 --- /dev/null +++ b/internal/vector/manager_test.go @@ -0,0 +1,391 @@ +package vector + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// soloEncoders wraps enc as a single-server EncoderSet, the shape most +// Manager tests need. +func soloEncoders(enc kitvec.EncodeFunc) EncoderSet { + return EncoderSet{Default: "test", ByName: map[string]ManagedEncoder{ + "test": {Encode: enc, Settings: EncodeSettings{BatchSize: 10}}, + }} +} + +// blockingEncoder returns an encoder that blocks until release is closed, +// letting tests observe a Manager while its build is still in flight. +func blockingEncoder(release <-chan struct{}) kitvec.EncodeFunc { + return func(_ context.Context, texts []string) ([][]float32, error) { + <-release + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } +} + +// waitFor polls cond until it returns true or the deadline passes, failing +// the test otherwise. Used instead of a fixed sleep to avoid flakiness. +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(time.Millisecond) + } + require.Fail(t, "timed out waiting for condition", msg) +} + +// generationIDByFingerprint looks up a generation's CLI-facing ordinal ID +// from its fingerprint, for tests that need to Activate/Retire a specific +// generation by ID. +func generationIDByFingerprint(t *testing.T, ix *Index, fp string) int64 { + t.Helper() + gens, err := ix.Generations(context.Background()) + require.NoError(t, err) + for _, g := range gens { + if g.Fingerprint == fp { + return g.ID + } + } + require.Fail(t, "generation not found for fingerprint", fp) + return 0 +} + +// countingEncoder returns a working encoder that counts its calls, so +// tests can tell which EncoderSet entry a build actually used. +func countingEncoder(calls *atomic.Int64) kitvec.EncodeFunc { + return func(_ context.Context, texts []string) ([][]float32, error) { + calls.Add(1) + out := make([][]float32, len(texts)) + for i := range texts { + out[i] = []float32{1, 0, 0} + } + return out, nil + } +} + +func TestManagerBuildUsingSelectsNamedEncoder(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + + var localCalls, remoteCalls atomic.Int64 + encoders := EncoderSet{Default: "local", ByName: map[string]ManagedEncoder{ + "local": {Encode: countingEncoder(&localCalls), Settings: EncodeSettings{BatchSize: 10}}, + "remote": {Encode: countingEncoder(&remoteCalls), Settings: EncodeSettings{BatchSize: 10}}, + }} + m := NewManager(ix, src, encoders, gen) + + started, err := m.TryBuild(context.Background(), BuildRequest{Using: "remote"}) + require.NoError(t, err) + require.True(t, started) + assert.Positive(t, remoteCalls.Load(), "build with Using must encode on the named server") + assert.Zero(t, localCalls.Load(), "the default server must stay idle") + + started, err = m.TryBuild(context.Background(), BuildRequest{FullRebuild: true}) + require.NoError(t, err) + require.True(t, started) + assert.Positive(t, localCalls.Load(), "a build without Using must encode on the default server") +} + +func TestManagerBuildUnknownUsingFailsBeforeStarting(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), gen) + + err := m.StartBuild(BuildRequest{Using: "nope"}) + require.ErrorContains(t, err, `no embeddings server named "nope"`) + assert.ErrorIs(t, err, ErrUnknownServer, + "callers map unknown-server errors to a client error via the sentinel") + assert.False(t, m.Status().Running, "a failed resolve must not leave the manager running") + + started, err := m.TryBuild(context.Background(), BuildRequest{Using: "nope"}) + require.ErrorContains(t, err, `no embeddings server named "nope"`) + assert.ErrorIs(t, err, ErrUnknownServer) + assert.False(t, started) +} + +func TestManagerStartBuildSetsRunningAndConcurrentStartReturnsErrBuildRunning(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + release := make(chan struct{}) + m := NewManager(ix, src, soloEncoders(blockingEncoder(release)), gen) + + require.NoError(t, m.StartBuild(BuildRequest{})) + waitFor(t, func() bool { return m.Status().Running }, "build never reported running") + + err := m.StartBuild(BuildRequest{}) + assert.ErrorIs(t, err, ErrBuildRunning) + + close(release) + waitFor(t, func() bool { return !m.Status().Running }, "build never finished") + assert.Empty(t, m.Status().LastError) +} + +func TestManagerTryBuildReturnsFalseWhileRunning(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + release := make(chan struct{}) + m := NewManager(ix, src, soloEncoders(blockingEncoder(release)), gen) + + require.NoError(t, m.StartBuild(BuildRequest{})) + waitFor(t, func() bool { return m.Status().Running }, "build never reported running") + + started, err := m.TryBuild(context.Background(), BuildRequest{}) + assert.False(t, started, "TryBuild must drop rather than queue while running") + assert.NoError(t, err) + + close(release) + waitFor(t, func() bool { return !m.Status().Running }, "build never finished") +} + +func TestManagerStatusTransitionsToLastResultOnCompletion(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), gen) + + require.NoError(t, m.StartBuild(BuildRequest{})) + waitFor(t, func() bool { return !m.Status().Running }, "build never finished") + + status := m.Status() + require.NotNil(t, status.LastResult) + assert.Equal(t, gen.Fingerprint(), status.LastResult.Fingerprint) + assert.True(t, status.LastResult.Activated) + assert.Empty(t, status.LastError) +} + +func TestManagerStatusSetsLastErrorOnEncoderFailure(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + failingEncoder := func(_ context.Context, _ []string) ([][]float32, error) { + return nil, fmt.Errorf("encoder rejected input") + } + m := NewManager(ix, src, soloEncoders(failingEncoder), gen) + + require.NoError(t, m.StartBuild(BuildRequest{})) + waitFor(t, func() bool { return !m.Status().Running }, "build never finished") + + status := m.Status() + assert.Contains(t, status.LastError, "encoder rejected input") + assert.Nil(t, status.LastResult) +} + +func TestManagerGenerationsDelegatesToIndex(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), gen) + + _, err := m.TryBuild(ctx, BuildRequest{}) + require.NoError(t, err) + + want, err := ix.Generations(ctx) + require.NoError(t, err) + got, err := m.Generations(ctx) + require.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestManagerActivateForceRefusalMatrix(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + genA := fakeGeneration("model-a") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), genA) + + _, err := m.TryBuild(ctx, BuildRequest{}) + require.NoError(t, err, "genA becomes active") + + genB := fakeGeneration("model-b") + fpB, err := ix.EnsureGeneration(ctx, genB, sqlitevec.StateBuilding) + require.NoError(t, err, "genB registered but never filled, so it has Missing > 0") + idB := generationIDByFingerprint(t, ix, fpB) + + err = m.Activate(ctx, idB, false) + require.Error(t, err, "refuses activation of an incompletely embedded generation") + assert.Contains(t, err.Error(), fmt.Sprintf("generation %d still has", idB)) + assert.Contains(t, err.Error(), "use --force") + + require.NoError(t, m.Activate(ctx, idB, true), "force overrides the refusal") + + active, ok, err := ix.ActiveFingerprint(ctx) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, fpB, active, "genB is now active") + + idA := generationIDByFingerprint(t, ix, genA.Fingerprint()) + gens, err := ix.Generations(ctx) + require.NoError(t, err) + var stateA string + for _, g := range gens { + if g.ID == idA { + stateA = g.State + } + } + assert.Equal(t, string(sqlitevec.StateRetired), stateA, "activating genB retires the old active genA") +} + +func TestManagerRetireRefusesActiveGenerationWithoutForce(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), gen) + + _, err := m.TryBuild(ctx, BuildRequest{}) + require.NoError(t, err) + id := generationIDByFingerprint(t, ix, gen.Fingerprint()) + + err = m.Retire(ctx, id, false) + require.Error(t, err, "refuses retiring the active generation without force") + + require.NoError(t, m.Retire(ctx, id, true), "force overrides the refusal") + + gens, err := ix.Generations(ctx) + require.NoError(t, err) + require.Len(t, gens, 1) + assert.Equal(t, string(sqlitevec.StateRetired), gens[0].State) +} + +// countActiveGenerations returns how many generations are currently in the +// active state, the invariant concurrent Activate calls must preserve (== 1 +// once any generation has been activated). +func countActiveGenerations(t *testing.T, ix *Index) int { + t.Helper() + gens, err := ix.Generations(context.Background()) + require.NoError(t, err) + active := 0 + for _, g := range gens { + if g.State == string(sqlitevec.StateActive) { + active++ + } + } + return active +} + +// TestManagerConcurrentActivateNeverLeavesTwoActive guards the +// retire-then-activate invariant: Activate must use the single-transaction +// activateGeneration primitive and serialize against other Activate calls, +// or two racing Activates on different generations can interleave their +// retire and activate steps and leave both generations active. +func TestManagerConcurrentActivateNeverLeavesTwoActive(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + genA := fakeGeneration("model-a") + genB := fakeGeneration("model-b") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), genA) + + _, err := ix.Build(ctx, src, fakeBuildEncoder(), genA, BuildOptions{}) + require.NoError(t, err) + _, err = ix.Build(ctx, src, fakeBuildEncoder(), genB, BuildOptions{}) + require.NoError(t, err, "both generations fully embedded; genB active, genA retired") + + idA := generationIDByFingerprint(t, ix, genA.Fingerprint()) + idB := generationIDByFingerprint(t, ix, genB.Fingerprint()) + + for i := range 25 { + var wg sync.WaitGroup + var errA, errB error + wg.Add(2) + go func() { + defer wg.Done() + errA = m.Activate(ctx, idA, false) + }() + go func() { + defer wg.Done() + errB = m.Activate(ctx, idB, false) + }() + wg.Wait() + require.NoError(t, errA, "iteration %d", i) + require.NoError(t, errB, "iteration %d", i) + require.Equal(t, 1, countActiveGenerations(t, ix), + "iteration %d: exactly one generation must be active after racing Activates", i) + } +} + +// TestManagerStartBuildRecoversPanickedEncoder guards the daemon against a +// panic in the caller-supplied encoder (a network client): StartBuild's +// detached goroutine must recover, record the panic in LastError, and clear +// the running state instead of crashing the process. +func TestManagerStartBuildRecoversPanickedEncoder(t *testing.T) { + ix := openTestIndex(t) + src := twoDocSource() + gen := fakeGeneration("fake-model") + panickingEncoder := func(_ context.Context, _ []string) ([][]float32, error) { + panic("encoder exploded") + } + m := NewManager(ix, src, soloEncoders(panickingEncoder), gen) + + require.NoError(t, m.StartBuild(BuildRequest{})) + waitFor(t, func() bool { return !m.Status().Running }, "build never finished after panic") + + status := m.Status() + assert.Contains(t, status.LastError, "panicked") + assert.Contains(t, status.LastError, "encoder exploded") + assert.Nil(t, status.LastResult) + + require.NoError(t, m.StartBuild(BuildRequest{}), + "manager must accept a new build after a panicked one") + waitFor(t, func() bool { return !m.Status().Running }, "second build never finished") +} + +// TestManagerActivateAndRetireUnknownIDPropagateNotFound guards the HTTP +// route mapping (embeddingsActionError in internal/server): Activate and +// Retire must propagate GenerationByID's ErrGenerationNotFound unwrapped +// enough for errors.Is to still match it, rather than losing the sentinel +// on the way up. +func TestManagerActivateAndRetireUnknownIDPropagateNotFound(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + m := NewManager(ix, src, soloEncoders(fakeBuildEncoder()), gen) + + err := m.Activate(ctx, 999, false) + assert.ErrorIs(t, err, ErrGenerationNotFound) + + err = m.Retire(ctx, 999, false) + assert.ErrorIs(t, err, ErrGenerationNotFound) +} + +func TestManagerActivateAndRetireRefuseWhileBuildRunning(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := twoDocSource() + gen := fakeGeneration("fake-model") + release := make(chan struct{}) + m := NewManager(ix, src, soloEncoders(blockingEncoder(release)), gen) + + require.NoError(t, m.StartBuild(BuildRequest{})) + waitFor(t, func() bool { return m.Status().Running }, "build never reported running") + + err := m.Activate(ctx, 1, true) + assert.ErrorIs(t, err, ErrBuildRunning) + + err = m.Retire(ctx, 1, true) + assert.ErrorIs(t, err, ErrBuildRunning) + + close(release) + waitFor(t, func() bool { return !m.Status().Running }, "build never finished") +} diff --git a/internal/vector/mirror.go b/internal/vector/mirror.go new file mode 100644 index 000000000..cd7a7727a --- /dev/null +++ b/internal/vector/mirror.go @@ -0,0 +1,578 @@ +package vector + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + "strings" + + "go.kenn.io/agentsview/internal/db" +) + +// refreshWatermarkKey is the vector_meta key holding the RFC3339 ended_at +// high-water mark of the most recent Refresh scan, used to restrict the +// next incremental (full=false) scan to newer sessions. +const refreshWatermarkKey = "refresh_watermark" + +// scopeIncludeAutomatedKey is the vector_meta key holding the +// include-automated scope ("true"/"false") the mirror was last refreshed +// under. Build compares it against the requested scope on every call: the +// scope is part of the mirror's identity, not the embedding fingerprint, so +// a change forces a full reconciliation scan rather than silently leaving +// now-out-of-scope rows (and their vectors) behind or missing newly-in-scope +// sessions an incremental scan's watermark would skip. +const scopeIncludeAutomatedKey = "scope_include_automated" + +// maxSQLVars caps bind variables per IN (...) clause to stay within +// SQLite's default SQLITE_MAX_VARIABLE_NUMBER (999), mirroring +// internal/db's constant of the same purpose: a pathological refresh (a +// large eviction batch) or a deep semantic overfetch can otherwise push a +// single-shot query over SQLite's limit. +const maxSQLVars = 500 + +// chunkKeys invokes fn once per maxSQLVars-sized slice of keys, for callers +// binding one key per IN (...) placeholder. +func chunkKeys(keys []string, fn func(chunk []string) error) error { + for start := 0; start < len(keys); start += maxSQLVars { + end := min(start+maxSQLVars, len(keys)) + if err := fn(keys[start:end]); err != nil { + return err + } + } + return nil +} + +// inPlaceholders returns a "(?,?,...)" string and []any args for a slice of +// string keys, for use inside an IN (...) clause. +func inPlaceholders(keys []string) (string, []any) { + args := make([]any, len(keys)) + for i, k := range keys { + args[i] = k + } + return "(" + strings.TrimSuffix(strings.Repeat("?,", len(keys)), ",") + ")", args +} + +// UnitSource is the slice of the archive the mirror needs (implemented by +// *db.DB): the stream of embedding-unit documents — individual user +// messages and runs of contiguous assistant messages. +type UnitSource interface { + ScanEmbeddableUnits(ctx context.Context, since string, includeAutomated bool, + fn func(db.EmbeddableUnit) error) (string, error) +} + +// RefreshStats summarizes one Refresh call: Upserted counts mirror rows +// inserted or changed (new identity or content_hash changed; this includes +// a doc_key reinserted after a same-scan slot eviction, see Refresh), +// Unchanged counts rows rescanned with an identical content_hash (e.g. an +// ordinal-only shift with no eviction involved), and Deleted counts mirror +// rows genuinely removed — a slot-evicted doc_key not reinserted anywhere +// else in the same scan, or, in full mode, an identity not seen in the scan +// at all. +type RefreshStats struct { + Upserted int + Deleted int + Unchanged int +} + +// DocKey computes the mirror's document identity for a unit: a source_uuid +// (the unit's first member's, for a run) keeps the key stable across +// ordinal-shifting rewrites (compaction, resync) and across later run +// members appending, splitting off, or changing; its absence falls back to +// a session+ordinal key. kind selects the prefix scheme: "run" units use +// "r:" (uuid) / "ro:" (ordinal fallback), "user" units use "u:" / "o:". +// +// The messages schema permits more than one message in a session to share a +// non-empty source_uuid, so occurrence disambiguates them: it is the 1-based +// count of how many times (sessionID, sourceUUID) has been seen so far in +// scan order, shared across unit kinds. The first occurrence keeps the +// plain ":" key; later occurrences append +// "#". Since the scan is ordered by (session_id, ordinal) of +// each unit's first member, occurrence assignment is deterministic and +// stable across resyncs. occurrence is ignored when sourceUUID is empty. +// +// sessionID and sourceUUID are percent-escaped (escapeDocKeyComponent) +// before joining so the ":" and "#" delimiters, and any literal "%", inside +// either component cannot be confused with the key's own structure — e.g. +// source_uuid "dup#2" at its first occurrence would otherwise collide with +// source_uuid "dup" at its second occurrence. +func DocKey(kind, sessionID, sourceUUID string, ordinal, occurrence int) string { + uuidPrefix, ordinalPrefix := "u:", "o:" + if kind == "run" { + uuidPrefix, ordinalPrefix = "r:", "ro:" + } + session := escapeDocKeyComponent(sessionID) + if sourceUUID != "" { + uuid := escapeDocKeyComponent(sourceUUID) + if occurrence > 1 { + return uuidPrefix + session + ":" + uuid + "#" + strconv.Itoa(occurrence) + } + return uuidPrefix + session + ":" + uuid + } + return ordinalPrefix + session + ":" + strconv.Itoa(ordinal) +} + +// escapeDocKeyComponent percent-encodes the characters DocKey uses as +// delimiters — ':', '#', and '%' itself — so a session_id or source_uuid +// containing them cannot be mistaken for key structure, keeping DocKey +// injective. +func escapeDocKeyComponent(s string) string { + if !strings.ContainsAny(s, "%:#") { + return s + } + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + switch r { + case '%', ':', '#': + fmt.Fprintf(&b, "%%%02X", r) + default: + b.WriteRune(r) + } + } + return b.String() +} + +// contentHash returns the mirror's content_hash for content: kit's +// sqlitevec store uses it as the revision column, so any change here +// invalidates the embedding stamp and marks the document pending. +func contentHash(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} + +// Refresh reconciles the vector_messages mirror against src. full=true +// scans the entire archive (since="") and additionally deletes mirror rows +// (and their vectors, via store.DeleteVectors) whose identity was not seen +// in the scan; full=false scans only sessions newer than the stored +// watermark (vector_meta key "refresh_watermark") and only upserts, +// leaving that reconciliation to a subsequent full refresh. includeAutomated +// is passed through to src.ScanEmbeddableUnits: false excludes automated +// sessions from the scan entirely, so their mirror rows are absent (and, in +// full mode, reconciled away) rather than merely unembedded. Either mode +// also resolves same-scan slot evictions (see evictSlotOccupant) once the +// scan completes: a UUID-keyed doc_key evicted from a (session_id, ordinal) +// slot it no longer occupies is deleted via store.DeleteVectors only if it +// was not reinserted elsewhere in the same scan, so a row that merely +// shifted (or was displaced in a shift cascade) keeps its embeddings. The +// watermark is advanced to the scan's max ended_at afterwards. +func (ix *Index) Refresh( + ctx context.Context, src UnitSource, full, includeAutomated bool, +) (RefreshStats, error) { + if err := ix.requireWritable(); err != nil { + return RefreshStats{}, err + } + + since := "" + if !full { + watermark, err := ix.refreshWatermark(ctx) + if err != nil { + return RefreshStats{}, err + } + since = watermark + } + + var stats RefreshStats + seen := make(map[string]struct{}) + occurrences := make(map[string]int) + evicted := make(map[string]struct{}) + sentinel, err := ix.parkingFloor(ctx) + if err != nil { + return RefreshStats{}, err + } + maxEnded, err := src.ScanEmbeddableUnits(ctx, since, includeAutomated, func(u db.EmbeddableUnit) error { + occurrence := 1 + if u.SourceUUID != "" { + occKey := u.SessionID + "\x00" + u.SourceUUID + occurrences[occKey]++ + occurrence = occurrences[occKey] + } + key := DocKey(u.Kind, u.SessionID, u.SourceUUID, u.Ordinal, occurrence) + unchanged, evictedKeys, err := ix.upsertMirrorRow(ctx, key, u, &sentinel) + if err != nil { + return fmt.Errorf("upserting mirror row %s: %w", key, err) + } + for _, k := range evictedKeys { + evicted[k] = struct{}{} + } + if unchanged { + stats.Unchanged++ + } else { + stats.Upserted++ + } + seen[key] = struct{}{} + return nil + }) + if err != nil { + return RefreshStats{}, fmt.Errorf("scanning embeddable units: %w", err) + } + + // finalizeEvictions must run before full-mode reconcileDeletions: an + // evicted key that never reappears anywhere in the scan is absent from + // seen too, so reconcileDeletions would otherwise also treat its (still + // present, sentinel-parked) row as a vanished identity and delete it a + // second time. Resolving evictions first means the row is gone by the + // time reconcileDeletions scans vector_messages, so it is never counted + // there. finalizeEvictions also guards against re-deleting an + // already-absent key on its own (see its doc comment), so this ordering + // and that guard together make Refresh's accounting robust regardless + // of which pass would otherwise see the row first. + finalized, err := ix.finalizeEvictions(ctx, evicted) + if err != nil { + return RefreshStats{}, err + } + stats.Deleted += finalized + + if full { + deleted, err := ix.reconcileDeletions(ctx, seen) + if err != nil { + return RefreshStats{}, err + } + stats.Deleted += deleted + } + + if maxEnded != "" { + if err := ix.setRefreshWatermark(ctx, maxEnded); err != nil { + return RefreshStats{}, err + } + } + + return stats, nil +} + +// upsertMirrorRow evicts any row occupying the same (session_id, ordinal) +// slot under a different doc_key, then upserts key's row from u. It returns +// whether the row's content_hash was unchanged (a no-op update, e.g. an +// ordinal-only shift) and the doc_key(s) the slot eviction displaced (0 or +// 1), for the caller to reconcile once the whole scan completes. sentinel +// is a per-Refresh-call counter evictSlotOccupant uses to park a displaced +// row at a unique negative ordinal; see evictSlotOccupant. +func (ix *Index) upsertMirrorRow( + ctx context.Context, key string, u db.EmbeddableUnit, sentinel *int, +) (unchanged bool, evicted []string, err error) { + evicted, err = ix.evictSlotOccupant(ctx, key, u.SessionID, u.Ordinal, sentinel) + if err != nil { + return false, nil, err + } + + var existingHash sql.NullString + err = ix.db.QueryRowContext(ctx, + `SELECT content_hash FROM vector_messages WHERE doc_key = ?`, key, + ).Scan(&existingHash) + if err != nil && err != sql.ErrNoRows { + return false, evicted, fmt.Errorf("reading existing content hash: %w", err) + } + + hash := contentHash(u.Content) + unchanged = existingHash.Valid && existingHash.String == hash + + offsets, err := marshalOffsets(u.Offsets) + if err != nil { + return false, evicted, err + } + + if _, err := ix.db.ExecContext(ctx, ` +INSERT INTO vector_messages (doc_key, session_id, source_uuid, ordinal, ordinal_end, + subordinate, offsets, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) +ON CONFLICT(doc_key) DO UPDATE SET + session_id = excluded.session_id, + ordinal = excluded.ordinal, + ordinal_end = excluded.ordinal_end, + subordinate = excluded.subordinate, + offsets = excluded.offsets, + content = excluded.content, + content_hash = excluded.content_hash`, + key, u.SessionID, u.SourceUUID, u.Ordinal, u.OrdinalEnd, + u.Subordinate, offsets, u.Content, hash, + ); err != nil { + return false, evicted, fmt.Errorf("upserting row: %w", err) + } + return unchanged, evicted, nil +} + +// marshalOffsets encodes a unit's member offsets for the mirror's offsets +// column. A nil slice (every user doc; see db.EmbeddableUnit.Offsets) is +// stored as the schema's canonical empty array "[]" rather than +// encoding/json's "null" for a nil slice, matching the column's DEFAULT and +// sparing readers a null case. +func marshalOffsets(offsets []db.UnitOffset) (string, error) { + if offsets == nil { + return "[]", nil + } + encoded, err := json.Marshal(offsets) + if err != nil { + return "", fmt.Errorf("marshaling unit offsets: %w", err) + } + return string(encoded), nil +} + +// evictSlotOccupant parks the mirror row of any doc_key occupying +// (sessionID, ordinal) under a key other than key at a unique negative +// ordinal, guarding the mirror's unique index before an upsert lands on +// that slot without deleting the row outright. Message ordinals are always +// >= 0, so a negative ordinal can never collide with a real one or with +// another parked row: sentinel is a counter shared across one Refresh +// scan, decremented per eviction to keep every parked ordinal distinct. +// +// The row is left in place, not deleted, so that if the same doc_key is +// reinserted later in the same scan (a stable UUID-keyed identity that +// merely shifted position in a cascade), upsertMirrorRow's ON CONFLICT(doc_key) +// path updates it in place and never touches the embed_gen column kit's +// SaveVectors stamped it with — a fresh INSERT would reset embed_gen to +// NULL and silently uncover the document. Whether an evicted key is +// genuinely gone or was reinserted is not decidable until the whole scan +// finishes, so store.DeleteVectors and the row's actual removal are +// deferred to Refresh's post-scan finalizeEvictions pass. +// parkingFloor returns the starting value for Refresh's parking sentinel: +// 0 when the mirror holds no parked rows, otherwise the most negative +// parked ordinal already present. Parking writes are individual autocommit +// updates, so a Refresh interrupted between evictSlotOccupant and +// finalizeEvictions leaves rows parked at negative ordinals; a later run +// restarting its sentinel at 0 could then park a row in the same session at +// an already-taken negative ordinal and fail the unique (session_id, +// ordinal) index — deterministically on every retry, wedging refreshes. +// Seeding below the leftover floor keeps every parked ordinal unique across +// runs; the leftovers themselves self-heal (reinserted rows overwrite their +// ordinal, full-mode reconciliation deletes vanished ones). +func (ix *Index) parkingFloor(ctx context.Context) (int, error) { + var floor sql.NullInt64 + if err := ix.db.QueryRowContext(ctx, + `SELECT MIN(ordinal) FROM vector_messages WHERE ordinal < 0`, + ).Scan(&floor); err != nil { + return 0, fmt.Errorf("reading parked-ordinal floor: %w", err) + } + if !floor.Valid { + return 0, nil + } + return int(floor.Int64), nil +} + +func (ix *Index) evictSlotOccupant( + ctx context.Context, key, sessionID string, ordinal int, sentinel *int, +) ([]string, error) { + rows, err := ix.db.QueryContext(ctx, + `SELECT doc_key FROM vector_messages + WHERE session_id = ? AND ordinal = ? AND doc_key != ?`, + sessionID, ordinal, key) + if err != nil { + return nil, fmt.Errorf("finding slot occupant: %w", err) + } + var evictKeys []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return nil, fmt.Errorf("scanning slot occupant: %w", err) + } + evictKeys = append(evictKeys, k) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, fmt.Errorf("iterating slot occupants: %w", err) + } + rows.Close() + + for _, k := range evictKeys { + *sentinel-- + if _, err := ix.db.ExecContext(ctx, + `UPDATE vector_messages SET ordinal = ? WHERE doc_key = ?`, *sentinel, k, + ); err != nil { + return nil, fmt.Errorf("evicting slot occupant %s: %w", k, err) + } + } + return evictKeys, nil +} + +// finalizeEvictions resolves every doc_key evictSlotOccupant displaced +// during one Refresh scan: a key whose ordinal is still negative (the +// sentinel evictSlotOccupant parked it at) once the scan is done was never +// reinserted, so it is genuinely gone — its vectors and stamps are deleted +// via store.DeleteVectors, and its mirror row is finally removed. kit's +// store keeps orphaned vectors occupying KNN LIMIT slots even though +// QueryGeneration filters them from hits, so this cleanup matters even +// though the row itself is inert. A key whose ordinal was overwritten back +// to a real (non-negative) value was reinserted under its own doc_key later +// in the same scan — it merely shifted position and keeps its row and +// embeddings untouched. +// +// A key already absent from vector_messages entirely (ok is false below) is +// skipped rather than deleted again: Refresh runs finalizeEvictions before +// full-mode reconcileDeletions specifically so this case shouldn't arise +// within one call, but the guard makes the accounting correct regardless of +// call order — an evicted key that never reappears in the scan is also +// absent from seen, so without this guard reconcileDeletions would delete +// the row once and finalizeEvictions would count deleting it again. +func (ix *Index) finalizeEvictions(ctx context.Context, evicted map[string]struct{}) (int, error) { + if len(evicted) == 0 { + return 0, nil + } + keys := make([]string, 0, len(evicted)) + for k := range evicted { + keys = append(keys, k) + } + ordinals, err := ix.currentOrdinals(ctx, keys) + if err != nil { + return 0, err + } + + var deleted int + for _, key := range keys { + ordinal, ok := ordinals[key] + if !ok { + continue // already removed from the mirror; nothing left to do + } + if ordinal >= 0 { + continue // reinserted under its own doc_key later in the same scan + } + if err := ix.store.DeleteVectors(ctx, key); err != nil { + return deleted, fmt.Errorf("deleting evicted vectors for %s: %w", key, err) + } + if _, err := ix.db.ExecContext(ctx, + `DELETE FROM vector_messages WHERE doc_key = ?`, key, + ); err != nil { + return deleted, fmt.Errorf("deleting evicted mirror row %s: %w", key, err) + } + deleted++ + } + return deleted, nil +} + +// currentOrdinals returns the current ordinal of each of keys that is still +// present in vector_messages; a key absent from the result was somehow +// already removed from the mirror. keys is queried in maxSQLVars-sized +// chunks: a large eviction batch in a single Refresh scan can otherwise +// exceed SQLite's bind-variable limit. +func (ix *Index) currentOrdinals(ctx context.Context, keys []string) (map[string]int, error) { + ordinals := make(map[string]int, len(keys)) + err := chunkKeys(keys, func(chunk []string) error { + placeholders, args := inPlaceholders(chunk) + rows, err := ix.db.QueryContext(ctx, + `SELECT doc_key, ordinal FROM vector_messages WHERE doc_key IN `+placeholders, args...) + if err != nil { + return fmt.Errorf("checking evicted doc_key ordinals: %w", err) + } + for rows.Next() { + var k string + var ordinal int + if err := rows.Scan(&k, &ordinal); err != nil { + rows.Close() + return fmt.Errorf("scanning evicted doc_key ordinal: %w", err) + } + ordinals[k] = ordinal + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("checking evicted doc_key ordinals: %w", err) + } + return rows.Close() + }) + if err != nil { + return nil, err + } + return ordinals, nil +} + +// reconcileDeletions deletes every mirror row (and its vectors) whose +// doc_key was not seen in a full scan. +func (ix *Index) reconcileDeletions( + ctx context.Context, seen map[string]struct{}, +) (int, error) { + rows, err := ix.db.QueryContext(ctx, `SELECT doc_key FROM vector_messages`) + if err != nil { + return 0, fmt.Errorf("listing mirror doc_keys: %w", err) + } + var vanished []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + rows.Close() + return 0, fmt.Errorf("scanning mirror doc_key: %w", err) + } + if _, ok := seen[key]; !ok { + vanished = append(vanished, key) + } + } + if err := rows.Err(); err != nil { + rows.Close() + return 0, fmt.Errorf("iterating mirror doc_keys: %w", err) + } + rows.Close() + + for _, key := range vanished { + if err := ix.store.DeleteVectors(ctx, key); err != nil { + return 0, fmt.Errorf("deleting vectors for %s: %w", key, err) + } + if _, err := ix.db.ExecContext(ctx, + `DELETE FROM vector_messages WHERE doc_key = ?`, key, + ); err != nil { + return 0, fmt.Errorf("deleting mirror row %s: %w", key, err) + } + } + return len(vanished), nil +} + +// refreshWatermark reads the stored refresh watermark, returning "" when +// none has been recorded yet. +func (ix *Index) refreshWatermark(ctx context.Context) (string, error) { + var value string + err := ix.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, refreshWatermarkKey, + ).Scan(&value) + if err == sql.ErrNoRows { + return "", nil + } + if err != nil { + return "", fmt.Errorf("reading refresh watermark: %w", err) + } + return value, nil +} + +// setRefreshWatermark advances the stored refresh watermark to value. +func (ix *Index) setRefreshWatermark(ctx context.Context, value string) error { + if _, err := ix.db.ExecContext(ctx, ` +INSERT INTO vector_meta (key, value) VALUES (?, ?) +ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + refreshWatermarkKey, value, + ); err != nil { + return fmt.Errorf("advancing refresh watermark: %w", err) + } + return nil +} + +// storedIncludeAutomatedScope reads the include-automated scope the mirror +// was last refreshed under. ok is false when no scope has ever been stored +// (the mirror's first build), in which case value is meaningless. +func (ix *Index) storedIncludeAutomatedScope(ctx context.Context) (value, ok bool, err error) { + var raw string + err = ix.db.QueryRowContext(ctx, + `SELECT value FROM vector_meta WHERE key = ?`, scopeIncludeAutomatedKey, + ).Scan(&raw) + if err == sql.ErrNoRows { + return false, false, nil + } + if err != nil { + return false, false, fmt.Errorf("reading stored include-automated scope: %w", err) + } + parsed, err := strconv.ParseBool(raw) + if err != nil { + return false, false, fmt.Errorf("parsing stored include-automated scope %q: %w", raw, err) + } + return parsed, true, nil +} + +// setIncludeAutomatedScope records value as the include-automated scope the +// mirror was most recently refreshed under. +func (ix *Index) setIncludeAutomatedScope(ctx context.Context, value bool) error { + if _, err := ix.db.ExecContext(ctx, ` +INSERT INTO vector_meta (key, value) VALUES (?, ?) +ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + scopeIncludeAutomatedKey, strconv.FormatBool(value), + ); err != nil { + return fmt.Errorf("storing include-automated scope: %w", err) + } + return nil +} diff --git a/internal/vector/mirror_test.go b/internal/vector/mirror_test.go new file mode 100644 index 000000000..f3dc85195 --- /dev/null +++ b/internal/vector/mirror_test.go @@ -0,0 +1,975 @@ +package vector + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// fakeUnitSource is a slice-backed UnitSource for mirror tests. It records +// the since/includeAutomated values it was called with and filters rows +// whose EndedAt is below since or whose automated flag is out of scope, +// mimicking db.ScanEmbeddableUnits's semantics. +type fakeUnitSource struct { + rows []fakeUnit + gotSince string + gotIncludeAutomated bool +} + +// fakeUnit pairs an EmbeddableUnit with the ended_at of its session, so the +// fake can compute a maxEnded watermark the way the real scan does. +// automated mimics sessions.is_automated: a false zero value means the row +// is never excluded regardless of includeAutomated. +type fakeUnit struct { + unit db.EmbeddableUnit + endedAt string + automated bool +} + +func (f *fakeUnitSource) ScanEmbeddableUnits( + _ context.Context, since string, includeAutomated bool, + fn func(db.EmbeddableUnit) error, +) (string, error) { + f.gotSince = since + f.gotIncludeAutomated = includeAutomated + var maxEnded string + for _, r := range f.rows { + if since != "" && r.endedAt < since { + continue + } + if r.automated && !includeAutomated { + continue + } + if err := fn(r.unit); err != nil { + return "", err + } + if r.endedAt > maxEnded { + maxEnded = r.endedAt + } + } + return maxEnded, nil +} + +// userDoc builds the single-message "user" unit shape most mirror and build +// tests use. +func userDoc(sessionID, sourceUUID string, ordinal int, content string) db.EmbeddableUnit { + return db.EmbeddableUnit{ + SessionID: sessionID, Kind: "user", SourceUUID: sourceUUID, + Ordinal: ordinal, OrdinalEnd: ordinal, Content: content, + } +} + +// runDoc builds a "run" unit spanning [start, end] with the given joined +// content and member offsets. +func runDoc( + sessionID, sourceUUID string, start, end int, + content string, offsets []db.UnitOffset, +) db.EmbeddableUnit { + return db.EmbeddableUnit{ + SessionID: sessionID, Kind: "run", SourceUUID: sourceUUID, + Ordinal: start, OrdinalEnd: end, Content: content, Offsets: offsets, + } +} + +func openTestIndex(t *testing.T) *Index { + t.Helper() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + ix, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ix.Close()) }) + return ix +} + +// vectorMessagesRow reads back one vector_messages row for assertions. +type vectorMessagesRow struct { + sessionID string + ordinal int + ordinalEnd int + subordinate bool + offsets string + content string + contentHash string +} + +func readMirrorRow(t *testing.T, ix *Index, docKey string) (vectorMessagesRow, bool) { + t.Helper() + var row vectorMessagesRow + err := ix.db.QueryRow( + `SELECT session_id, ordinal, ordinal_end, subordinate, offsets, content, content_hash + FROM vector_messages WHERE doc_key = ?`, docKey, + ).Scan(&row.sessionID, &row.ordinal, &row.ordinalEnd, &row.subordinate, + &row.offsets, &row.content, &row.contentHash) + if err != nil { + return vectorMessagesRow{}, false + } + return row, true +} + +func mirrorDocKeys(t *testing.T, ix *Index) []string { + t.Helper() + rows, err := ix.db.Query(`SELECT doc_key FROM vector_messages ORDER BY doc_key`) + require.NoError(t, err) + defer rows.Close() + var keys []string + for rows.Next() { + var k string + require.NoError(t, rows.Scan(&k)) + keys = append(keys, k) + } + require.NoError(t, rows.Err()) + return keys +} + +func TestDocKey(t *testing.T) { + assert.Equal(t, "u:sess-1:uuid-1", DocKey("user", "sess-1", "uuid-1", 5, 1)) + assert.Equal(t, "u:sess-1:uuid-1#2", DocKey("user", "sess-1", "uuid-1", 5, 2)) + assert.Equal(t, "u:sess-1:uuid-1#3", DocKey("user", "sess-1", "uuid-1", 5, 3)) + assert.Equal(t, "o:sess-1:5", DocKey("user", "sess-1", "", 5, 1)) + assert.Equal(t, "o:sess-1:5", DocKey("user", "sess-1", "", 5, 2), + "occurrence is ignored when source_uuid is empty") + + assert.Equal(t, "r:sess-1:uuid-1", DocKey("run", "sess-1", "uuid-1", 5, 1)) + assert.Equal(t, "r:sess-1:uuid-1#2", DocKey("run", "sess-1", "uuid-1", 5, 2)) + assert.Equal(t, "ro:sess-1:5", DocKey("run", "sess-1", "", 5, 1)) + assert.Equal(t, "ro:sess-1:5", DocKey("run", "sess-1", "", 5, 2), + "occurrence is ignored when source_uuid is empty") +} + +func TestRefreshInitialFullInsertsRowsWithCorrectDocKeys(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "", 1, "world"), endedAt: "2024-01-01T00:00:01Z"}, + }} + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 2, Deleted: 0, Unchanged: 0}, stats) + + keys := mirrorDocKeys(t, ix) + assert.Equal(t, []string{"o:s1:1", "u:s1:u1"}, keys) + + row, ok := readMirrorRow(t, ix, "u:s1:u1") + require.True(t, ok) + assert.Equal(t, "s1", row.sessionID) + assert.Equal(t, 0, row.ordinal) + assert.Equal(t, 0, row.ordinalEnd) + assert.False(t, row.subordinate) + assert.Equal(t, "[]", row.offsets, "user docs store empty offsets") + assert.Equal(t, "hello", row.content) + assert.NotEmpty(t, row.contentHash) +} + +// TestRefreshRunRowPersistsUnitColumns asserts a run unit's mirror row +// round-trips every v2 column: ordinal (start), ordinal_end, subordinate, +// and the offsets JSON, alongside content and content_hash. +func TestRefreshRunRowPersistsUnitColumns(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + offsets := []db.UnitOffset{ + {Ordinal: 1, RuneStart: 0, ByteStart: 0}, + {Ordinal: 2, RuneStart: 4, ByteStart: 5}, + } + run := runDoc("s1", "a1", 1, 2, "hé\n\nworld", offsets) + run.Subordinate = true + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u0", 0, "question"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: run, endedAt: "2024-01-01T00:00:01Z"}, + }} + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 2, Deleted: 0, Unchanged: 0}, stats) + assert.Equal(t, []string{"r:s1:a1", "u:s1:u0"}, mirrorDocKeys(t, ix)) + + row, ok := readMirrorRow(t, ix, "r:s1:a1") + require.True(t, ok) + assert.Equal(t, "s1", row.sessionID) + assert.Equal(t, 1, row.ordinal) + assert.Equal(t, 2, row.ordinalEnd) + assert.True(t, row.subordinate) + assert.Equal(t, "hé\n\nworld", row.content) + + var gotOffsets []db.UnitOffset + require.NoError(t, json.Unmarshal([]byte(row.offsets), &gotOffsets)) + assert.Equal(t, offsets, gotOffsets, "offsets JSON must round-trip through the column") + + userRow, ok := readMirrorRow(t, ix, "u:s1:u0") + require.True(t, ok) + assert.Equal(t, "[]", userRow.offsets) + assert.False(t, userRow.subordinate) +} + +// TestRefreshRunWithoutSourceUUIDFallsBackToOrdinalKey asserts a run whose +// first message predates source_uuid tracking gets the ro:: +// fallback key. +func TestRefreshRunWithoutSourceUUIDFallsBackToOrdinalKey(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: runDoc("s1", "", 4, 5, "a4\n\na5", + []db.UnitOffset{{Ordinal: 4}, {Ordinal: 5, RuneStart: 4, ByteStart: 4}}), + endedAt: "2024-01-01T00:00:00Z", + }, + }} + + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, []string{"ro:s1:4"}, mirrorDocKeys(t, ix)) +} + +func TestRefreshContentChangeUpdatesHash(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + before, ok := readMirrorRow(t, ix, "u:s1:u1") + require.True(t, ok) + + src.rows[0].unit.Content = "goodbye" + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 1, Deleted: 0, Unchanged: 0}, stats) + + after, ok := readMirrorRow(t, ix, "u:s1:u1") + require.True(t, ok) + assert.Equal(t, "goodbye", after.content) + assert.NotEqual(t, before.contentHash, after.contentHash) +} + +// TestRefreshTrailingAppendKeepsRunDocKeyAndReembedsOnlyIt covers the +// steady-state append shape: new assistant messages land on a session's +// trailing run. The run's doc_key (from its unchanged first message) must +// survive, its content_hash must change so it becomes pending re-embed, and +// an untouched run in another session must keep its embedding stamp. +func TestRefreshTrailingAppendKeepsRunDocKeyAndReembedsOnlyIt(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: runDoc("s1", "a1", 1, 2, "first\n\nsecond", []db.UnitOffset{ + {Ordinal: 1}, {Ordinal: 2, RuneStart: 7, ByteStart: 7}, + }), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: runDoc("s2", "b1", 0, 0, "other", []db.UnitOffset{{Ordinal: 0}}), + endedAt: "2024-01-01T00:00:01Z", + }, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + for _, key := range []string{"r:s1:a1", "r:s2:b1"} { + row, ok := readMirrorRow(t, ix, key) + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, key, row.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}})) + } + before, ok := readMirrorRow(t, ix, "r:s1:a1") + require.True(t, ok) + + // A third assistant message is appended to s1's trailing run. + src.rows[0].unit = runDoc("s1", "a1", 1, 3, "first\n\nsecond\n\nthird", []db.UnitOffset{ + {Ordinal: 1}, {Ordinal: 2, RuneStart: 7, ByteStart: 7}, + {Ordinal: 3, RuneStart: 15, ByteStart: 15}, + }) + src.rows[0].endedAt = "2024-01-02T00:00:00Z" + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 1, Deleted: 0, Unchanged: 1}, stats) + assert.ElementsMatch(t, []string{"r:s1:a1", "r:s2:b1"}, mirrorDocKeys(t, ix)) + + after, ok := readMirrorRow(t, ix, "r:s1:a1") + require.True(t, ok) + assert.Equal(t, 3, after.ordinalEnd, "trailing append extends ordinal_end") + assert.NotEqual(t, before.contentHash, after.contentHash, + "appended content must invalidate the hash so the run is re-embedded") + + pending, err := ix.store.PendingForGeneration(ctx, fingerprint, 100) + require.NoError(t, err) + var pendingDocs []string + for _, p := range pending { + pendingDocs = append(pendingDocs, p.Doc) + } + assert.Contains(t, pendingDocs, "r:s1:a1", + "the appended run must be pending re-embed") + assert.NotContains(t, pendingDocs, "r:s2:b1", + "an untouched run must keep its embedding stamp") +} + +// TestRefreshMidRunUserSplitCreatesSecondHalfUnderNewKey covers a rescan +// where a new embeddable user row lands mid-run: the run splits, the first +// half keeps the old doc_key (same first message) with a shrunken content +// and changed hash, and the second half plus the user row appear under new +// keys. Nothing is genuinely removed, so full-mode reconciliation must not +// delete anything. +func TestRefreshMidRunUserSplitCreatesSecondHalfUnderNewKey(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: runDoc("s1", "a0", 0, 3, "a0\n\na1\n\na2\n\na3", []db.UnitOffset{ + {Ordinal: 0}, {Ordinal: 1, RuneStart: 4, ByteStart: 4}, + {Ordinal: 2, RuneStart: 8, ByteStart: 8}, + {Ordinal: 3, RuneStart: 12, ByteStart: 12}, + }), + endedAt: "2024-01-01T00:00:00Z", + }, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + before, ok := readMirrorRow(t, ix, "r:s1:a0") + require.True(t, ok) + + // A user message surfaces at ordinal 2 on rescan, splitting the run. + src.rows = []fakeUnit{ + { + unit: runDoc("s1", "a0", 0, 1, "a0\n\na1", []db.UnitOffset{ + {Ordinal: 0}, {Ordinal: 1, RuneStart: 4, ByteStart: 4}, + }), + endedAt: "2024-01-02T00:00:00Z", + }, + {unit: userDoc("s1", "u2", 2, "question"), endedAt: "2024-01-02T00:00:00Z"}, + { + unit: runDoc("s1", "a3", 3, 3, "a3", []db.UnitOffset{{Ordinal: 3}}), + endedAt: "2024-01-02T00:00:00Z", + }, + } + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Zero(t, stats.Deleted, "a split rewrites and adds rows; nothing vanishes") + assert.Equal(t, []string{"r:s1:a0", "r:s1:a3", "u:s1:u2"}, mirrorDocKeys(t, ix)) + + firstHalf, ok := readMirrorRow(t, ix, "r:s1:a0") + require.True(t, ok) + assert.Equal(t, 0, firstHalf.ordinal) + assert.Equal(t, 1, firstHalf.ordinalEnd, "old key shrinks to the first half") + assert.Equal(t, "a0\n\na1", firstHalf.content) + assert.NotEqual(t, before.contentHash, firstHalf.contentHash, + "the shrunken first half must be re-embedded") + + secondHalf, ok := readMirrorRow(t, ix, "r:s1:a3") + require.True(t, ok) + assert.Equal(t, 3, secondHalf.ordinal) + assert.Equal(t, 3, secondHalf.ordinalEnd) + assert.Equal(t, "a3", secondHalf.content) +} + +// TestRefreshRunReplacingVanishedRunSlotEvictsVectorsBeforeRow asserts the +// two-phase eviction path still works over run documents: a new run landing +// on the (session_id, ordinal) slot of a vanished run evicts the old +// doc_key, and — since it is never reinserted in the scan — its vectors and +// stamps are deleted along with its mirror row. +func TestRefreshRunReplacingVanishedRunSlotEvictsVectorsBeforeRow(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + { + unit: runDoc("s1", "a2", 2, 3, "old\n\nrun", []db.UnitOffset{ + {Ordinal: 2}, {Ordinal: 3, RuneStart: 5, ByteStart: 5}, + }), + endedAt: "2024-01-01T00:00:00Z", + }, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + oldRow, ok := readMirrorRow(t, ix, "r:s1:a2") + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, "r:s1:a2", oldRow.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{0, 1, 0}}})) + + // The old run vanishes; a different run (new first-message uuid) now + // starts at the same ordinal. + src.rows = []fakeUnit{ + { + unit: runDoc("s1", "b2", 2, 4, "new\n\nrun\n\nhere", []db.UnitOffset{ + {Ordinal: 2}, {Ordinal: 3, RuneStart: 5, ByteStart: 5}, + {Ordinal: 4, RuneStart: 10, ByteStart: 10}, + }), + endedAt: "2024-01-02T00:00:00Z", + }, + } + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, 1, stats.Deleted, "the vanished run is evicted exactly once") + assert.Equal(t, []string{"r:s1:b2"}, mirrorDocKeys(t, ix)) + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps WHERE doc_key = ?`, "r:s1:a2", + ).Scan(&stampCount)) + assert.Zero(t, stampCount, "evicted run's stamps must be gone") + + var chunkCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_chunks WHERE doc_key = ?`, "r:s1:a2", + ).Scan(&chunkCount)) + assert.Zero(t, chunkCount, "evicted run's chunks must be gone") +} + +func TestRefreshOrdinalShiftOnUUIDRowKeepsHashStampSurvives(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + + row, ok := readMirrorRow(t, ix, "u:s1:u1") + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, "u:s1:u1", row.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}})) + + // Shift the ordinal without changing content: the hash must survive so + // the stamp (keyed by doc_key + revision) is not invalidated. + src.rows[0].unit.Ordinal = 3 + src.rows[0].unit.OrdinalEnd = 3 + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 0, Deleted: 0, Unchanged: 1}, stats) + + after, ok := readMirrorRow(t, ix, "u:s1:u1") + require.True(t, ok) + assert.Equal(t, 3, after.ordinal) + assert.Equal(t, row.contentHash, after.contentHash) + + pending, err := ix.store.PendingForGeneration(ctx, fingerprint, 100) + require.NoError(t, err) + for _, p := range pending { + assert.NotEqual(t, "u:s1:u1", p.Doc, "shifted doc should not be pending re-embed") + } +} + +func TestRefreshOrdinalShiftOntoStaleLegacySlotEvictsIt(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + // First refresh: a legacy o:-keyed row occupies (s1, 3). + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "", 3, "legacy"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"o:s1:3", "u:s1:u1"}, mirrorDocKeys(t, ix)) + + // Stamp the legacy row's vectors so eviction has something to clean up. + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + legacyRow, ok := readMirrorRow(t, ix, "o:s1:3") + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, "o:s1:3", legacyRow.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{0, 1, 0}}})) + + // The u1 unit's ordinal now shifts onto the legacy row's slot. + src.rows[1].unit.Ordinal = 3 + src.rows[1].unit.OrdinalEnd = 3 + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, 1, stats.Deleted, "legacy slot occupant evicted") + + assert.ElementsMatch(t, []string{"u:s1:u1"}, mirrorDocKeys(t, ix)) + row, ok := readMirrorRow(t, ix, "u:s1:u1") + require.True(t, ok) + assert.Equal(t, 3, row.ordinal) + + // The evicted doc_key's vectors must be deleted too, or they would + // permanently occupy KNN LIMIT slots: reconcileDeletions can never see + // the key, because its mirror row is already gone before full-mode + // reconciliation runs. + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps WHERE doc_key = ?`, "o:s1:3", + ).Scan(&stampCount)) + assert.Zero(t, stampCount, "evicted doc_key's stamps should be gone") + + var chunkCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_chunks WHERE doc_key = ?`, "o:s1:3", + ).Scan(&chunkCount)) + assert.Zero(t, chunkCount, "evicted doc_key's chunks should be gone") +} + +func TestRefreshFullDeletesVanishedIdentitiesAndVectors(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "u2", 1, "world"), endedAt: "2024-01-01T00:00:01Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + row, ok := readMirrorRow(t, ix, "u:s1:u2") + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, "u:s1:u2", row.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}})) + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps WHERE doc_key = ?`, "u:s1:u2", + ).Scan(&stampCount)) + require.Equal(t, 1, stampCount) + + // u2's unit vanishes from the archive. + src.rows = src.rows[:1] + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 0, Deleted: 1, Unchanged: 1}, stats) + + _, ok = readMirrorRow(t, ix, "u:s1:u2") + assert.False(t, ok, "mirror row for vanished identity should be gone") + + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps WHERE doc_key = ?`, "u:s1:u2", + ).Scan(&stampCount)) + assert.Zero(t, stampCount, "stamp for vanished identity should be gone") + + var chunkCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_chunks WHERE doc_key = ?`, "u:s1:u2", + ).Scan(&chunkCount)) + assert.Zero(t, chunkCount, "chunks for vanished identity should be gone") +} + +// TestRefreshFullEvictionOfVanishedOccupantCountsDeletedOnce covers the +// double-counting regression where a slot-evicted doc_key never reappears +// anywhere in a full-mode scan: it is absent from seen (nothing in the scan +// produced that key) while also being in evictSlotOccupant's evicted set. +// Before the fix, both finalizeEvictions and full-mode reconcileDeletions +// would independently delete the row and count it, double-counting +// RefreshStats.Deleted even though store.DeleteVectors is idempotent and the +// row is physically removed exactly once. +func TestRefreshFullEvictionOfVanishedOccupantCountsDeletedOnce(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + // First refresh: a legacy o:-keyed row occupies (s1, 3); u1 sits at (s1, 0). + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "", 3, "legacy"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"o:s1:3", "u:s1:u1"}, mirrorDocKeys(t, ix)) + + // Stamp the legacy row's vectors so eviction has something to clean up. + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + legacyRow, ok := readMirrorRow(t, ix, "o:s1:3") + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, "o:s1:3", legacyRow.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{0, 1, 0}}})) + + // The legacy unit vanishes from the archive entirely (unlike a mere + // slot shift, it is dropped from src.rows), and u1 shifts onto its old + // slot. The legacy doc_key is now both slot-evicted (evictSlotOccupant + // finds it occupying (s1, 3) via the DB) and never seen in this scan at + // all (it produced no row), so it is a candidate for both cleanup paths. + src.rows = []fakeUnit{ + {unit: userDoc("s1", "u1", 3, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + } + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, 1, stats.Deleted, + "vanished, slot-evicted occupant must be counted exactly once") + + assert.ElementsMatch(t, []string{"u:s1:u1"}, mirrorDocKeys(t, ix)) + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps WHERE doc_key = ?`, "o:s1:3", + ).Scan(&stampCount)) + assert.Zero(t, stampCount, "evicted doc_key's stamps should be gone") + + var chunkCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_chunks WHERE doc_key = ?`, "o:s1:3", + ).Scan(&chunkCount)) + assert.Zero(t, chunkCount, "evicted doc_key's chunks should be gone") +} + +func TestRefreshIncrementalUsesWatermark(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, "", src.gotSince, "full refresh should scan from the beginning") + + src.rows = append(src.rows, fakeUnit{ + unit: userDoc("s2", "u2", 0, "later"), endedAt: "2024-01-02T00:00:00Z", + }) + stats, err := ix.Refresh(ctx, src, false, true) + require.NoError(t, err) + assert.Equal(t, "2024-01-01T00:00:00Z", src.gotSince, + "incremental refresh should scan from the stored watermark") + // The fake mimics ScanEmbeddableUnits's inclusive s.ended_at >= since + // filter: the original s1/u1 row (endedAt == since) is re-scanned as + // unchanged, alongside the newly-added s2/u2 row. + assert.Equal(t, RefreshStats{Upserted: 1, Deleted: 0, Unchanged: 1}, stats, + "incremental refresh upserts only what the source rescans, never reconciles deletions") + + _, ok := readMirrorRow(t, ix, "u:s2:u2") + assert.True(t, ok) +} + +// TestRefreshDuplicateSourceUUIDGetsStableOccurrenceKeys asserts that two +// units in one session sharing a non-empty source_uuid (permitted by the +// messages schema) collapse into two distinct mirror rows rather than one, +// and that a second refresh reproduces the same occurrence-based keys so a +// stamped document is not spuriously evicted and re-embedded. +func TestRefreshDuplicateSourceUUIDGetsStableOccurrenceKeys(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "dup", 0, "first"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "dup", 1, "second"), endedAt: "2024-01-01T00:00:01Z"}, + }} + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 2, Deleted: 0, Unchanged: 0}, stats) + + keys := mirrorDocKeys(t, ix) + assert.Equal(t, []string{"u:s1:dup", "u:s1:dup#2"}, keys, + "duplicate source_uuid rows must collapse into distinct keys, not one") + + first, ok := readMirrorRow(t, ix, "u:s1:dup") + require.True(t, ok) + assert.Equal(t, "first", first.content) + second, ok := readMirrorRow(t, ix, "u:s1:dup#2") + require.True(t, ok) + assert.Equal(t, "second", second.content) + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, "u:s1:dup", first.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}})) + + // A second refresh must reproduce the same occurrence keys in the same + // scan order, or the stamped doc would appear to vanish and be + // re-embedded on every resync. + stats, err = ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 0, Deleted: 0, Unchanged: 2}, stats, + "stable keys across refreshes mean no churn") + + pending, err := ix.store.PendingForGeneration(ctx, fingerprint, 100) + require.NoError(t, err) + for _, p := range pending { + assert.NotEqual(t, "u:s1:dup", p.Doc, "stamped doc should not be pending re-embed") + } +} + +// TestRefreshDuplicateRunFirstMessageUUIDGetsOccurrenceSuffixes asserts run +// keys share the user docs' per-session occurrence machinery: a run whose +// first-message uuid collides with any earlier doc's uuid in the same +// session gets a deterministic #n suffix in (session_id, ordinal) scan +// order, stable across refreshes. +func TestRefreshDuplicateRunFirstMessageUUIDGetsOccurrenceSuffixes(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "dup", 0, "user first"), endedAt: "2024-01-01T00:00:00Z"}, + { + unit: runDoc("s1", "dup", 1, 2, "run one", []db.UnitOffset{ + {Ordinal: 1}, {Ordinal: 2, RuneStart: 4, ByteStart: 4}, + }), + endedAt: "2024-01-01T00:00:01Z", + }, + {unit: userDoc("s1", "", 3, "plain"), endedAt: "2024-01-01T00:00:02Z"}, + { + unit: runDoc("s1", "dup", 4, 4, "run two", []db.UnitOffset{{Ordinal: 4}}), + endedAt: "2024-01-01T00:00:03Z", + }, + }} + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 4, Deleted: 0, Unchanged: 0}, stats) + assert.Equal(t, + []string{"o:s1:3", "r:s1:dup#2", "r:s1:dup#3", "u:s1:dup"}, + mirrorDocKeys(t, ix), + "occurrence suffixes must be assigned in (session_id, ordinal) scan order") + + // A second refresh must reproduce the exact same keys. + stats, err = ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Equal(t, RefreshStats{Upserted: 0, Deleted: 0, Unchanged: 4}, stats, + "occurrence-suffixed run keys must be stable across refreshes") +} + +// TestRefreshCascadingOrdinalShiftReinsertsEvictedKeysWithoutLosingCoverage +// covers the two-phase eviction regression: three UUID-keyed rows all shift +// ordinal upward by one in a single refresh (e.g. a message inserted ahead +// of them during resync). Each row's move onto the next row's old slot +// evicts that row mid-scan, but the evicted row's own doc_key is stable and +// reappears moments later in the same scan at its new ordinal. The evicted +// rows' stamps and vectors must survive: only a slot eviction that is never +// reinserted anywhere in the scan should reach store.DeleteVectors. +func TestRefreshCascadingOrdinalShiftReinsertsEvictedKeysWithoutLosingCoverage(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u0", 0, "zero"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "u1", 1, "one"), endedAt: "2024-01-01T00:00:01Z"}, + {unit: userDoc("s1", "u2", 2, "two"), endedAt: "2024-01-01T00:00:02Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + gen := kitvec.Generation{Model: "fake-model", Dimensions: 3} + fingerprint, err := ix.EnsureGeneration(ctx, gen, sqlitevec.StateActive) + require.NoError(t, err) + keys := []string{"u:s1:u0", "u:s1:u1", "u:s1:u2"} + for _, key := range keys { + row, ok := readMirrorRow(t, ix, key) + require.True(t, ok) + require.NoError(t, ix.store.SaveVectors(ctx, fingerprint, key, row.contentHash, + []kitvec.ChunkVector{{ChunkIndex: 0, Vector: kitvec.Vector{1, 0, 0}}})) + } + + // Every existing row shifts up by one ordinal, cascading eviction + // across the whole session in scan order: u0's move onto slot 1 + // evicts u1's row, u1's move onto slot 2 evicts u2's row, then u1 and + // u2 are each reinserted under their own stable doc_key at their own + // turn later in this same scan. + for i := range src.rows { + src.rows[i].unit.Ordinal++ + src.rows[i].unit.OrdinalEnd++ + } + + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + assert.Zero(t, stats.Deleted, "cascaded rows are reinserted in-scan, not genuinely removed") + + for _, key := range keys { + row, ok := readMirrorRow(t, ix, key) + require.True(t, ok, "%s must still be in the mirror", key) + + var stampCount int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM message_vectors_stamps WHERE doc_key = ? AND revision = ?`, + key, row.contentHash, + ).Scan(&stampCount)) + assert.Equal(t, 1, stampCount, "%s's stamp must survive the cascading shift", key) + } + + pending, err := ix.store.PendingForGeneration(ctx, fingerprint, 100) + require.NoError(t, err) + assert.Empty(t, pending, "no document should need re-embedding after the cascading shift") +} + +// TestDocKeyInjectiveWithDelimiterCharacters covers pairs of inputs that +// would collide under DocKey's raw (unescaped) delimiters — a source_uuid +// containing a literal "#" suffix colliding with an occurrence suffix, +// and a ":" inside a session_id or source_uuid colliding across the +// session/uuid boundary — asserting escapeDocKeyComponent keeps the +// encoding injective. Kind is part of the identity too: the same components +// under "user" and "run" must produce distinct keys. +func TestDocKeyInjectiveWithDelimiterCharacters(t *testing.T) { + tests := []struct { + name string + aKind string + aSession, aUUID string + aOrdinal, aOccurrence int + bKind string + bSession, bUUID string + bOrdinal, bOccurrence int + }{ + { + name: "occurrence suffix vs literal hash-number in uuid", + aKind: "user", + aSession: "s1", + aUUID: "dup#2", + aOrdinal: 0, + aOccurrence: 1, + bKind: "user", + bSession: "s1", + bUUID: "dup", + bOrdinal: 1, + bOccurrence: 2, + }, + { + name: "colon in session vs session/uuid boundary", + aKind: "user", + aSession: "a:b", + aUUID: "c", + aOrdinal: 0, + aOccurrence: 1, + bKind: "user", + bSession: "a", + bUUID: "b:c", + bOrdinal: 0, + bOccurrence: 1, + }, + { + name: "colon in uuid vs session/uuid boundary", + aKind: "user", + aSession: "sess", + aUUID: "x:y", + aOrdinal: 0, + aOccurrence: 1, + bKind: "user", + bSession: "sess:x", + bUUID: "y", + bOrdinal: 0, + bOccurrence: 1, + }, + { + name: "literal percent-escape sequence vs raw delimiter", + aKind: "user", + aSession: "s1", + aUUID: "%3A", + aOrdinal: 0, + aOccurrence: 1, + bKind: "user", + bSession: "s1", + bUUID: ":", + bOrdinal: 0, + bOccurrence: 1, + }, + { + name: "same components under user vs run kinds", + aKind: "user", + aSession: "s1", + aUUID: "x", + aOrdinal: 0, + aOccurrence: 1, + bKind: "run", + bSession: "s1", + bUUID: "x", + bOrdinal: 0, + bOccurrence: 1, + }, + { + name: "run ordinal fallback vs user ordinal fallback", + aKind: "run", + aSession: "s1", + aUUID: "", + aOrdinal: 3, + aOccurrence: 1, + bKind: "user", + bSession: "s1", + bUUID: "", + bOrdinal: 3, + bOccurrence: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := DocKey(tt.aKind, tt.aSession, tt.aUUID, tt.aOrdinal, tt.aOccurrence) + b := DocKey(tt.bKind, tt.bSession, tt.bUUID, tt.bOrdinal, tt.bOccurrence) + assert.NotEqual(t, a, b, "distinct identities must not collide") + }) + } +} + +func TestRefreshReadOnlyIndexRejected(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + rw, err := Open(ctx, path, false, 4000) + require.NoError(t, err) + require.NoError(t, rw.Close()) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err) + defer ro.Close() + + _, err = ro.Refresh(ctx, &fakeUnitSource{}, true, true) + require.Error(t, err) +} + +// TestRefreshParkedLeftoverFromInterruptedRunDoesNotCollide simulates a +// Refresh interrupted between evictSlotOccupant and finalizeEvictions: a +// row left parked at ordinal -1 (parking writes are autocommit, so a crash +// or cancellation mid-scan leaves them behind). The next run's parking +// sentinel must start below that leftover — restarting at 0 would park a +// freshly evicted row in the same session at the taken -1 and fail the +// unique (session_id, ordinal) index, deterministically on every retry, +// wedging refreshes until a full rebuild. +func TestRefreshParkedLeftoverFromInterruptedRunDoesNotCollide(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + // A legacy occupant at (s1, 9), plus u1 and u2 elsewhere. + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "", 9, "legacy"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "u1", 0, "hello"), endedAt: "2024-01-01T00:00:00Z"}, + {unit: userDoc("s1", "u2", 5, "world"), endedAt: "2024-01-01T00:00:00Z"}, + }} + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + // Simulate the interrupted run's leftover: u2 parked at -1. + _, err = ix.db.Exec(`UPDATE vector_messages SET ordinal = -1 WHERE doc_key = 'u:s1:u2'`) + require.NoError(t, err) + + // Next run: u1 shifts onto the legacy occupant's slot, forcing a fresh + // eviction parking in the same session; u2 is rescanned and self-heals. + src.rows[1].unit.Ordinal = 9 + src.rows[1].unit.OrdinalEnd = 9 + stats, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err, "a parked leftover must not collide with new parking") + assert.Equal(t, 1, stats.Deleted, "the evicted legacy occupant is finalized") + + assert.ElementsMatch(t, []string{"u:s1:u1", "u:s1:u2"}, mirrorDocKeys(t, ix)) + row, ok := readMirrorRow(t, ix, "u:s1:u2") + require.True(t, ok) + assert.Equal(t, 5, row.ordinal, "the leftover parked row self-heals on rescan") + + var parked int + require.NoError(t, ix.db.QueryRow( + `SELECT COUNT(*) FROM vector_messages WHERE ordinal < 0`).Scan(&parked)) + assert.Zero(t, parked, "no parked rows survive a completed full refresh") +} diff --git a/internal/vector/search.go b/internal/vector/search.go new file mode 100644 index 000000000..d5f4c0d03 --- /dev/null +++ b/internal/vector/search.go @@ -0,0 +1,445 @@ +package vector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + + "go.kenn.io/agentsview/internal/db" + kitvec "go.kenn.io/kit/vector" +) + +// snippetMaxRunes bounds the length of a Hit's Snippet; longer chunks are +// truncated with a trailing ellipsis. +const snippetMaxRunes = 200 + +// Hit is one unit-level semantic search result, anchored to a specific +// message. For a run document Ordinal is the anchor: the member message +// whose rune span contains the matched chunk's center rune (see +// anchorMemberIndex), while OrdinalStart/OrdinalEnd span the whole run. For a +// user document all three ordinals are the message's own ordinal. +type Hit struct { + SessionID string + Ordinal int // anchor ordinal + OrdinalStart int + OrdinalEnd int + Subordinate bool + Score float32 + Snippet string +} + +// ErrNoActiveGeneration is returned by Search when the index has no active +// embedding generation to query — either nothing has ever been built, or a +// build is in progress but has not yet activated (see BuildingError). +var ErrNoActiveGeneration = errors.New("no active embedding generation") + +// BuildingError is returned by Search when only a building generation +// exists: a first build is in progress and has not yet activated, so no +// generation is queryable yet. +type BuildingError struct { + // Percent is the building generation's coverage of the current + // vector_messages mirror, 0-100. + Percent int +} + +func (e *BuildingError) Error() string { + return fmt.Sprintf("embedding index is building (%d%% complete)", e.Percent) +} + +// QueryEncodeError reports that Search failed because embedding the query +// text itself failed — the embeddings endpoint being down, slow, timing +// out, or erroring at search time — as distinct from ErrNoActiveGeneration +// and BuildingError, which both mean semantic search has nothing queryable +// yet. A QueryEncodeError means the index is otherwise ready; only this +// particular request failed, and it is generally worth retrying rather +// than reporting semantic search as unconfigured. +type QueryEncodeError struct { + Err error +} + +func (e *QueryEncodeError) Error() string { + return fmt.Sprintf("embed query: %v", e.Err) +} + +func (e *QueryEncodeError) Unwrap() error { return e.Err } + +// Search embeds query and returns up to limit message-level hits, best +// first. It returns ErrNoActiveGeneration when no live generation exists, +// and a *BuildingError (carrying the completion Percent) when only a +// building generation exists. +// +// Search deliberately queries only the active generation rather than +// kitvec.Search's default of every live (building + active) generation: a +// model or dimension change leaves a building generation coexisting with +// the active one, and kitvec.Search would encode the query once with enc +// and reuse that single vector for both, which errors (or, for a same- +// dimension model change, silently ranks against the wrong model's space) +// once their dimensions or embedding spaces differ. Encoding only for the +// caller-chosen generation and querying it directly with +// store.QueryGeneration sidesteps that; a system-level staleness gate +// elsewhere already rejects an encoder that no longer matches the active +// generation's fingerprint, so this mismatch cannot arise in practice once +// that gate is wired up. +// +// Search also returns ErrMirrorVersionMismatch, before touching any table, +// when ix was opened read-only against a vectors.db whose mirror schema +// version does not match this binary's (see prepareMirrorSchema): a +// read-only Index cannot reset the mirror itself, so it fails closed rather +// than risk misreading rows shaped by a different schema. +func (ix *Index) Search( + ctx context.Context, enc kitvec.EncodeFunc, query string, limit int, +) ([]Hit, error) { + if ix.versionMismatch { + return nil, ErrMirrorVersionMismatch + } + + active, hasActive, err := ix.ActiveFingerprint(ctx) + if err != nil { + return nil, err + } + if !hasActive { + return nil, ix.noActiveGenerationError(ctx) + } + + vectors, err := kitvec.EncodeBatched(ctx, enc, + []kitvec.Chunk{{Index: 0, Text: query}}, kitvec.BatchOptions{}) + if err != nil { + return nil, &QueryEncodeError{Err: err} + } + + hits, err := ix.store.QueryGeneration(ctx, active, vectors[0], limit) + if err != nil { + return nil, fmt.Errorf("search: %w", err) + } + hits = kitvec.RollupByDocument(hits) + if len(hits) > limit { + hits = hits[:limit] + } + + return ix.hydrateHits(ctx, hits) +} + +// noActiveGenerationError distinguishes an empty index (ErrNoActiveGeneration) +// from one with an in-progress first build (*BuildingError), for a caller +// that already knows there is no active generation. +func (ix *Index) noActiveGenerationError(ctx context.Context) error { + buildingFP, hasBuilding, err := ix.BuildingFingerprint(ctx) + if err != nil { + return err + } + if !hasBuilding { + return ErrNoActiveGeneration + } + percent, err := ix.buildingPercent(ctx, buildingFP) + if err != nil { + return err + } + return &BuildingError{Percent: percent} +} + +// buildingPercent reports fingerprint's coverage of the current +// vector_messages mirror as a 0-100 percentage, guarding the +// divide-by-zero case of an empty mirror. +func (ix *Index) buildingPercent(ctx context.Context, fingerprint string) (int, error) { + ordinal, err := ix.ordinalForFingerprint(ctx, fingerprint) + if err != nil { + return 0, err + } + info, err := ix.GenerationByID(ctx, ordinal) + if err != nil { + return 0, err + } + total := info.Embedded + info.Missing + if total == 0 { + return 0, nil + } + return int(info.Embedded * 100 / total), nil +} + +// mirrorDoc is the subset of a vector_messages row Search needs to hydrate a +// kit hit into an agentsview Hit. offsets is empty for user documents. +type mirrorDoc struct { + sessionID string + ordinal int + ordinalEnd int + subordinate bool + offsets []db.UnitOffset + content string +} + +// hydrateHits maps kit's doc-key-level hits to agentsview Hits: it looks up +// each doc_key's mirror row in one query, anchors run hits to a member +// ordinal, and computes a snippet by re-splitting content. Hits whose +// doc_key vanished from the mirror mid-flight (a concurrent Refresh +// reconciled it away) are dropped rather than erroring, since the search +// itself still succeeded. +func (ix *Index) hydrateHits(ctx context.Context, hits []kitvec.Hit[string]) ([]Hit, error) { + if len(hits) == 0 { + return nil, nil + } + + docKeys := make([]string, len(hits)) + for i, h := range hits { + docKeys[i] = h.Doc + } + docs, err := ix.lookupMirrorDocs(ctx, docKeys) + if err != nil { + return nil, err + } + + out := make([]Hit, 0, len(hits)) + for _, h := range hits { + doc, ok := docs[h.Doc] + if !ok { + continue + } + out = append(out, ix.resolveHit(h, doc)) + } + return out, nil +} + +// resolveHit builds the Hit for one matched document. A user document +// (empty offsets) anchors to its own mirror ordinal and snippets the whole +// matched chunk (already message-local); a run document anchors to the +// member whose rune span contains the matched chunk's center rune, and its +// snippet is sliced down to that anchor member's own text (see +// resolveRunHit) so downstream snippet centering can locate it inside the +// anchor message's content. +func (ix *Index) resolveHit(h kitvec.Hit[string], doc mirrorDoc) Hit { + hit := Hit{ + SessionID: doc.sessionID, + Ordinal: doc.ordinal, + OrdinalStart: doc.ordinal, + OrdinalEnd: doc.ordinalEnd, + Subordinate: doc.subordinate, + Score: h.Score, + Snippet: ix.snippet(doc.content, h.ChunkIndex), + } + if len(doc.offsets) > 0 { + hit.Ordinal, hit.Snippet = ix.resolveRunHit(doc.content, doc.offsets, h.ChunkIndex) + } + return hit +} + +// runMemberSeparatorRunes is the rune length of the "\n\n" separator +// db's runUnit joins run members with (see internal/db's runUnit). The +// separator belongs to no member's span, so a member's text within the +// joined content ends this many runes before the next member's RuneStart. +const runMemberSeparatorRunes = 2 + +// resolveRunHit computes a run hit's anchor ordinal and anchor-local +// snippet: the anchor is the member whose rune span contains the matched +// chunk's center rune (see anchorMemberIndex), and the snippet is the +// intersection of the chunk's rune window with that member's span — always +// a substring of the anchor message's own text, so the db layer's snippet +// centering (semanticSnippet) can locate it inside the anchor message's +// content. A degenerate/stale ChunkIndex whose re-split window misses the +// member entirely falls back to the anchor member's whole span text; the +// result is never text from a different member, and never a panic. +func (ix *Index) resolveRunHit( + content string, offsets []db.UnitOffset, chunkIndex int, +) (ordinal int, snippet string) { + runes := []rune(content) + start, end := chunkWindow(len(runes), chunkIndex, ix.split) + member := anchorMemberIndex(offsets, start, end) + + memberStart := min(offsets[member].RuneStart, len(runes)) + memberEnd := len(runes) + if member+1 < len(offsets) { + memberEnd = offsets[member+1].RuneStart - runMemberSeparatorRunes + } + memberEnd = min(max(memberEnd, memberStart), len(runes)) + + lo, hi := max(start, memberStart), min(end, memberEnd) + if lo >= hi { + lo, hi = memberStart, memberEnd + } + return offsets[member].Ordinal, truncateRunes(string(runes[lo:hi]), snippetMaxRunes) +} + +// chunkWindow returns the [start, end) rune window of content's +// chunkIndex'th chunk, mirroring kitvec.Split's semantics exactly: content +// that fits MaxRunes (or unbounded splitting) is one whole-content chunk; +// otherwise chunks start at multiples of the stride (MaxRunes minus the +// clamped overlap) and the final chunk is capped at the content's end, so +// end-start is the chunk's ACTUAL rune length, not always MaxRunes. +// TestChunkWindowMatchesKitSplit cross-checks this against kitvec.Split. +func chunkWindow(contentRunes, chunkIndex int, o kitvec.SplitOptions) (start, end int) { + if o.MaxRunes <= 0 || contentRunes <= o.MaxRunes { + return 0, contentRunes + } + overlap := min(max(o.Overlap, 0), o.MaxRunes-1) + stride := o.MaxRunes - overlap + start = chunkIndex * stride + end = min(start+o.MaxRunes, contentRunes) + return start, end +} + +// anchorMemberIndex returns the offsets index of the run member whose rune +// span contains the [start, end) chunk window's center rune, with the +// earlier member winning when the center falls in the separator between two +// members. offsets must be non-empty. +func anchorMemberIndex(offsets []db.UnitOffset, start, end int) int { + center := start + (end-start)/2 + anchor := 0 + for i, off := range offsets { + if off.RuneStart <= center { + anchor = i + } else { + break + } + } + return anchor +} + +// lookupMirrorDocs reads each of docKeys' mirror rows (session_id, ordinal +// range, subordinate flag, member offsets, content) from vector_messages, +// keyed by doc_key, in maxSQLVars-sized chunks: a deep semantic overfetch +// (large limit * over-fetch factor) can carry thousands of doc keys, well +// past what a single IN (...) clause can bind. A key with no matching row is +// simply absent from the result. Rows parked at a negative sentinel ordinal +// by a concurrent Refresh (see evictSlotOccupant) are excluded the same way: +// mid-refresh state must never hydrate into a hit with a negative ordinal. +func (ix *Index) lookupMirrorDocs(ctx context.Context, docKeys []string) (map[string]mirrorDoc, error) { + docs := make(map[string]mirrorDoc, len(docKeys)) + err := chunkKeys(docKeys, func(chunk []string) error { + placeholders, args := inPlaceholders(chunk) + rows, err := ix.db.QueryContext(ctx, ` +SELECT doc_key, session_id, ordinal, ordinal_end, subordinate, offsets, content + FROM vector_messages + WHERE ordinal >= 0 AND doc_key IN `+placeholders, args...) + if err != nil { + return fmt.Errorf("look up search hit documents: %w", err) + } + for rows.Next() { + var key, offsets string + var doc mirrorDoc + if err := rows.Scan(&key, &doc.sessionID, &doc.ordinal, + &doc.ordinalEnd, &doc.subordinate, &offsets, &doc.content); err != nil { + rows.Close() + return fmt.Errorf("scan search hit document: %w", err) + } + if err := json.Unmarshal([]byte(offsets), &doc.offsets); err != nil { + rows.Close() + return fmt.Errorf("parse offsets for search hit %s: %w", key, err) + } + docs[key] = doc + } + if err := rows.Err(); err != nil { + rows.Close() + return fmt.Errorf("look up search hit documents: %w", err) + } + return rows.Close() + }) + if err != nil { + return nil, err + } + return docs, nil +} + +// snippet re-splits content the same way Build did and returns the text of +// its chunkIndex'th chunk, truncated to snippetMaxRunes runes with a +// trailing ellipsis when truncated. A chunkIndex outside the re-split +// content's chunk count (content changed since embedding) yields an empty +// snippet rather than a panic. +func (ix *Index) snippet(content string, chunkIndex int) string { + chunks := kitvec.Split(content, ix.split) + if chunkIndex < 0 || chunkIndex >= len(chunks) { + return "" + } + return truncateRunes(chunks[chunkIndex].Text, snippetMaxRunes) +} + +// truncateRunes truncates s to at most maxRunes runes, appending an +// ellipsis when truncation occurs. It measures in runes so multi-byte +// characters are never torn apart. +func truncateRunes(s string, maxRunes int) string { + runes := []rune(s) + if len(runes) <= maxRunes { + return s + } + return string(runes[:maxRunes]) + "…" +} + +// ResolveMessageUnits maps each ref to the vector_messages unit containing +// it, returning a slice parallel to refs; a ref with no containing unit (its +// message lies outside the embeddable universe, or in a gap between units) +// yields a zero UnitRef. Each ref is a point lookup on the retained unique +// (session_id, ordinal) index — greatest unit ordinal <= ref ordinal, then a +// containment check against ordinal_end — via one prepared statement, so a +// batch of any size never approaches SQLite's bind-variable limit. Rows +// parked at a negative sentinel ordinal by a concurrent Refresh (see +// evictSlotOccupant) are skipped so a ref can never resolve into +// mid-refresh state and surface a negative ordinal range. +// +// Like Search and StaleActive, it fails closed with ErrMirrorVersionMismatch +// — before touching any table — when ix was opened read-only against a +// vectors.db written by a different mirror schema version. +func (ix *Index) ResolveMessageUnits( + ctx context.Context, refs []db.MessageRef, +) ([]db.UnitRef, error) { + if ix.versionMismatch { + return nil, ErrMirrorVersionMismatch + } + out := make([]db.UnitRef, len(refs)) + if len(refs) == 0 { + return out, nil + } + + stmt, err := ix.db.PrepareContext(ctx, ` +SELECT doc_key, ordinal, ordinal_end, subordinate + FROM vector_messages + WHERE session_id = ? AND ordinal >= 0 AND ordinal <= ? + ORDER BY ordinal DESC LIMIT 1`) + if err != nil { + return nil, fmt.Errorf("resolve message units: %w", err) + } + defer stmt.Close() + + for i, ref := range refs { + var unit db.UnitRef + err := stmt.QueryRowContext(ctx, ref.SessionID, ref.Ordinal).Scan( + &unit.DocKey, &unit.OrdinalStart, &unit.OrdinalEnd, &unit.Subordinate) + if errors.Is(err, sql.ErrNoRows) { + continue + } + if err != nil { + return nil, fmt.Errorf( + "resolve message unit (%s, %d): %w", ref.SessionID, ref.Ordinal, err) + } + if ref.Ordinal > unit.OrdinalEnd { + continue + } + unit.SessionID = ref.SessionID + out[i] = unit + } + return out, nil +} + +// StaleActive reports whether the active generation's fingerprint differs +// from want (the fingerprint of the current config) — the "index stale" +// signal surfaced by the db layer. It returns false when there is no active +// generation at all: Search already distinguishes that case with +// ErrNoActiveGeneration / BuildingError. +// +// Like Search, StaleActive fails closed with ErrMirrorVersionMismatch — +// before touching any table — when ix was opened read-only against a +// vectors.db written by a different mirror schema version: callers check +// staleness before searching, so without this gate a version-mismatched +// index would surface a raw SQL error (or a wrong staleness verdict) here +// and the sentinel in Search would never be reached. +func (ix *Index) StaleActive(ctx context.Context, want string) (bool, error) { + if ix.versionMismatch { + return false, ErrMirrorVersionMismatch + } + active, hasActive, err := ix.ActiveFingerprint(ctx) + if err != nil { + return false, err + } + if !hasActive { + return false, nil + } + return active != want, nil +} diff --git a/internal/vector/search_test.go b/internal/vector/search_test.go new file mode 100644 index 000000000..9906762fd --- /dev/null +++ b/internal/vector/search_test.go @@ -0,0 +1,711 @@ +package vector + +import ( + "context" + "path/filepath" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.kenn.io/agentsview/internal/db" + kitvec "go.kenn.io/kit/vector" + "go.kenn.io/kit/vector/sqlitevec" +) + +// fakeSearchEncoder maps distinct known texts to orthogonal 3-dimensional +// vectors so a query for one topic scores a perfect match against its own +// document and a clear non-match against the others. +func fakeSearchEncoder() func(_ context.Context, texts []string) ([][]float32, error) { + return func(_ context.Context, texts []string) ([][]float32, error) { + out := make([][]float32, len(texts)) + for i, text := range texts { + switch { + case strings.Contains(text, "alpha"): + out[i] = []float32{1, 0, 0} + case strings.Contains(text, "beta"): + out[i] = []float32{0, 1, 0} + default: + out[i] = []float32{0, 0, 1} + } + } + return out, nil + } +} + +// threeDocSearchSource returns three single-topic documents in one session, +// one each for "alpha", "beta", and a third ("gamma") topic. +func threeDocSearchSource() *fakeUnitSource { + return &fakeUnitSource{rows: []fakeUnit{ + { + unit: userDoc("s1", "u1", 0, "this message mentions alpha topic"), + endedAt: "2024-01-01T00:00:00Z", + }, + { + unit: userDoc("s1", "u2", 1, "this message mentions beta topic"), + endedAt: "2024-01-01T00:00:01Z", + }, + { + unit: userDoc("s1", "u3", 2, "this message mentions gamma topic"), + endedAt: "2024-01-01T00:00:02Z", + }, + }} +} + +func TestSearchReturnsBestMatchFirstWithSnippet(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := threeDocSearchSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeSearchEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + hits, err := ix.Search(ctx, fakeSearchEncoder(), "alpha", 10) + require.NoError(t, err) + require.NotEmpty(t, hits) + + best := hits[0] + assert.Equal(t, "s1", best.SessionID) + assert.Equal(t, 0, best.Ordinal) + assert.InDelta(t, 1.0, best.Score, 0.01) + assert.Contains(t, best.Snippet, "alpha") +} + +func TestSearchLimitCapsResults(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := threeDocSearchSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeSearchEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + hits, err := ix.Search(ctx, fakeSearchEncoder(), "alpha", 1) + require.NoError(t, err) + assert.Len(t, hits, 1) +} + +func TestSearchNoGenerationsReturnsErrNoActiveGeneration(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + _, err := ix.Search(ctx, fakeSearchEncoder(), "alpha", 10) + assert.ErrorIs(t, err, ErrNoActiveGeneration) +} + +func TestSearchBuildingOnlyReturnsBuildingError(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := threeDocSearchSource() + + _, err := ix.Refresh(ctx, src, true, true) + require.NoError(t, err) + + gen := fakeGeneration("fake-model") + _, err = ix.EnsureGeneration(ctx, gen, sqlitevec.StateBuilding) + require.NoError(t, err) + + _, err = ix.Search(ctx, fakeSearchEncoder(), "alpha", 10) + require.Error(t, err) + var buildingErr *BuildingError + require.ErrorAs(t, err, &buildingErr) + assert.Equal(t, 0, buildingErr.Percent, "nothing has been embedded yet") +} + +// TestSearchIgnoresBuildingGenerationOfDifferentDimension covers Search's +// active-generation-only query path: while a model/dimension change is in +// progress, a building generation of a different dimension coexists with +// the active one. kitvec.Search's default (query every live generation with +// one caller-supplied encoder) would try to run the active encoder's +// 3-dimensional query vector against the building generation's 5-dimensional +// vec0 table and fail. Search must encode once for, and query only, the +// active generation, so the building generation's differing dimension never +// matters and only active-generation hits come back. +func TestSearchIgnoresBuildingGenerationOfDifferentDimension(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := threeDocSearchSource() + activeGen := fakeGeneration("active-model") + + _, err := ix.Build(ctx, src, fakeSearchEncoder(), activeGen, BuildOptions{}) + require.NoError(t, err) + + buildingGen := kitvec.Generation{Model: "building-model", Dimensions: 5} + _, err = ix.EnsureGeneration(ctx, buildingGen, sqlitevec.StateBuilding) + require.NoError(t, err) + + hits, err := ix.Search(ctx, fakeSearchEncoder(), "alpha", 10) + require.NoError(t, err, "a building generation of a different dimension must not break search") + require.NotEmpty(t, hits) + assert.Equal(t, "s1", hits[0].SessionID) + assert.Equal(t, 0, hits[0].Ordinal) + assert.InDelta(t, 1.0, hits[0].Score, 0.01) +} + +func TestStaleActiveTrueWhenFingerprintsDiffer(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + src := threeDocSearchSource() + gen := fakeGeneration("fake-model") + + _, err := ix.Build(ctx, src, fakeSearchEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + stale, err := ix.StaleActive(ctx, "some-other-fingerprint") + require.NoError(t, err) + assert.True(t, stale) + + stale, err = ix.StaleActive(ctx, gen.Fingerprint()) + require.NoError(t, err) + assert.False(t, stale, "matching fingerprint is not stale") +} + +func TestStaleActiveFalseWhenNoActiveGeneration(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + stale, err := ix.StaleActive(ctx, "anything") + require.NoError(t, err) + assert.False(t, stale, "no active generation means nothing to compare") +} + +// anchorOrdinal is a test-only convenience over the production anchor +// pipeline: it maps a matched chunk back to the run member ordinal whose +// rune span contains the chunk's center rune, composing chunkWindow and +// anchorMemberIndex exactly the way resolveRunHit does. offsets must be +// non-empty (run documents only). +func anchorOrdinal(offsets []db.UnitOffset, contentRunes, chunkIndex int, o kitvec.SplitOptions) int { + start, end := chunkWindow(contentRunes, chunkIndex, o) + return offsets[anchorMemberIndex(offsets, start, end)].Ordinal +} + +// TestAnchorOrdinal pins the anchor policy: the anchor is the member message +// whose rune span contains the matched chunk's center rune, computed from +// the chunk's ACTUAL rune length (the final chunk is capped at the content's +// end, not at start+MaxRunes), with the earlier member winning when the +// center falls in the "\n\n" separator between two members. +func TestAnchorOrdinal(t *testing.T) { + // Three members joined with "\n\n": member 10 spans runes [0,5), + // separator [5,7), member 11 spans [7,12), separator [12,14), + // member 12 spans [14,19). + offsets := []db.UnitOffset{ + {Ordinal: 10, RuneStart: 0}, + {Ordinal: 11, RuneStart: 7}, + {Ordinal: 12, RuneStart: 14}, + } + tests := []struct { + name string + offsets []db.UnitOffset + contentRunes int + chunkIndex int + opts kitvec.SplitOptions + want int + }{ + { + name: "chunk fully inside first member", offsets: offsets, + // Window [0,4), center 2, inside member 10's span. + contentRunes: 19, chunkIndex: 0, + opts: kitvec.SplitOptions{MaxRunes: 4, Overlap: 0}, want: 10, + }, + { + name: "single chunk run centers whole content", offsets: offsets, + // Content fits one chunk: window [0,19), center 9, member 11. + contentRunes: 19, chunkIndex: 0, + opts: kitvec.SplitOptions{MaxRunes: 100, Overlap: 15}, want: 11, + }, + { + name: "center in separator anchors earlier member", offsets: offsets, + // Window [4,8), center 6 falls in the separator [5,7): the + // earlier member 10 wins the boundary tie. + contentRunes: 19, chunkIndex: 1, + opts: kitvec.SplitOptions{MaxRunes: 4, Overlap: 0}, want: 10, + }, + { + name: "center exactly at member start anchors that member", offsets: offsets, + // Window [0,14), center 7 == member 11's RuneStart: member 11's + // span contains rune 7. + contentRunes: 19, chunkIndex: 0, + opts: kitvec.SplitOptions{MaxRunes: 14, Overlap: 0}, want: 11, + }, + { + name: "short final chunk centers on actual length", + // Member 12 starts at rune 13; content is 15 runes. Stride is + // 10-1=9, so chunk 1's window is [9,15): actual length 6, + // center 12 -> member 11. Centering on MaxRunes instead + // ([9,19), center 14) would wrongly anchor member 12. + offsets: []db.UnitOffset{ + {Ordinal: 10, RuneStart: 0}, + {Ordinal: 11, RuneStart: 7}, + {Ordinal: 12, RuneStart: 13}, + }, + contentRunes: 15, chunkIndex: 1, + opts: kitvec.SplitOptions{MaxRunes: 10, Overlap: 1}, want: 11, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := anchorOrdinal(tt.offsets, tt.contentRunes, tt.chunkIndex, tt.opts) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestChunkWindowMatchesKitSplit cross-checks the anchor computation's +// deterministic window math against kitvec.Split itself: for every chunk kit +// actually produces, chunkWindow's [start,end) rune window must select +// exactly that chunk's text. If kit's stride semantics ever change, this +// test breaks instead of anchors silently drifting. +func TestChunkWindowMatchesKitSplit(t *testing.T) { + tests := []struct { + name string + content string + opts kitvec.SplitOptions + }{ + {"ascii multi chunk", strings.Repeat("abcde", 9), + kitvec.SplitOptions{MaxRunes: 10, Overlap: 2}}, + {"multi byte multi chunk", strings.Repeat("é", 25), + kitvec.SplitOptions{MaxRunes: 7, Overlap: 1}}, + {"single chunk", "short", + kitvec.SplitOptions{MaxRunes: 10, Overlap: 2}}, + {"short final chunk", strings.Repeat("x", 23), + kitvec.SplitOptions{MaxRunes: 10, Overlap: 1}}, + {"overlap exceeding max runes is clamped", strings.Repeat("y", 25), + kitvec.SplitOptions{MaxRunes: 10, Overlap: 50}}, + {"production overlap shape", strings.Repeat("word ", 100), + kitvec.SplitOptions{MaxRunes: 40, Overlap: ChunkOverlap(40)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + chunks := kitvec.Split(tt.content, tt.opts) + require.NotEmpty(t, chunks) + runes := []rune(tt.content) + for _, c := range chunks { + start, end := chunkWindow(len(runes), c.Index, tt.opts) + require.GreaterOrEqual(t, start, 0) + require.LessOrEqual(t, end, len(runes)) + assert.Equal(t, string(runes[start:end]), c.Text, + "chunk %d window mismatch", c.Index) + } + }) + } +} + +// openSmallChunkIndex opens a test index whose split options force +// multi-chunk documents at tiny content sizes (MaxRunes = maxInputChars, +// Overlap = ChunkOverlap(maxInputChars)). +func openSmallChunkIndex(t *testing.T, maxInputChars int) *Index { + t.Helper() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + ix, err := Open(ctx, path, false, maxInputChars) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ix.Close()) }) + return ix +} + +// seedMirrorRow inserts one vector_messages row shaped like Refresh would +// write it for u, so hydrate-level tests can exercise anchoring without a +// full Build. +func seedMirrorRow(t *testing.T, ix *Index, docKey string, u db.EmbeddableUnit) { + t.Helper() + offsets, err := marshalOffsets(u.Offsets) + require.NoError(t, err) + _, err = ix.db.Exec(` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, + subordinate, offsets, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + docKey, u.SessionID, u.Ordinal, u.OrdinalEnd, u.Subordinate, + offsets, u.Content, contentHash(u.Content)) + require.NoError(t, err) +} + +// TestHydrateHitsAnchorsRunChunks pins the full hydrate path for a run +// document: each matched chunk anchors to the member containing its center +// rune, the hit carries the run's ordinal range and subordinate flag, and +// the snippet is the intersection of the matched chunk's window with the +// ANCHOR member's own span — a substring of the anchor message's text, not +// run-level text spanning members — so the db layer's snippet centering can +// locate it inside the anchor message's content. +func TestHydrateHitsAnchorsRunChunks(t *testing.T) { + ix := openSmallChunkIndex(t, 10) // stride 10-1=9 + ctx := context.Background() + + content := "aaaaa\n\nbbbbb\n\nccccc" // 19 runes, chunks [0,10) and [9,19) + seedMirrorRow(t, ix, "r1", db.EmbeddableUnit{ + SessionID: "s1", Kind: "run", Ordinal: 5, OrdinalEnd: 7, + Subordinate: true, Content: content, + Offsets: []db.UnitOffset{ + {Ordinal: 5, RuneStart: 0, ByteStart: 0}, + {Ordinal: 6, RuneStart: 7, ByteStart: 7}, + {Ordinal: 7, RuneStart: 14, ByteStart: 14}, + }, + }) + + hits, err := ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "r1", ChunkIndex: 0, Score: 0.9}, + }) + require.NoError(t, err) + require.Len(t, hits, 1) + // Chunk 0's window [0,10) spans members 5 and 6 and centers on rune 5, + // in the separator after member 5: the earlier member anchors, and the + // snippet is clipped to member 5's own text rather than the whole + // cross-member chunk "aaaaa\n\nbbb". + assert.Equal(t, 5, hits[0].Ordinal, "anchor ordinal") + assert.Equal(t, 5, hits[0].OrdinalStart) + assert.Equal(t, 7, hits[0].OrdinalEnd) + assert.True(t, hits[0].Subordinate) + assert.Equal(t, "aaaaa", hits[0].Snippet, + "run snippet must be a substring of the anchor member's own text") + + hits, err = ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "r1", ChunkIndex: 1, Score: 0.8}, + }) + require.NoError(t, err) + require.Len(t, hits, 1) + // Chunk 1's window [9,19) centers on rune 14, member 7's first rune: + // the snippet is member 7's slice of the window, not "bbb\n\nccccc". + assert.Equal(t, 7, hits[0].Ordinal, "anchor ordinal") + assert.Equal(t, 5, hits[0].OrdinalStart) + assert.Equal(t, 7, hits[0].OrdinalEnd) + assert.Equal(t, "ccccc", hits[0].Snippet, + "run snippet must be a substring of the anchor member's own text") +} + +// TestHydrateHitsDegenerateChunkIndexFallsBackToAnchorSpan pins the +// fallback for a stale/degenerate ChunkIndex whose re-split window misses +// the content entirely (content changed since embedding): the snippet falls +// back to the anchor member's own span text — never a panic, never text +// from a different member. +func TestHydrateHitsDegenerateChunkIndexFallsBackToAnchorSpan(t *testing.T) { + ix := openSmallChunkIndex(t, 10) + ctx := context.Background() + + content := "aaaaa\n\nbbbbb\n\nccccc" + seedMirrorRow(t, ix, "r1", db.EmbeddableUnit{ + SessionID: "s1", Kind: "run", Ordinal: 5, OrdinalEnd: 7, + Content: content, + Offsets: []db.UnitOffset{ + {Ordinal: 5, RuneStart: 0, ByteStart: 0}, + {Ordinal: 6, RuneStart: 7, ByteStart: 7}, + {Ordinal: 7, RuneStart: 14, ByteStart: 14}, + }, + }) + + hits, err := ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "r1", ChunkIndex: 99, Score: 0.9}, + }) + require.NoError(t, err) + require.Len(t, hits, 1) + assert.Equal(t, 7, hits[0].Ordinal) + assert.Equal(t, "ccccc", hits[0].Snippet, + "an out-of-range chunk window must fall back to the anchor member's span text") +} + +// TestHydrateHitsCorruptOffsetsFailsWithDocKey seeds a mirror row whose +// offsets column holds invalid JSON and pins that hydration fails fast with +// the doc_key in the error rather than panicking or silently dropping the +// hit. +func TestHydrateHitsCorruptOffsetsFailsWithDocKey(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + _, err := ix.db.Exec(` +INSERT INTO vector_messages (doc_key, session_id, ordinal, ordinal_end, + subordinate, offsets, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + "r-corrupt", "s1", 0, 1, false, `{"not": "an array"`, "some content", "h1") + require.NoError(t, err) + + _, err = ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "r-corrupt", ChunkIndex: 0, Score: 0.9}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "r-corrupt", + "the error must name the doc_key whose offsets are corrupt") +} + +// TestHydrateHitsUserDocPassthrough pins that a user document (offsets "[]") +// passes its mirror ordinal through unchanged: Ordinal, OrdinalStart, and +// OrdinalEnd all equal the mirror row's ordinal and the anchor helper is +// never consulted. +func TestHydrateHitsUserDocPassthrough(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + seedMirrorRow(t, ix, "u1", db.EmbeddableUnit{ + SessionID: "s1", Kind: "user", Ordinal: 3, OrdinalEnd: 3, + Content: "a plain user question", + }) + + hits, err := ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "u1", ChunkIndex: 0, Score: 0.7}, + }) + require.NoError(t, err) + require.Len(t, hits, 1) + assert.Equal(t, 3, hits[0].Ordinal) + assert.Equal(t, 3, hits[0].OrdinalStart) + assert.Equal(t, 3, hits[0].OrdinalEnd) + assert.False(t, hits[0].Subordinate) + assert.Equal(t, "a plain user question", hits[0].Snippet) +} + +// TestHydrateHitsMultiByteSnippet pins that run snippets slice on rune +// boundaries: with every content rune multi-byte, byte-offset math would +// tear characters apart or select the wrong window. +func TestHydrateHitsMultiByteSnippet(t *testing.T) { + ix := openSmallChunkIndex(t, 10) // stride 9 + ctx := context.Background() + + content := strings.Repeat("é", 5) + "\n\n" + strings.Repeat("ü", 5) // 12 runes + seedMirrorRow(t, ix, "r1", db.EmbeddableUnit{ + SessionID: "s1", Kind: "run", Ordinal: 1, OrdinalEnd: 2, + Content: content, + Offsets: []db.UnitOffset{ + {Ordinal: 1, RuneStart: 0, ByteStart: 0}, + {Ordinal: 2, RuneStart: 7, ByteStart: 12}, + }, + }) + + hits, err := ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "r1", ChunkIndex: 1, Score: 0.9}, + }) + require.NoError(t, err) + require.Len(t, hits, 1) + // Chunk 1's window is [9,12): the last three ü runes, center rune 10 + // inside member 2's span. + assert.Equal(t, "üüü", hits[0].Snippet, + "run snippet must be a rune-sliced substring of the anchor member's own text") + assert.True(t, utf8.ValidString(hits[0].Snippet)) + assert.Equal(t, 2, hits[0].Ordinal) +} + +// TestSearchRunDocReturnsAnchoredHit pins Search end to end over a mixed +// mirror: a run document's hit is anchored to the member containing the +// matched content and carries its ordinal range and subordinate flag, while +// a user document's hit passes its own ordinal through. +func TestSearchRunDocReturnsAnchoredHit(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + + // "run first message" is 17 runes; member 2 starts at rune 19 after the + // "\n\n" separator. The whole run fits one chunk, whose center rune + // (39/2 = 19) is member 2's first rune. + run := db.EmbeddableUnit{ + SessionID: "s1", Kind: "run", SourceUUID: "a1", + Ordinal: 1, OrdinalEnd: 2, Subordinate: true, + Content: "run first message\n\nmentions alpha topic", + Offsets: []db.UnitOffset{ + {Ordinal: 1, RuneStart: 0, ByteStart: 0}, + {Ordinal: 2, RuneStart: 19, ByteStart: 19}, + }, + } + src := &fakeUnitSource{rows: []fakeUnit{ + {unit: userDoc("s1", "u1", 0, "this message mentions beta topic"), + endedAt: "2024-01-01T00:00:00Z"}, + {unit: run, endedAt: "2024-01-01T00:00:01Z"}, + }} + gen := fakeGeneration("fake-model") + _, err := ix.Build(ctx, src, fakeSearchEncoder(), gen, BuildOptions{}) + require.NoError(t, err) + + hits, err := ix.Search(ctx, fakeSearchEncoder(), "alpha", 10) + require.NoError(t, err) + require.NotEmpty(t, hits) + best := hits[0] + assert.Equal(t, "s1", best.SessionID) + assert.Equal(t, 2, best.Ordinal, "anchor: member containing the chunk center") + assert.Equal(t, 1, best.OrdinalStart) + assert.Equal(t, 2, best.OrdinalEnd) + assert.True(t, best.Subordinate) + assert.Equal(t, "mentions alpha topic", best.Snippet, + "snippet must be the anchor member's own slice of the chunk, not run-level text") + + hits, err = ix.Search(ctx, fakeSearchEncoder(), "beta", 10) + require.NoError(t, err) + require.NotEmpty(t, hits) + best = hits[0] + assert.Equal(t, 0, best.Ordinal) + assert.Equal(t, 0, best.OrdinalStart) + assert.Equal(t, 0, best.OrdinalEnd) + assert.False(t, best.Subordinate) +} + +// seedUnitRow inserts one vector_messages unit row directly, bypassing +// Build, so resolver tests can shape exact unit boundaries and gaps. +func seedUnitRow( + t *testing.T, ix *Index, docKey, sessionID string, start, end int, subordinate bool, +) { + t.Helper() + _, err := ix.db.Exec(` +INSERT INTO vector_messages + (doc_key, session_id, ordinal, ordinal_end, subordinate, content, content_hash) +VALUES (?, ?, ?, ?, ?, ?, ?)`, + docKey, sessionID, start, end, subordinate, "content "+docKey, "h-"+docKey) + require.NoError(t, err) +} + +// TestResolveMessageUnitsPointLookup pins the resolver's containment +// semantics over a mirror with a user unit, a multi-message run, a gap of +// non-embeddable ordinals, and a subordinate run. +func TestResolveMessageUnitsPointLookup(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + seedUnitRow(t, ix, "u:s:0", "s", 0, 0, false) + seedUnitRow(t, ix, "r:s:a", "s", 1, 3, false) + // Ordinals 4-5 are a gap: no unit covers them. + seedUnitRow(t, ix, "r:s:b", "s", 6, 7, true) + // Session t's first unit starts above ordinal 0. + seedUnitRow(t, ix, "r:t:a", "t", 5, 6, false) + + runA := db.UnitRef{DocKey: "r:s:a", SessionID: "s", OrdinalStart: 1, OrdinalEnd: 3} + runB := db.UnitRef{ + DocKey: "r:s:b", SessionID: "s", OrdinalStart: 6, OrdinalEnd: 7, Subordinate: true, + } + tests := []struct { + name string + ref db.MessageRef + want db.UnitRef + }{ + {"user unit own ordinal", db.MessageRef{SessionID: "s", Ordinal: 0}, + db.UnitRef{DocKey: "u:s:0", SessionID: "s"}}, + {"run first ordinal", db.MessageRef{SessionID: "s", Ordinal: 1}, runA}, + {"run interior ordinal", db.MessageRef{SessionID: "s", Ordinal: 2}, runA}, + {"run last ordinal", db.MessageRef{SessionID: "s", Ordinal: 3}, runA}, + {"gap after run", db.MessageRef{SessionID: "s", Ordinal: 4}, db.UnitRef{}}, + {"gap before next run", db.MessageRef{SessionID: "s", Ordinal: 5}, db.UnitRef{}}, + {"subordinate run", db.MessageRef{SessionID: "s", Ordinal: 7}, runB}, + {"past last unit", db.MessageRef{SessionID: "s", Ordinal: 99}, db.UnitRef{}}, + {"before first unit", db.MessageRef{SessionID: "t", Ordinal: 2}, db.UnitRef{}}, + {"unknown session", db.MessageRef{SessionID: "nope", Ordinal: 1}, db.UnitRef{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ix.ResolveMessageUnits(ctx, []db.MessageRef{tc.ref}) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, tc.want, got[0]) + }) + } +} + +// TestResolveMessageUnitsIgnoresParkedRows pins the mid-refresh read +// contract: Refresh parks a displaced row at a negative sentinel ordinal +// non-transactionally (evictSlotOccupant), so a concurrent resolver call can +// see it. The point lookup's ordinal-DESC seek would otherwise land on the +// parked row (its old ordinal_end still covers the ref) and emit a negative +// OrdinalStart; parked rows must be invisible to readers. +func TestResolveMessageUnitsIgnoresParkedRows(t *testing.T) { + ix := openTestIndex(t) + // Parked mid-refresh: ordinal moved to the sentinel, ordinal_end still + // holds its old value, so containment (2 <= 3) would pass. + seedUnitRow(t, ix, "r:s:parked", "s", -2, 3, false) + seedUnitRow(t, ix, "r:s:valid", "s", 5, 6, false) + + got, err := ix.ResolveMessageUnits(context.Background(), []db.MessageRef{ + {SessionID: "s", Ordinal: 2}, + {SessionID: "s", Ordinal: 5}, + }) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, db.UnitRef{}, got[0], + "a ref covered only by a parked row must stay unresolved, not surface a negative ordinal") + assert.Equal(t, db.UnitRef{ + DocKey: "r:s:valid", SessionID: "s", OrdinalStart: 5, OrdinalEnd: 6, + }, got[1], "valid rows must keep resolving alongside a parked one") +} + +// TestHydrateHitsIgnoresParkedRows pins the same mid-refresh contract on the +// hit-hydration path: a KNN hit whose doc_key points at a sentinel-parked +// mirror row must be dropped (like a vanished doc), never hydrated into a +// hit with a negative ordinal. +func TestHydrateHitsIgnoresParkedRows(t *testing.T) { + ix := openTestIndex(t) + ctx := context.Background() + seedMirrorRow(t, ix, "u-parked", db.EmbeddableUnit{ + SessionID: "s1", Kind: "user", Ordinal: -1, OrdinalEnd: 4, + Content: "parked mid-refresh", + }) + seedMirrorRow(t, ix, "u-valid", db.EmbeddableUnit{ + SessionID: "s1", Kind: "user", Ordinal: 6, OrdinalEnd: 6, + Content: "still visible", + }) + + hits, err := ix.hydrateHits(ctx, []kitvec.Hit[string]{ + {Doc: "u-parked", ChunkIndex: 0, Score: 0.9}, + {Doc: "u-valid", ChunkIndex: 0, Score: 0.8}, + }) + require.NoError(t, err) + require.Len(t, hits, 1, "the parked row's hit must be dropped") + assert.Equal(t, 6, hits[0].Ordinal) + assert.Equal(t, "still visible", hits[0].Snippet) +} + +// TestResolveMessageUnitsResultParallelToRefs pins that one call over a +// mixed batch keeps the result slice parallel to refs, with zero UnitRefs +// holding the positions of unresolvable refs. +func TestResolveMessageUnitsResultParallelToRefs(t *testing.T) { + ix := openTestIndex(t) + seedUnitRow(t, ix, "r:s:a", "s", 1, 3, true) + + got, err := ix.ResolveMessageUnits(context.Background(), []db.MessageRef{ + {SessionID: "s", Ordinal: 4}, + {SessionID: "s", Ordinal: 2}, + {SessionID: "missing", Ordinal: 2}, + }) + require.NoError(t, err) + require.Len(t, got, 3) + assert.Equal(t, db.UnitRef{}, got[0], "gap ref stays zero") + assert.Equal(t, db.UnitRef{ + DocKey: "r:s:a", SessionID: "s", OrdinalStart: 1, OrdinalEnd: 3, Subordinate: true, + }, got[1]) + assert.Equal(t, db.UnitRef{}, got[2], "unknown session stays zero") +} + +// TestResolveMessageUnitsEmptyRefs pins the empty-input shape: an empty, +// non-nil result and no query error. +func TestResolveMessageUnitsEmptyRefs(t *testing.T) { + ix := openTestIndex(t) + got, err := ix.ResolveMessageUnits(context.Background(), nil) + require.NoError(t, err) + assert.Empty(t, got) +} + +// TestResolveMessageUnitsManyRefs feeds well over SQLite's historical +// 999-bind-variable budget through the resolver in one call: the per-ref +// point-lookup implementation must handle any batch size without an +// unbounded IN list. +func TestResolveMessageUnitsManyRefs(t *testing.T) { + ix := openTestIndex(t) + const n = 1200 + seedUnitRow(t, ix, "r:s:wide", "s", 0, n-1, false) + + refs := make([]db.MessageRef, n) + for i := range refs { + refs[i] = db.MessageRef{SessionID: "s", Ordinal: i} + } + got, err := ix.ResolveMessageUnits(context.Background(), refs) + require.NoError(t, err) + require.Len(t, got, n) + for i, u := range got { + require.Equal(t, "r:s:wide", u.DocKey, "ref %d must resolve", i) + } +} + +// TestResolveMessageUnitsVersionMismatchGate pins the read gate: a read-only +// Index over a vectors.db stamped by a different mirror schema version must +// fail closed with ErrMirrorVersionMismatch before touching any table, the +// same contract Search and StaleActive already honor. +func TestResolveMessageUnitsVersionMismatchGate(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "vectors.db") + seedV2Mirror(t, path) + + ro, err := Open(ctx, path, true, 4000) + require.NoError(t, err, "read-only Open must succeed against a mismatched mirror") + defer ro.Close() + + _, err = ro.ResolveMessageUnits(ctx, []db.MessageRef{{SessionID: "s1", Ordinal: 0}}) + require.Error(t, err) + assert.ErrorIs(t, err, ErrMirrorVersionMismatch) +} diff --git a/internal/vector/varlimit_cgo_test.go b/internal/vector/varlimit_cgo_test.go new file mode 100644 index 000000000..eaebe00c2 --- /dev/null +++ b/internal/vector/varlimit_cgo_test.go @@ -0,0 +1,28 @@ +//go:build !windows && cgo + +package vector + +import ( + "database/sql" + "fmt" + "testing" + + sqlite3 "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +// setConnVarLimit lowers conn's SQLITE_LIMIT_VARIABLE_NUMBER through the +// mattn driver's raw-connection API; varlimit_modernc_test.go provides the +// equivalent for the modernc driver vectors.db uses on Windows or without +// cgo. +func setConnVarLimit(t *testing.T, conn *sql.Conn, limit int) { + t.Helper() + require.NoError(t, conn.Raw(func(dc any) error { + sc, ok := dc.(*sqlite3.SQLiteConn) + if !ok { + return fmt.Errorf("index conn is %T, want *sqlite3.SQLiteConn", dc) + } + sc.SetLimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, limit) + return nil + })) +} diff --git a/internal/vector/varlimit_modernc_test.go b/internal/vector/varlimit_modernc_test.go new file mode 100644 index 000000000..eadfc8ead --- /dev/null +++ b/internal/vector/varlimit_modernc_test.go @@ -0,0 +1,21 @@ +//go:build windows || !cgo + +package vector + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/require" + "modernc.org/sqlite" + sqlite3lib "modernc.org/sqlite/lib" +) + +// setConnVarLimit lowers conn's SQLITE_LIMIT_VARIABLE_NUMBER through the +// modernc driver's Limit API; varlimit_cgo_test.go provides the equivalent +// for the mattn driver vectors.db uses on Unix with cgo. +func setConnVarLimit(t *testing.T, conn *sql.Conn, limit int) { + t.Helper() + _, err := sqlite.Limit(conn, sqlite3lib.SQLITE_LIMIT_VARIABLE_NUMBER, limit) + require.NoError(t, err) +}