Skip to content

Commit ad65ba0

Browse files
committed
feat(data): first-class Data mode for project inventory and reclassification
Replace the Activity-page project-reclassification affordances with a dedicated Data mode: - New read-only Data API group on all stores (SQLite, PostgreSQL, DuckDB): project inventory with summary totals, per-machine project rules with governed counts, and archive-wide reclassification candidates selected by (display label, project key). - Svelte Data page: sortable/filterable inventory table, project workspace with the extracted ProjectReclassificationEditor (prefix edit, debounced preview, atomic apply, after-apply selection rules), and the relocated worktree mapping Rules view; Activity project labels become plain links into Data. - Remove the legacy Activity-scoped candidates endpoint, its SQLite-only db method, and the generated client method; regenerate the API client. - Parser: recognize the generic github.com worktree layout marker and bump dataVersion to force reparse; chunk project-identity label lookups on all backends to stay under bind-variable limits. - PostgreSQL: split push vs serve schema probes, exempt filtered pushes from the provenance backfill marker. - E2E: new data-mode Playwright spec covering the full writable flow and DuckDB-gated read-only variants; wired into e2e-duckdb. Additional branch commits: - docs: remove superpowers artifacts and document the Data page - docs: describe Data page relative to shipped Settings surface
1 parent 34d326c commit ad65ba0

184 files changed

Lines changed: 19149 additions & 3186 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/ci.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ jobs:
118118
run: |
119119
bash scripts/install_test.sh
120120
bash scripts/retry_test.sh
121+
bash scripts/e2e-server_test.sh
121122
bash scripts/check_desktop_release_health_test.sh
122123
bash scripts/check_desktop_release_health_from_event_test.sh
123124
bash desktop/scripts/test-repair-appimage-diricon.sh
@@ -362,3 +363,9 @@ jobs:
362363
env:
363364
E2E_PREBUILT_FIXTURE: /tmp/testfixture
364365
E2E_PREBUILT_SERVER: /tmp/agentsview
366+
367+
- name: Run DuckDB E2E tests
368+
run: make e2e-duckdb
369+
env:
370+
E2E_PREBUILT_FIXTURE: /tmp/testfixture
371+
E2E_PREBUILT_SERVER: /tmp/agentsview

.gitignore

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,9 @@ docs/.env*.local
6767
testdata/ssh/test_key
6868

6969
# Local data
70-
data/
71-
sessions/
72-
html/
70+
/data/
71+
/sessions/
72+
/html/
7373
.superset/
7474
.github/hooks/
7575
.superpowers/

Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,8 @@ e2e:
378378
# Run focused Playwright smoke tests against duckdb serve.
379379
e2e-duckdb:
380380
cd frontend && AGENTSVIEW_E2E_BACKEND=duckdb npx playwright test \
381-
e2e/duckdb-backend.spec.ts e2e/session-list.spec.ts --project=chromium
381+
e2e/duckdb-backend.spec.ts e2e/data-mode.spec.ts \
382+
e2e/session-list.spec.ts --project=chromium
382383

383384
# Vet
384385
vet: pricing-snapshot ensure-embed-dir

cmd/testfixture/main.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"go.kenn.io/agentsview/internal/db"
1414
duckdbsync "go.kenn.io/agentsview/internal/duckdb"
15+
"go.kenn.io/agentsview/internal/export"
1516
)
1617

1718
type sessionSpec struct {
@@ -118,6 +119,12 @@ func main() {
118119
log.Fatalf("creating recent-edits fixture: %v", err)
119120
}
120121

122+
if err := createProjectReclassificationFixture(
123+
database, base.Add(120*time.Hour),
124+
); err != nil {
125+
log.Fatalf("creating project-reclassification fixture: %v", err)
126+
}
127+
121128
fmt.Printf("Fixture DB written to %s\n", *out)
122129
if *duckDBOut != "" {
123130
if err := writeDuckDBMirror(database, *duckDBOut); err != nil {
@@ -127,6 +134,80 @@ func main() {
127134
}
128135
}
129136

137+
func createProjectReclassificationFixture(
138+
database *db.DB, start time.Time,
139+
) error {
140+
const (
141+
machine = "remote-example-host"
142+
project = "wrong_branch_label"
143+
worktreeRoot = "/srv/worktrees/github.com/example-org/sample-service/example-worktree"
144+
model = "claude-sonnet-4-20250514"
145+
)
146+
cwds := []struct {
147+
suffix string
148+
cwd string
149+
}{
150+
{suffix: "root", cwd: worktreeRoot},
151+
{suffix: "nested", cwd: worktreeRoot + "/cmd/server"},
152+
}
153+
ctx := context.Background()
154+
for index, item := range cwds {
155+
sessionID := "test-session-project-reclassification-" + item.suffix
156+
startedAt := start.Add(time.Duration(index) * time.Hour)
157+
endedAt := startedAt.Add(12 * time.Minute)
158+
firstMessage := "Inspect the sample service worktree."
159+
session := db.Session{
160+
ID: sessionID,
161+
Project: project,
162+
Machine: machine,
163+
Agent: "claude",
164+
StartedAt: new(startedAt.Format(time.RFC3339Nano)),
165+
EndedAt: new(endedAt.Format(time.RFC3339Nano)),
166+
MessageCount: 2,
167+
UserMessageCount: 1,
168+
FirstMessage: new(firstMessage),
169+
Cwd: item.cwd,
170+
}
171+
if err := database.UpsertSession(session); err != nil {
172+
return fmt.Errorf(
173+
"upserting project-reclassification session: %w", err,
174+
)
175+
}
176+
if err := database.InsertMessages(generateMessages(
177+
sessionID, session.MessageCount, startedAt, model,
178+
)); err != nil {
179+
return fmt.Errorf(
180+
"inserting project-reclassification messages: %w", err,
181+
)
182+
}
183+
if err := database.UpsertProjectIdentityObservation(
184+
ctx,
185+
export.ProjectIdentityObservation{
186+
SessionID: sessionID,
187+
Project: project,
188+
Machine: machine,
189+
RootPath: worktreeRoot,
190+
RepositoryPath: "/srv/worktrees/github.com/example-org/sample-service",
191+
WorktreeName: "example-worktree",
192+
WorktreeRootPath: worktreeRoot,
193+
WorktreeRelationship: export.WorktreeLinked,
194+
CheckoutState: export.CheckoutBranch,
195+
GitBranch: "example-worktree",
196+
ObservedAt: startedAt,
197+
},
198+
); err != nil {
199+
return fmt.Errorf(
200+
"upserting project-reclassification identity: %w", err,
201+
)
202+
}
203+
fmt.Printf(
204+
" %s: %d messages (project reclassification)\n",
205+
sessionID, session.MessageCount,
206+
)
207+
}
208+
return nil
209+
}
210+
130211
func writeDuckDBMirror(database *db.DB, path string) error {
131212
if err := os.Remove(path); err != nil &&
132213
!errors.Is(err, os.ErrNotExist) {

cmd/testfixture/main_test.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"path/filepath"
6+
"testing"
7+
"time"
8+
9+
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
12+
"go.kenn.io/agentsview/internal/db"
13+
)
14+
15+
func TestCreateProjectReclassificationFixture(t *testing.T) {
16+
database, err := db.Open(filepath.Join(t.TempDir(), "sessions.db"))
17+
require.NoError(t, err)
18+
t.Cleanup(func() { require.NoError(t, database.Close()) })
19+
20+
base := time.Date(2026, 7, 1, 10, 0, 0, 0, time.UTC)
21+
require.NoError(t, createProjectReclassificationFixture(database, base))
22+
23+
const (
24+
machine = "remote-example-host"
25+
wrongProject = "wrong_branch_label"
26+
worktreeRoot = "/srv/worktrees/github.com/example-org/sample-service/example-worktree"
27+
)
28+
wantCwds := map[string]string{
29+
"test-session-project-reclassification-root": worktreeRoot,
30+
"test-session-project-reclassification-nested": worktreeRoot + "/cmd/server",
31+
}
32+
for sessionID, wantCwd := range wantCwds {
33+
session, getErr := database.GetSession(context.Background(), sessionID)
34+
require.NoError(t, getErr)
35+
require.NotNil(t, session)
36+
assert.Equal(t, machine, session.Machine)
37+
assert.Equal(t, wrongProject, session.Project)
38+
assert.Equal(t, wantCwd, session.Cwd)
39+
}
40+
41+
snapshots, err := database.ListSessionProjectIdentitySnapshots(
42+
context.Background(),
43+
)
44+
require.NoError(t, err)
45+
require.Len(t, snapshots, 2)
46+
for _, snapshot := range snapshots {
47+
assert.Equal(t, machine, snapshot.Machine)
48+
assert.Equal(t, wrongProject, snapshot.Project)
49+
assert.Equal(t, worktreeRoot, snapshot.RootPath)
50+
assert.Equal(t, worktreeRoot, snapshot.WorktreeRootPath)
51+
assert.NotEmpty(t, snapshot.Key)
52+
}
53+
}

docs/activity.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,17 @@ allocates that total across usage rows in proportion to their catalog-price
106106
estimates. The per-model costs are therefore estimated attributions, not
107107
provider-reported model charges, but they still sum to the displayed total.
108108

109+
## Reclassify A Project
110+
111+
Worktree layouts the parser does not recognize can surface a branch or
112+
worktree directory name as a project. Each row in the **Project** breakdown
113+
links to that project on the [Data page](/data/), where the reclassification
114+
editor suggests a path prefix from the project's worktree evidence, previews
115+
the full-archive impact, and applies a
116+
[worktree project mapping](/configuration/#worktree-project-mappings) rule in
117+
one atomic step. Cleaning always evaluates the complete archive; the current
118+
Activity range and filters do not carry over.
119+
109120
## Activity Insight
110121

111122
At the bottom of the page, **Activity Insight** shows an existing global

docs/configuration.md

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -689,27 +689,43 @@ manual sync, and the periodic directory scan.
689689

690690
### Worktree Project Mappings
691691

692-
The parser infers a session's project from its `cwd`, which works for standard
693-
layouts but not custom worktree conventions like
694-
`~/code/{project}.worktrees/feat/<branch>/` — those sessions otherwise group
695-
under `<branch>` rather than `{project}`. As of 0.29.0, you can register manual
696-
**path-prefix → project** rules from the **Worktree Project Mappings** section
697-
in Settings:
698-
699-
![Worktree Project Mappings settings section](/assets/generated/screenshots/worktree-mappings.png)
692+
The parser infers a session's project from its `cwd`. It recognizes common
693+
worktree manager layouts, including the generic
694+
`worktrees/github.com/<owner>/<repository>/<worktree>` convention, where the
695+
repository segment becomes the project. Layouts it does not recognize — such
696+
as `~/code/{project}.worktrees/feat/<branch>/` — otherwise group sessions
697+
under `<branch>` rather than `{project}`. For those, register manual
698+
**path-prefix → project** rules from the **Rules** view on the
699+
[Data page](/data/#rules), or let the
700+
[reclassification editor](/data/#reclassify-a-project) create one from a
701+
project's worktree evidence:
702+
703+
![Worktree mapping rules on the Data page](/assets/generated/screenshots/worktree-mappings.png)
700704

701705
- Mappings are explicit; there is no auto-discovery.
706+
- Each rule is scoped to one machine. The machine selector manages rules for
707+
the local machine and for any remotely synced machine. Rules live in the
708+
writable archive that ingests that machine's sessions, which may be the
709+
source machine's local SQLite archive or a separate collector archive.
702710
- Each rule applies whenever a session's `cwd` falls under the configured
703711
prefix, on both new sessions as they sync and (via the **Apply** button)
704-
already-imported sessions.
712+
already-imported sessions. Prefixes match on directory boundaries, so
713+
`/worktrees/service` does not match `/worktrees/service-old`.
714+
- Enabled mappings run after parser inference, so an explicit rule always wins
715+
when the two disagree.
705716
- The default `explicit` layout maps every matching path to the project name
706717
stored on the rule. The `repo_dot_worktrees` layout derives the project from
707718
the first path segment under the prefix when it is named `<repo>.worktrees`,
708719
so a path like `/code/agentsview.worktrees/feature/frontend` resolves to
709720
project `agentsview`.
710-
- Rules are stored in a `worktree_project_mappings` SQLite table scoped to the
711-
host machine, so a mapping created on one machine does not leak into another
712-
machine's view of synced sessions.
721+
- Rules created from the Data reclassification editor record the mislabeled
722+
project they corrected, shown as the rule's **original label**. The value is
723+
informational and set once; to manually revert a reclassification, edit the
724+
rule's target back to that original label and apply again.
725+
- Disabling or deleting a rule does not rewrite sessions by itself. Sessions
726+
whose source files still exist revert to parser-derived names on a later
727+
reparse or full resync, while orphaned sessions keep their stored
728+
classification.
713729
- Excluded, trashed, and skipped session files are left alone.
714730

715731
Mappings only mutate the session's `project` field; the rest of the session

docs/data.md

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
---
2+
title: Data
3+
description: Project inventory, reclassification, and worktree mapping rules
4+
---
5+
6+
The **Data** page is where you inspect and clean project classification across
7+
the whole archive. It is the home of the worktree mapping rules that previously
8+
lived in the Settings **Worktree mappings** section.
9+
10+
Open it from the **Data** tab in the header, or follow a project link from the
11+
[Activity breakdown](/activity/#breakdowns). Deep links are stable:
12+
`/data?project_key=<key>` selects a project, and `/data?view=rules` opens the
13+
[Rules view](#rules).
14+
15+
## Project Inventory
16+
17+
The default view lists every project in the archive with its session, machine,
18+
agent, and working-directory counts plus first and last activity timestamps. A
19+
summary strip totals the projects, sessions, and the sessions currently governed
20+
by classification rules.
21+
22+
- The table is sortable by any column and filterable by project name.
23+
- Projects targeted by enabled rules carry a rule badge; projects recorded as a
24+
rule's original label carry an original-label badge.
25+
- Sessions whose stored project label is empty are grouped under a single
26+
"unknown" row.
27+
- Activity bounds come from session timestamps only; rows without any recorded
28+
timestamps show a no-activity state.
29+
30+
Selecting a row opens the project workspace. Unknown `project_key` deep links
31+
show the full inventory with a non-blocking notice.
32+
33+
## Reclassify A Project
34+
35+
The workspace embeds the reclassification editor, which works through the
36+
[worktree project mapping](/configuration/#worktree-project-mappings) system:
37+
38+
- It lists the worktrees that produced the project's sessions across the entire
39+
archive, grouped by machine and worktree evidence. A single group is
40+
preselected; several groups require an explicit choice, because different
41+
worktrees usually need different target projects.
42+
- The suggested **path prefix** covers the selected group's working directories.
43+
You can edit it — for example, shorten it to cover sibling worktrees of the
44+
same repository.
45+
- The **target project** typeahead suggests known projects and accepts a new
46+
name. When the server normalizes the name (for example `sample-service`
47+
becomes `sample_service`), the editor shows the stored form before you
48+
apply.
49+
- The **full archive impact** preview is live and authoritative: it counts
50+
matching sessions across all dates for that machine. A prefix that touches
51+
more than one existing project shows a warning with per-project counts —
52+
usually a sign the prefix is too broad. A prefix matching zero sessions
53+
cannot be applied.
54+
55+
**Apply** saves the rule and rewrites the matching sessions in one atomic step,
56+
then reloads the inventory. If the applied rule renamed the selected project,
57+
the selection follows the new name. If mappings changed between preview and
58+
apply, the apply is rejected and a fresh preview is required.
59+
60+
## Rules
61+
62+
The **Rules** toggle shows the worktree mapping rules for one machine at a time,
63+
with the same add, edit, apply, and delete controls that Settings previously
64+
offered — see
65+
[Worktree Project Mappings](/configuration/#worktree-project-mappings) for the
66+
full rule semantics. Each rule row also shows its **governed sessions** count
67+
(how many sessions the rule currently classifies) and the **original label**
68+
recorded when the rule was created through the reclassification editor. Rule
69+
targets link back to the corresponding inventory row.
70+
71+
## Read-Only Servers
72+
73+
On a read-only server (`pg serve` or `duckdb serve`) the inventory, the
74+
candidate evidence, and the Rules table remain fully readable, but the editor
75+
and rule mutations are replaced by a notice: classification rules are managed
76+
from the writable archive that ingests the machine's sessions.

docs/screenshots/extract-db.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ DELETE FROM session_project_identity_snapshots;
290290
DELETE FROM project_identity_observation_changes;
291291
DELETE FROM session_project_identity_snapshot_changes;
292292
DELETE FROM worktree_project_mappings;
293+
-- The mapping delete above journals tombstones, so clear that journal last.
294+
DELETE FROM worktree_project_mapping_changes;
293295
294296
-- Keep generated screenshots independent of the source machine's hostname.
295297
-- The PostgreSQL fixture relabels a subset as work-desktop after push so the

docs/screenshots/tests/screenshots.spec.ts

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1116,24 +1116,20 @@ test.describe('Settings', () => {
11161116
});
11171117

11181118
test('worktree project mappings section', async ({ page }) => {
1119-
await openSettings(page);
1120-
1121-
// SettingsSection renders a heading with the title prop;
1122-
// match the section that contains the "Worktree mappings"
1123-
// header text, the same pattern used for Remote Access above.
1124-
const worktreeSection = page.locator(
1125-
'.settings-section:has-text("Worktree mappings")'
1126-
);
1127-
await expect(worktreeSection.first()).toBeVisible({
1128-
timeout: 5_000,
1119+
// Mapping rules moved from Settings to the Data page's Rules view.
1120+
await page.goto('/data?view=rules');
1121+
1122+
const rulesView = page.locator('section.rules-view');
1123+
await expect(rulesView).toBeVisible({ timeout: 5_000 });
1124+
await rulesView.scrollIntoViewIfNeeded();
1125+
const mappingPath = rulesView.getByRole('textbox', {
1126+
name: 'Path prefix',
11291127
});
1130-
await worktreeSection.first().scrollIntoViewIfNeeded();
1131-
const mappingPath = worktreeSection.first().getByRole('textbox').first();
11321128
await mappingPath.fill('~/code/project.worktrees');
11331129
await expect(mappingPath).toHaveValue('~/code/project.worktrees');
11341130
await page.waitForTimeout(500);
11351131

1136-
await snapEl(worktreeSection.first(), 'worktree-mappings');
1132+
await snapEl(rulesView, 'worktree-mappings');
11371133
});
11381134
});
11391135

0 commit comments

Comments
 (0)