Skip to content

Commit 4723f89

Browse files
committed
pg: address review of --from-now
Fixes from the review of the previous commit. The flag was silently dropped on daemon-delegated pushes: when the local daemon owns the archive, `pg push` posts to /api/v1/push/pg, and neither the request nor the handler carried it, so the whole archive was backfilled anyway. It now travels in the request and maps into SyncOptions. Whether --full was requested cannot be decided inside Push: callers pass `cfg.Full || didResync`, so any automatic resync looked like explicit intent and disabled the flag. The precedence is resolved at the CLI instead, where the user's own flag is known. The boundary scoped session selection only, while other phases stayed archive-wide, so pre-boundary content was uploaded regardless. A bounded push now refuses when a vector source is attached, because embeddings carry raw text and cannot be bounded, and skips cursor usage events for the same reason a filtered push already skips them: the rows are global and unattributed. Freshness is now proven rather than inferred. The reset paths clear the watermark, so an empty watermark alone could not distinguish a genuinely new target from an established one whose marker was lost or whose first push failed part-way, and re-seeding there would permanently skip the history those resets exist to restore. Boundary state, marker presence, and whether a reset ran on this pass are all required. Also documents the flag and its interactions in the command reference.
1 parent 1278767 commit 4723f89

7 files changed

Lines changed: 96 additions & 51 deletions

File tree

cmd/agentsview/archive_write_backend.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,7 @@ func (b daemonArchiveWriteBackend) PGPush(
333333
SyncStateTarget: target.SyncStateTarget,
334334
MigrateLegacySyncState: target.MigrateLegacySyncState,
335335
NoVectors: cfg.NoVectors,
336+
FromNow: cfg.FromNow,
336337
ScopeVectorsToChangedSessions: cfg.
337338
ScopeVectorsToChangedSessions,
338339
LastReconciledVectorGeneration: cfg.
@@ -623,7 +624,7 @@ func (b *localArchiveWriteBackend) PGPush(
623624
ps, err := postgres.New(
624625
target.PG.URL, target.PG.Schema, b.database,
625626
target.PG.MachineName, target.PG.AllowInsecure,
626-
target.syncOptions(projects, excludeProjects, vectorSource, cfg.FromNow),
627+
target.syncOptions(projects, excludeProjects, vectorSource, cfg.FromNow && !cfg.Full),
627628
)
628629
if err != nil {
629630
return postgres.PushResult{}, err
@@ -870,7 +871,7 @@ func (b *localArchiveWriteBackend) PGPushWatch(
870871
s, cErr := postgres.New(
871872
target.PG.URL, target.PG.Schema, b.database,
872873
target.PG.MachineName, target.PG.AllowInsecure,
873-
target.syncOptions(projects, exclude, vectorSource, cfg.FromNow),
874+
target.syncOptions(projects, exclude, vectorSource, cfg.FromNow && !cfg.Full),
874875
)
875876
if cErr != nil {
876877
return nil, cErr

cmd/agentsview/daemon_push.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ type daemonPushRequest struct {
2525
// NoVectors mirrors the CLI --no-vectors flag into the daemon: it has no
2626
// per-invocation flag of its own, so the gate must travel in the request.
2727
NoVectors bool `json:"no_vectors,omitempty"`
28+
// FromNow carries the CLI --from-now flag. Without it a delegated push
29+
// silently backfills the whole archive, which is the opposite of what the
30+
// flag was asked to do.
31+
FromNow bool `json:"from_now,omitempty"`
2832
// ScopeVectorsToChangedSessions is set by change-triggered watch
2933
// pushes so the daemon's vector phase reads state only for the
3034
// changed relational sessions (see postgres.PushOptions).

docs/commands.md

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,7 @@ agentsview pg push [target] [flags]
563563
| -------------------- | ------- | -------------------------------------------------------------- |
564564
| `--full` | `false` | Force full local resync and re-push |
565565
| `--no-vectors` | `false` | Skip the semantic-search vector phase for this run |
566+
| `--from-now` | `false` | On a target's FIRST push, start at now instead of backfilling |
566567
| `--projects` | | Comma-separated projects to push (inclusive) |
567568
| `--exclude-projects` | | Comma-separated projects to exclude from push |
568569
| `--all-projects` | `false` | Ignore configured project filters for this run |
@@ -571,8 +572,14 @@ agentsview pg push [target] [flags]
571572
| `--debounce` | `30s` | Coalesce window after a change before pushing (`--watch` only) |
572573
| `--interval` | `15m` | Periodic floor push interval (`--watch` only) |
573574

574-
See [PostgreSQL Sync — Project Filtering](/pg-sync/#project-filtering) for
575-
details on how filtering interacts with the push watermark.
575+
`--from-now` bounds a target's very first push to sessions from that point on,
576+
for pushing into a database shared with other people where uploading the whole
577+
local archive would disclose unrelated work. It applies only when the target is
578+
provably new (no watermark, no boundary state, no push marker, and no reset on
579+
this run), is ignored with `--full`, and requires `--no-vectors` because the
580+
vector phase cannot be bounded. See
581+
[PostgreSQL Sync — Project Filtering](/pg-sync/#project-filtering) for details
582+
on how filtering interacts with the push watermark.
576583

577584
______________________________________________________________________
578585

internal/postgres/push.go

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -134,10 +134,6 @@ func (s *Sync) PushWithOptions(
134134
) (PushResult, error) {
135135
full := opts.Full
136136
start := time.Now()
137-
// The caller's own --full intent, kept because `full` is reassigned below
138-
// when reset detection forces a rebuild; PushFromNow must never override an
139-
// explicitly requested full push.
140-
requestedFull := full
141137
var result PushResult
142138
state := s.effectiveSyncState()
143139
aliasBackfillState := s.aliasBackfillSyncStateOrDefault()
@@ -296,6 +292,31 @@ func (s *Sync) PushWithOptions(
296292
}
297293
}
298294
}
295+
// Decide the from-now boundary here, after every reset check and before the
296+
// setup phases below. Freshness has to be proven, not assumed: a reset path
297+
// clears lastPush, so lastPush alone cannot distinguish a genuinely new
298+
// target from an established one whose marker was lost or whose first push
299+
// failed part-way. Re-seeding the boundary in those cases would permanently
300+
// skip history that should be restored, so require that nothing local
301+
// (watermark, boundary state) and nothing remote (this marker) has ever
302+
// recorded a push, and that no reset cleared state on this run.
303+
applyFromNow := pushFromNowApplies(
304+
s.pushFromNow, lastPush, boundaryState, markerExists, pushStateCleared,
305+
)
306+
// The boundary scopes SESSION selection only. Phases that are archive-wide
307+
// would still upload pre-boundary content, defeating the point, so a bounded
308+
// first push refuses or skips them rather than leaking quietly.
309+
// Scoped to this push, so a later unbounded push still syncs these phases.
310+
s.skipUnboundedPhases = applyFromNow
311+
defer func() { s.skipUnboundedPhases = false }()
312+
if applyFromNow && s.vectorSource != nil {
313+
return result, fmt.Errorf(
314+
"--from-now cannot bound the vector phase: it would upload " +
315+
"embeddings and raw text for sessions before the boundary; " +
316+
"re-run with --no-vectors",
317+
)
318+
}
319+
299320
if err := timedPushSetupStep("model pricing sync",
300321
func() error { return s.syncModelPricing(ctx) }); err != nil {
301322
return result, err
@@ -306,22 +327,18 @@ func (s *Sync) PushWithOptions(
306327
}
307328
cutoff := time.Now().UTC().Format(LocalSyncTimestampLayout)
308329

309-
// Start a brand-new target at "now" instead of backfilling the archive.
310-
// Applied HERE, after every reset check above has run: those checks key on
311-
// lastPush being non-empty and treat a watermark with no matching target
312-
// fingerprint or PG-side marker as corrupt local state, so a watermark
313-
// seeded any earlier (or from outside this process) would be wiped and the
314-
// push would fall back to a full backfill. finalizePushState records cutoff
315-
// at the end, so every later push for the target is normally incremental.
316-
if boundary, applied := pushFromNowBoundary(
317-
s.pushFromNow, requestedFull, lastPush, cutoff,
318-
); applied {
330+
// Seed the watermark so this target's history starts at the join point. The
331+
// decision was made above; it cannot be pre-seeded from outside the process
332+
// because the reset checks treat a watermark with no matching fingerprint or
333+
// PG-side marker as corrupt local state and clear it. finalizePushState
334+
// records cutoff at the end, so every later push is normally incremental.
335+
if applyFromNow {
319336
log.Printf(
320337
"pgsync: first push for this target starts at %s "+
321338
"(from-now); local history before it is not uploaded",
322339
cutoff,
323340
)
324-
lastPush = boundary
341+
lastPush = cutoff
325342
}
326343

327344
// Candidate selection shares ListSessionsForMirrorWindow with the
@@ -1375,18 +1392,23 @@ func persistPushTargetFingerprint(
13751392
return nil
13761393
}
13771394

1378-
// pushFromNowBoundary decides the lower bound of a from-now push. It applies
1379-
// only to a target with NO watermark yet, and never overrides an explicitly
1380-
// requested full push: narrowing an established target would silently create a
1381-
// gap in what the hub has, whereas bounding a target's very first push just
1382-
// means its history starts at the join point.
1383-
func pushFromNowBoundary(
1384-
enabled, requestedFull bool, lastPush, cutoff string,
1385-
) (string, bool) {
1386-
if !enabled || requestedFull || lastPush != "" {
1387-
return lastPush, false
1388-
}
1389-
return cutoff, true
1395+
// pushFromNowApplies reports whether a from-now boundary may be seeded.
1396+
//
1397+
// It requires PROOF that the target has never been pushed to, not merely an
1398+
// empty watermark: the reset paths above clear lastPush, so an established
1399+
// target whose PG marker was lost, whose fingerprint changed, or whose first
1400+
// push failed part-way would otherwise look brand new. Seeding a boundary there
1401+
// would permanently skip the history those resets exist to restore. Boundary
1402+
// state counts because a partial first push leaves fingerprints behind with no
1403+
// watermark, and stateCleared because a reset on this very run means state that
1404+
// did exist was just discarded.
1405+
func pushFromNowApplies(
1406+
enabled bool, lastPush, boundaryState string,
1407+
markerExists, stateCleared bool,
1408+
) bool {
1409+
return enabled &&
1410+
lastPush == "" && boundaryState == "" &&
1411+
!markerExists && !stateCleared
13901412
}
13911413

13921414
func pushTargetState(
@@ -3441,7 +3463,7 @@ func nilIfZero(n int) any {
34413463
func (s *Sync) syncCursorUsageEvents(ctx context.Context) error {
34423464
// Cursor admin rows are global and unattributed, so project-filtered pushes
34433465
// cannot sync them honestly.
3444-
if s.isFiltered() {
3466+
if s.isFiltered() || s.skipUnboundedPhases {
34453467
return nil
34463468
}
34473469

internal/postgres/push_test.go

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1823,33 +1823,34 @@ func TestShouldSkipSessionMessagesInBatchedPush(t *testing.T) {
18231823
), "full mode should not skip by fingerprint check")
18241824
}
18251825

1826-
// A from-now push must bound only a target's FIRST push, and must never
1827-
// silently narrow an established target or override an explicit full push.
1828-
func TestPushFromNowBoundary(t *testing.T) {
1829-
const cutoff = "2026-07-24 12:00:00"
1830-
const established = "2026-07-01 00:00:00"
1826+
// A from-now boundary may only be seeded when the target is PROVABLY new.
1827+
// The reset paths clear lastPush, so an empty watermark alone does not
1828+
// distinguish a fresh target from an established one whose marker was lost or
1829+
// whose first push failed part-way; seeding there would permanently skip the
1830+
// history those resets exist to restore.
1831+
func TestPushFromNowApplies(t *testing.T) {
18311832
cases := []struct {
18321833
name string
18331834
enabled bool
1834-
requestedFull bool
18351835
lastPush string
1836-
want string
1837-
wantApplied bool
1836+
boundaryState string
1837+
markerExists bool
1838+
stateCleared bool
1839+
want bool
18381840
}{
1839-
{"fresh target starts at now", true, false, "", cutoff, true},
1840-
{"disabled backfills as before", false, false, "", "", false},
1841-
{"explicit full push wins", true, true, "", "", false},
1842-
{"established target is untouched", true, false, established, established, false},
1843-
{"established target with full push is untouched", true, true, established, established, false},
1844-
{"disabled and established is untouched", false, false, established, established, false},
1841+
{name: "provably fresh target", enabled: true, want: true},
1842+
{name: "disabled", enabled: false, want: false},
1843+
{name: "established target has a watermark", enabled: true, lastPush: "2026-07-01 00:00:00", want: false},
1844+
{name: "partial first push left boundary state", enabled: true, boundaryState: "{}", want: false},
1845+
{name: "target already carries this push marker", enabled: true, markerExists: true, want: false},
1846+
{name: "a reset cleared state on this run", enabled: true, stateCleared: true, want: false},
18451847
}
18461848
for _, tc := range cases {
18471849
t.Run(tc.name, func(t *testing.T) {
1848-
got, applied := pushFromNowBoundary(
1849-
tc.enabled, tc.requestedFull, tc.lastPush, cutoff,
1850-
)
1851-
assert.Equal(t, tc.want, got)
1852-
assert.Equal(t, tc.wantApplied, applied)
1850+
assert.Equal(t, tc.want, pushFromNowApplies(
1851+
tc.enabled, tc.lastPush, tc.boundaryState,
1852+
tc.markerExists, tc.stateCleared,
1853+
))
18531854
})
18541855
}
18551856
}

internal/postgres/sync.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,11 @@ type Sync struct {
177177
// pushFromNow starts a fresh target at the current time rather than
178178
// backfilling the local archive. See SyncOptions.PushFromNow.
179179
pushFromNow bool
180+
181+
// skipUnboundedPhases is set for the duration of a push whose from-now
182+
// boundary applied, so archive-wide phases that the boundary cannot scope
183+
// are skipped instead of uploading pre-boundary content.
184+
skipUnboundedPhases bool
180185
// afterVectorApply is a full/scoped post-apply test hook.
181186
afterVectorApply func()
182187
// beforeVectorWitnessRecord is a generation-wide pre-witness test hook.

internal/server/huma_routes_push.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,10 @@ type daemonPushRequest struct {
159159
// NoVectors carries the CLI --no-vectors flag, which has no daemon-side
160160
// flag of its own, into the push handler's vector-source gate.
161161
NoVectors bool `json:"no_vectors,omitempty"`
162+
// FromNow carries the CLI --from-now flag. Without it a delegated push
163+
// silently backfills the whole archive, which is the opposite of what the
164+
// flag was asked to do.
165+
FromNow bool `json:"from_now,omitempty"`
162166
// ScopeVectorsToChangedSessions is set by change-triggered watch
163167
// pushes so the vector phase reads state only for the changed
164168
// relational sessions (see postgres.PushOptions).
@@ -365,6 +369,7 @@ func (s *Server) humaPGPush(
365369
SyncStateTarget: body.SyncStateTarget,
366370
MigrateLegacySyncState: body.MigrateLegacySyncState,
367371
VectorSource: vectorSource,
372+
PushFromNow: body.FromNow,
368373
},
369374
)
370375
if err != nil {

0 commit comments

Comments
 (0)