From ebdfd1f4998c0905a621c549e6ab1919592fd039 Mon Sep 17 00:00:00 2001 From: Chinmay-KB Date: Wed, 29 Jul 2026 17:10:48 +0530 Subject: [PATCH 1/9] feat(replay): consolidate interaction evidence --- docs/README.md | 2 +- .../production-replay-acceptance-0.4.13.md | 217 ++++ .../production-replay-acceptance-0.4.15.md | 65 ++ .../production-replay-acceptance.md | 21 +- ...uction-replay-run-2026-07-27-sdk-0.4.12.md | 177 +++ ...-001-sdk-interaction-consolidation-plan.md | 387 ++++++ packages/tugboat/CHANGELOG.md | 78 ++ packages/tugboat/README.md | 84 +- .../tugboat/lib/src/collector_mapper.dart | 3 + packages/tugboat/lib/src/controller.dart | 1034 ++++++++++++----- .../tugboat/lib/src/coordinate_space.dart | 25 + .../lib/src/interaction_transaction.dart | 294 +++++ packages/tugboat/lib/src/models.dart | 134 ++- packages/tugboat/lib/src/replay_config.dart | 40 +- packages/tugboat/lib/src/sdk_version.dart | 2 +- packages/tugboat/lib/tugboat.dart | 1 + packages/tugboat/pubspec.yaml | 2 +- .../tugboat/test/collector_mapper_test.dart | 41 + .../tugboat/test/coordinate_space_test.dart | 36 +- .../tugboat/test/helpers/json_roundtrip.dart | 3 +- .../helpers/replay_coherence_harness.dart | 8 + .../release_compatibility_matrix_test.dart | 6 +- .../replay/deferred_tap_emission_test.dart | 79 ++ .../replay/interaction_transaction_test.dart | 395 +++++++ .../replay/modal_capture_visual_test.dart | 1 + .../navigation_origin_contract_test.dart | 192 ++- ...ay_navigation_interaction_matrix_test.dart | 1 + .../replay_navigation_race_matrix_test.dart | 60 +- ...overlay_nested_navigation_matrix_test.dart | 1 + ...y_programmatic_navigation_matrix_test.dart | 13 +- .../replay/tap_coordinate_transform_test.dart | 5 + ...eplay_coherence_characterization_test.dart | 79 +- .../tugboat/test/scene_inventory_test.dart | 10 + .../tugboat/test/scroll_attribution_test.dart | 4 +- .../test/scroll_playground_live_test.dart | 1 + .../tugboat/test/tugboat_replay_test.dart | 5 +- .../test/viewport_semantic_map_test.dart | 33 +- 37 files changed, 3127 insertions(+), 412 deletions(-) create mode 100644 docs/integration/production-replay-acceptance-0.4.13.md create mode 100644 docs/integration/production-replay-acceptance-0.4.15.md create mode 100644 docs/integration/production-replay-run-2026-07-27-sdk-0.4.12.md create mode 100644 docs/plans/2026-07-28-001-sdk-interaction-consolidation-plan.md create mode 100644 packages/tugboat/lib/src/interaction_transaction.dart create mode 100644 packages/tugboat/test/replay/deferred_tap_emission_test.dart create mode 100644 packages/tugboat/test/replay/interaction_transaction_test.dart diff --git a/docs/README.md b/docs/README.md index 0526a81..24f64ad 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,7 +28,7 @@ verified in their own repositories. ## Current compatibility -- package version: `0.4.12`; +- package version: `0.4.13`; - session JSON schema: `7`; - fingerprint schema: `6`; - minimum Dart SDK: `3.9.2`; diff --git a/docs/integration/production-replay-acceptance-0.4.13.md b/docs/integration/production-replay-acceptance-0.4.13.md new file mode 100644 index 0000000..e44915a --- /dev/null +++ b/docs/integration/production-replay-acceptance-0.4.13.md @@ -0,0 +1,217 @@ +# Production replay acceptance: 0.4.12 → 0.4.13 + +Use this after shipping SDK **0.4.13** (and collector build passthrough) and +running Blend through a similar flow to the baseline session. + +## Baseline (locked) + +| Field | Value | +| --- | --- | +| Session | `session-1785142623932166` | +| SDK | `0.4.12` | +| App | `to.blend.mobile_app` | +| Build | `3.17.177+1472` | + +| Metric | Baseline | +| --- | ---: | +| Raw `tap` | 86 | +| Swipe-consumed taps | 63 | +| Settled taps | 23 | +| Truly orphaned taps | 0 | +| `tap_settled` `result=unknown` | 6 | +| unknown `superseded_route_epoch` | 2 | +| `missing_frame` with zero local/normalized | 1 | +| `capture_diagnostic` `missing_context_graph_build_identity` | 37 / 37 | + +## Manual run checklist + +1. Install / point Blend at tugboat **0.4.13** (includes same-turn claims, + deferred taps, session-end pointer fence, duplicate-down coalesce). +2. Confirm collector with event `build` passthrough is deployed (Gate 8). +3. Drive: scroll/flick on home, Get Pro / paywall, StageIt / sheet / chooser, + one rapid double-tap during settle. +4. Paste the new `sessionId` (+ Blend build) in chat for scoring. + +## Hard gates (all must PASS) + +1. **Identity** — `metadata.sdkVersion = '0.4.13'` on event groups. +2. **No phantom taps** — `swipe_consumed_taps = 0`. +3. **Settle coverage** — `settled / raw_taps >= 0.95`. +4. **No same-position bursts** — no UTC second with `tap_count >= 5` and + `distinct_positions = 1`. +5. **Unknown settles** — `unknown / tap_settled <= 0.10` **and** + `superseded_route_epoch` count = 0. +6. **After frames** — 100% of `navigated`/`changed` settles have `afterFrame`. +7. **Missing-frame geometry** — any `missing_frame` tap has + `normalizedX/Y ∈ [0,1]` and `boundaryWidth/Height > 0`. +8. **Diagnostics identity** — zero + `missing_context_graph_build_identity` on `capture_diagnostic` + (BLOCKED if collector not deployed). +9. **Swipe start geometry** — every `swipe` has `payload.startCaptureCoordinate`. + +Overall verdict: **ACCEPT** only if every gate PASSes; otherwise **REJECT** or +**BLOCKED** with the first failing gate id. + +## ClickHouse queries + +Replace `{newSession}` with the new session id. Service: pmkit ClickHouse. + +### Gate 1 — SDK version + +```sql +SELECT + argMax(metadata.sdkVersion::Nullable(String), receivedAt) AS sdkVersion, + count() AS rows +FROM pmkit.raw_events +WHERE sessionId = {newSession} +GROUP BY eventType +ORDER BY eventType +``` + +### Gates 2–3 — tap fate + +```sql +WITH events AS ( + SELECT id, + argMax(eventType, receivedAt) AS eventType, + argMax(metadata.relatedEventId::Nullable(String), receivedAt) AS related + FROM pmkit.raw_events + WHERE sessionId = {newSession} + GROUP BY id +), +taps AS (SELECT id FROM events WHERE eventType = 'tap'), +swipeRefs AS (SELECT related FROM events WHERE eventType = 'swipe' AND related IS NOT NULL), +settleRefs AS (SELECT related FROM events WHERE eventType = 'tap_settled' AND related IS NOT NULL) +SELECT + count() AS totalTaps, + countIf(id IN (SELECT related FROM swipeRefs)) AS consumedBySwipe, + countIf(id IN (SELECT related FROM settleRefs)) AS settled, + countIf( + id NOT IN (SELECT related FROM swipeRefs) + AND id NOT IN (SELECT related FROM settleRefs) + ) AS orphaned +FROM taps +``` + +### Gate 4 — bursts + +```sql +WITH taps AS ( + SELECT + argMax(triggeredAt, receivedAt) AS triggeredAt, + argMax(payload.x::Nullable(Float64), receivedAt) AS x, + argMax(payload.y::Nullable(Float64), receivedAt) AS y + FROM pmkit.raw_events + WHERE sessionId = {newSession} AND eventType = 'tap' + GROUP BY id +) +SELECT + toStartOfSecond(triggeredAt) AS sec, + count() AS tapCount, + uniqExact((round(x, 1), round(y, 1))) AS distinctPositions +FROM taps +GROUP BY sec +HAVING tapCount >= 5 AND distinctPositions = 1 +ORDER BY sec +``` + +### Gate 5 — unknown settles + +```sql +WITH settles AS ( + SELECT + argMax(result, receivedAt) AS result, + argMax(toJSONString(payload), receivedAt) AS payloadJson + FROM pmkit.raw_events + WHERE sessionId = {newSession} AND eventType = 'tap_settled' + GROUP BY id +) +SELECT + count() AS totalSettled, + countIf(result = 'unknown') AS unknownSettles, + countIf( + JSONExtractString(payloadJson, 'settleObservation', 'captureFailure') + = 'superseded_route_epoch' + ) AS supersededRouteEpoch +FROM settles +``` + +### Gate 6 — after frames on navigated/changed + +```sql +WITH settles AS ( + SELECT + argMax(result, receivedAt) AS result, + argMax(afterFrame, receivedAt) AS afterFrame + FROM pmkit.raw_events + WHERE sessionId = {newSession} AND eventType = 'tap_settled' + GROUP BY id +) +SELECT + countIf(result IN ('navigated', 'changed')) AS outcomeRows, + countIf(result IN ('navigated', 'changed') AND afterFrame IS NULL) AS missingAfter +FROM settles +``` + +### Gate 7 — missing_frame geometry + +```sql +WITH taps AS ( + SELECT argMax(toJSONString(payload), receivedAt) AS payloadJson + FROM pmkit.raw_events + WHERE sessionId = {newSession} AND eventType = 'tap' + GROUP BY id +) +SELECT + JSONExtractString(payloadJson, 'captureCoordinate', 'unavailableReason') AS reason, + JSONExtractFloat(payloadJson, 'captureCoordinate', 'normalizedX') AS nx, + JSONExtractFloat(payloadJson, 'captureCoordinate', 'normalizedY') AS ny, + JSONExtractFloat(payloadJson, 'captureCoordinate', 'boundaryWidth') AS bw, + JSONExtractFloat(payloadJson, 'captureCoordinate', 'boundaryHeight') AS bh +FROM taps +WHERE JSONExtractString(payloadJson, 'captureCoordinate', 'unavailableReason') + = 'missing_frame' +``` + +### Gate 8 — diagnostics enrichment + +```sql +WITH diags AS ( + SELECT argMax(toJSONString(payload), receivedAt) AS payloadJson + FROM pmkit.raw_events + WHERE sessionId = {newSession} AND eventType = 'capture_diagnostic' + GROUP BY id +) +SELECT + count() AS diagnostics, + countIf( + JSONExtractString(payloadJson, 'contextEnrichment', 'reason') + = 'missing_context_graph_build_identity' + ) AS missingBuildIdentity +FROM diags +``` + +### Gate 9 — swipe startCaptureCoordinate + +```sql +WITH swipes AS ( + SELECT argMax(toJSONString(payload), receivedAt) AS payloadJson + FROM pmkit.raw_events + WHERE sessionId = {newSession} AND eventType = 'swipe' + GROUP BY id +) +SELECT + count() AS swipes, + countIf(JSONHas(payloadJson, 'startCaptureCoordinate')) AS withStartCoord +FROM swipes +``` + +## Deliverable + +After scoring, write +`docs/integration/production-replay-compare-0.4.12-vs-0.4.13.md` with: + +- session ids + SDK versions +- side-by-side metric table +- per-gate PASS/FAIL with deciding counts +- single overall `ACCEPT` / `REJECT` / `BLOCKED` diff --git a/docs/integration/production-replay-acceptance-0.4.15.md b/docs/integration/production-replay-acceptance-0.4.15.md new file mode 100644 index 0000000..b017520 --- /dev/null +++ b/docs/integration/production-replay-acceptance-0.4.15.md @@ -0,0 +1,65 @@ +# Production replay acceptance: interaction consolidation (0.4.15) + +Use this after shipping SDK **0.4.15** (canonical interactions + delayed +reconciliation) and running Blend through the acceptance flow. + +## Baseline (locked) + +Prefer the nearest prior Blend session against SDK **0.4.12 / 0.4.13** for +side-by-side scoring. Record the new session id and Blend build before scoring. + +## What changed in the SDK + +| Concern | 0.4.13 behavior | 0.4.15 behavior | +| --- | --- | --- | +| Gesture identity | `tap` + `tap_settled` peers | one `interaction` (`stream: semantic`) + legacy projection | +| Claim window | microtask same-turn only | default 1,250 ms delayed reconciliation | +| Diagnostics | mixed into normal events | `stream: diagnostic` | +| Origin | frozen on pending tap | immutable `InteractionOrigin` on the transaction | +| Swipe state | refreshed at pointer-up | frozen to pointer-down origin | + +## Manual run checklist + +1. Point Blend at tugboat **0.4.15**. +2. Drive: home scroll/flick, Get Pro / paywall, full-screen navigation, modal + bottom sheet, asynchronous onboarding transition, rapid double-tap, + automatic redirect after a settled tap. +3. Paste the new `sessionId` (+ Blend build) for scoring. + +## Hard gates (all must PASS) + +1. **Identity** — `metadata.sdkVersion = '0.4.15'`. +2. **Canonical coverage** — one `stream: semantic` `interaction` per completed + user gesture (tap / swipe / scroll / cancelled). +3. **Origin correctness** — interaction `origin.route` / `origin.targetAnchor` + match the pointer-down screen/component, never the destination. +4. **Delayed attribution** — delayed navigation / bottom sheet inside 1,250 ms + has `attribution.kind = delayed_likely` (or `direct`) and + `result.status = navigated|changed`, with matching + `route_change.causedByInteractionId`. +5. **Automatic false-claim rate** — timer/auth redirects after the window, and + routes with competing pointers, stay `navigationOrigin = + automatic_or_unknown`. +6. **No semantic tap for scrolls/swipes** — completed scroll/swipe produces no + `stream: semantic` tap; one `interaction` with `gesture=scroll|swipe`. +7. **Diagnostic isolation** — enrichment selection of `stream: semantic` + excludes `capture_diagnostic`. +8. **Rage-tap precision** — three no-result taps on the same origin target flag + once; three scrolls or three successful navigation taps do not. + +## Soft / observational + +- Semantic event count per completed gesture should drop vs 0.4.0 raw + `tap`+`tap_settled`+scroll peer inflation. +- Legacy projection remains present until collector/graph cut over; do not + delete `tap`/`tap_settled` selection until two representative Blend flows pass + on canonical interactions alone. +- Instrument pending-to-success latency; retune `interactionClaimWindow` from + production evidence if 1,250 ms is too short/long. + +## Consumer follow-ups (separate PRs) + +- Collector / Context Graph enrichment select `stream: semantic` `interaction` + and map components via `origin.targetAnchor`. +- Build causal edges from `result` / `causedByInteractionId`. +- Update dashboard rage-tap detectors to the definition above. diff --git a/docs/integration/production-replay-acceptance.md b/docs/integration/production-replay-acceptance.md index 316e0d7..f11f1a8 100644 --- a/docs/integration/production-replay-acceptance.md +++ b/docs/integration/production-replay-acceptance.md @@ -12,14 +12,19 @@ database receipt alone as proof that a replay is correct. ## Current acceptance status -Production acceptance #13/#14 remains open. The SDK's route-epoch and frame -provenance behavior is an intended invariant, but rapid/nested modal chains and -programmatic/automatic navigation can still be absent or degraded in a -production replay. Record those observations as SDK capture gaps; do not infer -route/action coherence from the intended contract or repair the evidence in the -dashboard. Stored tap coordinates are global logical pixels and are not -capture-boundary-normalized for playback, so fractional overlay drift is also a -known limitation. +Interaction consolidation shipped in SDK **0.4.15** (canonical `interaction` +events, 1,250 ms delayed claim window, diagnostic stream isolation). Use +[`production-replay-acceptance-0.4.15.md`](./production-replay-acceptance-0.4.15.md) +for the Blend scoring gates. Collector/Context Graph migration onto +`stream: semantic` interactions remains a follow-up before legacy +`tap`/`tap_settled` projection can be removed. + +Production acceptance #13/#14 remains open for rapid/nested modal chains and +programmatic/automatic navigation gaps. Record those observations as SDK +capture gaps; do not infer route/action coherence from the intended contract or +repair the evidence in the dashboard. Stored tap coordinates are global logical +pixels and are not capture-boundary-normalized for playback, so fractional +overlay drift is also a known limitation. ## Roles and evidence diff --git a/docs/integration/production-replay-run-2026-07-27-sdk-0.4.12.md b/docs/integration/production-replay-run-2026-07-27-sdk-0.4.12.md new file mode 100644 index 0000000..89c5a22 --- /dev/null +++ b/docs/integration/production-replay-run-2026-07-27-sdk-0.4.12.md @@ -0,0 +1,177 @@ +# Production replay run report: SDK 0.4.12 + +Date: 2026-07-27 +Source: ClickHouse production tables (`pmkit.sessions`, `pmkit.raw_events`) +Session under analysis: `session-1785142623932166` +SDK version: `0.4.12` +App: `to.blend.mobile_app` +Blend version/build: `3.17.177+1472` +Device platform: Android + +## Verdict + +The session did ingest as SDK `0.4.12`, and the new capture diagnostics show +that route, modal, and fresh-frame capture are active in production. The run is +not yet a replay-quality pass. The remaining failures are concentrated around +tap volume, tap settlement coverage, missing settled frames, and at least one +missing-frame coordinate fallback. + +The next SDK fix should focus on tap deduplication / gesture coalescing and +raising `tap_settled` coverage for real taps. The dashboard should not paper +over these symptoms as a successful replay, because the SDK is still emitting +ambiguous interaction evidence. + +## Identity and timing + +| Field | Value | +| --- | --- | +| Session ID | `session-1785142623932166` | +| SDK version | `0.4.12` | +| App ID | `to.blend.mobile_app` | +| Blend build | `3.17.177+1472` | +| First event | `2026-07-27 08:57:03.932 UTC` | +| Last event | `2026-07-27 09:02:33.456 UTC` | +| Session received | `2026-07-27 08:57:08.619 UTC` | +| Total raw events | `249` | +| Unique event IDs | `249` | +| Referenced frames | `28` | + +Note: a later session, `session-1785142837528027`, was present in ClickHouse but +reported SDK `0.4.0`. It was excluded from this verdict. + +## Event summary + +| Event type | Rows | With before frame | With after frame | Notes | +| --- | ---: | ---: | ---: | --- | +| `session_start` | 1 | 0 | 0 | Correctly marked SDK `0.4.12` | +| `capture_diagnostic` | 37 | 0 | 29 | New diagnostic stream is present | +| `route_change` | 16 | 0 | 16 | Route evidence has destination frames | +| `tap` | 86 | 85 | 0 | Raw tap volume is high | +| `tap_settled` | 23 | 23 | 17 | Only 23 of 86 taps settled | +| `state_change` | 12 | 12 | 9 | Some changes lack after frames | +| `swipe` | 64 | 63 | 0 | Large gesture volume in the run | +| `scroll_start` | 3 | 1 | 0 | Limited scroll lifecycle coverage | +| `scroll_end` | 3 | 1 | 1 | Limited scroll lifecycle coverage | +| `app_inactive` | 2 | 0 | 0 | Lifecycle events captured | +| `app_foregrounded` | 2 | 0 | 0 | Lifecycle events captured | + +## What improved + +- SDK version propagation worked for this session. All event groups reported + `metadata.sdkVersion = 0.4.12`. +- Route observation was active. The run captured `/home`, + `/subscriptionPaywall`, `/stageit/recents`, `/aiStudio/upload`, + `/imageChooser`, `/stageit/image`, and `aiStudio/videos/anyImage`. +- Bottom sheet observation was active. The run captured + `ModalBottomSheetRoute`. +- Route captures consistently had `afterFrame` evidence: 16 route changes, + 16 with `afterFrame`. +- Capture diagnostics are now useful production evidence. The session included + fresh route, tap, lifecycle, scroll, and initial capture diagnostics. +- Tap coordinates are mostly emitted in capture-boundary local logical space: + 85 of 86 taps had `sourceSpace = boundaryLocalLogical`. + +## Remaining failures + +### 1. Tap settlement coverage is too low + +There were 86 raw `tap` events, but only 23 `tap_settled` events linked back to +those taps. That leaves 63 taps without a settled outcome. + +This is too weak for reliable replay. A user watching the replay will see many +tap markers that never get a corresponding visual or semantic outcome. + +### 2. Some settled taps still lack destination frames + +Among the 23 `tap_settled` events: + +| Settled result | Rows | Missing after frame | +| --- | ---: | ---: | +| `navigated` | 9 | 0 | +| `changed` | 8 | 0 | +| `unknown` | 6 | 6 | + +The `unknown` settled events are explicit failures for replay quality. They are +bounded, which is better than silently borrowing stale frames, but the user +experience is still degraded. + +### 3. Missing-frame coordinate fallback still happened + +One raw tap emitted: + +- `captureCoordinate.unavailableReason = missing_frame` +- `captureCoordinate.sourceSpace = globalLogical` +- zero frame width/height in the coordinate payload + +This is the exact class of evidence that can produce tap markers that appear to +point at nowhere or use fallback geometry. + +### 4. The SDK appears to over-record repeated taps + +Two bursts are suspicious: + +| Timestamp second | Tap rows | Distinct positions | +| --- | ---: | ---: | +| `2026-07-27 08:57:45 UTC` | 16 | 1 | +| `2026-07-27 08:57:52 UTC` | 38 | 1 | + +Both bursts recorded many taps at effectively the same position within one +second. This looks like repeated pointer/tap emission for one physical +interaction or a gesture sequence that should be coalesced before replay. + +This likely explains a major part of the replay feeling erratic: the player may +be faithfully rendering an event stream that is already too noisy. + +### 5. Context graph identity was missing for diagnostics + +All 37 `capture_diagnostic` events had +`contextEnrichment.reason = missing_context_graph_build_identity`. + +This does not invalidate SDK capture evidence, but it means the diagnostics were +not enriched against a resolved graph build. For acceptance, replay visual +coherence must still be judged separately from graph enrichment. + +## Route and modal evidence + +Observed route changes: + +| Route | Navigation | Rows | Frames | +| --- | --- | ---: | --- | +| `/home` | `route_push` | 1 | `frame-7` | +| `/subscriptionPaywall` | `route_push` | 2 | `frame-19`, `frame-271` | +| `/home` | `route_pop` | 1 | `frame-28` | +| `/stageit/recents` | `route_push` | 1 | `frame-173` | +| `/aiStudio/upload` | `route_replace` | 1 | `frame-186` | +| `ModalBottomSheetRoute` | `route_push` | 2 | `frame-196`, `frame-231` | +| `/stageit/results` | `route_pop` | 3 | `frame-202`, `frame-223`, `frame-322` | +| `/imageChooser` | `route_push` | 1 | `frame-210` | +| `/usageStatsError` | `route_push` | 1 | `frame-260` | +| `/stageit/image` | `route_push` | 1 | `frame-328` | +| `aiStudio/videos/anyImage` | `route_push` | 1 | `frame-336` | +| `/stageit/image` | `route_pop` | 1 | `frame-348` | + +Bottom sheets and paywalls were therefore not completely invisible to the SDK in +this run. The remaining issue is not route observation itself; it is the +quality and completeness of the interaction evidence around those routes. + +## Recommended follow-up issues + +1. Deduplicate repeated raw tap events before they reach replay. +2. Raise `tap_settled` coverage and explicitly classify taps that will not + settle. +3. Prevent `missing_frame` coordinate fallback from producing zero-size replay + coordinates without a clear degraded visual state. +4. Add a focused runtime acceptance flow for bottom sheets and paywalls in the + Blend app, using production collection and dashboard replay inspection. +5. Ensure capture diagnostics include or can resolve context graph build + identity, so replay-quality diagnostics and enrichment state can be separated + cleanly. + +## Acceptance status + +Rejected for production replay acceptance. + +Reason: although SDK `0.4.12` improved frame provenance, route capture, modal +capture, and diagnostic visibility, this session still contains too many +unsettled taps, repeated tap bursts, and a missing-frame coordinate fallback to +call the replay coherent. diff --git a/docs/plans/2026-07-28-001-sdk-interaction-consolidation-plan.md b/docs/plans/2026-07-28-001-sdk-interaction-consolidation-plan.md new file mode 100644 index 0000000..bef601e --- /dev/null +++ b/docs/plans/2026-07-28-001-sdk-interaction-consolidation-plan.md @@ -0,0 +1,387 @@ +--- +artifact_contract: ce-unified-plan/v1 +artifact_readiness: implementation-ready +execution: code +product_contract_source: ce-plan-bootstrap +created: 2026-07-28 +--- + +# SDK interaction consolidation: attribution, outcomes, and noise + +## Goal + +Make Tugboat emit one authoritative semantic interaction for every completed +user gesture. That interaction must retain the screen/modal and component that +were visible when the gesture began, classify the final gesture correctly, and +attach a causally supported visual result when one occurs shortly afterwards. + +The SDK must stop asking the collector, dashboard, or Context Graph to infer +these relationships from unrelated top-level callbacks. + +## Problem frame + +The current controller has deferred-tap and same-turn causal-claim machinery, +but it still finalizes many pointer-up interactions before delayed Flutter +navigation occurs. The result is a `tap_settled` with `unknown` while the route +or bottom sheet appears separately. In other paths, settlement refreshes +current context after navigation and risks attaching the interaction to the +destination rather than the origin component. + +The raw stream is also too broad for semantic enrichment: a single scroll can +produce a provisional tap, `scroll_start`, `swipe`, and `scroll_end`; a tap can +be represented by both `tap` and `tap_settled`; and `capture_diagnostic` +records are mixed into normal session activity. A production 0.4.0 session +(`session-1785242298527917`) had 570 records: 202 `tap`, 151 `tap_settled`, and +161 scroll-related records. This creates false candidates for enrichment and +inflates insight calculations such as rage taps. + +## Product contract + +### Requirements + +- **R1 — Immutable origin.** At pointer-down, capture and retain the origin + state anchor, route/modal identity, component target anchor, capture + coordinate transform, pre-interaction frame, and monotonic timestamp. No + later route/state refresh may overwrite those fields. +- **R2 — One canonical interaction.** Publish exactly one normal-stream + semantic record for each finalized user gesture: `tap`, `swipe`, `scroll`, or + `cancelled`. `tap_settled` must no longer be a second independently enriched + action. +- **R3 — Delayed causal result.** Hold a released tap in a bounded + reconciliation window. The first eligible visible route, modal, or state + successor in that window becomes its result, with origin and destination + identities preserved explicitly. +- **R4 — Conservative attribution.** A competing pointer, a classified + scroll/swipe, lifecycle interruption, an incompatible navigator/route epoch, + or an expired window prevents causal attribution. Such transitions stay + automatic; the origin interaction is finalized without a false result. +- **R5 — Gesture reclassification.** If a pointer becomes a scroll/swipe, + suppress its provisional tap from the normal stream and emit one completed + gesture summary with start/end positions, displacement, duration, and its + origin target. +- **R6 — Evidence, not competitors.** Route/state/frame observations remain + available as evidence and retain their own event records where required for + capture/replay, but refer to the canonical interaction through stable IDs. + They must not create additional enrichment candidates for the same gesture. +- **R7 — Insight-safe.** Rage tap and tap analytics operate on finalized tap + interactions only. Scrolls, swipes, cancelled pointers, and taps with a + successful route/modal/state result must not count as rage taps. +- **R8 — Diagnostic isolation.** Capture diagnostics are debug/health data, + not user actions. Keep them in a separately marked channel or aggregate them + into a session health summary so normal enrichment and insight queries ignore + them by default. +- **R9 — Bounded overhead.** Consolidation is in-memory only. It must not + persist every raw callback before classification, add screenshot/widget-tree + captures, or use unbounded pending state. + +### Non-goals + +- Do not use dashboard post-processing or ClickHouse joins to repair SDK + semantics. +- Do not infer a causal action from an arbitrary later automatic redirect. +- Do not remove raw diagnostic capability; isolate it from the semantic stream. +- Do not change privacy masking, frame encoding, or Context Graph matching + rules except to consume the new explicit origin/result fields. + +## Key technical decisions + +1. **Interaction transaction, not upload-side repair.** Replace the current + provisional-event/settlement representation with a bounded in-memory + transaction per pointer. The durable outbox receives only finalized semantic + events. This is the only layer that has reliable pointer, route, modal, and + widget-tree timing together. + +2. **Freeze origin at pointer-down.** The transaction owns an immutable + `InteractionOrigin` value rather than calling `_refreshStateAnchor()` at + settlement. It includes screen/modal route instance, target anchor, + coordinate space, before-frame reference, and `pointerGeneration`. + +3. **Bounded delayed reconciliation.** Replace the present + `_PendingInteractionClaim.sameTurnEligible` rule with a short configurable + reconciliation deadline after pointer-up. Eligibility additionally requires + same navigator, compatible route epoch, no competing eligible interaction, + and the first visible successor. Begin with a conservative 1,250 ms default; + expose it only as an internal constant until production timing data warrants + configuration. + +4. **Canonical `interaction` envelope with compatibility projection.** Add a + canonical semantic event shape (`type: interaction`, `gesture: tap|swipe| + scroll|cancelled`) and project legacy `tap`/`tap_settled` only behind a + temporary compatibility gate. The collector and graph should migrate to the + canonical shape before the legacy pair is removed. This avoids a breaking + ingestion cutover while ensuring one enrichment candidate per gesture. + +5. **Observation links are explicit.** Store `origin`, `result`, and + `evidenceEventIds` on the interaction. A route/state event also carries + `causedByInteractionId` when claimed. No consumer has to use time adjacency + to reconstruct the relationship. + +6. **Final-state gesture classification wins.** A move past the existing + gesture threshold irrevocably changes the transaction from tentative tap to + scroll/swipe. Its provisional tap is never emitted as an action; the + resulting summary keeps the same origin context. + +7. **Separate semantic and diagnostic streams.** Keep diagnostics available + for session health and support, but mark them `stream: diagnostic` and make + normal collector/graph queries select `stream: semantic` by default. + +## Target event model + +```text +interaction + id + gesture: tap | swipe | scroll | cancelled + origin: + stateAnchor, routeInstanceId, navigatorId, targetAnchor, + captureCoordinate, beforeFrame, atMs + result: + status: navigated | changed | unchanged | unknown | cancelled + route/state/modal identity, afterFrame, observedAtMs + attribution: + direct | delayed_likely | none + windowMs, rejectionReason? + evidenceEventIds: [route_change?, state_change?, scroll_end?] +``` + +An independently automatic route keeps `causedByInteractionId: null` and +`navigationOrigin: automatic_or_unknown`. It is not retroactively claimed when +the transaction window has ended or a guard failed. + +## Implementation units + +### U1 — Introduce immutable transaction data and lifecycle + +**Files** + +- `packages/tugboat/lib/src/controller.dart` +- `packages/tugboat/test/replay/deferred_tap_emission_test.dart` +- `packages/tugboat/test/replay/navigation_origin_contract_test.dart` +- `packages/tugboat/test/replay/interaction_transaction_test.dart` (new) + +**Change** + +- Replace `_PendingInteractionClaim`'s event-buffer-centric lifetime with an + `InteractionTransaction` that owns immutable origin data, gesture state, + release/deadline timestamps, evidence IDs, and terminal status. +- Create the transaction at pointer-down before any post-frame or route work. +- Refactor `recordPointerUp`, `_abandonPendingPointer`, and lifecycle/session + fences so every transaction reaches exactly one terminal state. +- Keep the existing pointer-generation and navigator/route-instance checks as + transaction guards rather than recomputing origin context. + +**Tests** + +- Pointer-down on a component, then route mutation before pointer-up: emitted + origin remains the original screen/component. +- Pointer-down in a modal and delayed modal dismissal: origin retains the + modal, result retains the underlying route. +- Lifecycle/session end, cancellation, and duplicate pointer-down produce no + stranded transaction or duplicate canonical interaction. + +### U2 — Reconcile delayed visual successors without false claims + +**Files** + +- `packages/tugboat/lib/src/controller.dart` +- `packages/tugboat/test/replay/replay_navigation_race_matrix_test.dart` +- `packages/tugboat/test/replay/replay_programmatic_navigation_matrix_test.dart` +- `packages/tugboat/test/replay/replay_navigation_interaction_matrix_test.dart` +- `packages/tugboat/test/replay/interaction_transaction_test.dart` (new) + +**Change** + +- Route `_resolveVisibleRouteChange`, `_tryClaimInteractionCause`, route + observer callbacks, and state-change capture through a single successor + matcher. +- At pointer-up, move a tap from active to reconciliation-pending instead of + emitting `unknown` immediately. +- Match only the first eligible visible successor before deadline; attach its + post-transition frame and `causedByInteractionId`; finalize as `navigated` + or `changed` with `attribution=direct|delayed_likely`. +- On deadline, finalize as `unchanged`/`unknown` against the frozen origin and + leave subsequent routes automatic. +- Record a machine-readable rejection reason for every unclaimed successor + (`expired`, `competing_pointer`, `gesture_reclassified`, `navigator_mismatch`, + `automatic_guard`). + +**Tests** + +- Same-turn push/pop/replace, delayed 100 ms/500 ms/1,200 ms navigation, and + delayed `ModalBottomSheetRoute` all produce one interaction with correct + origin and destination. +- A redirect after the deadline, a timer-driven route with no tap, and a route + following another pointer remain automatic. +- Two rapid taps can only claim their own first eligible successors; no route + is linked twice. +- Each `navigated`/`changed` interaction has an `afterFrame` when capture is + available; capture failure is explicit rather than silently changing origin. + +### U3 — Finalize gesture classification and collapse gesture noise + +**Files** + +- `packages/tugboat/lib/src/controller.dart` +- `packages/tugboat/lib/src/input_capture.dart` +- `packages/tugboat/test/replay/deferred_tap_emission_test.dart` +- `packages/tugboat/test/replay/replay_coherence_characterization_test.dart` +- `packages/tugboat/test/scroll_attribution_test.dart` + +**Change** + +- Refactor `markPendingTapAsSwipe` and scroll callbacks so they mutate one + transaction instead of emitting a causal tap plus `swipe`, `scroll_start`, + and `scroll_end` as semantic peers. +- Emit one finalized `interaction(gesture=scroll|swipe)` after scroll end or + pointer-up. Keep low-level scroll observations only as linked evidence or + diagnostic detail. +- Preserve the existing `startCaptureCoordinate` and transform metadata on the + finalized interaction so coordinate-based enrichment and replay markers stay + correct. + +**Tests** + +- A drag produces zero semantic taps and exactly one semantic scroll/swipe. +- A small movement below threshold remains one tap. +- Overscroll, pointer cancel, and interrupted drag produce one terminal + interaction with no provisional-tap leak. +- Coordinate transform, insets, and scaled capture fixtures retain the origin + target and correct normalized position after reclassification. + +### U4 — Publish canonical interactions and isolate diagnostics + +**Files** + +- `packages/tugboat/lib/src/models.dart` +- `packages/tugboat/lib/src/controller.dart` +- `packages/tugboat/lib/src/capture_sink.dart` +- `packages/tugboat/lib/src/outbox/outbox.dart` +- `packages/tugboat/lib/src/outbox/outbox_sink.dart` +- `packages/tugboat/lib/src/collector_http_sink.dart` +- `packages/tugboat/test/replay/replay_coherence_characterization_test.dart` +- `packages/tugboat/CHANGELOG.md` + +**Change** + +- Add the canonical interaction schema, event stream marker, origin/result + payloads, evidence IDs, and compatibility-version marker. +- Place the semantic-publication gate immediately before `_addEvent`. The + capture sink hub, outbox sink, and collector HTTP sink each serialize or + queue events immediately, so none can safely be made responsible for + consolidation. Enforce one terminal semantic event per transaction ID before + it reaches any sink. +- Move `_recordCaptureDiagnostic` to the diagnostic stream and define a compact + end-of-session health aggregate for production observability. +- Retain temporary legacy projection behind a documented feature/version gate; + it must point to the canonical interaction ID and be excluded from default + enrichment selection. + +**Tests** + +- Serialization round-trip preserves immutable origin and successor result. +- Outbox recovery never duplicates a finalized interaction or loses its + evidence IDs. +- Normal semantic event selection excludes diagnostics and legacy projections. +- A session with 10 gestures publishes 10 canonical semantic interactions, + regardless of raw pointer/route/scroll callback count. + +### U5 — Migrate enrichment, insights, and acceptance gates + +**Files** + +- `docs/integration/production-replay-acceptance-0.4.13.md` +- `docs/integration/production-replay-acceptance.md` +- `packages/tugboat/README.md` +- Context Graph/collector consumer repositories, in follow-up PRs after the + SDK schema lands + +**Change** + +- Make enrichment select canonical semantic interactions and map components + using `origin.targetAnchor`, never the current/destination route context. +- Build causal edges from `result`/`causedByInteractionId`, not arrival order. +- Define rage tap as repeated completed `gesture=tap` on the same origin target + within the insight window with no successful result. Exclude scroll/swipe, + cancellation, and `navigated`/`changed` interactions. +- Replace legacy acceptance metrics (`raw tap`, `swipe-consumed`, settled-pair + ratios) with canonical interaction coverage, origin/destination correctness, + delayed-success attribution, automatic-route false-claim rate, and semantic + event reduction. + +**Tests and production checks** + +- Fixture/replay ingestion maps a delayed navigation to the correct origin + component and destination screen. +- Rage-tap fixture with three no-result taps flags once; three scrolls or three + successful navigation taps do not. +- Production Blend flow covers home scroll, paywall, full-screen navigation, + modal bottom sheet, asynchronous onboarding transition, rapid double-tap, + and automatic redirect. + +## Sequencing and compatibility + +1. Land U1 and characterization tests first; no externally visible schema + change yet. +2. Land U2 and U3 in small commits with the race/gesture matrix. Release behind + an internal consolidation flag enabled for Blend only. +3. Land U4 with dual-write compatibility projection; collector/graph continue + reading legacy records during migration. +4. Land U5 consumer changes and update production acceptance queries. +5. Compare dual-written sessions. Remove legacy `tap` + `tap_settled` semantic + selection only after all acceptance gates pass for two representative Blend + flows and no consumer still relies on it. + +## Performance and safety budget + +- Maximum pending transactions: one per active pointer plus a small bounded + released queue; reject/flush oldest safely if the cap is reached. +- Default reconciliation deadline: 1,250 ms; implementation must record timing + distribution so the value can be tuned from production evidence. +- No widget-tree traversal, screenshot capture, disk I/O, or network call may + be introduced solely by consolidation. +- Use existing route/state/frame observations; references, not cloned frame + bytes, are stored in transactions. +- Expiry uses one controller sweep/clock hook, not unbounded per-event timers. + +## Definition of done + +- No finalized interaction ever changes its origin screen/modal/component after + pointer-down. +- Delayed user navigation and bottom sheets in the configured window produce a + single causal interaction with origin and destination fields. +- Automatic redirects and competing interactions are not falsely claimed. +- A completed scroll/swipe creates no semantic tap. +- Default downstream selection sees exactly one semantic action per completed + user gesture and excludes diagnostics. +- Rage tap uses finalized canonical taps and passes the positive/negative + fixtures above. +- Focused replay tests, full package analysis, formatting, and Blend production + acceptance all pass. + +## Risks and decisions to validate during implementation + +- A fixed reconciliation window may be too short for slow network-driven UI or + too long for automatic redirects. Instrument pending-to-success latency and + make the deadline data-driven after the Blend rollout. +- Some state changes are visual consequences rather than user-visible + destinations. Only first visible successor evidence should close a tap. +- Legacy collector and graph consumers may currently assume `tap_settled` is + the canonical enrichment record; dual-write and consumer migration are + mandatory before deleting that shape. +- Multi-pointer gestures and native/platform overlays need explicit + characterization before enabling causal attribution for them. +- `interactionClaimWindow` is currently constrained to same-turn eligibility + for compatibility. U2 must replace that policy deliberately rather than + merely changing configuration, and record the migration in the public + configuration documentation. + +## Verification contract + +1. Run focused replay matrices for U1–U4, then all package tests and static + analysis/formatting. +2. Build Blend against the local SDK and manually exercise the acceptance flow. +3. Query ClickHouse by SDK version and canonical interaction schema version. +4. Score origin correctness, delayed attribution, false claims, semantic event + count per completed gesture, diagnostic-stream isolation, and rage-tap + precision. +5. Publish a side-by-side report against a locked 0.4.0+ baseline before + removing legacy projection. diff --git a/packages/tugboat/CHANGELOG.md b/packages/tugboat/CHANGELOG.md index 8f9654d..94d27e6 100644 --- a/packages/tugboat/CHANGELOG.md +++ b/packages/tugboat/CHANGELOG.md @@ -1,3 +1,81 @@ +## 0.4.15 + +### Added + +- **Canonical `interaction` events** — each finalized gesture emits one + `stream: semantic` record with immutable `origin`, `result`, `attribution`, + and `evidenceEventIds` (`interactionSchema: 1`). Legacy `tap` / `tap_settled` + / `swipe` continue as dual-write peers on `stream: legacy_projection`. +- **Evidence stream** — `route_change`, `state_change`, `scroll_start`, + `scroll_end`, and `pointer_cancel` emit on `stream: evidence` so default + semantic enrichment selects only canonical interactions. +- **`enrichmentCandidate`** on collector-mapped events — false for evidence, + diagnostic, and legacy-projection records; true for canonical `interaction` + (and compat semantic tap/tap_settled/swipe when canonical emission is off). +- **Delayed reconciliation window** — `interactionClaimWindow` defaults to + 1,250 ms. A released tap can claim the first eligible visible route/modal + successor in that window (`interactionAttribution: delayed_likely`). Set the + window to `Duration.zero` to retain microtask-only same-turn claims. +- **Diagnostic stream isolation** — `capture_diagnostic` events carry + `stream: diagnostic` so enrichment/insight queries can ignore them by + default. Session health still aggregates outcome counts. +- **`causedByInteractionId`** on claimed `route_change` / `state_change` + (alongside existing `causeEventId`). + +### Fixed + +- **Swipe origin freeze** — swipe events retain the pointer-down state anchor + rather than refreshing live controller state at pointer-up. +- **Settle waits for delayed successors** — when the claim window is active, + tap settlement holds until a successor claims or the deadline expires instead + of finalizing `unknown` immediately. +- **Terminal cancelled interactions** — abandoning a pending/released + transaction (lifecycle, session end, supersede, post-up cancel) publishes a + canonical `gesture=cancelled` interaction instead of silently dropping it. + +## 0.4.14 + +### Fixed + +- **Automatic-navigation visual continuity** — when a route transition + supersedes an in-flight tap capture, `tap_settled` now waits for and attaches + the route's fresh frame as a non-causal `visual_successor`. A pointer-generation + fence prevents later user interactions from being attached to the earlier tap. + +## 0.4.13 + +### Fixed + +- **Deferred tap emission** — `tap` is sampled at pointer-down but only emitted + after gesture classification at pointer-up. Flick-scrolls no longer mint + phantom taps; swipes carry `startCaptureCoordinate` instead of + `relatedEventId` to a never-settled tap. +- **Same-turn interaction claims** — released pointer-up claims attribute + `route_change` only through the pointer-up turn (`same_turn`). A wall-clock + claim window incorrectly bound automatic redirects to taps; `interactionClaimWindow` + defaults to zero and no longer extends attribution. Pre-up claims still work + while the pointer is down (long-press → navigate). +- **Pre-up claim + swipe/cancel** — routes that claim before pointer-up no longer + force a normal interaction tap; the buffered tap publishes as `causal_only` + when the gesture finalizes as swipe/cancel (or at `route_change` publish). +- **Lifecycle claim fence** — backgrounding drops pending/released pointer claims + so resume/navigation cannot attribute a pre-background gesture. +- **Session-end claim fence** — pending pointers are abandoned without emitting + orphan `causal_only` taps when the claimed route was cancelled before publish; + further pointer-down/up/cancel is ignored. +- **Duplicate pointer-down** — a second down on the same pointer abandons the + prior pending claim instead of silently overwriting it. +- **Claim map hygiene** — `_claimsByTapEventId` entries are removed when a claim + emits, drops, or expires; `invalidatesRelatedTap` is always a boolean. +- **Monotonic deferred publish** — buffered taps / `tap_outside_tree` publish at + emission `atMs` with `sampledAtMs` preserving pointer-down time. +- **Gesture promotion** — `tap_gesture_resolved` (`promotesRelatedTap`) plus an + in-memory `replayRole` patch promote genuine taps that were first published as + `causal_only` for a pre-up route claim. +- **Missing-frame coordinates** — `unavailableReason: missing_frame` now keeps + boundary-local / normalized geometry when the boundary rect is known, instead + of zeroing local/normalized fields. + ## 0.4.12 ### Added diff --git a/packages/tugboat/README.md b/packages/tugboat/README.md index a833afb..184f814 100644 --- a/packages/tugboat/README.md +++ b/packages/tugboat/README.md @@ -5,7 +5,7 @@ checkpoints around meaningful interactions, compact structural anchors, route transitions, scrolling evidence, and optional viewport semantic maps. Capture can be sent to the local exploration WebSocket, the HTTP collector, or both. -The current package version is `0.4.12`. Session JSON uses schema version `7` +The current package version is `0.4.13`. Session JSON uses schema version `7` (readers still accept `6`), and structural fingerprints use fingerprint schema version `6`. @@ -73,6 +73,31 @@ Production acceptance for #13/#14 remains open: rapid or nested modal chains and programmatic/automatic navigation can still be absent or degraded. Treat those cases as an SDK capture gap, not as coherent replay evidence. +### Deferred taps and interaction claims + +`tap` is sampled at pointer-down but emitted only after gesture classification +(or when a claimed `route_change` must publish a cause). Consumers must read: + +| Field | Meaning | +| --- | --- | +| `data.replayRole` | `interaction` (real user tap) or `causal_only` (cause id for a route; not a replayable action) | +| `data.gestureFinal` | `tap`, `swipe`, `cancelled`, `superseded`, `session_end`, or `unresolved` | +| `data.sampledAtMs` | Pointer-down sample time when the event was published later | +| `data.captureCoordinate` | Boundary-local / normalized / raster transform for the before-frame | +| `route_change.data.navigationOrigin` | `interaction` or `automatic_or_unknown` | +| `route_change.data.causeEventId` | Claimed tap id when origin is `interaction` | +| `route_change.data.interactionAttribution` | Always `same_turn` when claimed | +| `swipe`/`pointer_cancel.data.invalidatesRelatedTap` | Boolean `true` when a related causal tap should be ignored for playback | +| `tap_gesture_resolved` | Promotes `relatedEventId` from `causal_only` → interaction (`promotesRelatedTap`) | + +Released pointer-up claims attribute a route only through the pointer-up turn +(sync `onTap` → `Navigator.push`). Timer/auth redirects after that turn stay +`automatic_or_unknown`. Backgrounding and `session_end` drop pending claims +without minting orphan `causal_only` taps for cancelled routes. + +**Downstream contract:** Context Graph and PMKit CLI must not treat +`replayRole: causal_only` taps as real actions or step openers. + ## Capture profiles and runtime state `TugboatReplayConfig.profile` controls whether the wrapper installs capture @@ -130,6 +155,8 @@ Call `TugboatReplay.clearDurableOutbox()` on logout/consent revocation. | --- | --- | --- | | `profile` | `dormant` | capture cost and exploration-only behavior | | `settleDelay` | 1 second | delay before post-interaction and post-route capture | +| `interactionClaimWindow` | 1,250 ms | released-tap window for delayed route/modal attribution; `Duration.zero` keeps microtask-only same-turn claims | +| `interactionPublishMode` | `dualWrite` | how finalized gestures are published: `legacyOnly`, `dualWrite` (canonical + legacy peers on `stream: legacy_projection`), or `canonicalOnly` | | `maxFrames` | 500 | in-memory frame bound | | `maxEvents` | 5000 | in-memory event bound | | `scrollCaptureInterval` | 2 seconds | interval for scroll checkpoint capture | @@ -146,6 +173,24 @@ Call `TugboatReplay.clearDurableOutbox()` on logout/consent revocation. | `viewportSemanticMode` | `tapResolutionOnly` | semantic engine and emission mode | | `viewportSemanticMapMaxNodes` | 120 | emitted map node budget | | `viewportSemanticMapMaxBytes` | 48000 | emitted map byte budget | +| `sinkFactories` | empty | extra `TugboatCaptureSinkFactory` adapters | +| `outbox` | disabled | durable HTTP outbox configuration | +| `screenshotBudget` | defaults | degraded-capture skip window / budget | + +### Resolver and exploration events + +When exploration is active, the controller may emit: + +| Event | Role | +| --- | --- | +| `scene_inventory` | Deduped actionable/image inventory for the settled state | +| `viewport_semantic_map` | Bounded semantic node map (mode-dependent) | +| `scroll_semantic_snapshot` | Semantic snapshot tied to scroll checkpoints | +| `action_window_set` / `action_window_cleared` | CLI exploration action-window fencing | +| `tap_outside_tree` | Pointer resolved no target; carries the same `replayRole` / `gestureFinal` as the paired tap when deferred | +| `tap_gesture_resolved` | Promotes a prior `causal_only` tap (`promotesRelatedTap: true`, `relatedEventId`) after the gesture finalizes as a real tap | + +Consumers that filter `replayRole: causal_only` must honor `tap_gesture_resolved` (or the patched in-memory tap) before suppressing genuine navigations. ## Privacy and payload boundary @@ -191,16 +236,27 @@ session is bounded by `maxFrames` and `maxEvents`; trimming marks it Emitted event types currently include: +- canonical: `interaction` (`stream: semantic`) — one finalized gesture with + immutable `origin`, `result`, `attribution`, and `evidenceEventIds`; +- legacy gesture peers (`stream: legacy_projection` when canonical is on): + `tap`, `tap_settled`, `swipe`, `tap_outside_tree`, `tap_gesture_resolved`; - lifecycle: `session_start`, `session_end`; -- input: `tap`, `tap_settled`, `swipe`, `pointer_cancel`, - `tap_outside_tree`; -- state/navigation: `state_change`, `route_change`; -- scrolling: `scroll_start`, `scroll_end`; +- input: `pointer_cancel` (`stream: evidence`); +- state/navigation evidence (`stream: evidence`): `state_change`, `route_change` + (claimed routes also carry `causedByInteractionId`); +- scrolling evidence (`stream: evidence`): `scroll_start`, `scroll_end`; +- diagnostics: `capture_diagnostic` (`stream: diagnostic`); - exploration: `scene_inventory`, `action_window_set`, `action_window_cleared`; - semantic-map modes: `viewport_semantic_map`, `scroll_semantic_snapshot`. +Default enrichment and insight selection should use `stream: semantic` +`interaction` records (`enrichmentCandidate: true` on collector payloads). +Rage-tap style insights must count finalized `gesture=tap` interactions with +no successful `navigated`/`changed` result; exclude scrolls, swipes, +cancellations, evidence, legacy projections, and diagnostics. + Frames can be triggered by initial startup, taps, scrolls, routes, lifecycle, or explicit controller calls. Capture requests are serialized and coalesced. The SDK first skips repeated state signatures, then uses a small dHash to avoid @@ -215,14 +271,16 @@ capture-boundary-normalized for replay playback. Do not interpret them as physical pixels or as coordinates relative to an individual widget. Fractional overlay drift is therefore still possible. -For a tap, `beforeFrame` is selected at pointer-down only if its provenance is -compatible with the tap's route epoch. At pointer-up, a single `tap_settled` -event links back to the `tap` via `relatedEventId` and is intended to contain -an `afterFrame` only when the settled capture is compatible with the observed -route. A missing attachment is explicit in `frameAttachment`/settle diagnostics -rather than a fallback to an unrelated frame. Acceptance remains open for the -navigation cases above, so consumers must still treat absence and degradation -as a capture gap. +For a tap, origin context (`stateAnchor`, target, `beforeFrame`, +`captureCoordinate`, route/navigator identity) is frozen at pointer-down into +an `InteractionTransaction`. After pointer-up, settlement waits for either the +first eligible visible successor inside `interactionClaimWindow` (default +1,250 ms) or the deadline. The canonical `interaction` event retains that +frozen origin and attaches destination/result fields when a successor claims. +Legacy `tap` + `tap_settled` remain dual-written for migration; `tap_settled` +links via `relatedEventId` / `interactionId`. A missing attachment is explicit +in `frameAttachment`/settle diagnostics rather than a fallback to an unrelated +frame. During local WebSocket exploration, connecting without an HTTP collector suppresses new Flutter screenshot capture for UI-thread performance. Events, diff --git a/packages/tugboat/lib/src/collector_mapper.dart b/packages/tugboat/lib/src/collector_mapper.dart index a8726ad..53515c3 100644 --- a/packages/tugboat/lib/src/collector_mapper.dart +++ b/packages/tugboat/lib/src/collector_mapper.dart @@ -13,6 +13,7 @@ Map mapTugboatEventToCollectorEvent({ final payload = { ...event.data, + 'stream': event.stream.wireName, if (event.relatedEventId != null) 'relatedEventId': event.relatedEventId, if (event.explorationRunId != null) 'explorationRunId': event.explorationRunId, @@ -26,6 +27,8 @@ Map mapTugboatEventToCollectorEvent({ if (sessionId != null) 'sessionId': sessionId, 'userId': userId, 'eventType': event.type, + 'stream': event.stream.wireName, + 'enrichmentCandidate': tugboatEventIsEnrichmentCandidate(event), if (event.explorationRunId != null) 'explorationRunId': event.explorationRunId, if (event.actionId != null) 'actionId': event.actionId, diff --git a/packages/tugboat/lib/src/controller.dart b/packages/tugboat/lib/src/controller.dart index 392521a..bd84178 100644 --- a/packages/tugboat/lib/src/controller.dart +++ b/packages/tugboat/lib/src/controller.dart @@ -12,6 +12,7 @@ import 'coordinate_space.dart'; import 'debug_logging.dart'; import 'exploration_sink.dart'; import 'health.dart'; +import 'interaction_transaction.dart'; import 'models.dart'; import 'outbox/outbox.dart'; import 'outbox/outbox_sink.dart'; @@ -27,57 +28,6 @@ export 'replay_config.dart' TugboatViewportSemanticPolicy, resolveViewportSemanticPolicy; -class _PendingTap { - _PendingTap({ - required this.eventId, - required this.targetAnchor, - required this.beforeState, - required this.beforeFrame, - required this.startPosition, - required this.startedAtMs, - required this.claim, - }); - - final String eventId; - final TugboatTargetAnchor? targetAnchor; - final TugboatStateAnchor? beforeState; - final String? beforeFrame; - final Offset startPosition; - final int startedAtMs; - final _PendingInteractionClaim claim; - bool suppressSettle = false; -} - -/// Immutable, single-use proof that a route observation may cite a tap cause. -class _PendingInteractionClaim { - _PendingInteractionClaim({ - required this.tapEventId, - required this.pointerId, - required this.captureSessionId, - required this.navigatorId, - required this.routeInstanceId, - required this.pointerGeneration, - }); - - final String tapEventId; - final int pointerId; - final String? captureSessionId; - final String? navigatorId; - final String? routeInstanceId; - final int pointerGeneration; - bool claimed = false; - bool cancelled = false; - - bool get isEligible => !claimed && !cancelled; -} - -class _PointerGestureState { - _PointerGestureState({required this.tapEventId}); - - final String tapEventId; - final List scrollStartEventIds = []; -} - class _ScrollTracker { _ScrollTracker({ required this.scrollableElement, @@ -466,6 +416,7 @@ class _VisibleRouteChange { this.visualObservationGeneration = 0, this.navigationOrigin = 'automatic_or_unknown', this.causeEventId, + this.interactionAttribution, }); final String? previousRoute; @@ -482,6 +433,10 @@ class _VisibleRouteChange { final String navigationOrigin; final String? causeEventId; + /// Wire form is [InteractionAttribution.claimWireName] (`same_turn` / + /// `delayed_likely`) when a claim succeeds. + final InteractionAttribution? interactionAttribution; + Map ownershipData() => { if (navigatorId != null) 'navigatorId': navigatorId, if (parentNavigatorId != null) 'parentNavigatorId': parentNavigatorId, @@ -492,6 +447,9 @@ class _VisibleRouteChange { 'visualObservationGeneration': visualObservationGeneration, 'navigationOrigin': navigationOrigin, if (causeEventId != null) 'causeEventId': causeEventId, + if (causeEventId != null) 'causedByInteractionId': causeEventId, + if (interactionAttribution != null) + 'interactionAttribution': interactionAttribution!.claimWireName, }; } @@ -819,8 +777,9 @@ class TugboatReplayController extends ChangeNotifier { final _NavigatorSurfaceRegistry _surfaces = _NavigatorSurfaceRegistry(); TugboatStateAnchor? _currentStateAnchor; String? _latestFrameId; - final Map _pendingTaps = {}; - final Map _releasedInteractionClaims = {}; + final InteractionRegistry _interactions = InteractionRegistry(); + bool _reconciliationSweepScheduled = false; + void Function()? _reconciliationSweepCancel; final Map _hashToFrameId = {}; final Map _frameProvenance = {}; final Map _frameReuseObservations = {}; @@ -858,7 +817,6 @@ class TugboatReplayController extends ChangeNotifier { static String _routeCaptureKey(String? navigatorId) => navigatorId ?? ''; final Map _scrollTrackers = {}; - final Map _activeGestures = {}; String? _lastCapturedStateSignature; final Set _emittedInventories = {}; SemanticsHandle? _semanticsHandle; @@ -1298,6 +1256,7 @@ class TugboatReplayController extends ChangeNotifier { final hub = _sinkHub; final ending = _endSession('dispose'); _disposed = true; + _clearReleasedInteractions(); _semanticsHandle?.dispose(); _semanticsHandle = null; _sinkHub = null; @@ -1324,6 +1283,10 @@ class TugboatReplayController extends ChangeNotifier { _cancelActiveTapSettles(cancellationReason); _cancelActiveRouteCapture(cancellationReason); _invalidateCaptureWork(cancellationReason); + // Routes cancelled above never publish causeEventId — do not mint an + // orphan causal_only tap for a claim that will never be referenced. + _abandonAllPendingPointers(publishClaimedTap: false); + _clearReleasedInteractions(); _captureLifecycleActive = false; _addEvent( @@ -1377,10 +1340,9 @@ class TugboatReplayController extends ChangeNotifier { _surfaces.clear(); _currentStateAnchor = null; _latestFrameId = null; - _pendingTaps.clear(); - _releasedInteractionClaims.clear(); + _clearReleasedInteractions(); + _interactions.clearAll(); _scrollTrackers.clear(); - _activeGestures.clear(); _hashToFrameId.clear(); _frameProvenance.clear(); _frameReuseObservations.clear(); @@ -1623,6 +1585,7 @@ class TugboatReplayController extends ChangeNotifier { id: _nextId('event'), atMs: atMs, type: 'capture_diagnostic', + stream: TugboatEventStream.diagnostic, afterFrame: resolution.frameId, data: { 'version': 1, @@ -2279,8 +2242,26 @@ class TugboatReplayController extends ChangeNotifier { } } + bool get _acceptsPointerInput => + !_disposed && + _session != null && + _captureLifecycleActive && + _endSessionFuture == null; + void recordPointerDown(Offset position, {int pointer = 0}) { - _releasedInteractionClaims.remove(pointer)?.cancelled = true; + if (!_acceptsPointerInput) return; + final previousClaim = _interactions.removeReleased(pointer); + if (previousClaim != null) { + previousClaim.cancelled = true; + previousClaim.rejectionReason ??= + InteractionRejectionReason.claimConsumed; + if (!previousClaim.tapEmitted) { + _interactions.forgetId(previousClaim.id); + } + } + if (_interactions.pendingAt(pointer) != null) { + _abandonPendingPointer(pointer, gestureFinal: 'superseded'); + } final resolver = _anchorResolver; TugboatTargetAnchor? target; TugboatStateAnchor? tapState = _currentStateAnchor; @@ -2338,47 +2319,50 @@ class TugboatReplayController extends ChangeNotifier { final beforeState = tapState; final eventId = _nextId('event'); - final claim = _PendingInteractionClaim( - tapEventId: eventId, - pointerId: pointer, - captureSessionId: _session?.id, - navigatorId: _currentNavigatorId, + final startedAtMs = atMs; + final origin = InteractionOrigin( + interactionId: eventId, + stateAnchor: beforeState, + route: _currentRoute, routeInstanceId: _currentRouteInstanceId, - pointerGeneration: ++_pointerGeneration, - ); - _pendingTaps[pointer] = _PendingTap( - eventId: eventId, + navigatorId: _currentNavigatorId, targetAnchor: target, - beforeState: beforeState, + captureCoordinate: captureCoordinate, beforeFrame: beforeFrame, + atMs: startedAtMs, startPosition: position, - startedAtMs: atMs, - claim: claim, + pointerGeneration: ++_pointerGeneration, + captureSessionId: _session?.id, ); - _activeGestures[pointer] = _PointerGestureState(tapEventId: eventId); - if (target == null) { - _addEvent( - TugboatEvent( - id: _nextId('event'), - atMs: atMs, - type: 'tap_outside_tree', - stateAnchor: beforeState, - beforeFrame: beforeFrame, - data: {'x': position.dx, 'y': position.dy, 'pointer': pointer}, - ), - ); - } - _addEvent( - TugboatEvent( - id: eventId, - atMs: atMs, - type: 'tap', - stateAnchor: beforeState, - targetAnchor: target, - beforeFrame: beforeFrame, - data: tapData, - ), + final tx = InteractionTransaction(origin: origin, pointerId: pointer); + final legacyStream = config.legacyGestureStream; + tx.bufferedOutside = target == null + ? TugboatEvent( + id: _nextId('event'), + atMs: startedAtMs, + type: 'tap_outside_tree', + stream: legacyStream, + stateAnchor: beforeState, + beforeFrame: beforeFrame, + data: { + 'x': position.dx, + 'y': position.dy, + 'pointer': pointer, + 'interactionId': eventId, + }, + ) + : null; + tx.bufferedTap = TugboatEvent( + id: eventId, + atMs: startedAtMs, + type: 'tap', + stream: legacyStream, + stateAnchor: beforeState, + targetAnchor: target, + beforeFrame: beforeFrame, + data: {...tapData, 'interactionId': eventId}, ); + _interactions.register(tx); if (viewportResolution != null && _viewportSemanticMapDebugLogsEnabled) { tugboatLogViewportSemanticTapResolution(position, viewportResolution); } @@ -2430,89 +2414,417 @@ class TugboatReplayController extends ChangeNotifier { ); } + void _emitBufferedTapFromClaim( + InteractionTransaction tx, { + required String gestureFinal, + required String replayRole, + }) { + if (tx.tapEmitted) return; + tx.tapEmitted = true; + final emittedAtMs = atMs; + final emitLegacy = config.emitLegacyInteractionProjection; + final outside = tx.bufferedOutside; + if (outside != null) { + if (emitLegacy) { + _addEvent( + outside.copyWith( + atMs: emittedAtMs, + data: { + ...outside.data, + 'gestureFinal': gestureFinal, + 'replayRole': replayRole, + 'sampledAtMs': outside.atMs, + }, + ), + ); + } + tx.bufferedOutside = null; + } + final tap = tx.bufferedTap; + if (tap != null) { + if (emitLegacy) { + _addEvent( + tap.copyWith( + atMs: emittedAtMs, + data: { + ...tap.data, + 'gestureFinal': gestureFinal, + 'replayRole': replayRole, + 'sampledAtMs': tap.atMs, + }, + ), + ); + } + tx.bufferedTap = null; + } + _interactions.forgetId(tx.id); + } + + /// Promotes a previously published `causal_only` tap once the gesture finalizes + /// as a real tap. Patches the in-memory session (and sibling `tap_outside_tree`) + /// and emits `tap_gesture_resolved` so already-flushed sinks can promote too. + void _promoteCausalTapToInteraction(String tapEventId) { + const promotion = { + 'gestureFinal': 'tap', + 'replayRole': 'interaction', + 'promotedFrom': 'causal_only', + }; + final session = _session; + if (session != null) { + Object? sampledAtMs; + for (var i = 0; i < session.events.length; i++) { + final event = session.events[i]; + if (event.id != tapEventId || event.type != 'tap') continue; + sampledAtMs = event.data['sampledAtMs']; + session.events[i] = event.withData(promotion); + break; + } + if (sampledAtMs != null) { + for (var i = 0; i < session.events.length; i++) { + final event = session.events[i]; + if (event.type != 'tap_outside_tree') continue; + if (event.data['sampledAtMs'] != sampledAtMs) continue; + if (event.data['replayRole'] != 'causal_only') continue; + session.events[i] = event.withData(promotion); + } + } + } + _addEvent( + TugboatEvent( + id: _nextId('event'), + atMs: atMs, + type: 'tap_gesture_resolved', + stream: config.legacyGestureStream, + relatedEventId: tapEventId, + data: { + 'gestureFinal': 'tap', + 'replayRole': 'interaction', + 'promotesRelatedTap': true, + 'interactionId': tapEventId, + }, + ), + ); + } + + /// Ensures a route_change causeEventId names a live tap before it is written. + void _ensureCauseTapPublished(String? causeEventId) { + if (causeEventId == null) return; + final tx = _interactions.byId(causeEventId); + if (tx == null || tx.tapEmitted) return; + // Still unresolved at route publish time — causal only until gesture ends. + _emitBufferedTapFromClaim( + tx, + gestureFinal: 'unresolved', + replayRole: 'causal_only', + ); + } + + void _releaseInteractionClaim(InteractionTransaction tx) { + final pointer = tx.pointerId; + tx.sameTurnEligible = true; + tx.releasedAtMs = atMs; + _interactions.release(tx); + final window = config.interactionClaimWindow; + if (window <= Duration.zero) { + // Microtask-only same-turn behaviour (characterization / rollback). + scheduleMicrotask(() { + if (!identical(_interactions.byPointer(pointer), tx)) return; + tx.sameTurnEligible = false; + _interactions.removeReleased(pointer); + if (!tx.claimed && !tx.tapEmitted) { + tx.rejectionReason ??= InteractionRejectionReason.expired; + _interactions.forgetId(tx.id); + } + }); + return; + } + tx.reconciliationDeadlineMs = atMs + window.inMilliseconds; + _ensureReconciliationSweepScheduled(); + } + + void _ensureReconciliationSweepScheduled() { + if (_reconciliationSweepScheduled) return; + if (!_interactions.hasReleased) return; + final earliest = _interactions.earliestReleasedDeadlineMs(); + if (earliest == null) return; + final now = atMs; + final delayMs = earliest - now; + final delay = Duration(milliseconds: delayMs < 0 ? 0 : delayMs); + _reconciliationSweepScheduled = true; + final scheduled = _scheduleDelay(delay); + _reconciliationSweepCancel = scheduled.cancel; + unawaited( + scheduled.done.then((_) { + _reconciliationSweepScheduled = false; + _reconciliationSweepCancel = null; + if (_disposed) return; + _sweepReleasedInteractions(); + if (_interactions.hasReleased) { + _ensureReconciliationSweepScheduled(); + } + }), + ); + } + + void _sweepReleasedInteractions() { + final now = atMs; + final expired = []; + for (final tx in _interactions.released) { + final deadline = tx.reconciliationDeadlineMs; + if (deadline == null) continue; + if (now < deadline) continue; + expired.add(tx); + } + for (final tx in expired) { + _expireReleasedInteraction(tx); + } + // Enforce cap by flushing oldest safely. + while (_interactions.releasedCount > + tugboatMaxReleasedInteractionTransactions) { + final oldest = _interactions.released.first; + _expireReleasedInteraction(oldest); + } + } + + void _expireReleasedInteraction(InteractionTransaction tx) { + tx.sameTurnEligible = false; + if (!tx.claimed && !tx.cancelled) { + tx.rejectionReason ??= InteractionRejectionReason.expired; + } + _interactions.removeReleased(tx.pointerId); + if (!tx.claimed && !tx.tapEmitted) { + _interactions.forgetId(tx.id); + } + } + + void _clearReleasedInteractions() { + _reconciliationSweepCancel?.call(); + _reconciliationSweepCancel = null; + _reconciliationSweepScheduled = false; + for (final tx in _interactions.takeAllReleased()) { + _finalizeAbandonedTransaction( + tx, + reason: InteractionRejectionReason.sessionEnd, + ); + if (!tx.tapEmitted) { + tx.bufferedTap = null; + tx.bufferedOutside = null; + _interactions.forgetId(tx.id); + } + } + } + + void _dropClaimBuffers(InteractionTransaction tx) { + tx.cancelled = true; + tx.bufferedTap = null; + tx.bufferedOutside = null; + _interactions.forgetId(tx.id); + } + + /// Ensures every transaction reaches exactly one terminal canonical state. + void _finalizeAbandonedTransaction( + InteractionTransaction tx, { + required InteractionRejectionReason reason, + }) { + if (tx.semanticPublished) return; + tx.cancelled = true; + tx.rejectionReason ??= reason; + tx.attribution = InteractionAttribution.none; + if (!tx.isSwipeOrScroll) { + tx.gesture = InteractionGesture.cancelled; + } + tx.resultStatus = InteractionResultStatus.cancelled; + tx.resultObservedAtMs ??= atMs; + _publishCanonicalInteraction(tx); + } + + void _abandonPendingPointer( + int pointer, { + required String gestureFinal, + bool publishClaimedTap = true, + }) { + final pending = _interactions.removePending(pointer); + if (pending == null) return; + if (pending.claimed && !pending.tapEmitted && publishClaimedTap) { + _emitBufferedTapFromClaim( + pending, + gestureFinal: gestureFinal, + replayRole: 'causal_only', + ); + } else if (!pending.tapEmitted) { + _dropClaimBuffers(pending); + } + final reason = switch (gestureFinal) { + 'superseded' => InteractionRejectionReason.claimConsumed, + 'session_end' => InteractionRejectionReason.sessionEnd, + _ => InteractionRejectionReason.lifecycle, + }; + _finalizeAbandonedTransaction(pending, reason: reason); + } + + void _abandonAllPendingPointers({bool publishClaimedTap = true}) { + for (final pointer in _interactions.takePendingPointers()) { + _abandonPendingPointer( + pointer, + gestureFinal: 'session_end', + publishClaimedTap: publishClaimedTap, + ); + } + } + void recordPointerCancel(Offset position, {int pointer = 0}) { - final pending = _pendingTaps.remove(pointer); - pending?.claim.cancelled = true; - _releasedInteractionClaims.remove(pointer)?.cancelled = true; - _activeGestures.remove(pointer); + if (!_acceptsPointerInput) return; + final pending = _interactions.removePending(pointer); + if (pending != null) { + pending.gesture = InteractionGesture.cancelled; + pending.rejectionReason ??= InteractionRejectionReason.lifecycle; + if (pending.claimed) { + _emitBufferedTapFromClaim( + pending, + gestureFinal: 'cancelled', + replayRole: 'causal_only', + ); + } else { + _dropClaimBuffers(pending); + } + pending.resultStatus = InteractionResultStatus.cancelled; + pending.resultObservedAtMs = atMs; + _publishCanonicalInteraction(pending); + } + final released = _interactions.removeReleased(pointer); + if (released != null) { + released.rejectionReason ??= InteractionRejectionReason.lifecycle; + if (!released.tapEmitted) { + _interactions.forgetId(released.id); + } + _finalizeAbandonedTransaction( + released, + reason: InteractionRejectionReason.lifecycle, + ); + } _addEvent( TugboatEvent( id: _nextId('event'), atMs: atMs, type: 'pointer_cancel', + stream: TugboatEventStream.evidence, stateAnchor: _currentStateAnchor, - data: {'x': position.dx, 'y': position.dy, 'pointer': pointer}, + data: { + 'x': position.dx, + 'y': position.dy, + 'pointer': pointer, + if (pending?.claimed == true) 'invalidatesRelatedTap': true, + if (pending != null) 'interactionId': pending.id, + }, + relatedEventId: pending?.claimed == true ? pending!.id : null, ), ); if (!_disposed) notifyListeners(); } void markPendingTapAsSwipe(int pointer) { - final pending = _pendingTaps[pointer]; + final pending = _interactions.pendingAt(pointer); if (pending != null) { - pending.suppressSettle = true; - pending.claim.cancelled = true; + pending.markSwipe(); + // Keep a claimed cause intact so route_change causeEventId stays valid. + if (!pending.claimed) { + pending.rejectionReason ??= + InteractionRejectionReason.gestureReclassified; + _dropClaimBuffers(pending); + } } } void recordPointerUp(Offset position, {int pointer = 0}) { - final pending = _pendingTaps.remove(pointer); - final gesture = _activeGestures.remove(pointer); + if (!_acceptsPointerInput) return; + final pending = _interactions.removePending(pointer); if (pending == null) return; - if (pending.suppressSettle) { - final delta = position - pending.startPosition; - final durationMs = atMs - pending.startedAtMs; + if (pending.isSwipeOrScroll) { + if (pending.claimed) { + _emitBufferedTapFromClaim( + pending, + gestureFinal: 'swipe', + replayRole: 'causal_only', + ); + } else { + _dropClaimBuffers(pending); + } + final origin = pending.origin; + final delta = position - origin.startPosition; + final durationMs = atMs - origin.atMs; final velocity = durationMs > 0 ? delta.distance / (durationMs / 1000) : 0.0; - final scrollStartEventId = gesture?.scrollStartEventIds.isNotEmpty == true - ? gesture!.scrollStartEventIds.first + final scrollStartEventId = pending.scrollStartEventIds.isNotEmpty + ? pending.scrollStartEventIds.first : null; final scrolled = scrollStartEventId != null; - _addEvent( - TugboatEvent( - id: _nextId('event'), - atMs: atMs, - type: 'swipe', - stateAnchor: _refreshStateAnchor(), - targetAnchor: pending.targetAnchor, - beforeFrame: pending.beforeFrame, - relatedEventId: pending.eventId, - result: scrolled - ? TugboatInteractionResult.changed - : TugboatInteractionResult.noVisibleChange, - data: { - 'x': position.dx, - 'y': position.dy, - 'startX': pending.startPosition.dx, - 'startY': pending.startPosition.dy, - 'deltaX': delta.dx, - 'deltaY': delta.dy, - 'direction': tugboatSwipeDirection(delta), - 'distance': delta.distance, - 'velocity': velocity, - 'durationMs': durationMs, - 'scrolled': scrolled, - if (scrollStartEventId != null) - 'scrollStartEventId': scrollStartEventId, - }, - ), - ); + final tapWasEmitted = pending.tapEmitted; + pending.gesture = scrolled + ? InteractionGesture.scroll + : InteractionGesture.swipe; + pending.resultStatus = scrolled + ? InteractionResultStatus.changed + : InteractionResultStatus.unchanged; + pending.resultObservedAtMs = atMs; + if (scrollStartEventId != null) pending.addEvidence(scrollStartEventId); + if (config.emitLegacyInteractionProjection) { + _addEvent( + TugboatEvent( + id: _nextId('event'), + atMs: atMs, + type: 'swipe', + stream: config.legacyGestureStream, + // R1: freeze to the origin state anchor rather than a live refresh. + stateAnchor: origin.stateAnchor, + targetAnchor: origin.targetAnchor, + beforeFrame: origin.beforeFrame, + relatedEventId: tapWasEmitted ? pending.id : null, + result: scrolled + ? TugboatInteractionResult.changed + : TugboatInteractionResult.noVisibleChange, + data: { + 'x': position.dx, + 'y': position.dy, + 'startX': origin.startPosition.dx, + 'startY': origin.startPosition.dy, + 'deltaX': delta.dx, + 'deltaY': delta.dy, + 'direction': tugboatSwipeDirection(delta), + 'distance': delta.distance, + 'velocity': velocity, + 'durationMs': durationMs, + 'scrolled': scrolled, + 'startCaptureCoordinate': origin.captureCoordinate.toJson(), + if (tapWasEmitted) 'invalidatesRelatedTap': true, + if (scrollStartEventId != null) + 'scrollStartEventId': scrollStartEventId, + 'interactionId': pending.id, + }, + ), + ); + } + _publishCanonicalInteraction(pending); if (!_disposed) notifyListeners(); return; } - // Gesture callbacks such as onTap run after the raw pointer-up listener - // within the same event-loop turn. Keep the single-use claim alive only - // through that turn so Navigator observers can attribute the transition - // without allowing later automatic navigation to borrow the tap. - _releasedInteractionClaims[pointer] = pending.claim; - scheduleMicrotask(() { - if (identical(_releasedInteractionClaims[pointer], pending.claim)) { - _releasedInteractionClaims.remove(pointer); - } - }); + if (!pending.tapEmitted) { + _emitBufferedTapFromClaim( + pending, + gestureFinal: 'tap', + replayRole: 'interaction', + ); + } else if (pending.claimed) { + // Route already published a causal-only tap; promote it for consumers that + // filter causal_only, and patch the in-memory event when still present. + _promoteCausalTapToInteraction(pending.id); + } + + // Keep the single-use claim alive through the pointer-up turn so sync + // onTap → Navigator can attribute without letting later redirects borrow. + _releaseInteractionClaim(pending); final work = _TapSettleWork(session: _session); _activeTapSettles.add(work); @@ -2521,13 +2833,13 @@ class TugboatReplayController extends ChangeNotifier { Future _resolveTapSettle( _TapSettleWork work, - _PendingTap pending, + InteractionTransaction pending, Offset position, _RouteCaptureWork? routeCaptureAtPointerUp, ) async { try { final initialRouteCapture = - routeCaptureAtPointerUp?.change.causeEventId == pending.eventId + routeCaptureAtPointerUp?.change.causeEventId == pending.id ? routeCaptureAtPointerUp : null; // Give a callback immediately after pointer-up the same settle boundary. @@ -2537,6 +2849,20 @@ class TugboatReplayController extends ChangeNotifier { await deadline.done; } if (!_isActiveTapSettle(work)) return; + // Hold finalization open through the reconciliation window so a delayed + // route/modal can claim before we publish unknown/unchanged. + if (config.interactionClaimWindow > Duration.zero && + !pending.claimed && + pending.reconciliationDeadlineMs != null) { + final remainingMs = pending.reconciliationDeadlineMs! - atMs; + if (remainingMs > 0) { + final deadline = _scheduleDelay(Duration(milliseconds: remainingMs)); + work.attachDeadlineCancellation(deadline.cancel); + await pending.awaitSuccessorOrDeadline(deadline.done); + deadline.cancel(); + } + } + if (!_isActiveTapSettle(work)) return; // A tap may only inherit a route barrier that was causally claimed by // that exact tap. In particular, an automatic navigation that starts // while this tap is waiting to settle is independent evidence: joining @@ -2545,14 +2871,14 @@ class TugboatReplayController extends ChangeNotifier { final currentRouteCapture = _activeRouteCapture; final routeCapture = initialRouteCapture ?? - (currentRouteCapture?.change.causeEventId == pending.eventId + (currentRouteCapture?.change.causeEventId == pending.id ? currentRouteCapture : null); _TapSettleObservation observation; if (routeCapture != null) { final routeBarrier = await _awaitRouteCaptureBarrier( routeCapture, - expectedCauseEventId: pending.eventId, + expectedCauseEventId: pending.id, ); if (!_isActiveTapSettle(work)) return; observation = _tapObservationFromRouteBarrier(routeBarrier); @@ -2563,7 +2889,7 @@ class TugboatReplayController extends ChangeNotifier { final capture = _requestCaptureCancellable( trigger: TugboatFrameTrigger.tap, settleDelay: Duration.zero, - relatedEventId: pending.eventId, + relatedEventId: pending.id, ); work.attachCaptureCancellation((reason) => capture.cancel(reason)); final captureResolution = await capture.resolution; @@ -2578,16 +2904,26 @@ class TugboatReplayController extends ChangeNotifier { provenance.context.routeEpoch == requestedRouteEpoch && provenance.context.route == requestedRoute; final replacementRoute = _activeRouteCapture; + final replacementIsCausal = + replacementRoute?.change.causeEventId == pending.id; + final replacementIsSafeVisualSuccessor = + replacementRoute != null && + _pointerGeneration == pending.origin.pointerGeneration; if (!compatibleFrame && replacementRoute != null && replacementRoute.epoch != requestedRouteEpoch && - replacementRoute.change.causeEventId == pending.eventId) { + (replacementIsCausal || replacementIsSafeVisualSuccessor)) { final routeBarrier = await _awaitRouteCaptureBarrier( replacementRoute, - expectedCauseEventId: pending.eventId, + expectedCauseEventId: replacementIsCausal ? pending.id : null, ); if (!_isActiveTapSettle(work)) return; - observation = _tapObservationFromRouteBarrier(routeBarrier); + observation = _tapObservationFromRouteBarrier( + routeBarrier, + navigationOutcome: replacementIsCausal + ? 'navigated' + : 'visual_successor', + ); } else { observation = _TapSettleObservation( routeEpoch: requestedRouteEpoch, @@ -2607,10 +2943,11 @@ class TugboatReplayController extends ChangeNotifier { } Future writeSettle() async { if (!_isActiveTapSettle(work)) return; - final beforeState = pending.beforeState; - final beforeFrame = pending.beforeFrame; - final tapEventId = pending.eventId; - final tapTargetAnchor = pending.targetAnchor; + final origin = pending.origin; + final beforeState = origin.stateAnchor; + final beforeFrame = origin.beforeFrame; + final tapEventId = pending.id; + final tapTargetAnchor = origin.targetAnchor; // Never read mutable controller state here: later route/capture work // may have advanced while this task waited on the serialized queue. final afterState = observation.afterState; @@ -2643,68 +2980,92 @@ class TugboatReplayController extends ChangeNotifier { ? beforeContentHash != afterContentHash : null; - _addEvent( - TugboatEvent( - id: _nextId('event'), - atMs: atMs, - type: 'tap_settled', - stateAnchor: afterState, - targetAnchor: tapTargetAnchor, - beforeFrame: beforeFrame, - afterFrame: afterFrame, - result: result, - relatedEventId: tapEventId, - data: { - 'x': position.dx, - 'y': position.dy, - 'settleObservation': { - 'version': 1, - 'routeEpoch': observation.routeEpoch, - if (observation.route != null) 'route': observation.route, - 'navigationOutcome': observation.navigationOutcome, - 'captureOutcome': observation.captureOutcome, - if (observation.captureFailure != null) - 'captureFailure': observation.captureFailure, - if (observation.routeEventId != null) - 'routeEventId': observation.routeEventId, - if (observation.captureRequestId != null) - 'captureRequestId': observation.captureRequestId, - 'semantic': { - 'changed': semanticChanged, - 'evidence': semanticAvailable - ? 'state_signature' - : 'unavailable', - 'reason': semanticChanged == null - ? 'unavailable' - : semanticChanged - ? 'state_signature_changed' - : 'same_signature', - }, - 'visual': { - 'changed': visualChanged, - 'evidence': visualAvailable ? 'content_hash' : 'unavailable', - 'reason': visualChanged == null - ? 'unavailable' - : visualChanged - ? 'frame_changed' - : 'same_frame', + if (config.emitLegacyInteractionProjection) { + _addEvent( + TugboatEvent( + id: _nextId('event'), + atMs: atMs, + type: 'tap_settled', + stream: config.legacyGestureStream, + stateAnchor: afterState, + targetAnchor: tapTargetAnchor, + beforeFrame: beforeFrame, + afterFrame: afterFrame, + result: result, + relatedEventId: tapEventId, + data: { + 'x': position.dx, + 'y': position.dy, + 'interactionId': tapEventId, + 'settleObservation': { + 'version': 1, + 'routeEpoch': observation.routeEpoch, + if (observation.route != null) 'route': observation.route, + 'navigationOutcome': observation.navigationOutcome, + 'captureOutcome': observation.captureOutcome, + if (observation.captureFailure != null) + 'captureFailure': observation.captureFailure, + if (observation.routeEventId != null) + 'routeEventId': observation.routeEventId, + if (observation.captureRequestId != null) + 'captureRequestId': observation.captureRequestId, + 'semantic': { + 'changed': semanticChanged, + 'evidence': semanticAvailable + ? 'state_signature' + : 'unavailable', + 'reason': semanticChanged == null + ? 'unavailable' + : semanticChanged + ? 'state_signature_changed' + : 'same_signature', + }, + 'visual': { + 'changed': visualChanged, + 'evidence': visualAvailable + ? 'content_hash' + : 'unavailable', + 'reason': visualChanged == null + ? 'unavailable' + : visualChanged + ? 'frame_changed' + : 'same_frame', + }, }, + if (afterFrame == null) + 'frameAttachment': { + 'after': 'unavailable', + 'reason': + observation.captureFailure ?? + observation.captureOutcome, + }, }, - if (afterFrame == null) - 'frameAttachment': { - 'after': 'unavailable', - 'reason': - observation.captureFailure ?? observation.captureOutcome, - }, - }, - ), - ); + ), + ); + } _maybeEmitStateChange( beforeState: beforeState, afterState: afterState, beforeFrame: beforeFrame, afterFrame: afterFrame, + causingTx: pending, ); + + pending.gesture = InteractionGesture.tap; + pending.resultStatus = InteractionResultStatus.fromSettle( + result: result, + navigationOutcome: observation.navigationOutcome, + degraded: observation.isDegraded, + ); + pending.afterFrame = afterFrame; + pending.resultStateAnchor = afterState; + pending.resultRoute = observation.route; + pending.resultObservedAtMs = atMs; + if (observation.routeEventId != null) { + pending.addEvidence(observation.routeEventId!); + } + _publishCanonicalInteraction(pending); + if (!_disposed) notifyListeners(); } @@ -2734,8 +3095,9 @@ class TugboatReplayController extends ChangeNotifier { identical(_session, work.session); _TapSettleObservation _tapObservationFromRouteBarrier( - ({_RouteCaptureWork work, _RouteCaptureResult result}) routeBarrier, - ) { + ({_RouteCaptureWork work, _RouteCaptureResult result}) routeBarrier, { + String navigationOutcome = 'navigated', + }) { final settledRoute = routeBarrier.work; final routeResult = routeBarrier.result; final frameId = routeResult.frameId; @@ -2753,7 +3115,9 @@ class TugboatReplayController extends ChangeNotifier { ? _stateObservedWithFrame(frameId) : routeResult.stateAnchor, afterFrame: validFrame ? frameId : null, - navigationOutcome: validFrame ? 'navigated' : 'navigation_unavailable', + navigationOutcome: validFrame + ? navigationOutcome + : 'navigation_unavailable', captureOutcome: validFrame ? 'captured' : routeResult.outcome == _RouteCaptureOutcome.timedOut @@ -2817,11 +3181,43 @@ class TugboatReplayController extends ChangeNotifier { } void _linkScrollStartToActiveGestures(String scrollStartEventId) { - for (final gesture in _activeGestures.values) { - gesture.scrollStartEventIds.add(scrollStartEventId); + for (final tx in _interactions.pending) { + if (!tx.scrollStartEventIds.contains(scrollStartEventId)) { + tx.scrollStartEventIds.add(scrollStartEventId); + } + tx.addEvidence(scrollStartEventId); } } + void _publishCanonicalInteraction(InteractionTransaction tx) { + if (tx.semanticPublished) return; + if (!config.emitCanonicalInteractions) return; + tx.semanticPublished = true; + _addEvent( + TugboatEvent( + id: _nextId('event'), + atMs: atMs, + type: 'interaction', + stateAnchor: tx.origin.stateAnchor, + targetAnchor: tx.origin.targetAnchor, + beforeFrame: tx.origin.beforeFrame, + afterFrame: tx.afterFrame, + result: tx.resultStatus?.asEventResult ?? TugboatInteractionResult.unknown, + data: { + 'interactionId': tx.id, + 'interactionSchema': tugboatInteractionSchemaVersion, + 'gesture': tx.gesture.name, + 'origin': tx.origin.toJson(), + 'result': tx.resultToJson(), + 'attribution': tx.attributionToJson( + windowMs: config.interactionClaimWindow.inMilliseconds, + ), + 'evidenceEventIds': List.from(tx.evidenceEventIds), + }, + ), + ); + } + Element? _scrollableElementFor(BuildContext? context) { if (context is! Element) return null; if (context.widget is Scrollable) return context; @@ -2971,6 +3367,7 @@ class TugboatReplayController extends ChangeNotifier { id: startEventId, atMs: atMs, type: 'scroll_start', + stream: TugboatEventStream.evidence, stateAnchor: _currentStateAnchor, targetAnchor: targetAnchor, beforeFrame: beforeFrame, @@ -3079,6 +3476,7 @@ class TugboatReplayController extends ChangeNotifier { id: _nextId('event'), atMs: atMs, type: 'scroll_end', + stream: TugboatEventStream.evidence, stateAnchor: tracker.startState, targetAnchor: tracker.targetAnchor, beforeFrame: tracker.beforeFrame, @@ -3139,6 +3537,7 @@ class TugboatReplayController extends ChangeNotifier { id: _nextId('event'), atMs: atMs, type: 'scroll_end', + stream: TugboatEventStream.evidence, stateAnchor: _stateObservedWithFrame(afterFrame) ?? _refreshStateAnchor(), targetAnchor: tracker.targetAnchor, @@ -3230,6 +3629,11 @@ class TugboatReplayController extends ChangeNotifier { prior?.supersededBy = work; _skipCapture = transition.transitionDuration > Duration.zero; _startRouteBarrierTimeout(work); + // Wake a reconciliation-pending settle only after this capture is visible + // in `_activeRouteCaptures`, otherwise settle can miss the causal barrier. + if (change.causeEventId != null) { + _interactions.byId(change.causeEventId!)?.signalSuccessorClaimed(); + } if (work.deadline <= Duration.zero) { unawaited(_enqueue('route_change', () => _finalizeRouteCapture(work))); } else { @@ -3338,30 +3742,51 @@ class TugboatReplayController extends ChangeNotifier { // This must not enqueue behind the blocked task that caused the timeout. // Dart's single isolate means the session mutation is still atomic with // respect to the next event-loop turn. + _ensureCauseTapPublished(change.causeEventId); + _emitRouteChange( + routeEventId: routeEventId, + change: change, + stateAnchor: observedState, + result: TugboatInteractionResult.unknown, + extraData: const {'captureOutcome': 'timed_out'}, + ); + work.complete( + _RouteCaptureResult( + _RouteCaptureOutcome.timedOut, + stateAnchor: observedState, + routeEventId: routeEventId, + ), + ); + if (!_disposed) notifyListeners(); + } + + /// Emits one canonical `route_change` on the evidence stream. + void _emitRouteChange({ + required String routeEventId, + required _VisibleRouteChange change, + required TugboatStateAnchor? stateAnchor, + required TugboatInteractionResult result, + String? afterFrame, + Map extraData = const {}, + }) { _addEvent( TugboatEvent( id: routeEventId, atMs: atMs, type: 'route_change', - stateAnchor: observedState, - result: TugboatInteractionResult.unknown, + stream: TugboatEventStream.evidence, + stateAnchor: stateAnchor, + afterFrame: afterFrame, + result: result, data: { if (change.previousRoute != null) 'fromRoute': change.previousRoute, if (change.destinationRoute != null) 'route': change.destinationRoute, 'navigation': change.navigation, - 'captureOutcome': 'timed_out', + ...extraData, ...change.ownershipData(), }, ), ); - work.complete( - _RouteCaptureResult( - _RouteCaptureOutcome.timedOut, - stateAnchor: observedState, - routeEventId: routeEventId, - ), - ); - if (!_disposed) notifyListeners(); } Future _awaitRouteDeadline( @@ -3411,27 +3836,18 @@ class TugboatReplayController extends ChangeNotifier { captureFailure = _lastCaptureFailure?.name; observedState = _snapshotStateAnchor(_currentStateAnchor); routeEventId = _nextId('event'); - _addEvent( - TugboatEvent( - id: routeEventId, - atMs: atMs, - type: 'route_change', - stateAnchor: observedState, - result: TugboatInteractionResult.navigated, - data: { - if (change.previousRoute != null) - 'fromRoute': change.previousRoute, - if (change.destinationRoute != null) - 'route': change.destinationRoute, - 'navigation': change.navigation, - 'captureOutcome': 'failed', - if (captureResult.captureFailure != null) - 'captureFailure': captureResult.captureFailure, - if (captureRequestId != null) - 'captureRequestId': captureRequestId, - ...change.ownershipData(), - }, - ), + _ensureCauseTapPublished(change.causeEventId); + _emitRouteChange( + routeEventId: routeEventId, + change: change, + stateAnchor: observedState, + result: TugboatInteractionResult.navigated, + extraData: { + 'captureOutcome': 'failed', + if (captureResult.captureFailure != null) + 'captureFailure': captureResult.captureFailure, + if (captureRequestId != null) 'captureRequestId': captureRequestId, + }, ); if (!_disposed) notifyListeners(); return; @@ -3445,32 +3861,24 @@ class TugboatReplayController extends ChangeNotifier { captureFailure = _lastCaptureFailure?.name; } if (!_isActiveRouteCapture(work)) return; - final previousRoute = change.previousRoute; - final destinationRoute = change.destinationRoute; observedState = _stateObservedWithFrame(afterFrame) ?? _currentStateAnchor; routeEventId = _nextId('event'); - _addEvent( - TugboatEvent( - id: routeEventId, - atMs: atMs, - type: 'route_change', - stateAnchor: observedState, - afterFrame: afterFrame, - result: TugboatInteractionResult.navigated, - data: { - if (previousRoute != null) 'fromRoute': previousRoute, - if (destinationRoute != null) 'route': destinationRoute, - 'navigation': change.navigation, - if (captureRequestId != null) 'captureRequestId': captureRequestId, - if (outcome == _RouteCaptureOutcome.failed) - 'captureOutcome': 'failed', - if (outcome == _RouteCaptureOutcome.failed && - captureResult.captureFailure != null) - 'captureFailure': captureResult.captureFailure, - ...change.ownershipData(), - }, - ), + _ensureCauseTapPublished(change.causeEventId); + _emitRouteChange( + routeEventId: routeEventId, + change: change, + stateAnchor: observedState, + afterFrame: afterFrame, + result: TugboatInteractionResult.navigated, + extraData: { + if (captureRequestId != null) 'captureRequestId': captureRequestId, + if (outcome == _RouteCaptureOutcome.failed) + 'captureOutcome': 'failed', + if (outcome == _RouteCaptureOutcome.failed && + captureResult.captureFailure != null) + 'captureFailure': captureResult.captureFailure, + }, ); _maybeEmitSceneInventory(); if (!_disposed) notifyListeners(); @@ -3530,6 +3938,10 @@ class TugboatReplayController extends ChangeNotifier { _cancelActiveTapSettles('lifecycle_deactivate'); _cancelActiveRouteCapture('lifecycle_deactivate'); _invalidateCaptureWork('lifecycle_deactivate'); + // Drop in-flight pointer claims so a later resume/navigation cannot + // attribute itself to a pre-background gesture. + _abandonAllPendingPointers(publishClaimedTap: false); + _clearReleasedInteractions(); _captureLifecycleActive = false; break; case AppLifecycleState.resumed: @@ -3663,7 +4075,7 @@ class TugboatReplayController extends ChangeNotifier { } _visualObservationGeneration++; - final causeEventId = _tryClaimInteractionCause( + final claimed = _tryClaimInteractionCause( navigatorId: navigatorId ?? _currentNavigatorId, ); return _VisibleRouteChange( @@ -3678,38 +4090,48 @@ class TugboatReplayController extends ChangeNotifier { stackRevision: stackRevision, overlayKind: transition.overlayKind, visualObservationGeneration: _visualObservationGeneration, - navigationOrigin: causeEventId == null + navigationOrigin: claimed == null ? 'automatic_or_unknown' : 'interaction', - causeEventId: causeEventId, + causeEventId: claimed?.id, + interactionAttribution: claimed?.attribution, ); } - /// Observer-time single-use claim. Returns the tap event ID only when exactly + /// Observer-time single-use claim. Returns the transaction only when exactly /// one unambiguous active pointer is eligible for this navigator/session. - String? _tryClaimInteractionCause({String? navigatorId}) { - final eligible = <_PendingInteractionClaim>[]; - for (final pending in _pendingTaps.values) { - if (pending.suppressSettle) continue; - final claim = pending.claim; - if (!claim.isEligible) continue; - if (claim.captureSessionId != _session?.id) continue; - eligible.add(claim); - } - for (final claim in _releasedInteractionClaims.values) { - if (!claim.isEligible) continue; - if (claim.captureSessionId != _session?.id) continue; - eligible.add(claim); - } - if (eligible.length != 1) return null; - final claim = eligible.single; + /// + /// Does not flush the buffered tap — that happens at route_change publish via + /// [_ensureCauseTapPublished], or earlier at pointer-up / swipe / cancel, so + /// a pre-up claim that later becomes a swipe stays `causal_only`. + InteractionTransaction? _tryClaimInteractionCause({String? navigatorId}) { + if (!_captureLifecycleActive || _endSessionFuture != null) return null; + final eligible = _interactions.eligibleForClaim( + nowMs: atMs, + sessionId: _session?.id, + ); + if (eligible.length != 1) { + if (eligible.length > 1) { + for (final tx in eligible) { + tx.rejectionReason ??= InteractionRejectionReason.competingPointer; + } + } + return null; + } + final tx = eligible.single; if (navigatorId != null && - claim.navigatorId != null && - claim.navigatorId != navigatorId) { + tx.origin.navigatorId != null && + tx.origin.navigatorId != navigatorId) { + tx.rejectionReason ??= InteractionRejectionReason.navigatorMismatch; return null; } - claim.claimed = true; - return claim.tapEventId; + tx.claimed = true; + final windowActive = config.interactionClaimWindow > Duration.zero; + final isPending = _interactions.pendingAt(tx.pointerId) != null; + tx.attribution = (isPending || !windowActive) + ? InteractionAttribution.direct + : InteractionAttribution.delayedLikely; + return tx; } void _maybeEmitStateChange({ @@ -3717,24 +4139,36 @@ class TugboatReplayController extends ChangeNotifier { required TugboatStateAnchor? afterState, required String? beforeFrame, required String? afterFrame, + InteractionTransaction? causingTx, }) { final beforeSignature = beforeState?.signature ?? ''; final afterSignature = afterState?.signature ?? ''; if (beforeSignature.isEmpty || afterSignature.isEmpty) return; if (beforeSignature == afterSignature) return; + final data = { + if (afterState?.subLabel != null) 'subLabel': afterState!.subLabel, + }; + // Route claim takes priority; only claim if not already claimed elsewhere. + if (causingTx != null && causingTx.isEligible) { + causingTx.claimed = true; + causingTx.attribution = InteractionAttribution.direct; + data['causedByInteractionId'] = causingTx.id; + } else if (causingTx != null && causingTx.claimed) { + data['causedByInteractionId'] = causingTx.id; + } + _addEvent( TugboatEvent( id: _nextId('event'), atMs: atMs, type: 'state_change', + stream: TugboatEventStream.evidence, stateAnchor: afterState, beforeFrame: beforeFrame, afterFrame: afterFrame, result: TugboatInteractionResult.changed, - data: { - if (afterState?.subLabel != null) 'subLabel': afterState!.subLabel, - }, + data: data, ), ); _maybeEmitSceneInventory(); diff --git a/packages/tugboat/lib/src/coordinate_space.dart b/packages/tugboat/lib/src/coordinate_space.dart index b1d9e8d..0258208 100644 --- a/packages/tugboat/lib/src/coordinate_space.dart +++ b/packages/tugboat/lib/src/coordinate_space.dart @@ -197,12 +197,37 @@ TugboatCaptureCoordinate buildCaptureCoordinate({ ); } if (framePixelWidth <= 0 || framePixelHeight <= 0 || frameId == null) { + final localX = globalX - boundaryOriginX; + final localY = globalY - boundaryOriginY; + if (localX < 0 || + localY < 0 || + localX > boundaryWidth || + localY > boundaryHeight) { + return TugboatCaptureCoordinate.unavailable( + unavailableReason: 'outside_boundary', + sourceSpace: TugboatCoordinateSourceSpace.boundaryLocalLogical, + boundaryOriginX: boundaryOriginX, + boundaryOriginY: boundaryOriginY, + boundaryWidth: boundaryWidth, + boundaryHeight: boundaryHeight, + localX: localX, + localY: localY, + boundaryTransformGeneration: boundaryTransformGeneration, + ); + } + final normalizedX = (localX / boundaryWidth).clamp(0.0, 1.0); + final normalizedY = (localY / boundaryHeight).clamp(0.0, 1.0); return TugboatCaptureCoordinate.unavailable( unavailableReason: 'missing_frame', + sourceSpace: TugboatCoordinateSourceSpace.boundaryLocalLogical, boundaryOriginX: boundaryOriginX, boundaryOriginY: boundaryOriginY, boundaryWidth: boundaryWidth, boundaryHeight: boundaryHeight, + localX: localX, + localY: localY, + normalizedX: normalizedX, + normalizedY: normalizedY, boundaryTransformGeneration: boundaryTransformGeneration, ); } diff --git a/packages/tugboat/lib/src/interaction_transaction.dart b/packages/tugboat/lib/src/interaction_transaction.dart new file mode 100644 index 0000000..dee4d2a --- /dev/null +++ b/packages/tugboat/lib/src/interaction_transaction.dart @@ -0,0 +1,294 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +import 'anchors.dart'; +import 'coordinate_space.dart'; +import 'models.dart'; + +/// Default post-pointer-up window for delayed causal route/modal attribution. +const Duration tugboatDefaultReconciliationWindow = Duration( + milliseconds: 1250, +); + +/// Maximum released transactions retained for delayed reconciliation. +const int tugboatMaxReleasedInteractionTransactions = 8; + +/// Immutable pointer-down origin for one user gesture. +class InteractionOrigin { + const InteractionOrigin({ + required this.interactionId, + required this.stateAnchor, + required this.route, + required this.routeInstanceId, + required this.navigatorId, + required this.targetAnchor, + required this.captureCoordinate, + required this.beforeFrame, + required this.atMs, + required this.startPosition, + required this.pointerGeneration, + required this.captureSessionId, + }); + + final String interactionId; + final TugboatStateAnchor? stateAnchor; + final String? route; + final String? routeInstanceId; + final String? navigatorId; + final TugboatTargetAnchor? targetAnchor; + final TugboatCaptureCoordinate captureCoordinate; + final String? beforeFrame; + final int atMs; + final Offset startPosition; + final int pointerGeneration; + final String? captureSessionId; + + Map toJson() => { + 'interactionId': interactionId, + if (stateAnchor != null) 'stateAnchor': stateAnchor!.toJson(), + if (route != null) 'route': route, + if (routeInstanceId != null) 'routeInstanceId': routeInstanceId, + if (navigatorId != null) 'navigatorId': navigatorId, + if (targetAnchor != null) 'targetAnchor': targetAnchor!.toJson(), + 'captureCoordinate': captureCoordinate.toJson(), + if (beforeFrame != null) 'beforeFrame': beforeFrame, + 'atMs': atMs, + 'startPosition': {'x': startPosition.dx, 'y': startPosition.dy}, + 'pointerGeneration': pointerGeneration, + if (captureSessionId != null) 'captureSessionId': captureSessionId, + }; +} + +enum InteractionGesture { tap, swipe, scroll, cancelled } + +enum InteractionResultStatus { + navigated, + changed, + unchanged, + unknown, + cancelled; + + TugboatInteractionResult get asEventResult => switch (this) { + InteractionResultStatus.navigated => TugboatInteractionResult.navigated, + InteractionResultStatus.changed => TugboatInteractionResult.changed, + InteractionResultStatus.unchanged => + TugboatInteractionResult.noVisibleChange, + InteractionResultStatus.cancelled || + InteractionResultStatus.unknown => TugboatInteractionResult.unknown, + }; + + static InteractionResultStatus fromSettle({ + required TugboatInteractionResult result, + String navigationOutcome = 'same_route', + bool degraded = false, + }) { + if (degraded) return InteractionResultStatus.unknown; + if (navigationOutcome == 'navigated') { + return InteractionResultStatus.navigated; + } + return switch (result) { + TugboatInteractionResult.navigated => InteractionResultStatus.navigated, + TugboatInteractionResult.changed => InteractionResultStatus.changed, + TugboatInteractionResult.noVisibleChange => + InteractionResultStatus.unchanged, + TugboatInteractionResult.unknown => InteractionResultStatus.unknown, + }; + } +} + +enum InteractionAttribution { + direct, + delayedLikely, + none; + + String get wireName => switch (this) { + InteractionAttribution.direct => 'direct', + InteractionAttribution.delayedLikely => 'delayed_likely', + InteractionAttribution.none => 'none', + }; + + /// Route_change compatibility string (`same_turn` | `delayed_likely`). + String get claimWireName => switch (this) { + InteractionAttribution.direct => 'same_turn', + InteractionAttribution.delayedLikely => 'delayed_likely', + InteractionAttribution.none => 'same_turn', + }; +} + +enum InteractionRejectionReason { + expired, + competingPointer, + gestureReclassified, + navigatorMismatch, + automaticGuard, + claimConsumed, + lifecycle, + sessionEnd, +} + +/// Bounded in-memory transaction for one pointer gesture. +class InteractionTransaction { + InteractionTransaction({required this.origin, required this.pointerId}); + + final InteractionOrigin origin; + final int pointerId; + + InteractionGesture gesture = InteractionGesture.tap; + bool claimed = false; + bool cancelled = false; + bool tapEmitted = false; + bool semanticPublished = false; + bool sameTurnEligible = true; + + int? releasedAtMs; + int? reconciliationDeadlineMs; + + TugboatEvent? bufferedTap; + TugboatEvent? bufferedOutside; + + final List evidenceEventIds = []; + final List scrollStartEventIds = []; + + InteractionResultStatus? resultStatus; + InteractionAttribution attribution = InteractionAttribution.none; + InteractionRejectionReason? rejectionReason; + String? resultRoute; + String? resultRouteInstanceId; + String? afterFrame; + int? resultObservedAtMs; + TugboatStateAnchor? resultStateAnchor; + + Completer? _successorSignal; + + String get id => origin.interactionId; + + bool get isSwipeOrScroll => + gesture == InteractionGesture.swipe || gesture == InteractionGesture.scroll; + + bool get isEligible => !claimed && !cancelled && !semanticPublished; + + bool isWithinReconciliationWindow(int nowMs) { + if (!sameTurnEligible) return false; + final deadline = reconciliationDeadlineMs; + if (deadline == null) return sameTurnEligible; + return nowMs <= deadline; + } + + void signalSuccessorClaimed() { + final signal = _successorSignal; + if (signal != null && !signal.isCompleted) signal.complete(); + } + + Future awaitSuccessorOrDeadline(Future deadline) { + final signal = Completer(); + _successorSignal = signal; + return Future.any([deadline, signal.future]).whenComplete(() { + if (identical(_successorSignal, signal)) _successorSignal = null; + }); + } + + void addEvidence(String eventId) { + if (!evidenceEventIds.contains(eventId)) evidenceEventIds.add(eventId); + } + + void markSwipe() { + gesture = InteractionGesture.swipe; + } + + Map resultToJson() => { + 'status': (resultStatus ?? InteractionResultStatus.unknown).name, + if (resultRoute != null) 'route': resultRoute, + if (resultRouteInstanceId != null) 'routeInstanceId': resultRouteInstanceId, + if (resultStateAnchor != null) 'stateAnchor': resultStateAnchor!.toJson(), + if (afterFrame != null) 'afterFrame': afterFrame, + if (resultObservedAtMs != null) 'observedAtMs': resultObservedAtMs, + }; + + Map attributionToJson({int? windowMs}) => { + 'kind': attribution.wireName, + if (windowMs != null) 'windowMs': windowMs, + if (rejectionReason != null) 'rejectionReason': rejectionReason!.name, + }; +} + +/// Single index for pending/released interaction transactions. +class InteractionRegistry { + final Map _pending = {}; + final Map _released = {}; + final Map _byId = {}; + + Iterable get pending => _pending.values; + Iterable get released => _released.values; + bool get hasPending => _pending.isNotEmpty; + bool get hasReleased => _released.isNotEmpty; + int get releasedCount => _released.length; + + InteractionTransaction? byPointer(int pointer) => + _pending[pointer] ?? _released[pointer]; + + InteractionTransaction? pendingAt(int pointer) => _pending[pointer]; + + InteractionTransaction? byId(String id) => _byId[id]; + + void register(InteractionTransaction tx) { + _byId[tx.id] = tx; + _pending[tx.pointerId] = tx; + } + + InteractionTransaction? removePending(int pointer) => _pending.remove(pointer); + + void release(InteractionTransaction tx) { + _pending.remove(tx.pointerId); + _released[tx.pointerId] = tx; + } + + InteractionTransaction? removeReleased(int pointer) => + _released.remove(pointer); + + void forgetId(String id) => _byId.remove(id); + + void clearAll() { + _pending.clear(); + _released.clear(); + _byId.clear(); + } + + List takeAllReleased() { + final values = List.from(_released.values); + _released.clear(); + return values; + } + + List takePendingPointers() => List.from(_pending.keys); + + List eligibleForClaim({ + required int nowMs, + required String? sessionId, + }) { + final eligible = []; + for (final tx in _pending.values) { + if (tx.isSwipeOrScroll) continue; + if (!tx.isEligible) continue; + if (tx.origin.captureSessionId != sessionId) continue; + eligible.add(tx); + } + for (final tx in _released.values) { + if (!tx.isEligible) continue; + if (!tx.isWithinReconciliationWindow(nowMs)) continue; + if (tx.origin.captureSessionId != sessionId) continue; + eligible.add(tx); + } + return eligible; + } + + int? earliestReleasedDeadlineMs() { + int? earliest; + for (final tx in _released.values) { + final deadline = tx.reconciliationDeadlineMs; + if (deadline == null) continue; + if (earliest == null || deadline < earliest) earliest = deadline; + } + return earliest; + } +} diff --git a/packages/tugboat/lib/src/models.dart b/packages/tugboat/lib/src/models.dart index b86b09e..4fd8854 100644 --- a/packages/tugboat/lib/src/models.dart +++ b/packages/tugboat/lib/src/models.dart @@ -6,8 +6,80 @@ import 'package:flutter/widgets.dart'; import 'anchors.dart'; import 'collector_config.dart'; -/// Current session JSON schema. Writers emit this; readers accept 6 and 7. -const int tugboatSessionSchemaVersion = 7; +/// Current session JSON schema. Writers emit this; readers accept 6–8. +const int tugboatSessionSchemaVersion = 8; + +/// Event selection channel for enrichment / insight / replay consumers. +enum TugboatEventStream { + /// Default enrichment stream — prefer `type: interaction`. + semantic, + + /// Route/state/scroll/pointer observations linked to interactions. + evidence, + + /// Capture health and support diagnostics. + diagnostic, + + /// Temporary dual-write of legacy `tap` / `tap_settled` / `swipe` peers. + legacyProjection; + + String get wireName => switch (this) { + TugboatEventStream.semantic => 'semantic', + TugboatEventStream.evidence => 'evidence', + TugboatEventStream.diagnostic => 'diagnostic', + TugboatEventStream.legacyProjection => 'legacy_projection', + }; + + static TugboatEventStream parse(String? raw) { + switch (raw) { + case 'evidence': + return TugboatEventStream.evidence; + case 'diagnostic': + return TugboatEventStream.diagnostic; + case 'legacy_projection': + return TugboatEventStream.legacyProjection; + case 'semantic': + case null: + default: + return TugboatEventStream.semantic; + } + } +} + +/// How finalized gestures are published to sinks. +enum TugboatInteractionPublishMode { + /// Only legacy `tap` / `tap_settled` / `swipe` on the semantic stream. + legacyOnly, + + /// Canonical `interaction` plus legacy peers on [TugboatEventStream.legacyProjection]. + dualWrite, + + /// Canonical `interaction` only. + canonicalOnly, +} + +/// Wire-compatible string aliases for tests and docs. +const String tugboatEventStreamSemantic = 'semantic'; +const String tugboatEventStreamEvidence = 'evidence'; +const String tugboatEventStreamDiagnostic = 'diagnostic'; +const String tugboatEventStreamLegacyProjection = 'legacy_projection'; + +const int tugboatInteractionSchemaVersion = 1; + +/// Whether [event] is a default enrichment / insight candidate. +bool tugboatEventIsEnrichmentCandidate(TugboatEvent event) { + switch (event.stream) { + case TugboatEventStream.diagnostic: + case TugboatEventStream.evidence: + case TugboatEventStream.legacyProjection: + return false; + case TugboatEventStream.semantic: + if (event.type == 'interaction') return true; + return event.type == 'tap' || + event.type == 'tap_settled' || + event.type == 'swipe'; + } +} class TugboatRect { const TugboatRect(this.x, this.y, this.width, this.height); @@ -109,6 +181,7 @@ class TugboatEvent { required this.id, required this.atMs, required this.type, + this.stream = TugboatEventStream.semantic, this.sessionId, this.captureSessionId, this.activationRequestId, @@ -126,6 +199,7 @@ class TugboatEvent { final String id; final int atMs; final String type; + final TugboatEventStream stream; /// Legacy alias for [captureSessionId]. final String? sessionId; @@ -143,10 +217,15 @@ class TugboatEvent { String? get effectiveCaptureSessionId => captureSessionId ?? sessionId; + bool get isSemanticStream => stream == TugboatEventStream.semantic; + + bool get isEnrichmentCandidate => tugboatEventIsEnrichmentCandidate(this); + Map toJson() => { 'id': id, 'atMs': atMs, 'type': type, + 'stream': stream.wireName, if (sessionId != null) 'sessionId': sessionId, if (captureSessionId != null) 'captureSessionId': captureSessionId, if (activationRequestId != null) 'activationRequestId': activationRequestId, @@ -161,26 +240,55 @@ class TugboatEvent { if (actionId != null) 'actionId': actionId, }; - TugboatEvent withExplorationContext({ + TugboatEvent copyWith({ + String? id, + int? atMs, + String? type, + TugboatEventStream? stream, String? sessionId, String? captureSessionId, String? activationRequestId, + TugboatStateAnchor? stateAnchor, + TugboatTargetAnchor? targetAnchor, + String? beforeFrame, + String? afterFrame, + TugboatInteractionResult? result, + String? relatedEventId, + Map? data, String? explorationRunId, String? actionId, }) => TugboatEvent( - id: id, - atMs: atMs, - type: type, + id: id ?? this.id, + atMs: atMs ?? this.atMs, + type: type ?? this.type, + stream: stream ?? this.stream, + sessionId: sessionId ?? this.sessionId, + captureSessionId: captureSessionId ?? this.captureSessionId, + activationRequestId: activationRequestId ?? this.activationRequestId, + stateAnchor: stateAnchor ?? this.stateAnchor, + targetAnchor: targetAnchor ?? this.targetAnchor, + beforeFrame: beforeFrame ?? this.beforeFrame, + afterFrame: afterFrame ?? this.afterFrame, + result: result ?? this.result, + relatedEventId: relatedEventId ?? this.relatedEventId, + data: data ?? this.data, + explorationRunId: explorationRunId ?? this.explorationRunId, + actionId: actionId ?? this.actionId, + ); + + TugboatEvent withData(Map updates) => + copyWith(data: {...data, ...updates}); + + TugboatEvent withExplorationContext({ + String? sessionId, + String? captureSessionId, + String? activationRequestId, + String? explorationRunId, + String? actionId, + }) => copyWith( sessionId: sessionId ?? this.sessionId, captureSessionId: captureSessionId ?? this.captureSessionId, activationRequestId: activationRequestId ?? this.activationRequestId, - stateAnchor: stateAnchor, - targetAnchor: targetAnchor, - beforeFrame: beforeFrame, - afterFrame: afterFrame, - result: result, - relatedEventId: relatedEventId, - data: data, explorationRunId: explorationRunId ?? this.explorationRunId, actionId: actionId ?? this.actionId, ); diff --git a/packages/tugboat/lib/src/replay_config.dart b/packages/tugboat/lib/src/replay_config.dart index fccc077..6bc7dde 100644 --- a/packages/tugboat/lib/src/replay_config.dart +++ b/packages/tugboat/lib/src/replay_config.dart @@ -1,5 +1,7 @@ import 'capture_profile.dart'; import 'collector_config.dart'; +import 'interaction_transaction.dart' show tugboatDefaultReconciliationWindow; +import 'models.dart'; import 'outbox/outbox.dart'; import 'screenshot_mask_level.dart'; import 'sinks/capture_sink.dart' show TugboatCaptureSinkFactory; @@ -24,20 +26,12 @@ class TugboatViewportSemanticPolicy { holdPersistentSemanticsHandle: false, ); - /// Whether maps are built (for tap resolution and/or emission). final bool engineEnabled; - - /// Whether `viewport_semantic_map` / `scroll_semantic_snapshot` events emit. final bool emitEvents; - - /// Whether diagnostic prints are enabled. final bool debugLogs; - - /// Hold Flutter [SemanticsHandle] for the whole session (exploration only). final bool holdPersistentSemanticsHandle; } -/// Derives [TugboatViewportSemanticPolicy] from profile + mode. TugboatViewportSemanticPolicy resolveViewportSemanticPolicy({ required TugboatCaptureProfile profile, required TugboatViewportSemanticMode mode, @@ -70,7 +64,6 @@ TugboatViewportSemanticPolicy resolveViewportSemanticPolicy({ ); } -/// Rolling screenshot budget policy. class TugboatScreenshotBudgetConfig { const TugboatScreenshotBudgetConfig({ this.window = const Duration(seconds: 5), @@ -90,6 +83,8 @@ class TugboatReplayConfig { const TugboatReplayConfig({ this.profile = TugboatCaptureProfile.dormant, this.settleDelay = const Duration(seconds: 1), + this.interactionClaimWindow = tugboatDefaultReconciliationWindow, + this.interactionPublishMode = TugboatInteractionPublishMode.dualWrite, this.maxFrames = 500, this.maxEvents = 5000, this.scrollCaptureInterval = const Duration(seconds: 2), @@ -113,6 +108,27 @@ class TugboatReplayConfig { final TugboatCaptureProfile profile; final Duration settleDelay; + + /// Released-tap window for delayed route/modal attribution. + /// + /// Default is [tugboatDefaultReconciliationWindow] (1,250 ms). Set to + /// [Duration.zero] for microtask-only same-turn claims. + final Duration interactionClaimWindow; + + /// Canonical vs legacy gesture publication policy. + final TugboatInteractionPublishMode interactionPublishMode; + + bool get emitCanonicalInteractions => + interactionPublishMode != TugboatInteractionPublishMode.legacyOnly; + + bool get emitLegacyInteractionProjection => + interactionPublishMode != TugboatInteractionPublishMode.canonicalOnly; + + TugboatEventStream get legacyGestureStream => + interactionPublishMode == TugboatInteractionPublishMode.dualWrite + ? TugboatEventStream.legacyProjection + : TugboatEventStream.semantic; + final int maxFrames; final int maxEvents; final Duration scrollCaptureInterval; @@ -151,6 +167,8 @@ class TugboatReplayConfig { TugboatReplayConfig copyWith({ TugboatCaptureProfile? profile, Duration? settleDelay, + Duration? interactionClaimWindow, + TugboatInteractionPublishMode? interactionPublishMode, int? maxFrames, int? maxEvents, Duration? scrollCaptureInterval, @@ -174,6 +192,10 @@ class TugboatReplayConfig { return TugboatReplayConfig( profile: profile ?? this.profile, settleDelay: settleDelay ?? this.settleDelay, + interactionClaimWindow: + interactionClaimWindow ?? this.interactionClaimWindow, + interactionPublishMode: + interactionPublishMode ?? this.interactionPublishMode, maxFrames: maxFrames ?? this.maxFrames, maxEvents: maxEvents ?? this.maxEvents, scrollCaptureInterval: diff --git a/packages/tugboat/lib/src/sdk_version.dart b/packages/tugboat/lib/src/sdk_version.dart index fb3e18a..95379a8 100644 --- a/packages/tugboat/lib/src/sdk_version.dart +++ b/packages/tugboat/lib/src/sdk_version.dart @@ -1,3 +1,3 @@ // Keep this in sync with packages/tugboat/pubspec.yaml. The SDK version test // reads pubspec.yaml directly so release bumps fail fast if this drifts. -const tugboatSdkVersion = '0.4.12'; +const tugboatSdkVersion = '0.4.15'; diff --git a/packages/tugboat/lib/tugboat.dart b/packages/tugboat/lib/tugboat.dart index 8c8ecba..d50bf8b 100644 --- a/packages/tugboat/lib/tugboat.dart +++ b/packages/tugboat/lib/tugboat.dart @@ -31,6 +31,7 @@ export 'src/health.dart' TugboatSanitizedFailure; export 'src/lifecycle.dart' show TugboatLifecycleState, TugboatLifecycleNotifier; +export 'src/interaction_transaction.dart' show tugboatDefaultReconciliationWindow; export 'src/models.dart'; export 'src/coordinate_space.dart' show diff --git a/packages/tugboat/pubspec.yaml b/packages/tugboat/pubspec.yaml index 40f64c7..0e3ab20 100644 --- a/packages/tugboat/pubspec.yaml +++ b/packages/tugboat/pubspec.yaml @@ -1,7 +1,7 @@ name: tugboat description: >- Screenshot-based session replay with compact interaction anchors for Tugboat. -version: 0.4.12 +version: 0.4.15 repository: https://github.com/blendto/tugboat-flutter issue_tracker: https://github.com/blendto/tugboat-flutter/issues homepage: https://github.com/blendto/tugboat-flutter diff --git a/packages/tugboat/test/collector_mapper_test.dart b/packages/tugboat/test/collector_mapper_test.dart index 7399764..c5d4033 100644 --- a/packages/tugboat/test/collector_mapper_test.dart +++ b/packages/tugboat/test/collector_mapper_test.dart @@ -72,6 +72,9 @@ void main() { expect(mapped['sessionId'], 'sess_123'); expect(mapped['userId'], 'user_1'); expect(mapped['eventType'], 'tap'); + expect(mapped['stream'], tugboatEventStreamSemantic); + // Compat path: semantic tap without canonical dual-write remains eligible. + expect(mapped['enrichmentCandidate'], isTrue); expect(mapped['beforeFrame'], 'frame-3'); expect((mapped['stateAnchor'] as Map)['signature'], '23f17a629520d522'); expect((mapped['targetAnchor'] as Map)['fingerprint'], '9eadb7c56ae836bc'); @@ -89,6 +92,44 @@ void main() { }); }); + test('marks legacy projection and evidence as non-enrichment candidates', () { + final sessionStartedAt = DateTime.utc(2026, 6, 19); + final legacy = mapTugboatEventToCollectorEvent( + event: const TugboatEvent( + id: 'event-legacy', + atMs: 1, + type: 'tap_settled', + stream: TugboatEventStream.legacyProjection, + ), + sessionStartedAt: sessionStartedAt, + collectorConfig: collectorConfig, + ); + final evidence = mapTugboatEventToCollectorEvent( + event: const TugboatEvent( + id: 'event-route', + atMs: 2, + type: 'route_change', + stream: TugboatEventStream.evidence, + ), + sessionStartedAt: sessionStartedAt, + collectorConfig: collectorConfig, + ); + final interaction = mapTugboatEventToCollectorEvent( + event: const TugboatEvent( + id: 'event-interaction', + atMs: 3, + type: 'interaction', + stream: TugboatEventStream.semantic, + ), + sessionStartedAt: sessionStartedAt, + collectorConfig: collectorConfig, + ); + + expect(legacy['enrichmentCandidate'], isFalse); + expect(evidence['enrichmentCandidate'], isFalse); + expect(interaction['enrichmentCandidate'], isTrue); + }); + test('omits sessionId when not provided so the sink can stamp at send', () { final mapped = mapTugboatEventToCollectorEvent( event: TugboatEvent(id: 'event-1', atMs: 0, type: 'tap'), diff --git a/packages/tugboat/test/coordinate_space_test.dart b/packages/tugboat/test/coordinate_space_test.dart index 004b7e3..43a8013 100644 --- a/packages/tugboat/test/coordinate_space_test.dart +++ b/packages/tugboat/test/coordinate_space_test.dart @@ -84,9 +84,37 @@ void main() { expect(coord.projectToRaster(), isNull); }); - test('missing frame yields unavailable transform', () { + test('missing frame yields positioned unavailable transform', () { final coord = buildCaptureCoordinate( - globalX: 10, + globalX: 40, + globalY: 60, + boundaryOriginX: 0, + boundaryOriginY: 0, + boundaryWidth: 100, + boundaryHeight: 200, + framePixelWidth: 0, + framePixelHeight: 0, + frameId: null, + boundaryTransformGeneration: 1, + ); + expect(coord.unavailableReason, 'missing_frame'); + expect(coord.isAvailable, isFalse); + expect( + coord.sourceSpace, + TugboatCoordinateSourceSpace.boundaryLocalLogical, + ); + expect(coord.localX, 40); + expect(coord.localY, 60); + expect(coord.normalizedX, 0.4); + expect(coord.normalizedY, 0.3); + expect(coord.boundaryWidth, 100); + expect(coord.boundaryHeight, 200); + expect(coord.projectToRaster(), isNull); + }); + + test('missing frame outside boundary prefers outside_boundary', () { + final coord = buildCaptureCoordinate( + globalX: -5, globalY: 10, boundaryOriginX: 0, boundaryOriginY: 0, @@ -97,7 +125,9 @@ void main() { frameId: null, boundaryTransformGeneration: 1, ); - expect(coord.unavailableReason, 'missing_frame'); + expect(coord.unavailableReason, 'outside_boundary'); + expect(coord.localX, -5); + expect(coord.localY, 10); }); test('golden fixture freezes the consumer contract', () { diff --git a/packages/tugboat/test/helpers/json_roundtrip.dart b/packages/tugboat/test/helpers/json_roundtrip.dart index ffbc880..9f7513a 100644 --- a/packages/tugboat/test/helpers/json_roundtrip.dart +++ b/packages/tugboat/test/helpers/json_roundtrip.dart @@ -84,6 +84,7 @@ extension TugboatEventTestJson on TugboatEvent { id: json['id'] as String, atMs: json['atMs'] as int, type: json['type'] as String, + stream: TugboatEventStream.parse(json['stream'] as String?), sessionId: json['sessionId'] as String?, captureSessionId: json['captureSessionId'] as String?, activationRequestId: json['activationRequestId'] as String?, @@ -112,7 +113,7 @@ extension TugboatEventTestJson on TugboatEvent { extension TugboatSessionTestJson on TugboatSession { static TugboatSession fromJson(Map json) { final version = json['schemaVersion'] as int?; - if (version != 6 && version != 7) { + if (version != 6 && version != 7 && version != 8) { throw const FormatException( 'Unsupported Tugboat session schema version.', ); diff --git a/packages/tugboat/test/helpers/replay_coherence_harness.dart b/packages/tugboat/test/helpers/replay_coherence_harness.dart index eb9d6b4..e3475a0 100644 --- a/packages/tugboat/test/helpers/replay_coherence_harness.dart +++ b/packages/tugboat/test/helpers/replay_coherence_harness.dart @@ -208,12 +208,19 @@ class ControllableCaptureExecutor { class ReplayCoherenceHarness { ReplayCoherenceHarness({ this.settleDelay = Duration.zero, + + /// Characterization defaults to microtask-only claims so automatic routes + /// during settle stay independent. Production defaults to 1,250 ms delayed + /// reconciliation — pass that window explicitly when testing delayed + /// attribution. + this.interactionClaimWindow = Duration.zero, this.maxFrames = 300, this.screenshotBudget = TugboatScreenshotBudgetConfig.defaults, GlobalKey? boundaryKey, }) : boundaryKey = boundaryKey ?? GlobalKey(); final Duration settleDelay; + final Duration interactionClaimWindow; final int maxFrames; final TugboatScreenshotBudgetConfig screenshotBudget; final GlobalKey boundaryKey; @@ -258,6 +265,7 @@ class ReplayCoherenceHarness { config: TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: settleDelay, + interactionClaimWindow: interactionClaimWindow, maxFrames: maxFrames, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, diff --git a/packages/tugboat/test/integration/release_compatibility_matrix_test.dart b/packages/tugboat/test/integration/release_compatibility_matrix_test.dart index a57a8ed..9148716 100644 --- a/packages/tugboat/test/integration/release_compatibility_matrix_test.dart +++ b/packages/tugboat/test/integration/release_compatibility_matrix_test.dart @@ -24,6 +24,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, ), child: child!, @@ -75,6 +76,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, ), child: child!, @@ -113,7 +115,7 @@ void main() { }); test('v6 session JSON remains readable alongside v7 writers', () { - expect(tugboatSessionSchemaVersion, 7); + expect(tugboatSessionSchemaVersion, 8); }); test( @@ -133,6 +135,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.dormant, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, ), child: child!, @@ -180,6 +183,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: true, capturePixelRatio: 1, screenshotMaskLevel: TugboatScreenshotMaskLevel.allTextAndMedia, diff --git a/packages/tugboat/test/replay/deferred_tap_emission_test.dart b/packages/tugboat/test/replay/deferred_tap_emission_test.dart new file mode 100644 index 0000000..07f4486 --- /dev/null +++ b/packages/tugboat/test/replay/deferred_tap_emission_test.dart @@ -0,0 +1,79 @@ +import 'package:flutter_test/flutter_test.dart'; + +import '../helpers/replay_coherence_harness.dart'; + +void main() { + test('pointer-down buffers tap until pointer-up', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(20, 30)); + expect(harness.controller.session!.ofType('tap'), isEmpty); + + harness.controller.recordPointerUp(const Offset(20, 30)); + final tap = harness.controller.session!.ofType('tap').single; + expect(tap.data['x'], 20.0); + expect(tap.data['y'], 30.0); + expect(tap.data['captureCoordinate'], isA()); + }); + + test( + 'swipe emits without phantom tap and carries startCaptureCoordinate', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(10, 100)); + harness.controller.markPendingTapAsSwipe(0); + harness.controller.recordPointerUp(const Offset(10, 40)); + + expect(harness.controller.session!.ofType('tap'), isEmpty); + final swipe = harness.controller.session!.ofType('swipe').single; + expect(swipe.relatedEventId, isNull); + expect(swipe.data['startCaptureCoordinate'], isA()); + final start = Map.from( + swipe.data['startCaptureCoordinate']! as Map, + ); + expect( + start.containsKey('unavailableReason') || start['sourceSpace'] != null, + isTrue, + ); + }, + ); + + test('cancel drops buffered tap', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(4, 4)); + harness.controller.recordPointerCancel(const Offset(4, 4)); + + expect(harness.controller.session!.ofType('tap'), isEmpty); + expect(harness.controller.session!.ofType('pointer_cancel'), hasLength(1)); + }); + + test( + 'claimed-then-swiped tap still emits so causeEventId resolves', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(8, 8)); + await harness.controller.route('route_push', harness.route('/claimed')); + await harness.pumpMicrotasks(); + + final tap = harness.controller.session!.ofType('tap').single; + harness.controller.markPendingTapAsSwipe(0); + harness.controller.recordPointerUp(const Offset(8, 80)); + + expect(harness.controller.session!.ofType('tap'), hasLength(1)); + final swipe = harness.controller.session!.ofType('swipe').single; + expect(swipe.relatedEventId, tap.id); + expect(swipe.data['startCaptureCoordinate'], isA()); + }, + ); +} diff --git a/packages/tugboat/test/replay/interaction_transaction_test.dart b/packages/tugboat/test/replay/interaction_transaction_test.dart new file mode 100644 index 0000000..d8fb0dc --- /dev/null +++ b/packages/tugboat/test/replay/interaction_transaction_test.dart @@ -0,0 +1,395 @@ +import 'dart:convert'; + +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:tugboat/tugboat.dart'; + +import '../helpers/replay_coherence_harness.dart'; + +Map _roundTrip(Map json) => + Map.from(jsonDecode(jsonEncode(json)) as Map); + +extension on TugboatSession { + List semanticOfType(String type) => events + .where((e) => e.type == type && e.stream == TugboatEventStream.semantic) + .toList(growable: false); + + List ofStream(TugboatEventStream stream) => + events.where((e) => e.stream == stream).toList(growable: false); +} + +void main() { + group('InteractionTransaction origin freeze (U1)', () { + test( + 'origin screen/component survive route mutation before pointer-up', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.debugSetCurrentRoute('/origin'); + harness.controller.debugFreezeStateAnchor = true; + harness.controller.debugSetCurrentStateAnchor( + const TugboatStateAnchor( + signature: 'origin-sig', + signatureConfidence: 'high', + signatureParts: {'route': '/origin'}, + ), + ); + + harness.controller.recordPointerDown(const Offset(12, 34)); + await harness.controller.route('route_push', harness.route('/dest')); + await harness.flushScheduler(); + harness.controller.recordPointerUp(const Offset(12, 34)); + await harness.flushScheduler(); + + final interaction = harness.controller.session! + .semanticOfType('interaction') + .single; + final origin = Map.from( + interaction.data['origin']! as Map, + ); + expect(origin['route'], '/origin'); + final state = Map.from(origin['stateAnchor']! as Map); + expect(state['signature'], 'origin-sig'); + expect(interaction.targetAnchor?.fingerprint, isNull); + }, + ); + + test('lifecycle cancel finalizes without stranded transaction', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(4, 4)); + harness.controller.recordPointerCancel(const Offset(4, 4)); + await harness.flushScheduler(); + + final interactions = harness.controller.session!.semanticOfType( + 'interaction', + ); + expect(interactions, hasLength(1)); + expect(interactions.single.data['gesture'], 'cancelled'); + expect(harness.controller.session!.ofType('tap'), isEmpty); + }); + + test('duplicate pointer-down cancels prior and keeps one tap', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(10, 10)); + harness.controller.recordPointerDown(const Offset(20, 20)); + harness.controller.recordPointerUp(const Offset(20, 20)); + await harness.flushScheduler(); + + final interactions = harness.controller.session!.semanticOfType( + 'interaction', + ); + expect(interactions, hasLength(2)); + expect( + interactions.map((e) => e.data['gesture']), + containsAll(['cancelled', 'tap']), + ); + final tap = interactions.singleWhere((e) => e.data['gesture'] == 'tap'); + final origin = Map.from(tap.data['origin']! as Map); + expect( + Map.from(origin['startPosition']! as Map)['x'], + 20.0, + ); + }); + + test( + 'lifecycle clear of released claim publishes cancelled interaction', + () async { + final harness = ReplayCoherenceHarness( + interactionClaimWindow: const Duration(milliseconds: 1250), + ); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(4, 4)); + harness.controller.recordPointerUp(const Offset(4, 4)); + // Still reconciling — backgrounding must terminalize the released tx. + harness.controller.recordAppLifecycleState(AppLifecycleState.paused); + await harness.flushScheduler(); + + final cancelled = harness.controller.session! + .semanticOfType('interaction') + .where((e) => e.data['gesture'] == 'cancelled'); + expect(cancelled, isNotEmpty); + }, + ); + }); + + group('Delayed reconciliation (U2)', () { + test( + 'delayed route within claim window attributes as delayed_likely', + () async { + final harness = ReplayCoherenceHarness( + interactionClaimWindow: const Duration(milliseconds: 1250), + ); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + harness.controller.recordPointerUp(const Offset(12, 34)); + await harness.pumpMicrotasks(); + + harness.scheduler.advance(const Duration(milliseconds: 500)); + await harness.controller.route('route_push', harness.route('/delayed')); + await harness.flushScheduler(); + + final change = harness.controller.session! + .ofType('route_change') + .lastWhere((e) => e.data['route'] == '/delayed'); + expect(change.data['navigationOrigin'], 'interaction'); + expect(change.data['interactionAttribution'], 'delayed_likely'); + expect(change.data['causedByInteractionId'], isNotNull); + expect( + change.data['causeEventId'], + change.data['causedByInteractionId'], + ); + + final interaction = harness.controller.session! + .semanticOfType('interaction') + .single; + final attribution = Map.from( + interaction.data['attribution']! as Map, + ); + expect(attribution['kind'], anyOf('direct', 'delayed_likely')); + final result = Map.from( + interaction.data['result']! as Map, + ); + expect(result['status'], anyOf('navigated', 'changed', 'unknown')); + }, + ); + + test('route after claim window remains automatic', () async { + final harness = ReplayCoherenceHarness( + interactionClaimWindow: const Duration(milliseconds: 1250), + ); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + harness.controller.recordPointerUp(const Offset(12, 34)); + await harness.pumpMicrotasks(); + + harness.scheduler.advance(const Duration(milliseconds: 1300)); + await harness.pumpMicrotasks(); + await harness.controller.route('route_push', harness.route('/late')); + await harness.flushScheduler(); + + final change = harness.controller.session! + .ofType('route_change') + .lastWhere((e) => e.data['route'] == '/late'); + expect(change.data['navigationOrigin'], 'automatic_or_unknown'); + expect(change.data['causeEventId'], isNull); + expect(change.data['causedByInteractionId'], isNull); + }); + + test('two rapid taps cannot claim the same route twice', () async { + final harness = ReplayCoherenceHarness( + interactionClaimWindow: const Duration(milliseconds: 1250), + ); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(10, 10), pointer: 1); + harness.controller.recordPointerUp(const Offset(10, 10), pointer: 1); + harness.controller.recordPointerDown(const Offset(20, 20), pointer: 2); + harness.controller.recordPointerUp(const Offset(20, 20), pointer: 2); + + await harness.controller.route('route_push', harness.route('/only-one')); + await harness.flushScheduler(); + + final change = harness.controller.session! + .ofType('route_change') + .lastWhere((e) => e.data['route'] == '/only-one'); + expect(change.data['navigationOrigin'], 'automatic_or_unknown'); + expect(change.data['causeEventId'], isNull); + }); + }); + + group('Gesture classification (U3)', () { + test('swipe emits zero semantic taps and one interaction', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(10, 100)); + harness.controller.markPendingTapAsSwipe(0); + harness.controller.recordPointerUp(const Offset(10, 40)); + await harness.flushScheduler(); + + expect(harness.controller.session!.ofType('tap'), isEmpty); + expect(harness.controller.session!.ofType('tap_settled'), isEmpty); + final interactions = harness.controller.session!.semanticOfType( + 'interaction', + ); + expect(interactions, hasLength(1)); + expect(interactions.single.data['gesture'], anyOf('swipe', 'scroll')); + final origin = Map.from( + interactions.single.data['origin']! as Map, + ); + expect(origin['captureCoordinate'], isA()); + }); + + test('sub-slop movement remains one tap interaction', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(10, 10)); + harness.controller.recordPointerUp(const Offset(12, 11)); + await harness.flushScheduler(); + + final interactions = harness.controller.session!.semanticOfType( + 'interaction', + ); + expect(interactions, hasLength(1)); + expect(interactions.single.data['gesture'], 'tap'); + }); + }); + + group('Canonical publish and diagnostic isolation (U4)', () { + test('serialization round-trip preserves origin and result', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(8, 8)); + harness.controller.recordPointerUp(const Offset(8, 8)); + await harness.flushScheduler(); + + final interaction = harness.controller.session! + .semanticOfType('interaction') + .single; + final json = _roundTrip(interaction.toJson()); + expect(json['type'], 'interaction'); + expect(json['stream'], tugboatEventStreamSemantic); + final data = Map.from(json['data']! as Map); + expect(data['interactionSchema'], tugboatInteractionSchemaVersion); + expect(data['origin'], isA()); + expect(data['result'], isA()); + expect(data['attribution'], isA()); + }); + + test('legacy peers are dual-written on legacy_projection stream', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(8, 8)); + harness.controller.recordPointerUp(const Offset(8, 8)); + await harness.flushScheduler(); + + final taps = harness.controller.session!.ofType('tap'); + final settles = harness.controller.session!.ofType('tap_settled'); + expect(taps, hasLength(1)); + expect(settles, hasLength(1)); + expect(taps.single.stream, TugboatEventStream.legacyProjection); + expect(settles.single.stream, TugboatEventStream.legacyProjection); + expect(taps.single.data['interactionId'], isNotNull); + expect( + settles.single.data['interactionId'], + taps.single.data['interactionId'], + ); + + final semantic = harness.controller.session!.ofStream( + TugboatEventStream.semantic, + ); + expect(semantic.where((e) => e.type == 'tap'), isEmpty); + expect(semantic.where((e) => e.type == 'tap_settled'), isEmpty); + expect(semantic.where((e) => e.type == 'interaction'), hasLength(1)); + }); + + test('ten gestures publish ten canonical semantic interactions', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + for (var i = 0; i < 10; i++) { + final point = Offset(10.0 + i, 10.0); + harness.controller.recordPointerDown(point); + harness.controller.recordPointerUp(point); + } + await harness.flushScheduler(); + + expect( + harness.controller.session!.semanticOfType('interaction'), + hasLength(10), + ); + }); + + test('capture diagnostics use diagnostic stream', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + final request = harness.controller.debugRequestCapture( + trigger: TugboatFrameTrigger.manual, + force: true, + ); + await request.resolution; + await harness.flushScheduler(); + + final diagnostics = harness.controller.session!.ofType( + 'capture_diagnostic', + ); + expect(diagnostics, isNotEmpty); + expect( + diagnostics.every((e) => e.stream == TugboatEventStream.diagnostic), + isTrue, + ); + expect( + harness.controller.session! + .ofStream(TugboatEventStream.semantic) + .where((e) => e.type == 'capture_diagnostic'), + isEmpty, + ); + }); + + test( + 'route and state evidence are not semantic enrichment peers', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(8, 8)); + await harness.controller.route('route_push', harness.route('/dest')); + harness.controller.recordPointerUp(const Offset(8, 8)); + await harness.flushScheduler(); + + final routes = harness.controller.session!.ofType('route_change'); + expect(routes, isNotEmpty); + expect( + routes.every((e) => e.stream == TugboatEventStream.evidence), + isTrue, + ); + expect(routes.every((e) => !e.isEnrichmentCandidate), isTrue); + + final semantic = harness.controller.session!.ofStream( + TugboatEventStream.semantic, + ); + expect(semantic.where((e) => e.type == 'route_change'), isEmpty); + expect(semantic.where((e) => e.type == 'interaction'), hasLength(1)); + expect( + semantic + .singleWhere((e) => e.type == 'interaction') + .isEnrichmentCandidate, + isTrue, + ); + expect( + harness.controller.session! + .ofType('tap') + .single + .isEnrichmentCandidate, + isFalse, + ); + }, + ); + }); +} diff --git a/packages/tugboat/test/replay/modal_capture_visual_test.dart b/packages/tugboat/test/replay/modal_capture_visual_test.dart index 8767163..7eda2c9 100644 --- a/packages/tugboat/test/replay/modal_capture_visual_test.dart +++ b/packages/tugboat/test/replay/modal_capture_visual_test.dart @@ -357,6 +357,7 @@ const _openNestedSheet = Key('modal-open-nested-sheet'); const _config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: true, capturePixelRatio: 1, screenshotMaskLevel: TugboatScreenshotMaskLevel.explicitOnly, diff --git a/packages/tugboat/test/replay/navigation_origin_contract_test.dart b/packages/tugboat/test/replay/navigation_origin_contract_test.dart index f3bdc9b..3bab1d9 100644 --- a/packages/tugboat/test/replay/navigation_origin_contract_test.dart +++ b/packages/tugboat/test/replay/navigation_origin_contract_test.dart @@ -1,5 +1,7 @@ +import 'dart:async'; import 'dart:convert'; +import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:tugboat/tugboat.dart'; @@ -48,26 +50,52 @@ void main() { addTearDown(harness.dispose); harness.controller.recordPointerDown(const Offset(12, 34)); - final tap = harness.controller.session!.ofType('tap').single; + expect(harness.controller.session!.ofType('tap'), isEmpty); await harness.controller.route('route_push', harness.route('/dest')); await harness.flushScheduler(); + final tap = harness.controller.session!.ofType('tap').single; final change = harness.controller.session! .ofType('route_change') .lastWhere((e) => e.data['route'] == '/dest'); expect(change.data['navigationOrigin'], 'interaction'); expect(change.data['causeEventId'], tap.id); + expect(change.data['interactionAttribution'], 'same_turn'); }); - test('timer redirect after tap settle has no causal event id', () async { + test( + 'same-turn claim attributes navigation during pointer-up turn', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + harness.controller.recordPointerUp(const Offset(12, 34)); + // Still in the pointer-up turn — sync onTap → Navigator can claim. + await harness.controller.route('route_push', harness.route('/async')); + await harness.flushScheduler(); + + final tap = harness.controller.session!.ofType('tap').single; + final change = harness.controller.session! + .ofType('route_change') + .lastWhere((e) => e.data['route'] == '/async'); + expect(change.data['navigationOrigin'], 'interaction'); + expect(change.data['causeEventId'], tap.id); + expect(change.data['interactionAttribution'], 'same_turn'); + }, + ); + + test('timer redirect after pointer-up turn has no causal event id', () async { final harness = ReplayCoherenceHarness(); await harness.setUp(); addTearDown(harness.dispose); harness.controller.recordPointerDown(const Offset(12, 34)); harness.controller.recordPointerUp(const Offset(12, 34)); - await harness.flushScheduler(); + // Expire the released same-turn claim. + await harness.pumpMicrotasks(); await harness.controller.route('route_push', harness.route('/redirect')); await harness.flushScheduler(); @@ -79,6 +107,161 @@ void main() { expect(change.data['causeEventId'], isNull); }); + test('pre-up claimed route then swipe keeps tap causal_only', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 80)); + await harness.controller.route('route_push', harness.route('/claimed')); + await harness.flushScheduler(); + expect(harness.controller.session!.ofType('tap'), isNotEmpty); + + harness.controller.markPendingTapAsSwipe(0); + harness.controller.recordPointerUp(const Offset(40, 80)); + await harness.flushScheduler(); + + final tap = harness.controller.session!.ofType('tap').single; + expect(tap.data['replayRole'], 'causal_only'); + expect(tap.data['gestureFinal'], anyOf('swipe', 'unresolved')); + final swipe = harness.controller.session!.ofType('swipe').single; + expect(swipe.data['invalidatesRelatedTap'], isTrue); + }); + + test('pointer events after session_end are ignored', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + await harness.controller.endSession(); + final before = harness.controller.session!.events.length; + + harness.controller.recordPointerUp(const Offset(12, 34)); + harness.controller.recordPointerDown(const Offset(50, 50)); + harness.controller.recordPointerCancel(const Offset(50, 50)); + + expect(harness.controller.session!.events.length, before); + expect(harness.controller.session!.ofType('swipe'), isEmpty); + }); + + test( + 'session_end does not emit orphan causal_only for cancelled route claim', + () async { + final harness = ReplayCoherenceHarness( + settleDelay: const Duration(milliseconds: 100), + ); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + // Claim during down; do not await — cancel via session_end before publish. + unawaited( + harness.controller.route('route_push', harness.route('/claimed')), + ); + await harness.pumpMicrotasks(); + await harness.controller.endSession(); + await harness.flushScheduler(); + + expect(harness.controller.session!.ofType('tap'), isEmpty); + expect( + harness.controller.session! + .ofType('route_change') + .where((e) => e.data['route'] == '/claimed'), + isEmpty, + ); + }, + ); + + test( + 'backgrounding drops pending claims so resume cannot attribute them', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + harness.controller.recordAppLifecycleState(AppLifecycleState.paused); + harness.controller.recordAppLifecycleState(AppLifecycleState.resumed); + + await harness.controller.route('route_push', harness.route('/after')); + await harness.flushScheduler(); + + final change = harness.controller.session! + .ofType('route_change') + .lastWhere((e) => e.data['route'] == '/after'); + expect(change.data['navigationOrigin'], 'automatic_or_unknown'); + expect(change.data['causeEventId'], isNull); + expect(harness.controller.session!.ofType('tap'), isEmpty); + }, + ); + + test('duplicate pointer-down abandons the prior pending claim', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(10, 10)); + final firstBuffered = harness.controller.session!.ofType( + 'tap', + ); // still deferred + expect(firstBuffered, isEmpty); + + harness.controller.recordPointerDown(const Offset(20, 20)); + harness.controller.recordPointerUp(const Offset(20, 20)); + await harness.flushScheduler(); + + final taps = harness.controller.session!.ofType('tap'); + expect(taps, hasLength(1)); + expect(taps.single.data['x'], 20); + expect(taps.single.data['y'], 20); + }); + + test('deferred tap publish uses emission-time atMs for chronology', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + harness.scheduler.advance(const Duration(milliseconds: 40)); + await harness.controller.route('route_push', harness.route('/dest')); + await harness.flushScheduler(); + + final events = harness.controller.session!.events; + for (var i = 1; i < events.length; i++) { + expect(events[i].atMs, greaterThanOrEqualTo(events[i - 1].atMs)); + } + final tap = harness.controller.session!.ofType('tap').single; + expect(tap.data['sampledAtMs'], isA()); + expect(tap.atMs, greaterThanOrEqualTo(tap.data['sampledAtMs'] as int)); + }); + + test('pointer-up promotes causal_only tap after pre-up claim', () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + await harness.controller.route('route_push', harness.route('/dest')); + await harness.flushScheduler(); + expect( + harness.controller.session!.ofType('tap').single.data['replayRole'], + 'causal_only', + ); + + harness.controller.recordPointerUp(const Offset(12, 34)); + await harness.flushScheduler(); + + final tap = harness.controller.session!.ofType('tap').single; + expect(tap.data['replayRole'], 'interaction'); + expect(tap.data['promotedFrom'], 'causal_only'); + final resolved = harness.controller.session! + .ofType('tap_gesture_resolved') + .single; + expect(resolved.relatedEventId, tap.id); + expect(resolved.data['promotesRelatedTap'], isTrue); + }); + test('cancelled pointer cannot claim a route', () async { final harness = ReplayCoherenceHarness(); await harness.setUp(); @@ -86,6 +269,7 @@ void main() { harness.controller.recordPointerDown(const Offset(12, 34)); harness.controller.recordPointerCancel(const Offset(12, 34)); + expect(harness.controller.session!.ofType('tap'), isEmpty); await harness.controller.route('route_push', harness.route('/x')); await harness.flushScheduler(); @@ -130,10 +314,10 @@ void main() { addTearDown(harness.dispose); harness.controller.recordPointerDown(const Offset(12, 34)); - final tap = harness.controller.session!.ofType('tap').single; await harness.controller.route('route_push', harness.route('/first')); await harness.pumpMicrotasks(); + final tap = harness.controller.session!.ofType('tap').single; // Second navigation has no eligible unclaimed tap — cause was consumed. await harness.controller.route('route_push', harness.route('/second')); diff --git a/packages/tugboat/test/replay/replay_navigation_interaction_matrix_test.dart b/packages/tugboat/test/replay/replay_navigation_interaction_matrix_test.dart index d87266b..f7f398d 100644 --- a/packages/tugboat/test/replay/replay_navigation_interaction_matrix_test.dart +++ b/packages/tugboat/test/replay/replay_navigation_interaction_matrix_test.dart @@ -137,6 +137,7 @@ class _NavigationFixture { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: true, capturePixelRatio: 1, ), diff --git a/packages/tugboat/test/replay/replay_navigation_race_matrix_test.dart b/packages/tugboat/test/replay/replay_navigation_race_matrix_test.dart index 5c40d07..424a1e5 100644 --- a/packages/tugboat/test/replay/replay_navigation_race_matrix_test.dart +++ b/packages/tugboat/test/replay/replay_navigation_race_matrix_test.dart @@ -9,6 +9,7 @@ import '../helpers/replay_coherence_harness.dart'; const _config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: true, capturePixelRatio: 1, ); @@ -303,6 +304,59 @@ void main() { expect(CoherenceInvariants.hasNoStrandedCaptureWork(harness), isTrue); }); + test( + 'automatic route superseding a tap capture supplies visual successor', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); + harness.seedRouteState(route: '/home', signature: 'home'); + harness.capturer.blockNext = true; + + harness.controller.recordPointerDown(const Offset(12, 12)); + harness.controller.recordPointerUp(const Offset(12, 12)); + await harness.pumpQueueWork(); + expect(harness.capturer.blockedCount, 1); + + // The tap claim has expired, so this route remains automatic rather than + // borrowing causal attribution from the preceding interaction. + await harness.pumpMicrotasks(); + final route = harness.controller.route( + 'route_push', + harness.route('/automatic'), + ); + harness.controller.debugSetCurrentStateAnchor( + const TugboatStateAnchor( + signature: 'automatic', + signatureParts: {'route': '/automatic'}, + ), + ); + + harness.capturer.completeBlocked(); + await harness.flushScheduler(); + await route; + + final session = harness.controller.session!; + final tap = _ofType(session, 'tap').single; + final settle = _ofType(session, 'tap_settled').single; + final change = _ofType(session, 'route_change').single; + final observation = Map.from( + settle.data['settleObservation']! as Map, + ); + + expect(change.data['navigationOrigin'], 'automatic_or_unknown'); + expect(change.data['causeEventId'], isNull); + expect(change.afterFrame, isNotNull); + expect(settle.relatedEventId, tap.id); + expect(settle.afterFrame, change.afterFrame); + expect(observation['navigationOutcome'], 'visual_successor'); + expect(observation['captureOutcome'], 'captured'); + expect(observation['routeEventId'], change.id); + _expectEveryDiagnosticRequestIsResolvedOnce(session); + expect(CoherenceInvariants.hasNoStrandedCaptureWork(harness), isTrue); + }, + ); + testWidgets('real swipe input overlapping Navigator push stays swipe-only', ( tester, ) async { @@ -333,13 +387,14 @@ void main() { await _drain(tester); final session = controller.session!; - final tap = _ofType(session, 'tap').single; + expect(_ofType(session, 'tap'), isEmpty); final scrollStart = _ofType(session, 'scroll_start').single; final swipe = _ofType(session, 'swipe').single; final scrollEnd = _ofType(session, 'scroll_end').single; final change = _ofType(session, 'route_change').single; expect(_ofType(session, 'tap_settled'), isEmpty); - expect(swipe.relatedEventId, tap.id); + expect(swipe.relatedEventId, isNull); + expect(swipe.data['startCaptureCoordinate'], isA()); expect(swipe.data['scrolled'], isTrue); expect(scrollEnd.relatedEventId, scrollStart.id); expect(scrollEnd.afterFrame, isNull); @@ -353,7 +408,6 @@ void main() { CoherenceInvariants.hasChronologicalChain( events: session.events, orderedEventIds: [ - tap.id, scrollStart.id, swipe.id, scrollEnd.id, diff --git a/packages/tugboat/test/replay/replay_overlay_nested_navigation_matrix_test.dart b/packages/tugboat/test/replay/replay_overlay_nested_navigation_matrix_test.dart index cef0247..0264ace 100644 --- a/packages/tugboat/test/replay/replay_overlay_nested_navigation_matrix_test.dart +++ b/packages/tugboat/test/replay/replay_overlay_nested_navigation_matrix_test.dart @@ -191,6 +191,7 @@ class _OverlayFixture { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: true, capturePixelRatio: 1, ), diff --git a/packages/tugboat/test/replay/replay_programmatic_navigation_matrix_test.dart b/packages/tugboat/test/replay/replay_programmatic_navigation_matrix_test.dart index edcddb4..d93fa66 100644 --- a/packages/tugboat/test/replay/replay_programmatic_navigation_matrix_test.dart +++ b/packages/tugboat/test/replay/replay_programmatic_navigation_matrix_test.dart @@ -78,8 +78,9 @@ void main() { harness.seedRouteState(route: '/source', signature: 'source'); harness.controller.recordPointerDown(const Offset(8, 8)); - final tap = harness.controller.session!.ofType('tap').single; harness.controller.recordPointerUp(const Offset(8, 8)); + final tap = harness.controller.session!.ofType('tap').single; + await harness.pumpMicrotasks(); // The pointer event turn has ended, so this navigation must remain @@ -123,12 +124,12 @@ void main() { addTearDown(harness.dispose); harness.controller.recordPointerDown(const Offset(8, 8)); - final tap = harness.controller.session!.ofType('tap').single; final tappedRoute = harness.controller.route( 'route_push', harness.route('/tapped'), ); harness.controller.recordPointerUp(const Offset(8, 8)); + final tap = harness.controller.session!.ofType('tap').single; // Supersede the causally claimed route before its terminal frame is // available. The redirect has no pointer cause and must not become the @@ -166,9 +167,13 @@ void main() { addTearDown(harness.dispose); harness.controller.recordPointerDown(const Offset(3, 3)); - final tap = harness.controller.session!.ofType('tap').single; - await harness.controller.route('route_push', harness.route('/tapped')); + final tappedRoute = harness.controller.route( + 'route_push', + harness.route('/tapped'), + ); harness.controller.recordPointerUp(const Offset(3, 3)); + final tap = harness.controller.session!.ofType('tap').single; + await tappedRoute; await harness.pumpQueueWork(); await harness.controller.route('route_push', harness.route('/redirect')); diff --git a/packages/tugboat/test/replay/tap_coordinate_transform_test.dart b/packages/tugboat/test/replay/tap_coordinate_transform_test.dart index 2315885..bdc085a 100644 --- a/packages/tugboat/test/replay/tap_coordinate_transform_test.dart +++ b/packages/tugboat/test/replay/tap_coordinate_transform_test.dart @@ -18,6 +18,7 @@ void main() { config: TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: true, capturePixelRatio: capturePixelRatio, screenshotMaskLevel: TugboatScreenshotMaskLevel.explicitOnly, @@ -67,6 +68,7 @@ void main() { ); final center = tester.getCenter(find.byKey(const Key('target'))); controller.recordPointerDown(center); + controller.recordPointerUp(center); final tap = controller.session!.events.where((e) => e.type == 'tap').last; expect(tap.data['x'], center.dx); expect(tap.data['y'], center.dy); @@ -92,6 +94,7 @@ void main() { final controller = await mount(tester); // Far outside the capture boundary / screen. controller.recordPointerDown(const Offset(-80, -80)); + controller.recordPointerUp(const Offset(-80, -80)); final tap = controller.session!.events.where((e) => e.type == 'tap').last; final coord = Map.from( tap.data['captureCoordinate']! as Map, @@ -112,6 +115,7 @@ void main() { await tester.pump(); final center = tester.getCenter(find.byKey(const Key('target'))); controller.recordPointerDown(center); + controller.recordPointerUp(center); final tap = controller.session!.events.where((e) => e.type == 'tap').last; final coord = TugboatCaptureCoordinate.fromJson( Map.from(tap.data['captureCoordinate']! as Map), @@ -144,6 +148,7 @@ void main() { final center = tester.getCenter(find.byKey(const Key('target'))); controller.recordPointerDown(center); + controller.recordPointerUp(center); final tap = controller.session!.events.where((e) => e.type == 'tap').last; final coord = TugboatCaptureCoordinate.fromJson( Map.from(tap.data['captureCoordinate']! as Map), diff --git a/packages/tugboat/test/replay_coherence_characterization_test.dart b/packages/tugboat/test/replay_coherence_characterization_test.dart index 9019c74..1817dea 100644 --- a/packages/tugboat/test/replay_coherence_characterization_test.dart +++ b/packages/tugboat/test/replay_coherence_characterization_test.dart @@ -258,38 +258,41 @@ void main() { ); }); - test('automatic route during tap readback stays independent', () async { - final harness = ReplayCoherenceHarness(); - await harness.setUp(); - addTearDown(harness.dispose); + test( + 'automatic route during tap readback supplies visual successor', + () async { + final harness = ReplayCoherenceHarness(); + await harness.setUp(); + addTearDown(harness.dispose); - harness.seedRouteState(route: '/scan', signature: 'sig-scan'); - harness.capturer.blockNext = true; - harness.controller.recordPointerDown(const Offset(10, 10)); - harness.controller.recordPointerUp(const Offset(10, 10)); - await harness.pumpQueueWork(); - expect(harness.capturer.blockedCount, 1); + harness.seedRouteState(route: '/scan', signature: 'sig-scan'); + harness.capturer.blockNext = true; + harness.controller.recordPointerDown(const Offset(10, 10)); + harness.controller.recordPointerUp(const Offset(10, 10)); + await harness.pumpQueueWork(); + expect(harness.capturer.blockedCount, 1); - final routeFuture = harness.controller.route( - 'route_push', - harness.route('/home'), - ); - harness.capturer.completeBlocked('stale-tap-frame'); - await harness.flushScheduler(); - await routeFuture; + final routeFuture = harness.controller.route( + 'route_push', + harness.route('/home'), + ); + harness.capturer.completeBlocked('stale-tap-frame'); + await harness.flushScheduler(); + await routeFuture; - final session = harness.controller.session!; - final routeChange = session.ofType('route_change').single; - final settle = session.ofType('tap_settled').single; - final observation = Map.from( - settle.data['settleObservation']! as Map, - ); - expect(settle.afterFrame, isNull); - expect(settle.result, isNot(TugboatInteractionResult.navigated)); - expect(observation['navigationOutcome'], 'same_route'); - expect(observation['routeEventId'], isNull); - expect(routeChange.afterFrame, isNotNull); - }); + final session = harness.controller.session!; + final routeChange = session.ofType('route_change').single; + final settle = session.ofType('tap_settled').single; + final observation = Map.from( + settle.data['settleObservation']! as Map, + ); + expect(settle.afterFrame, routeChange.afterFrame); + expect(settle.result, isNot(TugboatInteractionResult.navigated)); + expect(observation['navigationOutcome'], 'visual_successor'); + expect(observation['routeEventId'], routeChange.id); + expect(routeChange.afterFrame, isNotNull); + }, + ); test( 'automatic successors cannot replace a tap-caused route barrier', @@ -857,6 +860,7 @@ void main() { ); harness.controller.recordPointerDown(const Offset(4, 4)); + harness.controller.recordPointerUp(const Offset(4, 4)); final tap = harness.controller.session!.ofType('tap').single; expect(tap.beforeFrame, isNull); expect(tap.data['frameAttachment'], { @@ -1056,6 +1060,7 @@ void main() { ); harness.controller.recordPointerDown(const Offset(8, 8)); + harness.controller.recordPointerUp(const Offset(8, 8)); final tap = harness.controller.session!.ofType('tap').single; expect(tap.beforeFrame, isNull, reason: transition.$1); expect(tap.beforeFrame, isNot(origin), reason: transition.$1); @@ -1079,6 +1084,7 @@ void main() { contentHash: 'first-pixels', ); harness.controller.recordPointerDown(const Offset(1, 1), pointer: 1); + harness.controller.recordPointerUp(const Offset(1, 1), pointer: 1); final retainedTap = harness.controller.session!.ofType('tap').single; expect(retainedTap.beforeFrame, first); @@ -1096,6 +1102,7 @@ void main() { expect(harness.controller.debugReuseFrameForCurrentRoute(first), isNull); harness.controller.recordPointerDown(const Offset(2, 2), pointer: 2); + harness.controller.recordPointerUp(const Offset(2, 2), pointer: 2); final latestTap = harness.controller.session!.ofType('tap').last; expect(latestTap.beforeFrame, second); }); @@ -2006,22 +2013,16 @@ void main() { await harness.recordClassifiedSwipe(start); final session = harness.controller.session!; - final tap = session.ofType('tap').single; + expect(session.ofType('tap'), isEmpty); final swipe = session.ofType('swipe').single; final eventTypes = session.events.map((event) => event.type).toList(); expect(session.ofType('tap_settled'), isEmpty); - expect(eventTypes.indexOf('tap'), lessThan(eventTypes.indexOf('swipe'))); - expect(swipe.relatedEventId, tap.id); + expect(eventTypes, contains('swipe')); + expect(swipe.relatedEventId, isNull); expect(swipe.beforeFrame, originFrame); - expect(swipe.stateAnchor?.signature, tap.stateAnchor?.signature); - expect(tap.targetAnchor, isNotNull); expect(swipe.targetAnchor, isNotNull); - expect(swipe.targetAnchor!.fingerprint, tap.targetAnchor!.fingerprint); - expect( - swipe.targetAnchor!.canonicalPath, - tap.targetAnchor!.canonicalPath, - ); + expect(swipe.data['startCaptureCoordinate'], isA()); expect(swipe.data['startX'], closeTo(start.dx, 0.01)); expect(swipe.data['startY'], closeTo(start.dy, 0.01)); expect(swipe.data['scrolled'], isFalse); diff --git a/packages/tugboat/test/scene_inventory_test.dart b/packages/tugboat/test/scene_inventory_test.dart index f71426d..b743ae7 100644 --- a/packages/tugboat/test/scene_inventory_test.dart +++ b/packages/tugboat/test/scene_inventory_test.dart @@ -184,6 +184,7 @@ void main() { const config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ); @@ -204,6 +205,7 @@ void main() { final controller = TugboatReplay.controller!; final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final tapEvents = controller.session!.events @@ -246,6 +248,7 @@ void main() { const config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ); @@ -269,6 +272,7 @@ void main() { final controller = TugboatReplay.controller!; final tapPoint = const Offset(20, 20); controller.recordPointerDown(tapPoint); + controller.recordPointerUp(tapPoint); await tester.pump(); final tapEvent = controller.session!.events @@ -302,6 +306,7 @@ void main() { const config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ); @@ -322,6 +327,7 @@ void main() { final controller = TugboatReplay.controller!; final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final afterFirstTap = controller.session!.events @@ -329,6 +335,7 @@ void main() { .length; controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final afterSecondTap = controller.session!.events @@ -343,6 +350,7 @@ void main() { const config = TugboatReplayConfig( profile: TugboatCaptureProfile.productionLean, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ); @@ -363,6 +371,7 @@ void main() { final controller = TugboatReplay.controller!; final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final eventTypes = controller.session!.events.map((event) => event.type); @@ -374,6 +383,7 @@ void main() { const config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ); diff --git a/packages/tugboat/test/scroll_attribution_test.dart b/packages/tugboat/test/scroll_attribution_test.dart index 4bfa74c..ca94e7f 100644 --- a/packages/tugboat/test/scroll_attribution_test.dart +++ b/packages/tugboat/test/scroll_attribution_test.dart @@ -5,6 +5,7 @@ import 'package:tugboat/tugboat.dart'; const _scrollTestConfig = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, scrollCaptureInterval: Duration(milliseconds: 50), captureScrollSamples: true, @@ -104,7 +105,8 @@ void main() { expect(swipes, isNotEmpty); expect(swipes.first.data['scrolled'], isFalse); expect(swipes.first.result, TugboatInteractionResult.noVisibleChange); - expect(swipes.first.relatedEventId, isNotNull); + expect(swipes.first.relatedEventId, isNull); + expect(swipes.first.data['startCaptureCoordinate'], isA()); expect(settled, isEmpty); expect(scrollStarts, isEmpty); }); diff --git a/packages/tugboat/test/scroll_playground_live_test.dart b/packages/tugboat/test/scroll_playground_live_test.dart index 2ef7676..eef501f 100644 --- a/packages/tugboat/test/scroll_playground_live_test.dart +++ b/packages/tugboat/test/scroll_playground_live_test.dart @@ -9,6 +9,7 @@ void main() { const config = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, scrollCaptureInterval: Duration(milliseconds: 50), captureScrollSamples: true, diff --git a/packages/tugboat/test/tugboat_replay_test.dart b/packages/tugboat/test/tugboat_replay_test.dart index 4f9a16e..37a9084 100644 --- a/packages/tugboat/test/tugboat_replay_test.dart +++ b/packages/tugboat/test/tugboat_replay_test.dart @@ -14,6 +14,7 @@ import 'helpers/json_roundtrip.dart'; const _testConfig = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, scrollCaptureInterval: Duration(milliseconds: 50), captureScrollSamples: true, @@ -384,6 +385,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration(milliseconds: 50), + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), @@ -439,6 +441,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration(milliseconds: 50), + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), @@ -864,7 +867,7 @@ void main() { ); final json = jsonDecode(session.toPrettyJson()) as Map; - expect(json['schemaVersion'], 7); + expect(json['schemaVersion'], 8); expect(json.containsKey('routes'), isFalse); expect(json['events'], [isNot(contains('route'))]); expect(json['frames'], [containsPair('captureMicros', 12345)]); diff --git a/packages/tugboat/test/viewport_semantic_map_test.dart b/packages/tugboat/test/viewport_semantic_map_test.dart index 5dadc78..f39711b 100644 --- a/packages/tugboat/test/viewport_semantic_map_test.dart +++ b/packages/tugboat/test/viewport_semantic_map_test.dart @@ -6,6 +6,7 @@ import 'package:tugboat/src/anchors.dart'; const _semanticMapConfig = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, viewportSemanticMode: TugboatViewportSemanticMode.full, @@ -14,6 +15,7 @@ const _semanticMapConfig = TugboatReplayConfig( const _semanticMapConfigWithLogs = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, viewportSemanticMode: TugboatViewportSemanticMode.fullWithDebugLogs, @@ -22,6 +24,7 @@ const _semanticMapConfigWithLogs = TugboatReplayConfig( const _scrollSemanticMapConfig = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, scrollCaptureInterval: Duration.zero, captureScrollSamples: true, @@ -136,6 +139,7 @@ void main() { final controller = TugboatReplay.controller!; final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final tapEvent = controller.session!.events @@ -184,16 +188,16 @@ void main() { final textNode = nodes.singleWhere((node) => node['role'] == 'text'); final bounds = textNode['boundsNorm'] as Map; final scaffoldSize = tester.getSize(find.byType(Scaffold)); - controller.recordPointerDown( - Offset( - ((bounds['left'] as num).toDouble() + - (bounds['width'] as num).toDouble() / 2) * - scaffoldSize.width, - ((bounds['top'] as num).toDouble() + - (bounds['height'] as num).toDouble() / 2) * - scaffoldSize.height, - ), + final tapPoint = Offset( + ((bounds['left'] as num).toDouble() + + (bounds['width'] as num).toDouble() / 2) * + scaffoldSize.width, + ((bounds['top'] as num).toDouble() + + (bounds['height'] as num).toDouble() / 2) * + scaffoldSize.height, ); + controller.recordPointerDown(tapPoint); + controller.recordPointerUp(tapPoint); await tester.pump(); final tapEvent = controller.session!.events @@ -224,6 +228,7 @@ void main() { final controller = TugboatReplay.controller!; final bottomRight = tester.getBottomRight(find.byType(Scaffold)); controller.recordPointerDown(bottomRight - const Offset(2, 2)); + controller.recordPointerUp(bottomRight - const Offset(2, 2)); await tester.pump(); final tapEvent = controller.session!.events @@ -286,6 +291,7 @@ void main() { find.byKey(const ValueKey('custom-cta')), ); controller.recordPointerDown(ctaCenter); + controller.recordPointerUp(ctaCenter); await tester.pump(); final tapEvent = controller.session!.events @@ -307,6 +313,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.dormant, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), @@ -334,6 +341,7 @@ void main() { final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final tapEvent = controller.session!.events @@ -351,6 +359,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.productionLean, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), @@ -368,6 +377,7 @@ void main() { final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final mapEvents = controller.session!.events @@ -400,6 +410,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.productionLean, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, viewportSemanticMode: TugboatViewportSemanticMode.full, @@ -422,6 +433,7 @@ void main() { final controller = TugboatReplay.controller!; controller.recordPointerDown(tester.getCenter(find.text('Go'))); + controller.recordPointerUp(tester.getCenter(find.text('Go'))); await tester.pump(); final mapEvents = controller.session!.events @@ -449,6 +461,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.productionLean, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), @@ -469,6 +482,7 @@ void main() { final controller = TugboatReplay.controller!; controller.recordPointerDown(tester.getCenter(find.text('Go'))); + controller.recordPointerUp(tester.getCenter(find.text('Go'))); await tester.pump(); final mapEvents = controller.session!.events @@ -626,6 +640,7 @@ void main() { final tapCenter = tester.getCenter(find.text('Go')); controller.recordPointerDown(tapCenter); + controller.recordPointerUp(tapCenter); await tester.pump(); final afterInventoryCount = controller.session!.events From cdf00273441c56eef25343dbe500f721d05406d9 Mon Sep 17 00:00:00 2001 From: Chinmay-KB Date: Wed, 29 Jul 2026 17:21:31 +0530 Subject: [PATCH 2/9] fix(replay): honor interaction stream lifecycle --- docs/README.md | 4 +-- packages/tugboat/README.md | 2 +- packages/tugboat/lib/src/controller.dart | 29 ++++++++++++------- .../helpers/replay_coherence_harness.dart | 3 ++ .../replay/interaction_transaction_test.dart | 26 +++++++++++++++++ 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/docs/README.md b/docs/README.md index 24f64ad..ea28de3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,8 +28,8 @@ verified in their own repositories. ## Current compatibility -- package version: `0.4.13`; -- session JSON schema: `7`; +- package version: `0.4.15`; +- session JSON schema: `8`; - fingerprint schema: `6`; - minimum Dart SDK: `3.9.2`; - minimum Flutter SDK: `3.35.0`. diff --git a/packages/tugboat/README.md b/packages/tugboat/README.md index 184f814..bfca542 100644 --- a/packages/tugboat/README.md +++ b/packages/tugboat/README.md @@ -5,7 +5,7 @@ checkpoints around meaningful interactions, compact structural anchors, route transitions, scrolling evidence, and optional viewport semantic maps. Capture can be sent to the local exploration WebSocket, the HTTP collector, or both. -The current package version is `0.4.13`. Session JSON uses schema version `7` +The current package version is `0.4.15`. Session JSON uses schema version `8` (readers still accept `6`), and structural fingerprints use fingerprint schema version `6`. diff --git a/packages/tugboat/lib/src/controller.dart b/packages/tugboat/lib/src/controller.dart index bd84178..a3f244c 100644 --- a/packages/tugboat/lib/src/controller.dart +++ b/packages/tugboat/lib/src/controller.dart @@ -2464,6 +2464,7 @@ class TugboatReplayController extends ChangeNotifier { /// as a real tap. Patches the in-memory session (and sibling `tap_outside_tree`) /// and emits `tap_gesture_resolved` so already-flushed sinks can promote too. void _promoteCausalTapToInteraction(String tapEventId) { + if (!config.emitLegacyInteractionProjection) return; const promotion = { 'gestureFinal': 'tap', 'replayRole': 'interaction', @@ -2597,15 +2598,14 @@ class TugboatReplayController extends ChangeNotifier { } } - void _clearReleasedInteractions() { + void _clearReleasedInteractions({ + InteractionRejectionReason reason = InteractionRejectionReason.sessionEnd, + }) { _reconciliationSweepCancel?.call(); _reconciliationSweepCancel = null; _reconciliationSweepScheduled = false; for (final tx in _interactions.takeAllReleased()) { - _finalizeAbandonedTransaction( - tx, - reason: InteractionRejectionReason.sessionEnd, - ); + _finalizeAbandonedTransaction(tx, reason: reason); if (!tx.tapEmitted) { tx.bufferedTap = null; tx.bufferedOutside = null; @@ -2662,11 +2662,14 @@ class TugboatReplayController extends ChangeNotifier { _finalizeAbandonedTransaction(pending, reason: reason); } - void _abandonAllPendingPointers({bool publishClaimedTap = true}) { + void _abandonAllPendingPointers({ + bool publishClaimedTap = true, + String gestureFinal = 'session_end', + }) { for (final pointer in _interactions.takePendingPointers()) { _abandonPendingPointer( pointer, - gestureFinal: 'session_end', + gestureFinal: gestureFinal, publishClaimedTap: publishClaimedTap, ); } @@ -3202,7 +3205,8 @@ class TugboatReplayController extends ChangeNotifier { targetAnchor: tx.origin.targetAnchor, beforeFrame: tx.origin.beforeFrame, afterFrame: tx.afterFrame, - result: tx.resultStatus?.asEventResult ?? TugboatInteractionResult.unknown, + result: + tx.resultStatus?.asEventResult ?? TugboatInteractionResult.unknown, data: { 'interactionId': tx.id, 'interactionSchema': tugboatInteractionSchemaVersion, @@ -3940,8 +3944,13 @@ class TugboatReplayController extends ChangeNotifier { _invalidateCaptureWork('lifecycle_deactivate'); // Drop in-flight pointer claims so a later resume/navigation cannot // attribute itself to a pre-background gesture. - _abandonAllPendingPointers(publishClaimedTap: false); - _clearReleasedInteractions(); + _abandonAllPendingPointers( + publishClaimedTap: false, + gestureFinal: 'lifecycle', + ); + _clearReleasedInteractions( + reason: InteractionRejectionReason.lifecycle, + ); _captureLifecycleActive = false; break; case AppLifecycleState.resumed: diff --git a/packages/tugboat/test/helpers/replay_coherence_harness.dart b/packages/tugboat/test/helpers/replay_coherence_harness.dart index e3475a0..77ba828 100644 --- a/packages/tugboat/test/helpers/replay_coherence_harness.dart +++ b/packages/tugboat/test/helpers/replay_coherence_harness.dart @@ -214,6 +214,7 @@ class ReplayCoherenceHarness { /// reconciliation — pass that window explicitly when testing delayed /// attribution. this.interactionClaimWindow = Duration.zero, + this.interactionPublishMode = TugboatInteractionPublishMode.dualWrite, this.maxFrames = 300, this.screenshotBudget = TugboatScreenshotBudgetConfig.defaults, GlobalKey? boundaryKey, @@ -221,6 +222,7 @@ class ReplayCoherenceHarness { final Duration settleDelay; final Duration interactionClaimWindow; + final TugboatInteractionPublishMode interactionPublishMode; final int maxFrames; final TugboatScreenshotBudgetConfig screenshotBudget; final GlobalKey boundaryKey; @@ -266,6 +268,7 @@ class ReplayCoherenceHarness { profile: TugboatCaptureProfile.exploration, settleDelay: settleDelay, interactionClaimWindow: interactionClaimWindow, + interactionPublishMode: interactionPublishMode, maxFrames: maxFrames, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, diff --git a/packages/tugboat/test/replay/interaction_transaction_test.dart b/packages/tugboat/test/replay/interaction_transaction_test.dart index d8fb0dc..ef34103 100644 --- a/packages/tugboat/test/replay/interaction_transaction_test.dart +++ b/packages/tugboat/test/replay/interaction_transaction_test.dart @@ -118,6 +118,32 @@ void main() { .semanticOfType('interaction') .where((e) => e.data['gesture'] == 'cancelled'); expect(cancelled, isNotEmpty); + final attribution = Map.from( + cancelled.last.data['attribution']! as Map, + ); + expect(attribution['rejectionReason'], 'lifecycle'); + }, + ); + + test( + 'canonical-only mode does not emit legacy promotion evidence', + () async { + final harness = ReplayCoherenceHarness( + interactionPublishMode: TugboatInteractionPublishMode.canonicalOnly, + ); + await harness.setUp(); + addTearDown(harness.dispose); + + harness.controller.recordPointerDown(const Offset(12, 34)); + await harness.controller.route('route_push', harness.route('/dest')); + await harness.flushScheduler(); + harness.controller.recordPointerUp(const Offset(12, 34)); + await harness.flushScheduler(); + + expect( + harness.controller.session!.ofType('tap_gesture_resolved'), + isEmpty, + ); }, ); }); From 25a32efc598780ebc99655637ac2d4bdfc0463ca Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 29 Jul 2026 06:35:36 +0000 Subject: [PATCH 3/9] feat(replay): capture privacy-safe control values on interactions Record which option or state was chosen for radios, dropdowns, toggles, sliders, and chips on tap/tap_settled/swipe events. Free-text option strings are hashed; bools, numbers, enums, and short developer tokens are retained. Also harden Radio role inspection against typed callbacks. Co-authored-by: Chinmay Kabi --- docs/design/capture-and-fingerprint.md | 12 + packages/tugboat/lib/src/anchor_resolver.dart | 27 ++ packages/tugboat/lib/src/anchors.dart | 1 + packages/tugboat/lib/src/control_value.dart | 351 ++++++++++++++++++ packages/tugboat/lib/src/controller.dart | 38 ++ .../lib/src/interaction_transaction.dart | 10 +- packages/tugboat/lib/src/widget_roles.dart | 10 +- packages/tugboat/lib/tugboat.dart | 4 + packages/tugboat/test/control_value_test.dart | 339 +++++++++++++++++ 9 files changed, 783 insertions(+), 9 deletions(-) create mode 100644 packages/tugboat/lib/src/control_value.dart create mode 100644 packages/tugboat/test/control_value_test.dart diff --git a/docs/design/capture-and-fingerprint.md b/docs/design/capture-and-fingerprint.md index cc89fb3..6b92e22 100644 --- a/docs/design/capture-and-fingerprint.md +++ b/docs/design/capture-and-fingerprint.md @@ -235,6 +235,18 @@ Developer-authored identity strings can still be emitted: - widget type names or configured `widgetNames` replacements; - canonical structural paths. +Interaction events may also carry a privacy-safe `controlValue` payload for +valued controls (checkbox, switch, radio, slider, dropdown / menu item, chip): + +- bools and numbers are emitted literally; +- enums and short developer-token strings are emitted as tokens; +- free-text option strings are hashed (`str:`) with length only. + +`tap` includes the value sampled at pointer-down. `tap_settled` includes +`before` / `after` snapshots so toggle flips and post-callback radio/dropdown +selections are visible. Slider drags that become `swipe` events also carry the +value sampled at pointer-up. + Bounds, pointer coordinates, scroll metrics, and masked screenshot pixels are also capture data. Apps must treat tags, route names, and subview labels as telemetry and avoid putting user data in them. diff --git a/packages/tugboat/lib/src/anchor_resolver.dart b/packages/tugboat/lib/src/anchor_resolver.dart index 915389b..5c23972 100644 --- a/packages/tugboat/lib/src/anchor_resolver.dart +++ b/packages/tugboat/lib/src/anchor_resolver.dart @@ -169,6 +169,33 @@ class AnchorResolver { ); } + /// Privacy-safe control value under [globalPosition], if any. + /// + /// Samples toggle/radio/dropdown/slider widget state at hit time. Free-text + /// option labels are hashed; bools, numbers, enums, and short developer + /// tokens are retained. + TugboatControlValue? controlValueAt(Offset globalPosition) { + final rootContext = rootKey.currentContext; + final rootRender = rootContext?.findRenderObject(); + if (rootRender is! RenderBox || rootContext is! Element) return null; + + final tokenMap = _tokenMapFor(rootContext, rootRender); + if (tokenMap == null) return null; + + final result = BoxHitTestResult(); + final localPosition = rootRender.globalToLocal(globalPosition); + rootRender.hitTest(result, position: localPosition); + + for (final entry in result.path) { + if (entry.target is! RenderObject) continue; + final element = tokenMap.renderElements[entry.target as RenderObject]; + if (element == null || tugboatIsCaptureChrome(element.widget)) continue; + final value = tugboatControlValueForElement(element); + if (value != null) return value; + } + return null; + } + /// Builds inventory and resolves a tap target from one token-map walk. ({TugboatSceneInventory? inventory, TugboatTargetAnchor? target}) buildTapContext({ diff --git a/packages/tugboat/lib/src/anchors.dart b/packages/tugboat/lib/src/anchors.dart index f4051c0..cf295b3 100644 --- a/packages/tugboat/lib/src/anchors.dart +++ b/packages/tugboat/lib/src/anchors.dart @@ -12,6 +12,7 @@ import 'semantics_flags_compat.dart'; part 'anchor_fingerprint.dart'; part 'anchor_models.dart'; part 'widget_roles.dart'; +part 'control_value.dart'; part 'anchor_resolver.dart'; part 'anchor_scene_inventory.dart'; part 'anchor_viewport_semantics.dart'; diff --git a/packages/tugboat/lib/src/control_value.dart b/packages/tugboat/lib/src/control_value.dart new file mode 100644 index 0000000..2c9dddf --- /dev/null +++ b/packages/tugboat/lib/src/control_value.dart @@ -0,0 +1,351 @@ +part of 'anchors.dart'; + +/// Schema version for privacy-safe control value payloads. +const int tugboatControlValueSchemaVersion = 1; + +final RegExp _developerTokenPattern = RegExp(r'^[A-Za-z0-9_./:-]{1,64}$'); + +/// Encodes a single control scalar without retaining free-text labels. +class TugboatEncodedControlScalar { + const TugboatEncodedControlScalar._({ + required this.kind, + this.value, + this.length, + }); + + /// `null`, `bool`, `number`, or `token`. + final String kind; + + /// Literal bool/num, or a privacy-safe token string. + final Object? value; + + /// Original string length when [kind] is `token` derived from a String. + final int? length; + + factory TugboatEncodedControlScalar.encode(Object? raw) { + if (raw == null) { + return const TugboatEncodedControlScalar._(kind: 'null'); + } + if (raw is bool) { + return TugboatEncodedControlScalar._(kind: 'bool', value: raw); + } + if (raw is num) { + return TugboatEncodedControlScalar._(kind: 'number', value: raw); + } + if (raw is Enum) { + return TugboatEncodedControlScalar._( + kind: 'token', + value: '${raw.runtimeType}.${raw.name}', + ); + } + if (raw is String) { + if (_developerTokenPattern.hasMatch(raw)) { + return TugboatEncodedControlScalar._(kind: 'token', value: raw); + } + return TugboatEncodedControlScalar._( + kind: 'token', + value: 'str:${tugboatLabelHash(raw)}', + length: raw.length, + ); + } + final text = raw.toString(); + return TugboatEncodedControlScalar._( + kind: 'token', + value: '${raw.runtimeType}:${tugboatLabelHash(text)}', + length: text.length, + ); + } + + Map toJson() => { + 'kind': kind, + if (kind != 'null') 'value': value, + if (length != null) 'length': length, + }; + + @override + bool operator ==(Object other) => + other is TugboatEncodedControlScalar && + kind == other.kind && + value == other.value && + length == other.length; + + @override + int get hashCode => Object.hash(kind, value, length); +} + +/// Privacy-safe snapshot of an interactive control's value at sample time. +/// +/// Free-text option labels are hashed. Bools, numbers, enums, and short +/// developer-token strings are retained so taps on radios, dropdowns, +/// toggles, and sliders remain interpretable without storing arbitrary UI copy. +class TugboatControlValue { + const TugboatControlValue({ + required this.role, + this.widgetType, + this.value, + this.groupValue, + this.selected, + this.index, + this.start, + this.end, + this.schemaVersion = tugboatControlValueSchemaVersion, + }); + + final int schemaVersion; + + /// Control role (`checkbox`, `switch`, `radio`, `slider`, `dropdown`, + /// `dropdownItem`, `menuItem`, `chip`). + final String role; + final String? widgetType; + + /// Primary sampled value (option identity, toggle state, slider position). + final TugboatEncodedControlScalar? value; + + /// Current group selection for radio controls. + final TugboatEncodedControlScalar? groupValue; + + /// Whether this option is the active selection (radios/chips). + final bool? selected; + + /// Zero-based index among sibling options when known. + final int? index; + + /// Range slider start (inclusive). + final TugboatEncodedControlScalar? start; + + /// Range slider end (inclusive). + final TugboatEncodedControlScalar? end; + + Map toJson() => { + 'schemaVersion': schemaVersion, + 'role': role, + if (widgetType != null && widgetType!.isNotEmpty) 'widgetType': widgetType, + if (value != null) 'value': value!.toJson(), + if (groupValue != null) 'groupValue': groupValue!.toJson(), + if (selected != null) 'selected': selected, + if (index != null) 'index': index, + if (start != null) 'start': start!.toJson(), + if (end != null) 'end': end!.toJson(), + }; + + @override + bool operator ==(Object other) => + other is TugboatControlValue && + schemaVersion == other.schemaVersion && + role == other.role && + widgetType == other.widgetType && + value == other.value && + groupValue == other.groupValue && + selected == other.selected && + index == other.index && + start == other.start && + end == other.end; + + @override + int get hashCode => Object.hash( + schemaVersion, + role, + widgetType, + value, + groupValue, + selected, + index, + start, + end, + ); +} + +/// Reads a privacy-safe control value from [widget], or null when unsupported. +TugboatControlValue? tugboatControlValueForWidget(Widget widget, {int? index}) { + final widgetType = widget.runtimeType.toString(); + + if (widget is Checkbox) { + return TugboatControlValue( + role: 'checkbox', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + selected: widget.value == true, + index: index, + ); + } + if (widget is CheckboxListTile) { + return TugboatControlValue( + role: 'checkbox', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + selected: widget.value == true, + index: index, + ); + } + if (widget is Switch) { + return TugboatControlValue( + role: 'switch', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + selected: widget.value, + index: index, + ); + } + if (widget is CupertinoSwitch) { + return TugboatControlValue( + role: 'switch', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + selected: widget.value, + index: index, + ); + } + if (widget is SwitchListTile) { + return TugboatControlValue( + role: 'switch', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + selected: widget.value, + index: index, + ); + } + if (widget is Radio || widget is RadioListTile) { + // Typed Radio / RadioListTile cannot be read through a promoted + // Radio view; keep access dynamic like role detection. + final dynamic radio = widget; + final option = radio.value; + final group = radio.groupValue; + return TugboatControlValue( + role: 'radio', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(option), + groupValue: TugboatEncodedControlScalar.encode(group), + selected: option == group, + index: index, + ); + } + if (widget is Slider) { + return TugboatControlValue( + role: 'slider', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + index: index, + ); + } + if (widget is CupertinoSlider) { + return TugboatControlValue( + role: 'slider', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.value), + index: index, + ); + } + if (widget is RangeSlider) { + return TugboatControlValue( + role: 'slider', + widgetType: widgetType, + start: TugboatEncodedControlScalar.encode(widget.values.start), + end: TugboatEncodedControlScalar.encode(widget.values.end), + index: index, + ); + } + if (widget is DropdownButton) { + final dynamic dropdown = widget; + return TugboatControlValue( + role: 'dropdown', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(dropdown.value), + index: index, + ); + } + if (widget is DropdownMenuItem) { + final dynamic item = widget; + return TugboatControlValue( + role: 'dropdownItem', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(item.value), + index: index, + ); + } + if (widget is PopupMenuItem) { + final dynamic item = widget; + return TugboatControlValue( + role: 'menuItem', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(item.value), + index: index, + ); + } + if (widget is FilterChip) { + return TugboatControlValue( + role: 'chip', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.selected), + selected: widget.selected, + index: index, + ); + } + if (widget is ChoiceChip) { + return TugboatControlValue( + role: 'chip', + widgetType: widgetType, + value: TugboatEncodedControlScalar.encode(widget.selected), + selected: widget.selected, + index: index, + ); + } + return null; +} + +/// Walks [hitElement] and its ancestors for the deepest valued control. +TugboatControlValue? tugboatControlValueForElement(Element hitElement) { + TugboatControlValue? found; + + void consider(Element element) { + if (found != null) return; + final index = _optionIndexAmongSiblings(element); + found = tugboatControlValueForWidget(element.widget, index: index); + } + + consider(hitElement); + hitElement.visitAncestorElements((ancestor) { + consider(ancestor); + return found == null; + }); + return found; +} + +int? _optionIndexAmongSiblings(Element element) { + final self = tugboatControlValueForWidget(element.widget); + if (self == null) return null; + const optionRoles = {'radio', 'dropdownItem', 'menuItem', 'chip'}; + if (!optionRoles.contains(self.role)) return null; + + // Walk up until a parent exposes multiple same-role options among its + // descendants, then return this element's ordinal among those options. + Element? parent; + element.visitAncestorElements((ancestor) { + parent = ancestor; + return false; + }); + while (parent != null) { + final options = []; + void collect(Element node) { + final value = tugboatControlValueForWidget(node.widget); + if (value != null && value.role == self.role) { + options.add(node); + return; + } + node.visitChildElements(collect); + } + + parent!.visitChildElements(collect); + if (options.length > 1) { + final index = options.indexWhere((option) => identical(option, element)); + return index >= 0 ? index : null; + } + + Element? next; + parent!.visitAncestorElements((ancestor) { + next = ancestor; + return false; + }); + parent = next; + } + return null; +} diff --git a/packages/tugboat/lib/src/controller.dart b/packages/tugboat/lib/src/controller.dart index a3f244c..53b419e 100644 --- a/packages/tugboat/lib/src/controller.dart +++ b/packages/tugboat/lib/src/controller.dart @@ -2266,6 +2266,7 @@ class TugboatReplayController extends ChangeNotifier { TugboatTargetAnchor? target; TugboatStateAnchor? tapState = _currentStateAnchor; TugboatSceneInventory? tapInventory; + TugboatControlValue? controlValue; if (resolver != null && config.profile != TugboatCaptureProfile.dormant) { final tapContext = resolver.buildTapContext( @@ -2276,6 +2277,7 @@ class TugboatReplayController extends ChangeNotifier { ); target = tapContext.target; tapInventory = tapContext.inventory; + controlValue = resolver.controlValueAt(position); if (tapInventory != null) { _currentStateAnchor = tapInventory.stateAnchor; tapState = tapInventory.stateAnchor; @@ -2283,6 +2285,7 @@ class TugboatReplayController extends ChangeNotifier { } } else { target = resolver?.targetAt(position, route: _currentRoute); + controlValue = resolver?.controlValueAt(position); } // Resolve after the tap context so a stale settled map can be refreshed @@ -2315,6 +2318,7 @@ class TugboatReplayController extends ChangeNotifier { }, if (viewportResolution != null) 'viewportSemanticResolution': viewportResolution.toJson(), + if (controlValue != null) 'controlValue': controlValue.toJson(), }; final beforeState = tapState; @@ -2333,6 +2337,7 @@ class TugboatReplayController extends ChangeNotifier { startPosition: position, pointerGeneration: ++_pointerGeneration, captureSessionId: _session?.id, + controlValue: controlValue, ); final tx = InteractionTransaction(origin: origin, pointerId: pointer); final legacyStream = config.legacyGestureStream; @@ -2764,6 +2769,9 @@ class TugboatReplayController extends ChangeNotifier { : null; final scrolled = scrollStartEventId != null; final tapWasEmitted = pending.tapEmitted; + final controlValue = + _anchorResolver?.controlValueAt(position) ?? + pending.origin.controlValue; pending.gesture = scrolled ? InteractionGesture.scroll : InteractionGesture.swipe; @@ -2803,6 +2811,7 @@ class TugboatReplayController extends ChangeNotifier { if (tapWasEmitted) 'invalidatesRelatedTap': true, if (scrollStartEventId != null) 'scrollStartEventId': scrollStartEventId, + if (controlValue != null) 'controlValue': controlValue.toJson(), 'interactionId': pending.id, }, ), @@ -2982,6 +2991,16 @@ class TugboatReplayController extends ChangeNotifier { final visualChanged = visualAvailable ? beforeContentHash != afterContentHash : null; + // Sample after the host onChanged callback has run. Prefer the live + // control under the pointer; fall back to the tap-time snapshot for + // ephemeral menu items that disappear when the overlay closes. + final afterControlValue = + _anchorResolver?.controlValueAt(position) ?? + pending.origin.controlValue; + final controlValuePayload = _controlValueSettlePayload( + before: pending.origin.controlValue, + after: afterControlValue, + ); if (config.emitLegacyInteractionProjection) { _addEvent( @@ -3042,6 +3061,8 @@ class TugboatReplayController extends ChangeNotifier { observation.captureFailure ?? observation.captureOutcome, }, + if (controlValuePayload != null) + 'controlValue': controlValuePayload, }, ), ); @@ -3139,6 +3160,23 @@ class TugboatReplayController extends ChangeNotifier { _activeTapSettles.clear(); } + /// Builds a before/after control-value payload for `tap_settled`. + Map? _controlValueSettlePayload({ + required TugboatControlValue? before, + required TugboatControlValue? after, + }) { + if (before == null && after == null) return null; + final role = after?.role ?? before!.role; + final widgetType = after?.widgetType ?? before?.widgetType; + return { + 'schemaVersion': tugboatControlValueSchemaVersion, + 'role': role, + if (widgetType != null && widgetType.isNotEmpty) 'widgetType': widgetType, + if (before != null) 'before': before.toJson(), + if (after != null) 'after': after.toJson(), + }; + } + TugboatInteractionResult _computeTapSettleResult({ required TugboatStateAnchor? beforeState, required TugboatStateAnchor? afterState, diff --git a/packages/tugboat/lib/src/interaction_transaction.dart b/packages/tugboat/lib/src/interaction_transaction.dart index dee4d2a..9dc308a 100644 --- a/packages/tugboat/lib/src/interaction_transaction.dart +++ b/packages/tugboat/lib/src/interaction_transaction.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:flutter/widgets.dart'; import 'anchors.dart'; +import 'control_value.dart'; import 'coordinate_space.dart'; import 'models.dart'; @@ -29,6 +30,7 @@ class InteractionOrigin { required this.startPosition, required this.pointerGeneration, required this.captureSessionId, + this.controlValue, }); final String interactionId; @@ -43,6 +45,7 @@ class InteractionOrigin { final Offset startPosition; final int pointerGeneration; final String? captureSessionId; + final TugboatControlValue? controlValue; Map toJson() => { 'interactionId': interactionId, @@ -57,6 +60,7 @@ class InteractionOrigin { 'startPosition': {'x': startPosition.dx, 'y': startPosition.dy}, 'pointerGeneration': pointerGeneration, if (captureSessionId != null) 'captureSessionId': captureSessionId, + if (controlValue != null) 'controlValue': controlValue!.toJson(), }; } @@ -164,7 +168,8 @@ class InteractionTransaction { String get id => origin.interactionId; bool get isSwipeOrScroll => - gesture == InteractionGesture.swipe || gesture == InteractionGesture.scroll; + gesture == InteractionGesture.swipe || + gesture == InteractionGesture.scroll; bool get isEligible => !claimed && !cancelled && !semanticPublished; @@ -236,7 +241,8 @@ class InteractionRegistry { _pending[tx.pointerId] = tx; } - InteractionTransaction? removePending(int pointer) => _pending.remove(pointer); + InteractionTransaction? removePending(int pointer) => + _pending.remove(pointer); void release(InteractionTransaction tx) { _pending.remove(tx.pointerId); diff --git a/packages/tugboat/lib/src/widget_roles.dart b/packages/tugboat/lib/src/widget_roles.dart index 24b162c..38aac71 100644 --- a/packages/tugboat/lib/src/widget_roles.dart +++ b/packages/tugboat/lib/src/widget_roles.dart @@ -188,13 +188,9 @@ WidgetRole? tugboatRoleForWidget(Widget widget) { ); } if (widget is Radio || widget is RadioListTile) { - final enabled = switch (widget) { - // ignore: deprecated_member_use - Radio w => w.onChanged != null, - // ignore: deprecated_member_use - RadioListTile w => w.onChanged != null, - _ => false, - }; + // Same generic-callback cast hazard as DropdownButton: reading onChanged + // through RadioListTile / Radio can throw at runtime. + final enabled = (widget as dynamic).onChanged != null; return WidgetRole( 'radio', enabled, diff --git a/packages/tugboat/lib/tugboat.dart b/packages/tugboat/lib/tugboat.dart index d50bf8b..c2e713b 100644 --- a/packages/tugboat/lib/tugboat.dart +++ b/packages/tugboat/lib/tugboat.dart @@ -5,6 +5,10 @@ export 'src/anchors.dart' TugboatNormalizedBounds, TugboatStateAnchor, TugboatTargetAnchor, + TugboatEncodedControlScalar, + TugboatControlValue, + tugboatControlValueSchemaVersion, + tugboatControlValueForWidget, tugboatIconLabel, tugboatIconHash, tugboatLabelHash; diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart new file mode 100644 index 0000000..65f3d38 --- /dev/null +++ b/packages/tugboat/test/control_value_test.dart @@ -0,0 +1,339 @@ +import 'package:flutter/cupertino.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:tugboat/tugboat.dart'; + +const _testConfig = TugboatReplayConfig( + profile: TugboatCaptureProfile.exploration, + settleDelay: Duration.zero, + enableGlobalPointerCapture: false, + capturePixelRatio: 1.0, +); + +Future _waitForCaptures(WidgetTester tester) async { + await tester.pump(); + await tester.runAsync(() async { + await Future.delayed(const Duration(milliseconds: 300)); + }); + await tester.pump(); +} + +Map? _controlValueFrom(TugboatEvent event) { + final raw = event.data['controlValue']; + if (raw is Map) return raw; + if (raw is Map) return Map.from(raw); + return null; +} + +void main() { + setUp(TugboatReplay.resetForTest); + tearDown(TugboatReplay.resetForTest); + + group('tugboatControlValueForWidget', () { + test('encodes bool and number literals', () { + final checkbox = tugboatControlValueForWidget( + Checkbox(value: true, onChanged: (_) {}), + ); + expect(checkbox?.role, 'checkbox'); + expect(checkbox?.value?.kind, 'bool'); + expect(checkbox?.value?.value, isTrue); + + final slider = tugboatControlValueForWidget( + Slider(value: 0.4, onChanged: (_) {}), + ); + expect(slider?.role, 'slider'); + expect(slider?.value?.kind, 'number'); + expect(slider?.value?.value, 0.4); + }); + + test('hashes free-text option strings and keeps developer tokens', () { + final freeText = TugboatEncodedControlScalar.encode('Secret Option Name'); + expect(freeText.kind, 'token'); + expect(freeText.value, startsWith('str:')); + expect(freeText.value, isNot(contains('Secret'))); + expect(freeText.length, 'Secret Option Name'.length); + + final token = TugboatEncodedControlScalar.encode('usd'); + expect(token.kind, 'token'); + expect(token.value, 'usd'); + }); + + test('reads radio option identity and group selection', () { + final radio = tugboatControlValueForWidget( + // ignore: deprecated_member_use + Radio( + value: 2, + // ignore: deprecated_member_use + groupValue: 1, + // ignore: deprecated_member_use + onChanged: (_) {}, + ), + ); + expect(radio?.role, 'radio'); + expect(radio?.value?.value, 2); + expect(radio?.groupValue?.value, 1); + expect(radio?.selected, isFalse); + }); + }); + + testWidgets('switch tap emits before/after control values', (tester) async { + var enabled = false; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return Switch( + key: const Key('notify-switch'), + value: enabled, + onChanged: (next) => setState(() => enabled = next), + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('notify-switch'))); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final tap = session.events.firstWhere((e) => e.type == 'tap'); + final settled = session.events.firstWhere((e) => e.type == 'tap_settled'); + + final tapValue = _controlValueFrom(tap); + expect(tapValue?['role'], 'switch'); + expect((tapValue?['value'] as Map)['value'], isFalse); + + final settledValue = _controlValueFrom(settled); + expect(settledValue?['role'], 'switch'); + expect((settledValue?['before'] as Map)['value'], isA()); + expect( + ((settledValue?['before'] as Map)['value'] as Map)['value'], + isFalse, + ); + expect(((settledValue?['after'] as Map)['value'] as Map)['value'], isTrue); + expect(enabled, isTrue); + }); + + testWidgets('radio tap records which option was selected', (tester) async { + int? selected = 1; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return Column( + children: [ + // ignore: deprecated_member_use + RadioListTile( + key: const Key('radio-1'), + title: const Text('One'), + value: 1, + // ignore: deprecated_member_use + groupValue: selected, + // ignore: deprecated_member_use + onChanged: (next) => setState(() => selected = next), + ), + // ignore: deprecated_member_use + RadioListTile( + key: const Key('radio-2'), + title: const Text('Two'), + value: 2, + // ignore: deprecated_member_use + groupValue: selected, + // ignore: deprecated_member_use + onChanged: (next) => setState(() => selected = next), + ), + ], + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('radio-2'))); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final tap = session.events.firstWhere((e) => e.type == 'tap'); + final tapValue = _controlValueFrom(tap)!; + expect(tapValue['role'], 'radio'); + expect((tapValue['value'] as Map)['value'], 2); + expect((tapValue['groupValue'] as Map)['value'], 1); + expect(tapValue['selected'], isFalse); + expect(tapValue['index'], 1); + + final settled = session.events.firstWhere((e) => e.type == 'tap_settled'); + final settledValue = _controlValueFrom(settled)!; + expect(((settledValue['after'] as Map)['value'] as Map)['value'], 2); + expect(((settledValue['after'] as Map)['groupValue'] as Map)['value'], 2); + expect((settledValue['after'] as Map)['selected'], isTrue); + expect(selected, 2); + }); + + testWidgets('dropdown item tap records the chosen option value', ( + tester, + ) async { + var selected = 1; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return DropdownButton( + key: const Key('plan-dropdown'), + value: selected, + items: const [ + DropdownMenuItem(value: 1, child: Text('Starter')), + DropdownMenuItem(value: 2, child: Text('Pro')), + ], + onChanged: (next) { + if (next != null) setState(() => selected = next); + }, + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('plan-dropdown'))); + await tester.pumpAndSettle(); + await tester.tap(find.text('Pro').last); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final itemTaps = session.events + .where((e) => e.type == 'tap') + .map(_controlValueFrom) + .where((value) => value?['role'] == 'dropdownItem') + .toList(); + expect(itemTaps, isNotEmpty); + expect((itemTaps.last!['value'] as Map)['value'], 2); + expect(selected, 2); + }); + + testWidgets('slider drag swipe records numeric value', (tester) async { + var value = 0.0; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return Slider( + key: const Key('volume-slider'), + value: value, + onChanged: (next) => setState(() => value = next), + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.drag( + find.byKey(const Key('volume-slider')), + const Offset(80, 0), + ); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final swipes = session.events.where((e) => e.type == 'swipe').toList(); + expect(swipes, isNotEmpty); + final controlValue = _controlValueFrom(swipes.last); + expect(controlValue?['role'], 'slider'); + expect((controlValue?['value'] as Map)['kind'], 'number'); + expect((controlValue?['value'] as Map)['value'], isA()); + expect(value, greaterThan(0)); + }); + + testWidgets('cupertino switch values are captured', (tester) async { + var enabled = true; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return CupertinoSwitch( + key: const Key('cupertino-switch'), + value: enabled, + onChanged: (next) => setState(() => enabled = next), + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('cupertino-switch'))); + await _waitForCaptures(tester); + + final tap = TugboatReplay.controller!.session!.events.firstWhere( + (e) => e.type == 'tap', + ); + final tapValue = _controlValueFrom(tap); + expect(tapValue?['role'], 'switch'); + expect((tapValue?['value'] as Map)['value'], isTrue); + }); + + testWidgets('free-text dropdown values stay hashed in session json', ( + tester, + ) async { + var selected = 'alpha-code'; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return DropdownButton( + value: selected, + items: const [ + DropdownMenuItem( + value: 'alpha-code', + child: Text('Alpha'), + ), + DropdownMenuItem( + value: 'Visible Secret City Name', + child: Text('Beta'), + ), + ], + onChanged: (next) { + if (next != null) setState(() => selected = next); + }, + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byType(DropdownButton)); + await tester.pumpAndSettle(); + await tester.tap(find.text('Beta').last); + await _waitForCaptures(tester); + + final json = TugboatReplay.controller!.session!.toJson().toString(); + expect(json, isNot(contains('Visible Secret City Name'))); + expect(json, contains('str:')); + }); +} From a391a3b3dd7e0b91b50473232c61fce6186d9f82 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 29 Jul 2026 06:50:33 +0000 Subject: [PATCH 4/9] feat(replay): sample semantic value/label with control values Extend per-interaction controlValue capture so custom hit targets can report Flutter semantic value/label tokens when typed widget state is unavailable. Standard controls still prefer Material/Cupertino state and merge semantics as supplemental fields under sources. Co-authored-by: Chinmay Kabi --- docs/design/capture-and-fingerprint.md | 18 +- packages/tugboat/lib/src/anchor_resolver.dart | 95 +++++- packages/tugboat/lib/src/control_value.dart | 272 ++++++++++++++++-- .../lib/src/semantics_flags_compat.dart | 35 +++ packages/tugboat/lib/tugboat.dart | 3 + packages/tugboat/test/control_value_test.dart | 82 +++++- 6 files changed, 467 insertions(+), 38 deletions(-) diff --git a/docs/design/capture-and-fingerprint.md b/docs/design/capture-and-fingerprint.md index 6b92e22..92cd5ba 100644 --- a/docs/design/capture-and-fingerprint.md +++ b/docs/design/capture-and-fingerprint.md @@ -236,20 +236,30 @@ Developer-authored identity strings can still be emitted: - canonical structural paths. Interaction events may also carry a privacy-safe `controlValue` payload for -valued controls (checkbox, switch, radio, slider, dropdown / menu item, chip): +valued controls (checkbox, switch, radio, slider, dropdown / menu item, chip) +and for hit targets that expose Flutter semantic annotations: - bools and numbers are emitted literally; +- numeric strings from semantics (for example `"15"`) are parsed as numbers; - enums and short developer-token strings are emitted as tokens; -- free-text option strings are hashed (`str:`) with length only. +- free-text option / semantic label strings are hashed (`str:`) with + length only. `tap` includes the value sampled at pointer-down. `tap_settled` includes `before` / `after` snapshots so toggle flips and post-callback radio/dropdown selections are visible. Slider drags that become `swipe` events also carry the value sampled at pointer-up. +When a typed widget value is unavailable (custom GestureDetector rows, bottom +sheets, etc.), the SDK still samples `SemanticsProperties` / live semantics +nodes under the pointer and records `semanticValue` / `semanticLabel` with the +same encoding rules. Standard controls may include both widget state and +semantic annotations under `sources: ["semantics","widget"]`. + Bounds, pointer coordinates, scroll metrics, and masked screenshot pixels are -also capture data. Apps must treat tags, route names, and subview labels as -telemetry and avoid putting user data in them. +also capture data. Apps must treat tags, route names, subview labels, and +semantic value/label tokens as telemetry and avoid putting raw user PII in +them. ## Screenshot pipeline diff --git a/packages/tugboat/lib/src/anchor_resolver.dart b/packages/tugboat/lib/src/anchor_resolver.dart index 5c23972..96056ee 100644 --- a/packages/tugboat/lib/src/anchor_resolver.dart +++ b/packages/tugboat/lib/src/anchor_resolver.dart @@ -171,9 +171,10 @@ class AnchorResolver { /// Privacy-safe control value under [globalPosition], if any. /// - /// Samples toggle/radio/dropdown/slider widget state at hit time. Free-text - /// option labels are hashed; bools, numbers, enums, and short developer - /// tokens are retained. + /// Samples standard Material/Cupertino control state and, when present, + /// Flutter semantic value/label annotations on the hit target. Free-text + /// strings are hashed; bools, numbers, enums, numeric strings, and short + /// developer tokens are retained. TugboatControlValue? controlValueAt(Offset globalPosition) { final rootContext = rootKey.currentContext; final rootRender = rootContext?.findRenderObject(); @@ -182,20 +183,92 @@ class AnchorResolver { final tokenMap = _tokenMapFor(rootContext, rootRender); if (tokenMap == null) return null; - final result = BoxHitTestResult(); - final localPosition = rootRender.globalToLocal(globalPosition); - rootRender.hitTest(result, position: localPosition); + final pipelineOwner = + rootRender.owner ?? RendererBinding.instance.rootPipelineOwner; + final semanticsAlreadyEnabled = + pipelineOwner.semanticsOwner != null || + RendererBinding.instance.rootPipelineOwner.semanticsOwner != null; + final semanticsHandle = semanticsAlreadyEnabled + ? null + : SemanticsBinding.instance.ensureSemantics(); + try { + if (!semanticsAlreadyEnabled) { + pipelineOwner.flushSemantics(); + } - for (final entry in result.path) { - if (entry.target is! RenderObject) continue; - final element = tokenMap.renderElements[entry.target as RenderObject]; - if (element == null || tugboatIsCaptureChrome(element.widget)) continue; - final value = tugboatControlValueForElement(element); + final result = BoxHitTestResult(); + final localPosition = rootRender.globalToLocal(globalPosition); + rootRender.hitTest(result, position: localPosition); + + for (final entry in result.path) { + if (entry.target is! RenderObject) continue; + final element = tokenMap.renderElements[entry.target as RenderObject]; + if (element == null || tugboatIsCaptureChrome(element.widget)) continue; + final value = tugboatControlValueForElement(element); + if (value != null) return value; + } + + // Fall back to the semantics tree for custom hit targets that only + // expose value/label through accessibility annotations. + return _controlValueFromSemanticsHit( + globalPosition: globalPosition, + rootContext: rootContext, + rootRender: rootRender, + ); + } finally { + semanticsHandle?.dispose(); + } + } + + TugboatControlValue? _controlValueFromSemanticsHit({ + required Offset globalPosition, + required Element rootContext, + required RenderBox rootRender, + }) { + final pipelineOwner = + rootRender.owner ?? RendererBinding.instance.rootPipelineOwner; + final semanticsOwner = + pipelineOwner.semanticsOwner ?? + RendererBinding.instance.rootPipelineOwner.semanticsOwner; + if (semanticsOwner == null) return null; + pipelineOwner.flushSemantics(); + final rootNode = semanticsOwner.rootSemanticsNode; + if (rootNode == null) return null; + + final devicePixelRatio = View.maybeOf(rootContext)?.devicePixelRatio ?? 1.0; + final physical = globalPosition * devicePixelRatio; + final hits = []; + _collectSemanticsHits(rootNode, physical, hits, Matrix4.identity()); + for (final node in hits.reversed) { + final value = tugboatControlValueFromSemanticsNode(node); if (value != null) return value; } return null; } + void _collectSemanticsHits( + SemanticsNode node, + Offset physicalGlobal, + List hits, + Matrix4 transformToRoot, + ) { + final transform = node.transform; + final nextTransform = transform == null + ? transformToRoot + : (transformToRoot.clone()..multiply(transform)); + final inverted = Matrix4.tryInvert(nextTransform); + if (inverted != null) { + final local = MatrixUtils.transformPoint(inverted, physicalGlobal); + if (node.rect.contains(local)) { + hits.add(node); + } + } + node.visitChildren((child) { + _collectSemanticsHits(child, physicalGlobal, hits, nextTransform); + return true; + }); + } + /// Builds inventory and resolves a tap target from one token-map walk. ({TugboatSceneInventory? inventory, TugboatTargetAnchor? target}) buildTapContext({ diff --git a/packages/tugboat/lib/src/control_value.dart b/packages/tugboat/lib/src/control_value.dart index 2c9dddf..6f0bd8e 100644 --- a/packages/tugboat/lib/src/control_value.dart +++ b/packages/tugboat/lib/src/control_value.dart @@ -1,7 +1,7 @@ part of 'anchors.dart'; /// Schema version for privacy-safe control value payloads. -const int tugboatControlValueSchemaVersion = 1; +const int tugboatControlValueSchemaVersion = 2; final RegExp _developerTokenPattern = RegExp(r'^[A-Za-z0-9_./:-]{1,64}$'); @@ -39,13 +39,21 @@ class TugboatEncodedControlScalar { ); } if (raw is String) { - if (_developerTokenPattern.hasMatch(raw)) { - return TugboatEncodedControlScalar._(kind: 'token', value: raw); + final trimmed = raw.trim(); + if (trimmed.isEmpty) { + return const TugboatEncodedControlScalar._(kind: 'null'); + } + final asNum = num.tryParse(trimmed); + if (asNum != null) { + return TugboatEncodedControlScalar._(kind: 'number', value: asNum); + } + if (_developerTokenPattern.hasMatch(trimmed)) { + return TugboatEncodedControlScalar._(kind: 'token', value: trimmed); } return TugboatEncodedControlScalar._( kind: 'token', - value: 'str:${tugboatLabelHash(raw)}', - length: raw.length, + value: 'str:${tugboatLabelHash(trimmed)}', + length: trimmed.length, ); } final text = raw.toString(); @@ -75,36 +83,47 @@ class TugboatEncodedControlScalar { /// Privacy-safe snapshot of an interactive control's value at sample time. /// -/// Free-text option labels are hashed. Bools, numbers, enums, and short -/// developer-token strings are retained so taps on radios, dropdowns, -/// toggles, and sliders remain interpretable without storing arbitrary UI copy. +/// Prefer typed widget state for standard Material/Cupertino controls. When +/// the hit target exposes Flutter semantics, [semanticValue] / [semanticLabel] +/// are attached as well so custom rows (e.g. GestureDetector lists) can still +/// report developer-authored semantic tokens. +/// +/// Free-text strings are hashed. Bools, numbers, enums, numeric strings, and +/// short developer-token strings are retained. class TugboatControlValue { const TugboatControlValue({ required this.role, this.widgetType, + this.sources = const ['widget'], this.value, this.groupValue, this.selected, this.index, this.start, this.end, + this.semanticValue, + this.semanticLabel, this.schemaVersion = tugboatControlValueSchemaVersion, }); final int schemaVersion; /// Control role (`checkbox`, `switch`, `radio`, `slider`, `dropdown`, - /// `dropdownItem`, `menuItem`, `chip`). + /// `dropdownItem`, `menuItem`, `chip`, `button`, `semantic`, …). final String role; final String? widgetType; - /// Primary sampled value (option identity, toggle state, slider position). + /// Provenance markers such as `widget` and/or `semantics`. + final List sources; + + /// Primary sampled value (option identity, toggle state, slider position, + /// or best-effort semantic value when no typed widget value exists). final TugboatEncodedControlScalar? value; /// Current group selection for radio controls. final TugboatEncodedControlScalar? groupValue; - /// Whether this option is the active selection (radios/chips). + /// Whether this option is the active selection (radios/chips/semantics). final bool? selected; /// Zero-based index among sibling options when known. @@ -116,16 +135,63 @@ class TugboatControlValue { /// Range slider end (inclusive). final TugboatEncodedControlScalar? end; + /// Encoded [SemanticsData.value] / [SemanticsProperties.value] when present. + final TugboatEncodedControlScalar? semanticValue; + + /// Encoded [SemanticsData.label] / [SemanticsProperties.label] when present. + final TugboatEncodedControlScalar? semanticLabel; + + bool get hasPayload => + value != null || + groupValue != null || + selected != null || + start != null || + end != null || + semanticValue != null || + semanticLabel != null; + + TugboatControlValue copyWith({ + String? role, + String? widgetType, + List? sources, + TugboatEncodedControlScalar? value, + TugboatEncodedControlScalar? groupValue, + bool? selected, + int? index, + TugboatEncodedControlScalar? start, + TugboatEncodedControlScalar? end, + TugboatEncodedControlScalar? semanticValue, + TugboatEncodedControlScalar? semanticLabel, + }) { + return TugboatControlValue( + schemaVersion: schemaVersion, + role: role ?? this.role, + widgetType: widgetType ?? this.widgetType, + sources: sources ?? this.sources, + value: value ?? this.value, + groupValue: groupValue ?? this.groupValue, + selected: selected ?? this.selected, + index: index ?? this.index, + start: start ?? this.start, + end: end ?? this.end, + semanticValue: semanticValue ?? this.semanticValue, + semanticLabel: semanticLabel ?? this.semanticLabel, + ); + } + Map toJson() => { 'schemaVersion': schemaVersion, 'role': role, if (widgetType != null && widgetType!.isNotEmpty) 'widgetType': widgetType, + if (sources.isNotEmpty) 'sources': sources, if (value != null) 'value': value!.toJson(), if (groupValue != null) 'groupValue': groupValue!.toJson(), if (selected != null) 'selected': selected, if (index != null) 'index': index, if (start != null) 'start': start!.toJson(), if (end != null) 'end': end!.toJson(), + if (semanticValue != null) 'semanticValue': semanticValue!.toJson(), + if (semanticLabel != null) 'semanticLabel': semanticLabel!.toJson(), }; @override @@ -134,24 +200,30 @@ class TugboatControlValue { schemaVersion == other.schemaVersion && role == other.role && widgetType == other.widgetType && + _listEquals(sources, other.sources) && value == other.value && groupValue == other.groupValue && selected == other.selected && index == other.index && start == other.start && - end == other.end; + end == other.end && + semanticValue == other.semanticValue && + semanticLabel == other.semanticLabel; @override int get hashCode => Object.hash( schemaVersion, role, widgetType, + Object.hashAll(sources), value, groupValue, selected, index, start, end, + semanticValue, + semanticLabel, ); } @@ -292,22 +364,184 @@ TugboatControlValue? tugboatControlValueForWidget(Widget widget, {int? index}) { return null; } -/// Walks [hitElement] and its ancestors for the deepest valued control. +/// Builds a control-value snapshot from explicit [SemanticsProperties]. +TugboatControlValue? tugboatControlValueFromSemanticsProperties( + SemanticsProperties properties, { + String? widgetType, + int? index, + String? roleHint, +}) { + final label = properties.label; + final valueText = properties.value; + final selected = properties.selected; + final checked = properties.checked; + final toggled = properties.toggled; + + final semanticValue = (valueText != null && valueText.trim().isNotEmpty) + ? TugboatEncodedControlScalar.encode(valueText) + : null; + final semanticLabel = (label != null && label.trim().isNotEmpty) + ? TugboatEncodedControlScalar.encode(label) + : null; + + if (semanticValue == null && + semanticLabel == null && + selected == null && + checked == null && + toggled == null) { + return null; + } + + final role = + roleHint ?? + (properties.slider == true + ? 'slider' + : properties.button == true + ? 'button' + : checked != null + ? 'checkbox' + : toggled != null + ? 'switch' + : 'semantic'); + + return TugboatControlValue( + role: role, + widgetType: widgetType, + sources: const ['semantics'], + value: + semanticValue ?? + (checked != null + ? TugboatEncodedControlScalar.encode(checked) + : toggled != null + ? TugboatEncodedControlScalar.encode(toggled) + : selected != null + ? TugboatEncodedControlScalar.encode(selected) + : null), + selected: selected ?? checked ?? toggled, + index: index, + semanticValue: semanticValue, + semanticLabel: semanticLabel, + ); +} + +/// Builds a control-value snapshot from a live [SemanticsNode]. +TugboatControlValue? tugboatControlValueFromSemanticsNode( + SemanticsNode node, { + String? roleHint, +}) { + final data = node.getSemanticsData(); + final flags = data.flagsCollection; + final checked = semanticsCheckedFromFlags(flags); + final toggled = semanticsToggledFromFlags(flags); + final selected = semanticsSelectedFromFlags(flags); + + final semanticValue = data.value.trim().isNotEmpty + ? TugboatEncodedControlScalar.encode(data.value) + : null; + final semanticLabel = data.label.trim().isNotEmpty + ? TugboatEncodedControlScalar.encode(data.label) + : null; + + if (semanticValue == null && + semanticLabel == null && + checked == null && + toggled == null && + selected == null) { + return null; + } + + final role = + roleHint ?? + (flags.isButton + ? 'button' + : checked != null + ? 'checkbox' + : toggled != null + ? 'switch' + : data.role != SemanticsRole.none + ? data.role.name + : 'semantic'); + + return TugboatControlValue( + role: role, + sources: const ['semantics'], + value: + semanticValue ?? + (checked != null + ? TugboatEncodedControlScalar.encode(checked) + : toggled != null + ? TugboatEncodedControlScalar.encode(toggled) + : selected != null + ? TugboatEncodedControlScalar.encode(selected) + : null), + selected: selected ?? checked ?? toggled, + semanticValue: semanticValue, + semanticLabel: semanticLabel, + ); +} + +/// Merges typed widget state with semantic annotations. +TugboatControlValue? tugboatMergeControlValues( + TugboatControlValue? widgetValue, + TugboatControlValue? semanticsValue, +) { + if (widgetValue == null) return semanticsValue; + if (semanticsValue == null) { + return widgetValue.sources.contains('widget') + ? widgetValue + : widgetValue.copyWith(sources: const ['widget']); + } + + final sources = { + ...widgetValue.sources, + ...semanticsValue.sources, + 'widget', + 'semantics', + }.toList()..sort(); + + return widgetValue.copyWith( + sources: sources, + value: widgetValue.value ?? semanticsValue.value, + selected: widgetValue.selected ?? semanticsValue.selected, + semanticValue: semanticsValue.semanticValue ?? widgetValue.semanticValue, + semanticLabel: semanticsValue.semanticLabel ?? widgetValue.semanticLabel, + ); +} + +/// Walks [hitElement] and its ancestors for widget + semantic control values. TugboatControlValue? tugboatControlValueForElement(Element hitElement) { - TugboatControlValue? found; + TugboatControlValue? widgetValue; + TugboatControlValue? semanticsValue; void consider(Element element) { - if (found != null) return; - final index = _optionIndexAmongSiblings(element); - found = tugboatControlValueForWidget(element.widget, index: index); + final index = widgetValue == null + ? _optionIndexAmongSiblings(element) + : null; + widgetValue ??= tugboatControlValueForWidget(element.widget, index: index); + // Explicit Semantics widgets are preferred over live nodes for labels. + if (semanticsValue == null && element.widget is Semantics) { + semanticsValue = tugboatControlValueFromSemanticsProperties( + (element.widget as Semantics).properties, + widgetType: element.widget.runtimeType.toString(), + ); + } + if (semanticsValue == null) { + final node = element.renderObject?.debugSemantics; + if (node != null) { + semanticsValue = tugboatControlValueFromSemanticsNode(node); + } + } } consider(hitElement); hitElement.visitAncestorElements((ancestor) { consider(ancestor); - return found == null; + return widgetValue == null || semanticsValue == null; }); - return found; + + final merged = tugboatMergeControlValues(widgetValue, semanticsValue); + if (merged == null || !merged.hasPayload) return null; + return merged; } int? _optionIndexAmongSiblings(Element element) { diff --git a/packages/tugboat/lib/src/semantics_flags_compat.dart b/packages/tugboat/lib/src/semantics_flags_compat.dart index 6365205..fee65fc 100644 --- a/packages/tugboat/lib/src/semantics_flags_compat.dart +++ b/packages/tugboat/lib/src/semantics_flags_compat.dart @@ -14,3 +14,38 @@ bool? semanticsEnabledFromFlags(SemanticsFlags flags) { } return enabled.toBoolOrNull() as bool?; } + +/// Reads checked state across Flutter SDK versions. +bool? semanticsCheckedFromFlags(SemanticsFlags flags) { + final dynamic state = flags; + final checked = state.isChecked; + if (checked is bool) { + if (state.hasCheckedState == true) return checked; + return null; + } + // Flutter 3.36+: CheckedState enum (none / isTrue / isFalse / mixed). + try { + if (checked.toString().endsWith('.none')) return null; + if (checked.toString().endsWith('.mixed')) return null; + return checked == CheckedState.isTrue; + } catch (_) { + return null; + } +} + +/// Reads toggled state across Flutter SDK versions. +bool? semanticsToggledFromFlags(SemanticsFlags flags) { + final dynamic state = flags; + final toggled = state.isToggled; + if (toggled is bool) return toggled; + return toggled.toBoolOrNull() as bool?; +} + +/// Reads selected state across Flutter SDK versions. +bool? semanticsSelectedFromFlags(SemanticsFlags flags) { + final dynamic state = flags; + final selected = state.isSelected; + if (selected is bool) return selected; + // Flutter 3.36+: Tristate. + return selected.toBoolOrNull() as bool?; +} diff --git a/packages/tugboat/lib/tugboat.dart b/packages/tugboat/lib/tugboat.dart index c2e713b..5ddf08f 100644 --- a/packages/tugboat/lib/tugboat.dart +++ b/packages/tugboat/lib/tugboat.dart @@ -9,6 +9,9 @@ export 'src/anchors.dart' TugboatControlValue, tugboatControlValueSchemaVersion, tugboatControlValueForWidget, + tugboatControlValueFromSemanticsProperties, + tugboatControlValueFromSemanticsNode, + tugboatMergeControlValues, tugboatIconLabel, tugboatIconHash, tugboatLabelHash; diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart index 65f3d38..bdc4499 100644 --- a/packages/tugboat/test/control_value_test.dart +++ b/packages/tugboat/test/control_value_test.dart @@ -1,5 +1,6 @@ import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/semantics.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:tugboat/tugboat.dart'; @@ -307,10 +308,7 @@ void main() { return DropdownButton( value: selected, items: const [ - DropdownMenuItem( - value: 'alpha-code', - child: Text('Alpha'), - ), + DropdownMenuItem(value: 'alpha-code', child: Text('Alpha')), DropdownMenuItem( value: 'Visible Secret City Name', child: Text('Beta'), @@ -336,4 +334,80 @@ void main() { expect(json, isNot(contains('Visible Secret City Name'))); expect(json, contains('str:')); }); + + test('semantic properties encode value and label tokens', () { + final snapshot = tugboatControlValueFromSemanticsProperties( + const SemanticsProperties( + button: true, + value: '15', + label: 'Duration fifteen seconds', + selected: true, + ), + ); + expect(snapshot?.role, 'button'); + expect(snapshot?.sources, ['semantics']); + expect(snapshot?.value?.kind, 'number'); + expect(snapshot?.value?.value, 15); + expect(snapshot?.semanticValue?.value, 15); + expect(snapshot?.semanticLabel?.value, startsWith('str:')); + expect(snapshot?.selected, isTrue); + }); + + testWidgets('custom gesture detector list captures semantic value/label', ( + tester, + ) async { + String? selected; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return Column( + children: [ + Semantics( + button: true, + value: '15', + label: 'Duration 15 seconds', + selected: selected == '15', + child: GestureDetector( + key: const Key('duration-15'), + onTap: () => setState(() => selected = '15'), + child: const Text('15 seconds'), + ), + ), + Semantics( + button: true, + value: '30', + label: 'Duration 30 seconds', + selected: selected == '30', + child: GestureDetector( + key: const Key('duration-30'), + onTap: () => setState(() => selected = '30'), + child: const Text('30 seconds'), + ), + ), + ], + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('duration-30'))); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final tap = session.events.firstWhere((e) => e.type == 'tap'); + final tapValue = _controlValueFrom(tap)!; + expect(tapValue['sources'], contains('semantics')); + expect((tapValue['semanticValue'] as Map)['value'], 30); + expect((tapValue['value'] as Map)['value'], 30); + expect((tapValue['semanticLabel'] as Map)['value'], startsWith('str:')); + expect(tapValue.toString(), isNot(contains('Duration 30 seconds'))); + expect(selected, '30'); + }); } From ff7a22d63816b90ef88c317a6b361ad23128dfdb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 29 Jul 2026 06:59:42 +0000 Subject: [PATCH 5/9] feat(replay): emit semanticAnnotation on all interactions Attach privacy-safe semantic identifier/label/value to tap, settle, swipe, and scroll events whenever Flutter semantics expose them. Merge ancestor/descendant nodes so Material button roles pick up child labels. Co-authored-by: Chinmay Kabi --- docs/design/capture-and-fingerprint.md | 8 + packages/tugboat/lib/src/anchor_resolver.dart | 136 ++++++++-- packages/tugboat/lib/src/control_value.dart | 236 ++++++++++++++++++ packages/tugboat/lib/src/controller.dart | 23 ++ .../lib/src/interaction_transaction.dart | 3 + packages/tugboat/lib/tugboat.dart | 5 + packages/tugboat/test/control_value_test.dart | 95 +++++++ 7 files changed, 486 insertions(+), 20 deletions(-) diff --git a/docs/design/capture-and-fingerprint.md b/docs/design/capture-and-fingerprint.md index 92cd5ba..3684131 100644 --- a/docs/design/capture-and-fingerprint.md +++ b/docs/design/capture-and-fingerprint.md @@ -256,6 +256,14 @@ nodes under the pointer and records `semanticValue` / `semanticLabel` with the same encoding rules. Standard controls may include both widget state and semantic annotations under `sources: ["semantics","widget"]`. +Independently, every interaction event (`tap`, `tap_settled`, `swipe`, +`scroll_start`, `scroll_end`) may carry a top-level `semanticAnnotation` +payload whenever Flutter semantics expose an identifier, label, value, or +selection flag on the target. This covers ordinary buttons and scrollables as +well as valued controls. The field is named `semanticAnnotation` to avoid +colliding with `tap_settled.data.settleObservation.semantic` (state-signature +change evidence). + Bounds, pointer coordinates, scroll metrics, and masked screenshot pixels are also capture data. Apps must treat tags, route names, subview labels, and semantic value/label tokens as telemetry and avoid putting raw user PII in diff --git a/packages/tugboat/lib/src/anchor_resolver.dart b/packages/tugboat/lib/src/anchor_resolver.dart index 96056ee..da48f98 100644 --- a/packages/tugboat/lib/src/anchor_resolver.dart +++ b/packages/tugboat/lib/src/anchor_resolver.dart @@ -183,19 +183,7 @@ class AnchorResolver { final tokenMap = _tokenMapFor(rootContext, rootRender); if (tokenMap == null) return null; - final pipelineOwner = - rootRender.owner ?? RendererBinding.instance.rootPipelineOwner; - final semanticsAlreadyEnabled = - pipelineOwner.semanticsOwner != null || - RendererBinding.instance.rootPipelineOwner.semanticsOwner != null; - final semanticsHandle = semanticsAlreadyEnabled - ? null - : SemanticsBinding.instance.ensureSemantics(); - try { - if (!semanticsAlreadyEnabled) { - pipelineOwner.flushSemantics(); - } - + return _withSemanticsEnabled(rootRender, () { final result = BoxHitTestResult(); final localPosition = rootRender.globalToLocal(globalPosition); rootRender.hitTest(result, position: localPosition); @@ -215,6 +203,69 @@ class AnchorResolver { rootContext: rootContext, rootRender: rootRender, ); + }); + } + + /// Privacy-safe semantic annotation under [globalPosition], if any. + /// + /// Used for all interactions (buttons, custom rows, scrolls) so enrichment + /// can see identifier/label/value whenever Flutter semantics expose them. + TugboatSemanticAnnotation? semanticAnnotationAt(Offset globalPosition) { + final rootContext = rootKey.currentContext; + final rootRender = rootContext?.findRenderObject(); + if (rootRender is! RenderBox || rootContext is! Element) return null; + + final tokenMap = _tokenMapFor(rootContext, rootRender); + if (tokenMap == null) return null; + + return _withSemanticsEnabled(rootRender, () { + final result = BoxHitTestResult(); + final localPosition = rootRender.globalToLocal(globalPosition); + rootRender.hitTest(result, position: localPosition); + + for (final entry in result.path) { + if (entry.target is! RenderObject) continue; + final element = tokenMap.renderElements[entry.target as RenderObject]; + if (element == null || tugboatIsCaptureChrome(element.widget)) continue; + final annotation = tugboatSemanticAnnotationForElement(element); + if (annotation != null) return annotation; + } + + return _semanticAnnotationFromSemanticsHit( + globalPosition: globalPosition, + rootContext: rootContext, + rootRender: rootRender, + ); + }); + } + + /// Privacy-safe semantic annotation for an [element] already in the tree. + TugboatSemanticAnnotation? semanticAnnotationForElement(Element element) { + final rootContext = rootKey.currentContext; + final rootRender = rootContext?.findRenderObject(); + if (rootRender is! RenderBox) { + return tugboatSemanticAnnotationForElement(element); + } + return _withSemanticsEnabled( + rootRender, + () => tugboatSemanticAnnotationForElement(element), + ); + } + + T _withSemanticsEnabled(RenderBox rootRender, T Function() body) { + final pipelineOwner = + rootRender.owner ?? RendererBinding.instance.rootPipelineOwner; + final semanticsAlreadyEnabled = + pipelineOwner.semanticsOwner != null || + RendererBinding.instance.rootPipelineOwner.semanticsOwner != null; + final semanticsHandle = semanticsAlreadyEnabled + ? null + : SemanticsBinding.instance.ensureSemantics(); + try { + if (!semanticsAlreadyEnabled) { + pipelineOwner.flushSemantics(); + } + return body(); } finally { semanticsHandle?.dispose(); } @@ -224,26 +275,71 @@ class AnchorResolver { required Offset globalPosition, required Element rootContext, required RenderBox rootRender, + }) { + final node = _deepestSemanticsNodeAt( + globalPosition: globalPosition, + rootContext: rootContext, + rootRender: rootRender, + ); + if (node == null) return null; + return tugboatControlValueFromSemanticsNode(node); + } + + TugboatSemanticAnnotation? _semanticAnnotationFromSemanticsHit({ + required Offset globalPosition, + required Element rootContext, + required RenderBox rootRender, + }) { + final hits = _semanticsNodesAt( + globalPosition: globalPosition, + rootContext: rootContext, + rootRender: rootRender, + ); + TugboatSemanticAnnotation? merged; + // hits are root→leaf; reverse so deeper nodes win, ancestors fill gaps. + for (final node in hits.reversed) { + final next = tugboatSemanticAnnotationFromNode(node); + if (next == null) continue; + merged = merged == null + ? next + : tugboatMergeSemanticAnnotations(merged, next); + } + return merged; + } + + SemanticsNode? _deepestSemanticsNodeAt({ + required Offset globalPosition, + required Element rootContext, + required RenderBox rootRender, + }) { + final hits = _semanticsNodesAt( + globalPosition: globalPosition, + rootContext: rootContext, + rootRender: rootRender, + ); + return hits.isEmpty ? null : hits.last; + } + + List _semanticsNodesAt({ + required Offset globalPosition, + required Element rootContext, + required RenderBox rootRender, }) { final pipelineOwner = rootRender.owner ?? RendererBinding.instance.rootPipelineOwner; final semanticsOwner = pipelineOwner.semanticsOwner ?? RendererBinding.instance.rootPipelineOwner.semanticsOwner; - if (semanticsOwner == null) return null; + if (semanticsOwner == null) return const []; pipelineOwner.flushSemantics(); final rootNode = semanticsOwner.rootSemanticsNode; - if (rootNode == null) return null; + if (rootNode == null) return const []; final devicePixelRatio = View.maybeOf(rootContext)?.devicePixelRatio ?? 1.0; final physical = globalPosition * devicePixelRatio; final hits = []; _collectSemanticsHits(rootNode, physical, hits, Matrix4.identity()); - for (final node in hits.reversed) { - final value = tugboatControlValueFromSemanticsNode(node); - if (value != null) return value; - } - return null; + return hits; } void _collectSemanticsHits( diff --git a/packages/tugboat/lib/src/control_value.dart b/packages/tugboat/lib/src/control_value.dart index 6f0bd8e..e1aed82 100644 --- a/packages/tugboat/lib/src/control_value.dart +++ b/packages/tugboat/lib/src/control_value.dart @@ -3,6 +3,9 @@ part of 'anchors.dart'; /// Schema version for privacy-safe control value payloads. const int tugboatControlValueSchemaVersion = 2; +/// Schema version for per-interaction semantic annotations. +const int tugboatSemanticAnnotationSchemaVersion = 1; + final RegExp _developerTokenPattern = RegExp(r'^[A-Za-z0-9_./:-]{1,64}$'); /// Encodes a single control scalar without retaining free-text labels. @@ -81,6 +84,239 @@ class TugboatEncodedControlScalar { int get hashCode => Object.hash(kind, value, length); } +/// Privacy-safe semantic annotation for any interaction target. +/// +/// Attached to taps, settles, swipes, and scrolls whenever Flutter semantics +/// expose an identifier, label, value, or selection flag under the target. +class TugboatSemanticAnnotation { + const TugboatSemanticAnnotation({ + this.role, + this.identifier, + this.label, + this.value, + this.selected, + this.checked, + this.toggled, + this.schemaVersion = tugboatSemanticAnnotationSchemaVersion, + }); + + final int schemaVersion; + final String? role; + + /// Developer-authored semantics identifier when set. + final TugboatEncodedControlScalar? identifier; + + /// Encoded semantics label (hashed when free-text). + final TugboatEncodedControlScalar? label; + + /// Encoded semantics value (numbers/tokens retained). + final TugboatEncodedControlScalar? value; + + final bool? selected; + final bool? checked; + final bool? toggled; + + bool get hasPayload => + (role != null && role!.isNotEmpty) || + identifier != null || + label != null || + value != null || + selected != null || + checked != null || + toggled != null; + + Map toJson() => { + 'schemaVersion': schemaVersion, + if (role != null && role!.isNotEmpty) 'role': role, + if (identifier != null) 'identifier': identifier!.toJson(), + if (label != null) 'label': label!.toJson(), + if (value != null) 'value': value!.toJson(), + if (selected != null) 'selected': selected, + if (checked != null) 'checked': checked, + if (toggled != null) 'toggled': toggled, + }; + + @override + bool operator ==(Object other) => + other is TugboatSemanticAnnotation && + schemaVersion == other.schemaVersion && + role == other.role && + identifier == other.identifier && + label == other.label && + value == other.value && + selected == other.selected && + checked == other.checked && + toggled == other.toggled; + + @override + int get hashCode => Object.hash( + schemaVersion, + role, + identifier, + label, + value, + selected, + checked, + toggled, + ); +} + +/// Builds a semantic annotation from explicit [SemanticsProperties]. +TugboatSemanticAnnotation? tugboatSemanticAnnotationFromProperties( + SemanticsProperties properties, { + String? roleHint, +}) { + final identifierText = properties.identifier; + final labelText = properties.label; + final valueText = properties.value; + final selected = properties.selected; + final checked = properties.checked; + final toggled = properties.toggled; + + final identifier = + (identifierText != null && identifierText.trim().isNotEmpty) + ? TugboatEncodedControlScalar.encode(identifierText) + : null; + final label = (labelText != null && labelText.trim().isNotEmpty) + ? TugboatEncodedControlScalar.encode(labelText) + : null; + final value = (valueText != null && valueText.trim().isNotEmpty) + ? TugboatEncodedControlScalar.encode(valueText) + : null; + + final role = + roleHint ?? + (properties.slider == true + ? 'slider' + : properties.button == true + ? 'button' + : properties.link == true + ? 'link' + : properties.textField == true + ? 'textField' + : properties.header == true + ? 'header' + : checked != null + ? 'checkbox' + : toggled != null + ? 'switch' + : null); + + final annotation = TugboatSemanticAnnotation( + role: role, + identifier: identifier, + label: label, + value: value, + selected: selected, + checked: checked, + toggled: toggled, + ); + return annotation.hasPayload ? annotation : null; +} + +/// Builds a semantic annotation from a live [SemanticsNode]. +TugboatSemanticAnnotation? tugboatSemanticAnnotationFromNode( + SemanticsNode node, { + String? roleHint, +}) { + final data = node.getSemanticsData(); + final flags = data.flagsCollection; + final checked = semanticsCheckedFromFlags(flags); + final toggled = semanticsToggledFromFlags(flags); + final selected = semanticsSelectedFromFlags(flags); + + final identifier = data.identifier.trim().isNotEmpty + ? TugboatEncodedControlScalar.encode(data.identifier) + : null; + final label = data.label.trim().isNotEmpty + ? TugboatEncodedControlScalar.encode(data.label) + : null; + final value = data.value.trim().isNotEmpty + ? TugboatEncodedControlScalar.encode(data.value) + : null; + + final role = + roleHint ?? + (flags.isButton + ? 'button' + : flags.isLink + ? 'link' + : flags.isTextField + ? 'textField' + : flags.isHeader + ? 'header' + : checked != null + ? 'checkbox' + : toggled != null + ? 'switch' + : data.role != SemanticsRole.none + ? data.role.name + : null); + + final annotation = TugboatSemanticAnnotation( + role: role, + identifier: identifier, + label: label, + value: value, + selected: selected, + checked: checked, + toggled: toggled, + ); + return annotation.hasPayload ? annotation : null; +} + +/// Merges two annotations, preferring [primary] fields and filling gaps. +TugboatSemanticAnnotation tugboatMergeSemanticAnnotations( + TugboatSemanticAnnotation primary, + TugboatSemanticAnnotation fallback, +) { + return TugboatSemanticAnnotation( + role: (primary.role != null && primary.role!.isNotEmpty) + ? primary.role + : fallback.role, + identifier: primary.identifier ?? fallback.identifier, + label: primary.label ?? fallback.label, + value: primary.value ?? fallback.value, + selected: primary.selected ?? fallback.selected, + checked: primary.checked ?? fallback.checked, + toggled: primary.toggled ?? fallback.toggled, + ); +} + +/// Walks [hitElement] and ancestors, merging semantic fields. +/// +/// Child/deeper nodes win for concrete fields; ancestors fill gaps so a +/// Material button role can combine with a child Text label. +TugboatSemanticAnnotation? tugboatSemanticAnnotationForElement( + Element hitElement, +) { + TugboatSemanticAnnotation? merged; + + void consider(Element element) { + TugboatSemanticAnnotation? next; + if (element.widget is Semantics) { + next = tugboatSemanticAnnotationFromProperties( + (element.widget as Semantics).properties, + ); + } + next ??= () { + final node = element.renderObject?.debugSemantics; + return node == null ? null : tugboatSemanticAnnotationFromNode(node); + }(); + if (next == null) return; + merged = merged == null + ? next + : tugboatMergeSemanticAnnotations(merged!, next); + } + + consider(hitElement); + hitElement.visitAncestorElements((ancestor) { + consider(ancestor); + return true; + }); + return merged; +} + /// Privacy-safe snapshot of an interactive control's value at sample time. /// /// Prefer typed widget state for standard Material/Cupertino controls. When diff --git a/packages/tugboat/lib/src/controller.dart b/packages/tugboat/lib/src/controller.dart index 53b419e..2b15ce6 100644 --- a/packages/tugboat/lib/src/controller.dart +++ b/packages/tugboat/lib/src/controller.dart @@ -43,6 +43,7 @@ class _ScrollTracker { required this.depth, required this.maxScrollExtent, this.pageStart, + this.semantic, }); final Element scrollableElement; @@ -58,6 +59,7 @@ class _ScrollTracker { final int depth; final double maxScrollExtent; final double? pageStart; + final TugboatSemanticAnnotation? semantic; int overscrollCount = 0; DateTime? lastSampleAt; } @@ -2267,6 +2269,7 @@ class TugboatReplayController extends ChangeNotifier { TugboatStateAnchor? tapState = _currentStateAnchor; TugboatSceneInventory? tapInventory; TugboatControlValue? controlValue; + TugboatSemanticAnnotation? semantic; if (resolver != null && config.profile != TugboatCaptureProfile.dormant) { final tapContext = resolver.buildTapContext( @@ -2278,6 +2281,7 @@ class TugboatReplayController extends ChangeNotifier { target = tapContext.target; tapInventory = tapContext.inventory; controlValue = resolver.controlValueAt(position); + semantic = resolver.semanticAnnotationAt(position); if (tapInventory != null) { _currentStateAnchor = tapInventory.stateAnchor; tapState = tapInventory.stateAnchor; @@ -2286,6 +2290,7 @@ class TugboatReplayController extends ChangeNotifier { } else { target = resolver?.targetAt(position, route: _currentRoute); controlValue = resolver?.controlValueAt(position); + semantic = resolver?.semanticAnnotationAt(position); } // Resolve after the tap context so a stale settled map can be refreshed @@ -2319,6 +2324,7 @@ class TugboatReplayController extends ChangeNotifier { if (viewportResolution != null) 'viewportSemanticResolution': viewportResolution.toJson(), if (controlValue != null) 'controlValue': controlValue.toJson(), + if (semantic != null) 'semanticAnnotation': semantic.toJson(), }; final beforeState = tapState; @@ -2338,6 +2344,7 @@ class TugboatReplayController extends ChangeNotifier { pointerGeneration: ++_pointerGeneration, captureSessionId: _session?.id, controlValue: controlValue, + semantic: semantic, ); final tx = InteractionTransaction(origin: origin, pointerId: pointer); final legacyStream = config.legacyGestureStream; @@ -2772,6 +2779,9 @@ class TugboatReplayController extends ChangeNotifier { final controlValue = _anchorResolver?.controlValueAt(position) ?? pending.origin.controlValue; + final semantic = + _anchorResolver?.semanticAnnotationAt(position) ?? + pending.origin.semantic; pending.gesture = scrolled ? InteractionGesture.scroll : InteractionGesture.swipe; @@ -2812,6 +2822,7 @@ class TugboatReplayController extends ChangeNotifier { if (scrollStartEventId != null) 'scrollStartEventId': scrollStartEventId, if (controlValue != null) 'controlValue': controlValue.toJson(), + if (semantic != null) 'semanticAnnotation': semantic.toJson(), 'interactionId': pending.id, }, ), @@ -3001,6 +3012,9 @@ class TugboatReplayController extends ChangeNotifier { before: pending.origin.controlValue, after: afterControlValue, ); + final semanticAnnotation = + _anchorResolver?.semanticAnnotationAt(position) ?? + pending.origin.semantic; if (config.emitLegacyInteractionProjection) { _addEvent( @@ -3063,6 +3077,8 @@ class TugboatReplayController extends ChangeNotifier { }, if (controlValuePayload != null) 'controlValue': controlValuePayload, + if (semanticAnnotation != null) + 'semanticAnnotation': semanticAnnotation.toJson(), }, ), ); @@ -3315,6 +3331,9 @@ class TugboatReplayController extends ChangeNotifier { if (tracker.sectionLabel != null) { data['sectionLabel'] = tracker.sectionLabel; } + if (tracker.semantic != null) { + data['semanticAnnotation'] = tracker.semantic!.toJson(); + } if (overscrollCount != null && overscrollCount > 0) { data['overscrollCount'] = overscrollCount; } @@ -3364,6 +3383,9 @@ class TugboatReplayController extends ChangeNotifier { _refreshStateAnchor(); final targetAnchor = _resolveScrollableAnchor(scrollableElement); final sectionLabel = _sectionLabelFor(scrollableElement); + final semantic = _anchorResolver?.semanticAnnotationForElement( + scrollableElement, + ); final attachmentContext = _captureContext(TugboatFrameTrigger.scroll); final beforeFrame = _compatibleFrameFor(attachmentContext); final unavailableReason = _unavailableAttachmentReason(attachmentContext); @@ -3384,6 +3406,7 @@ class TugboatReplayController extends ChangeNotifier { depth: depth, maxScrollExtent: metrics.maxScrollExtent, pageStart: pageStart, + semantic: semantic, ); _scrollTrackers[scrollableElement] = tracker; diff --git a/packages/tugboat/lib/src/interaction_transaction.dart b/packages/tugboat/lib/src/interaction_transaction.dart index 9dc308a..32a9b43 100644 --- a/packages/tugboat/lib/src/interaction_transaction.dart +++ b/packages/tugboat/lib/src/interaction_transaction.dart @@ -31,6 +31,7 @@ class InteractionOrigin { required this.pointerGeneration, required this.captureSessionId, this.controlValue, + this.semantic, }); final String interactionId; @@ -46,6 +47,7 @@ class InteractionOrigin { final int pointerGeneration; final String? captureSessionId; final TugboatControlValue? controlValue; + final TugboatSemanticAnnotation? semantic; Map toJson() => { 'interactionId': interactionId, @@ -61,6 +63,7 @@ class InteractionOrigin { 'pointerGeneration': pointerGeneration, if (captureSessionId != null) 'captureSessionId': captureSessionId, if (controlValue != null) 'controlValue': controlValue!.toJson(), + if (semantic != null) 'semanticAnnotation': semantic!.toJson(), }; } diff --git a/packages/tugboat/lib/tugboat.dart b/packages/tugboat/lib/tugboat.dart index 5ddf08f..604a996 100644 --- a/packages/tugboat/lib/tugboat.dart +++ b/packages/tugboat/lib/tugboat.dart @@ -7,10 +7,15 @@ export 'src/anchors.dart' TugboatTargetAnchor, TugboatEncodedControlScalar, TugboatControlValue, + TugboatSemanticAnnotation, tugboatControlValueSchemaVersion, + tugboatSemanticAnnotationSchemaVersion, tugboatControlValueForWidget, tugboatControlValueFromSemanticsProperties, tugboatControlValueFromSemanticsNode, + tugboatSemanticAnnotationFromProperties, + tugboatSemanticAnnotationFromNode, + tugboatMergeSemanticAnnotations, tugboatMergeControlValues, tugboatIconLabel, tugboatIconHash, diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart index bdc4499..0f9f9a7 100644 --- a/packages/tugboat/test/control_value_test.dart +++ b/packages/tugboat/test/control_value_test.dart @@ -26,6 +26,13 @@ Map? _controlValueFrom(TugboatEvent event) { return null; } +Map? _semanticAnnotationFrom(TugboatEvent event) { + final raw = event.data['semanticAnnotation']; + if (raw is Map) return raw; + if (raw is Map) return Map.from(raw); + return null; +} + void main() { setUp(TugboatReplay.resetForTest); tearDown(TugboatReplay.resetForTest); @@ -368,6 +375,7 @@ void main() { children: [ Semantics( button: true, + identifier: 'duration-15', value: '15', label: 'Duration 15 seconds', selected: selected == '15', @@ -379,6 +387,7 @@ void main() { ), Semantics( button: true, + identifier: 'duration-30', value: '30', label: 'Duration 30 seconds', selected: selected == '30', @@ -408,6 +417,92 @@ void main() { expect((tapValue['value'] as Map)['value'], 30); expect((tapValue['semanticLabel'] as Map)['value'], startsWith('str:')); expect(tapValue.toString(), isNot(contains('Duration 30 seconds'))); + + final annotation = _semanticAnnotationFrom(tap)!; + expect(annotation['role'], 'button'); + expect((annotation['identifier'] as Map)['value'], 'duration-30'); + expect((annotation['value'] as Map)['value'], 30); + expect((annotation['label'] as Map)['value'], startsWith('str:')); expect(selected, '30'); }); + + testWidgets('button taps emit semanticAnnotation labels', (tester) async { + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: FilledButton( + key: const Key('generate-cta'), + onPressed: () {}, + child: const Text('Generate'), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('generate-cta'))); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final tap = session.events.firstWhere((e) => e.type == 'tap'); + final settled = session.events.firstWhere((e) => e.type == 'tap_settled'); + final tapSemantic = _semanticAnnotationFrom(tap); + final settledSemantic = _semanticAnnotationFrom(settled); + + expect(tapSemantic, isNotNull); + expect(tapSemantic?['role'], 'button'); + expect((tapSemantic?['label'] as Map)['value'], 'Generate'); + expect(settledSemantic, isNotNull); + expect((settledSemantic?['label'] as Map)['value'], 'Generate'); + }); + + testWidgets('scroll events carry semanticAnnotation when present', ( + tester, + ) async { + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: Semantics( + identifier: 'preset-list', + label: 'Preset options', + child: ListView( + key: const Key('preset-list'), + children: [ + for (var i = 0; i < 30; i++) ListTile(title: Text('Preset $i')), + ], + ), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.drag( + find.byKey(const Key('preset-list')), + const Offset(0, -200), + ); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + final scrollEvents = session.events + .where((e) => e.type == 'scroll_start' || e.type == 'scroll_end') + .toList(); + expect(scrollEvents, isNotEmpty); + final annotated = scrollEvents + .map(_semanticAnnotationFrom) + .whereType>() + .toList(); + expect(annotated, isNotEmpty); + expect( + annotated.any((annotation) { + final identifier = annotation['identifier']; + return identifier is Map && identifier['value'] == 'preset-list'; + }), + isTrue, + ); + }); } From d56b3315e8915d219e11809fb33ad35f1b5e7f20 Mon Sep 17 00:00:00 2001 From: Chinmay-KB Date: Wed, 29 Jul 2026 17:21:09 +0530 Subject: [PATCH 6/9] fix(replay): harden control value capture --- docs/README.md | 2 +- docs/design/capture-and-fingerprint.md | 26 +- packages/tugboat/CHANGELOG.md | 15 + packages/tugboat/README.md | 2 +- packages/tugboat/lib/src/anchor_resolver.dart | 248 +++++---- packages/tugboat/lib/src/anchors.dart | 2 + packages/tugboat/lib/src/control_value.dart | 83 +-- packages/tugboat/lib/src/controller.dart | 111 +++- .../lib/src/interaction_transaction.dart | 16 +- packages/tugboat/lib/src/sdk_version.dart | 2 +- .../lib/src/semantics_flags_compat.dart | 14 +- packages/tugboat/lib/tugboat.dart | 1 + packages/tugboat/pubspec.yaml | 2 +- packages/tugboat/test/control_value_test.dart | 475 ++++++++++++++++-- .../test/semantics_flags_compat_test.dart | 67 ++- 15 files changed, 852 insertions(+), 214 deletions(-) diff --git a/docs/README.md b/docs/README.md index ea28de3..ac68cda 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,7 +28,7 @@ verified in their own repositories. ## Current compatibility -- package version: `0.4.15`; +- package version: `0.4.16`; - session JSON schema: `8`; - fingerprint schema: `6`; - minimum Dart SDK: `3.9.2`; diff --git a/docs/design/capture-and-fingerprint.md b/docs/design/capture-and-fingerprint.md index 3684131..3c2f822 100644 --- a/docs/design/capture-and-fingerprint.md +++ b/docs/design/capture-and-fingerprint.md @@ -240,21 +240,25 @@ valued controls (checkbox, switch, radio, slider, dropdown / menu item, chip) and for hit targets that expose Flutter semantic annotations: - bools and numbers are emitted literally; -- numeric strings from semantics (for example `"15"`) are parsed as numbers; -- enums and short developer-token strings are emitted as tokens; -- free-text option / semantic label strings are hashed (`str:`) with - length only. - -`tap` includes the value sampled at pointer-down. `tap_settled` includes -`before` / `after` snapshots so toggle flips and post-callback radio/dropdown -selections are visible. Slider drags that become `swipe` events also carry the -value sampled at pointer-up. +- enums are emitted as tokens; +- arbitrary strings, including numeric strings and single-word values, are + hashed with a session-scoped secret (`str:`) and omit raw length; +- semantics identifiers are hashed by default. Apps may retain an explicitly + developer-authored short token by prefixing it with `tugboat:`; the prefix is + removed from the emitted value. + +`tap` includes a `controlValue` snapshot sampled at pointer-down. +`tap_settled` uses the distinct `controlValueTransition` contract with +`before` / `after` snapshots. Its post-callback sample stays bound to the +original hit element, so later taps, route changes, or dismissed overlays +cannot donate unrelated control state. Slider drags that become `swipe` events +carry a `controlValue` snapshot sampled at pointer-up. When a typed widget value is unavailable (custom GestureDetector rows, bottom sheets, etc.), the SDK still samples `SemanticsProperties` / live semantics nodes under the pointer and records `semanticValue` / `semanticLabel` with the -same encoding rules. Standard controls may include both widget state and -semantic annotations under `sources: ["semantics","widget"]`. +same untrusted-string encoding rules. Standard controls may include both widget +state and semantic annotations under `sources: ["semantics","widget"]`. Independently, every interaction event (`tap`, `tap_settled`, `swipe`, `scroll_start`, `scroll_end`) may carry a top-level `semanticAnnotation` diff --git a/packages/tugboat/CHANGELOG.md b/packages/tugboat/CHANGELOG.md index 94d27e6..98b502f 100644 --- a/packages/tugboat/CHANGELOG.md +++ b/packages/tugboat/CHANGELOG.md @@ -1,3 +1,18 @@ +## 0.4.16 + +### Added + +- **Privacy-safe interaction metadata** — valued controls and semantic + annotations can enrich tap, settle, swipe, and scroll events without + retaining arbitrary semantic text. + +### Changed + +- **Causal control-value transitions** — settled control values are captured + from the original interaction target and use a distinct transition payload. +- **Cross-SDK semantics flags** — checked-state capture compiles on the + package's declared Flutter 3.35 minimum and newer enum-based SDKs. + ## 0.4.15 ### Added diff --git a/packages/tugboat/README.md b/packages/tugboat/README.md index bfca542..256ffa6 100644 --- a/packages/tugboat/README.md +++ b/packages/tugboat/README.md @@ -5,7 +5,7 @@ checkpoints around meaningful interactions, compact structural anchors, route transitions, scrolling evidence, and optional viewport semantic maps. Capture can be sent to the local exploration WebSocket, the HTTP collector, or both. -The current package version is `0.4.15`. Session JSON uses schema version `8` +The current package version is `0.4.16`. Session JSON uses schema version `8` (readers still accept `6`), and structural fingerprints use fingerprint schema version `6`. diff --git a/packages/tugboat/lib/src/anchor_resolver.dart b/packages/tugboat/lib/src/anchor_resolver.dart index da48f98..4b7bf14 100644 --- a/packages/tugboat/lib/src/anchor_resolver.dart +++ b/packages/tugboat/lib/src/anchor_resolver.dart @@ -54,12 +54,38 @@ class _VisitAcc { final bool hasTokenizedActionableDescendant; } +/// Metadata sampled from one interaction target. +/// +/// The resolver retains the concrete hit element privately so a post-callback +/// sample can stay bound to the original target instead of re-hit-testing a +/// coordinate that may now belong to another route or overlay. +class TugboatInteractionMetadata { + const TugboatInteractionMetadata._({ + required Element? element, + this.controlValue, + this.semanticAnnotation, + }) : _element = element; + + final Element? _element; + final TugboatControlValue? controlValue; + final TugboatSemanticAnnotation? semanticAnnotation; + + Object? get resampleTargetIdentity => _element; + + TugboatInteractionMetadata detached() => TugboatInteractionMetadata._( + element: null, + controlValue: controlValue, + semanticAnnotation: semanticAnnotation, + ); +} + /// Builds target anchors from hit-test results. class AnchorResolver { AnchorResolver({required this.rootKey, this.widgetNames = const {}}); final GlobalKey rootKey; final Map widgetNames; + List _controlValueHashKey = _newControlValueHashKey(); _TokenMap? _cachedTokenMap; int? _cachedFrameId; @@ -69,6 +95,10 @@ class AnchorResolver { int _frameEpoch = 0; bool _frameCallbackScheduled = false; + void rotateControlValueHashKey() { + _controlValueHashKey = _newControlValueHashKey(); + } + void invalidateTokenMapCache() { _cachedTokenMap = null; _cachedFrameId = null; @@ -169,13 +199,12 @@ class AnchorResolver { ); } - /// Privacy-safe control value under [globalPosition], if any. + /// Samples control and semantic metadata with one hit test and semantics + /// session. /// - /// Samples standard Material/Cupertino control state and, when present, - /// Flutter semantic value/label annotations on the hit target. Free-text - /// strings are hashed; bools, numbers, enums, numeric strings, and short - /// developer tokens are retained. - TugboatControlValue? controlValueAt(Offset globalPosition) { + /// The returned sample can be passed to [resampleInteractionMetadata] after + /// the host callback runs to read updated state from the same target. + TugboatInteractionMetadata? interactionMetadataAt(Offset globalPosition) { final rootContext = rootKey.currentContext; final rootRender = rootContext?.findRenderObject(); if (rootRender is! RenderBox || rootContext is! Element) return null; @@ -183,60 +212,92 @@ class AnchorResolver { final tokenMap = _tokenMapFor(rootContext, rootRender); if (tokenMap == null) return null; - return _withSemanticsEnabled(rootRender, () { - final result = BoxHitTestResult(); - final localPosition = rootRender.globalToLocal(globalPosition); - rootRender.hitTest(result, position: localPosition); + return _withControlValueHashKey( + _controlValueHashKey, + () => _withSemanticsEnabled(rootRender, () { + final result = BoxHitTestResult(); + final localPosition = rootRender.globalToLocal(globalPosition); + rootRender.hitTest(result, position: localPosition); + return _interactionMetadataFromHitTest( + globalPosition: globalPosition, + result: result, + tokenMap: tokenMap, + rootContext: rootContext, + rootRender: rootRender, + ); + }), + ); + } - for (final entry in result.path) { - if (entry.target is! RenderObject) continue; - final element = tokenMap.renderElements[entry.target as RenderObject]; - if (element == null || tugboatIsCaptureChrome(element.widget)) continue; - final value = tugboatControlValueForElement(element); - if (value != null) return value; + TugboatInteractionMetadata _interactionMetadataFromHitTest({ + required Offset globalPosition, + required BoxHitTestResult result, + required _TokenMap tokenMap, + required Element rootContext, + required RenderBox rootRender, + }) { + Element? sampledElement; + TugboatControlValue? controlValue; + TugboatSemanticAnnotation? semanticAnnotation; + for (final entry in result.path) { + if (entry.target is! RenderObject) continue; + final element = tokenMap.renderElements[entry.target as RenderObject]; + if (element == null || tugboatIsCaptureChrome(element.widget)) continue; + final nextControl = controlValue == null + ? tugboatControlValueForElement(element) + : null; + final nextSemantic = semanticAnnotation == null + ? tugboatSemanticAnnotationForElement(element) + : null; + if (nextControl != null || nextSemantic != null) { + sampledElement ??= element; + controlValue ??= nextControl; + semanticAnnotation ??= nextSemantic; } + if (controlValue != null && semanticAnnotation != null) break; + } - // Fall back to the semantics tree for custom hit targets that only - // expose value/label through accessibility annotations. - return _controlValueFromSemanticsHit( + if (controlValue == null || semanticAnnotation == null) { + final hits = _semanticsNodesAt( globalPosition: globalPosition, rootContext: rootContext, rootRender: rootRender, ); - }); + if (controlValue == null && hits.isNotEmpty) { + controlValue = tugboatControlValueFromSemanticsNode(hits.last); + } + semanticAnnotation ??= _semanticAnnotationFromHits(hits); + } + + return TugboatInteractionMetadata._( + element: sampledElement, + controlValue: controlValue, + semanticAnnotation: semanticAnnotation, + ); } - /// Privacy-safe semantic annotation under [globalPosition], if any. - /// - /// Used for all interactions (buttons, custom rows, scrolls) so enrichment - /// can see identifier/label/value whenever Flutter semantics expose them. - TugboatSemanticAnnotation? semanticAnnotationAt(Offset globalPosition) { + /// Re-samples state from the exact element captured by + /// [interactionMetadataAt]. + TugboatInteractionMetadata? resampleInteractionMetadata( + TugboatInteractionMetadata sample, + ) { + final element = sample._element; + if (element == null || !element.mounted) return null; final rootContext = rootKey.currentContext; final rootRender = rootContext?.findRenderObject(); - if (rootRender is! RenderBox || rootContext is! Element) return null; - final tokenMap = _tokenMapFor(rootContext, rootRender); - if (tokenMap == null) return null; - - return _withSemanticsEnabled(rootRender, () { - final result = BoxHitTestResult(); - final localPosition = rootRender.globalToLocal(globalPosition); - rootRender.hitTest(result, position: localPosition); - - for (final entry in result.path) { - if (entry.target is! RenderObject) continue; - final element = tokenMap.renderElements[entry.target as RenderObject]; - if (element == null || tugboatIsCaptureChrome(element.widget)) continue; - final annotation = tugboatSemanticAnnotationForElement(element); - if (annotation != null) return annotation; - } + TugboatInteractionMetadata readElement() => TugboatInteractionMetadata._( + element: null, + controlValue: tugboatControlValueForElement(element), + semanticAnnotation: tugboatSemanticAnnotationForElement(element), + ); - return _semanticAnnotationFromSemanticsHit( - globalPosition: globalPosition, - rootContext: rootContext, - rootRender: rootRender, - ); - }); + return _withControlValueHashKey( + _controlValueHashKey, + () => rootRender is RenderBox + ? _withSemanticsEnabled(rootRender, readElement) + : readElement(), + ); } /// Privacy-safe semantic annotation for an [element] already in the tree. @@ -246,9 +307,12 @@ class AnchorResolver { if (rootRender is! RenderBox) { return tugboatSemanticAnnotationForElement(element); } - return _withSemanticsEnabled( - rootRender, - () => tugboatSemanticAnnotationForElement(element), + return _withControlValueHashKey( + _controlValueHashKey, + () => _withSemanticsEnabled( + rootRender, + () => tugboatSemanticAnnotationForElement(element), + ), ); } @@ -271,30 +335,9 @@ class AnchorResolver { } } - TugboatControlValue? _controlValueFromSemanticsHit({ - required Offset globalPosition, - required Element rootContext, - required RenderBox rootRender, - }) { - final node = _deepestSemanticsNodeAt( - globalPosition: globalPosition, - rootContext: rootContext, - rootRender: rootRender, - ); - if (node == null) return null; - return tugboatControlValueFromSemanticsNode(node); - } - - TugboatSemanticAnnotation? _semanticAnnotationFromSemanticsHit({ - required Offset globalPosition, - required Element rootContext, - required RenderBox rootRender, - }) { - final hits = _semanticsNodesAt( - globalPosition: globalPosition, - rootContext: rootContext, - rootRender: rootRender, - ); + TugboatSemanticAnnotation? _semanticAnnotationFromHits( + List hits, + ) { TugboatSemanticAnnotation? merged; // hits are root→leaf; reverse so deeper nodes win, ancestors fill gaps. for (final node in hits.reversed) { @@ -307,19 +350,6 @@ class AnchorResolver { return merged; } - SemanticsNode? _deepestSemanticsNodeAt({ - required Offset globalPosition, - required Element rootContext, - required RenderBox rootRender, - }) { - final hits = _semanticsNodesAt( - globalPosition: globalPosition, - rootContext: rootContext, - rootRender: rootRender, - ); - return hits.isEmpty ? null : hits.last; - } - List _semanticsNodesAt({ required Offset globalPosition, required Element rootContext, @@ -366,7 +396,11 @@ class AnchorResolver { } /// Builds inventory and resolves a tap target from one token-map walk. - ({TugboatSceneInventory? inventory, TugboatTargetAnchor? target}) + ({ + TugboatSceneInventory? inventory, + TugboatTargetAnchor? target, + TugboatInteractionMetadata? metadata, + }) buildTapContext({ required Offset tapPosition, required String? route, @@ -376,11 +410,31 @@ class AnchorResolver { final rootContext = rootKey.currentContext; final rootRender = rootContext?.findRenderObject(); if (rootRender is! RenderBox || rootContext is! Element) { - return (inventory: null, target: null); + return (inventory: null, target: null, metadata: null); } final tokenMap = _tokenMapFor(rootContext, rootRender); - if (tokenMap == null) return (inventory: null, target: null); + if (tokenMap == null) { + return (inventory: null, target: null, metadata: null); + } + final hitTest = BoxHitTestResult(); + rootRender.hitTest( + hitTest, + position: rootRender.globalToLocal(tapPosition), + ); + final metadata = _withControlValueHashKey( + _controlValueHashKey, + () => _withSemanticsEnabled( + rootRender, + () => _interactionMetadataFromHitTest( + globalPosition: tapPosition, + result: hitTest, + tokenMap: tokenMap, + rootContext: rootContext, + rootRender: rootRender, + ), + ), + ); final stateAnchor = _stateAnchorFromTokenMap( tokenMap: tokenMap, route: route, @@ -388,7 +442,7 @@ class AnchorResolver { modalOpen: modalOpen, ); if (stateAnchor.signature.isEmpty) { - return (inventory: null, target: null); + return (inventory: null, target: null, metadata: metadata); } var target = _targetAtWithTokenMap( @@ -396,6 +450,7 @@ class AnchorResolver { route: route, tokenMap: tokenMap, rootRender: rootRender, + hitTest: hitTest, ); var inventory = _buildSceneInventoryFromTokenMap( tokenMap: tokenMap, @@ -418,7 +473,7 @@ class AnchorResolver { tokenMap: tokenMap, rootRender: rootRender, ); - return (inventory: inventory, target: target); + return (inventory: inventory, target: target, metadata: metadata); } /// Resolves a [TugboatTargetAnchor] for the [Scrollable] element that emitted @@ -465,11 +520,14 @@ class AnchorResolver { required String? route, required _TokenMap tokenMap, required RenderBox rootRender, + BoxHitTestResult? hitTest, }) { final viewport = rootRender.size; - final result = BoxHitTestResult(); - final localPosition = rootRender.globalToLocal(globalPosition); - rootRender.hitTest(result, position: localPosition); + final result = hitTest ?? BoxHitTestResult(); + if (hitTest == null) { + final localPosition = rootRender.globalToLocal(globalPosition); + rootRender.hitTest(result, position: localPosition); + } TugboatTargetAnchor? roleOnly; TugboatTargetAnchor? fallback; diff --git a/packages/tugboat/lib/src/anchors.dart b/packages/tugboat/lib/src/anchors.dart index cf295b3..e4109d8 100644 --- a/packages/tugboat/lib/src/anchors.dart +++ b/packages/tugboat/lib/src/anchors.dart @@ -1,4 +1,6 @@ +import 'dart:async'; import 'dart:convert'; +import 'dart:math'; import 'package:crypto/crypto.dart'; import 'package:flutter/cupertino.dart'; diff --git a/packages/tugboat/lib/src/control_value.dart b/packages/tugboat/lib/src/control_value.dart index e1aed82..2f91be0 100644 --- a/packages/tugboat/lib/src/control_value.dart +++ b/packages/tugboat/lib/src/control_value.dart @@ -3,18 +3,39 @@ part of 'anchors.dart'; /// Schema version for privacy-safe control value payloads. const int tugboatControlValueSchemaVersion = 2; +/// Schema version for `tap_settled.controlValueTransition`. +const int tugboatControlValueTransitionSchemaVersion = 1; + /// Schema version for per-interaction semantic annotations. const int tugboatSemanticAnnotationSchemaVersion = 1; final RegExp _developerTokenPattern = RegExp(r'^[A-Za-z0-9_./:-]{1,64}$'); +const String _developerTokenPrefix = 'tugboat:'; +const Symbol _controlValueHashKeyZoneKey = #tugboatControlValueHashKey; +final List _defaultControlValueHashKey = _newControlValueHashKey(); + +List _newControlValueHashKey() { + final random = Random.secure(); + return List.generate(32, (_) => random.nextInt(256), growable: false); +} + +T _withControlValueHashKey(List key, T Function() body) { + return runZoned(body, zoneValues: {_controlValueHashKeyZoneKey: key}); +} + +String _controlValueHash(String value) { + final key = + Zone.current[_controlValueHashKeyZoneKey] as List? ?? + _defaultControlValueHashKey; + return Hmac( + sha256, + key, + ).convert(utf8.encode(value)).toString().substring(0, 16); +} /// Encodes a single control scalar without retaining free-text labels. class TugboatEncodedControlScalar { - const TugboatEncodedControlScalar._({ - required this.kind, - this.value, - this.length, - }); + const TugboatEncodedControlScalar._({required this.kind, this.value}); /// `null`, `bool`, `number`, or `token`. final String kind; @@ -22,9 +43,6 @@ class TugboatEncodedControlScalar { /// Literal bool/num, or a privacy-safe token string. final Object? value; - /// Original string length when [kind] is `token` derived from a String. - final int? length; - factory TugboatEncodedControlScalar.encode(Object? raw) { if (raw == null) { return const TugboatEncodedControlScalar._(kind: 'null'); @@ -32,9 +50,16 @@ class TugboatEncodedControlScalar { if (raw is bool) { return TugboatEncodedControlScalar._(kind: 'bool', value: raw); } - if (raw is num) { + if (raw is num && raw.isFinite) { return TugboatEncodedControlScalar._(kind: 'number', value: raw); } + if (raw is num) { + final text = raw.toString(); + return TugboatEncodedControlScalar._( + kind: 'token', + value: 'num:${_controlValueHash(text)}', + ); + } if (raw is Enum) { return TugboatEncodedControlScalar._( kind: 'token', @@ -46,42 +71,43 @@ class TugboatEncodedControlScalar { if (trimmed.isEmpty) { return const TugboatEncodedControlScalar._(kind: 'null'); } - final asNum = num.tryParse(trimmed); - if (asNum != null) { - return TugboatEncodedControlScalar._(kind: 'number', value: asNum); - } - if (_developerTokenPattern.hasMatch(trimmed)) { - return TugboatEncodedControlScalar._(kind: 'token', value: trimmed); - } return TugboatEncodedControlScalar._( kind: 'token', - value: 'str:${tugboatLabelHash(trimmed)}', - length: trimmed.length, + value: 'str:${_controlValueHash(trimmed)}', ); } final text = raw.toString(); return TugboatEncodedControlScalar._( kind: 'token', - value: '${raw.runtimeType}:${tugboatLabelHash(text)}', - length: text.length, + value: '${raw.runtimeType}:${_controlValueHash(text)}', ); } + /// Retains only identifiers explicitly prefixed with `tugboat:`. + static TugboatEncodedControlScalar encodeDeveloperToken(String raw) { + final trimmed = raw.trim(); + if (trimmed.startsWith(_developerTokenPrefix)) { + final token = trimmed.substring(_developerTokenPrefix.length); + if (_developerTokenPattern.hasMatch(token)) { + return TugboatEncodedControlScalar._(kind: 'token', value: token); + } + } + return TugboatEncodedControlScalar.encode(trimmed); + } + Map toJson() => { 'kind': kind, if (kind != 'null') 'value': value, - if (length != null) 'length': length, }; @override bool operator ==(Object other) => other is TugboatEncodedControlScalar && kind == other.kind && - value == other.value && - length == other.length; + value == other.value; @override - int get hashCode => Object.hash(kind, value, length); + int get hashCode => Object.hash(kind, value); } /// Privacy-safe semantic annotation for any interaction target. @@ -175,7 +201,7 @@ TugboatSemanticAnnotation? tugboatSemanticAnnotationFromProperties( final identifier = (identifierText != null && identifierText.trim().isNotEmpty) - ? TugboatEncodedControlScalar.encode(identifierText) + ? TugboatEncodedControlScalar.encodeDeveloperToken(identifierText) : null; final label = (labelText != null && labelText.trim().isNotEmpty) ? TugboatEncodedControlScalar.encode(labelText) @@ -226,7 +252,7 @@ TugboatSemanticAnnotation? tugboatSemanticAnnotationFromNode( final selected = semanticsSelectedFromFlags(flags); final identifier = data.identifier.trim().isNotEmpty - ? TugboatEncodedControlScalar.encode(data.identifier) + ? TugboatEncodedControlScalar.encodeDeveloperToken(data.identifier) : null; final label = data.label.trim().isNotEmpty ? TugboatEncodedControlScalar.encode(data.label) @@ -324,8 +350,9 @@ TugboatSemanticAnnotation? tugboatSemanticAnnotationForElement( /// are attached as well so custom rows (e.g. GestureDetector lists) can still /// report developer-authored semantic tokens. /// -/// Free-text strings are hashed. Bools, numbers, enums, numeric strings, and -/// short developer-token strings are retained. +/// Bools, finite numbers, and enums are retained. Every ordinary string, +/// including numeric and short token-shaped strings, is hashed. Only explicit +/// developer identifiers use [TugboatEncodedControlScalar.encodeDeveloperToken]. class TugboatControlValue { const TugboatControlValue({ required this.role, diff --git a/packages/tugboat/lib/src/controller.dart b/packages/tugboat/lib/src/controller.dart index 2b15ce6..3efdbef 100644 --- a/packages/tugboat/lib/src/controller.dart +++ b/packages/tugboat/lib/src/controller.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:collection'; import 'dart:typed_data'; import 'package:flutter/semantics.dart'; @@ -28,6 +29,20 @@ export 'replay_config.dart' TugboatViewportSemanticPolicy, resolveViewportSemanticPolicy; +class _PostCallbackMetadataCapture { + TugboatInteractionMetadata? value; + bool _ambiguous = false; + + void markAmbiguous() { + _ambiguous = true; + value = null; + } + + void complete(TugboatInteractionMetadata? metadata) { + if (!_ambiguous) value = metadata; + } +} + class _ScrollTracker { _ScrollTracker({ required this.scrollableElement, @@ -808,6 +823,8 @@ class TugboatReplayController extends ChangeNotifier { {}; String? _latestRouteCaptureKey; final Set<_TapSettleWork> _activeTapSettles = <_TapSettleWork>{}; + final Map _pendingPostCallbackMetadata = + HashMap.identity(); /// Most recently started route-capture work (any Navigator). _RouteCaptureWork? get _activeRouteCapture { @@ -1323,6 +1340,7 @@ class TugboatReplayController extends ChangeNotifier { _clock ..reset() ..start(); + _anchorResolver?.rotateControlValueHashKey(); _session = TugboatSession( id: 'session-${DateTime.now().microsecondsSinceEpoch}', startedAt: DateTime.now(), @@ -1344,6 +1362,7 @@ class TugboatReplayController extends ChangeNotifier { _latestFrameId = null; _clearReleasedInteractions(); _interactions.clearAll(); + _pendingPostCallbackMetadata.clear(); _scrollTrackers.clear(); _hashToFrameId.clear(); _frameProvenance.clear(); @@ -2268,6 +2287,7 @@ class TugboatReplayController extends ChangeNotifier { TugboatTargetAnchor? target; TugboatStateAnchor? tapState = _currentStateAnchor; TugboatSceneInventory? tapInventory; + TugboatInteractionMetadata? metadata; TugboatControlValue? controlValue; TugboatSemanticAnnotation? semantic; @@ -2280,8 +2300,9 @@ class TugboatReplayController extends ChangeNotifier { ); target = tapContext.target; tapInventory = tapContext.inventory; - controlValue = resolver.controlValueAt(position); - semantic = resolver.semanticAnnotationAt(position); + metadata = tapContext.metadata; + controlValue = metadata?.controlValue; + semantic = metadata?.semanticAnnotation; if (tapInventory != null) { _currentStateAnchor = tapInventory.stateAnchor; tapState = tapInventory.stateAnchor; @@ -2289,8 +2310,9 @@ class TugboatReplayController extends ChangeNotifier { } } else { target = resolver?.targetAt(position, route: _currentRoute); - controlValue = resolver?.controlValueAt(position); - semantic = resolver?.semanticAnnotationAt(position); + metadata = resolver?.interactionMetadataAt(position); + controlValue = metadata?.controlValue; + semantic = metadata?.semanticAnnotation; } // Resolve after the tap context so a stale settled map can be refreshed @@ -2346,7 +2368,12 @@ class TugboatReplayController extends ChangeNotifier { controlValue: controlValue, semantic: semantic, ); - final tx = InteractionTransaction(origin: origin, pointerId: pointer); + final tx = InteractionTransaction( + origin: origin, + pointerId: pointer, + metadata: metadata?.detached(), + resampleTarget: metadata, + ); final legacyStream = config.legacyGestureStream; tx.bufferedOutside = target == null ? TugboatEvent( @@ -2754,6 +2781,7 @@ class TugboatReplayController extends ChangeNotifier { if (!_acceptsPointerInput) return; final pending = _interactions.removePending(pointer); if (pending == null) return; + final resampleTarget = pending.takeResampleTarget(); if (pending.isSwipeOrScroll) { if (pending.claimed) { @@ -2776,12 +2804,13 @@ class TugboatReplayController extends ChangeNotifier { : null; final scrolled = scrollStartEventId != null; final tapWasEmitted = pending.tapEmitted; + final metadata = resampleTarget == null + ? null + : _anchorResolver?.resampleInteractionMetadata(resampleTarget); final controlValue = - _anchorResolver?.controlValueAt(position) ?? - pending.origin.controlValue; + metadata?.controlValue ?? pending.metadata?.controlValue; final semantic = - _anchorResolver?.semanticAnnotationAt(position) ?? - pending.origin.semantic; + metadata?.semanticAnnotation ?? pending.metadata?.semanticAnnotation; pending.gesture = scrolled ? InteractionGesture.scroll : InteractionGesture.swipe; @@ -2851,7 +2880,38 @@ class TugboatReplayController extends ChangeNotifier { final work = _TapSettleWork(session: _session); _activeTapSettles.add(work); - unawaited(_resolveTapSettle(work, pending, position, _activeRouteCapture)); + final postCallbackMetadata = _capturePostCallbackMetadata(resampleTarget); + unawaited( + _resolveTapSettle( + work, + pending, + position, + _activeRouteCapture, + postCallbackMetadata, + ), + ); + } + + _PostCallbackMetadataCapture _capturePostCallbackMetadata( + TugboatInteractionMetadata? before, + ) { + final capture = _PostCallbackMetadataCapture(); + final targetIdentity = before?.resampleTargetIdentity; + if (before == null || targetIdentity == null) return capture; + final overlapping = _pendingPostCallbackMetadata[targetIdentity]; + if (overlapping != null) { + overlapping.markAmbiguous(); + capture.markAmbiguous(); + } + _pendingPostCallbackMetadata[targetIdentity] = capture; + WidgetsBinding.instance.addPostFrameCallback((_) { + if (identical(_pendingPostCallbackMetadata[targetIdentity], capture)) { + _pendingPostCallbackMetadata.remove(targetIdentity); + } + if (_disposed) return; + capture.complete(_anchorResolver?.resampleInteractionMetadata(before)); + }); + return capture; } Future _resolveTapSettle( @@ -2859,6 +2919,7 @@ class TugboatReplayController extends ChangeNotifier { InteractionTransaction pending, Offset position, _RouteCaptureWork? routeCaptureAtPointerUp, + _PostCallbackMetadataCapture postCallbackMetadata, ) async { try { final initialRouteCapture = @@ -3002,19 +3063,17 @@ class TugboatReplayController extends ChangeNotifier { final visualChanged = visualAvailable ? beforeContentHash != afterContentHash : null; - // Sample after the host onChanged callback has run. Prefer the live - // control under the pointer; fall back to the tap-time snapshot for - // ephemeral menu items that disappear when the overlay closes. - final afterControlValue = - _anchorResolver?.controlValueAt(position) ?? - pending.origin.controlValue; - final controlValuePayload = _controlValueSettlePayload( - before: pending.origin.controlValue, - after: afterControlValue, + // The first post-callback frame samples the pointer-down target. + // Publication can happen much later without borrowing another + // interaction's state. If no frame ran, omit the after value. + final afterMetadata = postCallbackMetadata.value; + final controlValueTransition = _controlValueTransitionPayload( + before: pending.metadata?.controlValue, + after: afterMetadata?.controlValue, ); final semanticAnnotation = - _anchorResolver?.semanticAnnotationAt(position) ?? - pending.origin.semantic; + afterMetadata?.semanticAnnotation ?? + pending.metadata?.semanticAnnotation; if (config.emitLegacyInteractionProjection) { _addEvent( @@ -3075,8 +3134,8 @@ class TugboatReplayController extends ChangeNotifier { observation.captureFailure ?? observation.captureOutcome, }, - if (controlValuePayload != null) - 'controlValue': controlValuePayload, + if (controlValueTransition != null) + 'controlValueTransition': controlValueTransition, if (semanticAnnotation != null) 'semanticAnnotation': semanticAnnotation.toJson(), }, @@ -3176,8 +3235,8 @@ class TugboatReplayController extends ChangeNotifier { _activeTapSettles.clear(); } - /// Builds a before/after control-value payload for `tap_settled`. - Map? _controlValueSettlePayload({ + /// Builds a before/after control-value transition for `tap_settled`. + Map? _controlValueTransitionPayload({ required TugboatControlValue? before, required TugboatControlValue? after, }) { @@ -3185,7 +3244,7 @@ class TugboatReplayController extends ChangeNotifier { final role = after?.role ?? before!.role; final widgetType = after?.widgetType ?? before?.widgetType; return { - 'schemaVersion': tugboatControlValueSchemaVersion, + 'schemaVersion': tugboatControlValueTransitionSchemaVersion, 'role': role, if (widgetType != null && widgetType.isNotEmpty) 'widgetType': widgetType, if (before != null) 'before': before.toJson(), diff --git a/packages/tugboat/lib/src/interaction_transaction.dart b/packages/tugboat/lib/src/interaction_transaction.dart index 32a9b43..4af1681 100644 --- a/packages/tugboat/lib/src/interaction_transaction.dart +++ b/packages/tugboat/lib/src/interaction_transaction.dart @@ -3,7 +3,6 @@ import 'dart:async'; import 'package:flutter/widgets.dart'; import 'anchors.dart'; -import 'control_value.dart'; import 'coordinate_space.dart'; import 'models.dart'; @@ -136,10 +135,17 @@ enum InteractionRejectionReason { /// Bounded in-memory transaction for one pointer gesture. class InteractionTransaction { - InteractionTransaction({required this.origin, required this.pointerId}); + InteractionTransaction({ + required this.origin, + required this.pointerId, + this.metadata, + TugboatInteractionMetadata? resampleTarget, + }) : _resampleTarget = resampleTarget; final InteractionOrigin origin; final int pointerId; + final TugboatInteractionMetadata? metadata; + TugboatInteractionMetadata? _resampleTarget; InteractionGesture gesture = InteractionGesture.tap; bool claimed = false; @@ -204,6 +210,12 @@ class InteractionTransaction { gesture = InteractionGesture.swipe; } + TugboatInteractionMetadata? takeResampleTarget() { + final target = _resampleTarget; + _resampleTarget = null; + return target; + } + Map resultToJson() => { 'status': (resultStatus ?? InteractionResultStatus.unknown).name, if (resultRoute != null) 'route': resultRoute, diff --git a/packages/tugboat/lib/src/sdk_version.dart b/packages/tugboat/lib/src/sdk_version.dart index 95379a8..d124cc6 100644 --- a/packages/tugboat/lib/src/sdk_version.dart +++ b/packages/tugboat/lib/src/sdk_version.dart @@ -1,3 +1,3 @@ // Keep this in sync with packages/tugboat/pubspec.yaml. The SDK version test // reads pubspec.yaml directly so release bumps fail fast if this drifts. -const tugboatSdkVersion = '0.4.15'; +const tugboatSdkVersion = '0.4.16'; diff --git a/packages/tugboat/lib/src/semantics_flags_compat.dart b/packages/tugboat/lib/src/semantics_flags_compat.dart index fee65fc..026d8ca 100644 --- a/packages/tugboat/lib/src/semantics_flags_compat.dart +++ b/packages/tugboat/lib/src/semantics_flags_compat.dart @@ -24,13 +24,13 @@ bool? semanticsCheckedFromFlags(SemanticsFlags flags) { return null; } // Flutter 3.36+: CheckedState enum (none / isTrue / isFalse / mixed). - try { - if (checked.toString().endsWith('.none')) return null; - if (checked.toString().endsWith('.mixed')) return null; - return checked == CheckedState.isTrue; - } catch (_) { - return null; - } + final name = checked.toString().split('.').last; + return switch (name) { + 'isTrue' => true, + 'isFalse' => false, + 'none' || 'mixed' => null, + _ => null, + }; } /// Reads toggled state across Flutter SDK versions. diff --git a/packages/tugboat/lib/tugboat.dart b/packages/tugboat/lib/tugboat.dart index 604a996..ddb4b67 100644 --- a/packages/tugboat/lib/tugboat.dart +++ b/packages/tugboat/lib/tugboat.dart @@ -9,6 +9,7 @@ export 'src/anchors.dart' TugboatControlValue, TugboatSemanticAnnotation, tugboatControlValueSchemaVersion, + tugboatControlValueTransitionSchemaVersion, tugboatSemanticAnnotationSchemaVersion, tugboatControlValueForWidget, tugboatControlValueFromSemanticsProperties, diff --git a/packages/tugboat/pubspec.yaml b/packages/tugboat/pubspec.yaml index 0e3ab20..94fc392 100644 --- a/packages/tugboat/pubspec.yaml +++ b/packages/tugboat/pubspec.yaml @@ -1,7 +1,7 @@ name: tugboat description: >- Screenshot-based session replay with compact interaction anchors for Tugboat. -version: 0.4.15 +version: 0.4.16 repository: https://github.com/blendto/tugboat-flutter issue_tracker: https://github.com/blendto/tugboat-flutter/issues homepage: https://github.com/blendto/tugboat-flutter diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart index 0f9f9a7..cd2d3c2 100644 --- a/packages/tugboat/test/control_value_test.dart +++ b/packages/tugboat/test/control_value_test.dart @@ -1,6 +1,6 @@ import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; -import 'package:flutter/semantics.dart'; +import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:tugboat/tugboat.dart'; @@ -11,6 +11,59 @@ const _testConfig = TugboatReplayConfig( capturePixelRatio: 1.0, ); +class _SemanticsOnlyControl extends LeafRenderObjectWidget { + const _SemanticsOnlyControl({super.key, required this.value}); + + final String value; + + @override + _SemanticsOnlyRenderBox createRenderObject(BuildContext context) => + _SemanticsOnlyRenderBox(value); + + @override + void updateRenderObject( + BuildContext context, + _SemanticsOnlyRenderBox renderObject, + ) { + renderObject.value = value; + } +} + +class _SemanticsOnlyRenderBox extends RenderBox { + _SemanticsOnlyRenderBox(this._value); + + String _value; + + set value(String next) { + if (_value == next) return; + _value = next; + markNeedsSemanticsUpdate(); + } + + @override + bool get sizedByParent => true; + + @override + void performResize() { + size = constraints.constrain(const Size(120, 48)); + } + + @override + bool hitTestSelf(Offset position) => true; + + @override + void describeSemanticsConfiguration(SemanticsConfiguration config) { + super.describeSemanticsConfiguration(config); + config + ..isSemanticBoundary = true + ..isButton = true + ..textDirection = TextDirection.ltr + ..label = 'Semantics only control' + ..value = _value + ..onTap = () {}; + } +} + Future _waitForCaptures(WidgetTester tester) async { await tester.pump(); await tester.runAsync(() async { @@ -26,6 +79,13 @@ Map? _controlValueFrom(TugboatEvent event) { return null; } +Map? _controlValueTransitionFrom(TugboatEvent event) { + final raw = event.data['controlValueTransition']; + if (raw is Map) return raw; + if (raw is Map) return Map.from(raw); + return null; +} + Map? _semanticAnnotationFrom(TugboatEvent event) { final raw = event.data['semanticAnnotation']; if (raw is Map) return raw; @@ -54,16 +114,42 @@ void main() { expect(slider?.value?.value, 0.4); }); - test('hashes free-text option strings and keeps developer tokens', () { + test('hashes every untrusted string scalar', () { final freeText = TugboatEncodedControlScalar.encode('Secret Option Name'); expect(freeText.kind, 'token'); expect(freeText.value, startsWith('str:')); expect(freeText.value, isNot(contains('Secret'))); - expect(freeText.length, 'Secret Option Name'.length); - final token = TugboatEncodedControlScalar.encode('usd'); - expect(token.kind, 'token'); - expect(token.value, 'usd'); + final oneWordName = TugboatEncodedControlScalar.encode('Alice'); + expect(oneWordName.value, startsWith('str:')); + expect(oneWordName.value, isNot('Alice')); + expect(oneWordName.value, isNot('str:${tugboatLabelHash('Alice')}')); + + final numericPii = TugboatEncodedControlScalar.encode('123456'); + expect(numericPii.value, startsWith('str:')); + expect(numericPii.value, isNot(123456)); + + final implicitIdentifier = + TugboatEncodedControlScalar.encodeDeveloperToken('123456'); + expect(implicitIdentifier.value, startsWith('str:')); + + final explicitIdentifier = + TugboatEncodedControlScalar.encodeDeveloperToken( + 'tugboat:duration-30', + ); + expect(explicitIdentifier.value, 'duration-30'); + }); + + test('keeps encoded numbers JSON-safe', () { + for (final value in [ + double.nan, + double.infinity, + double.negativeInfinity, + ]) { + final encoded = TugboatEncodedControlScalar.encode(value); + expect(encoded.kind, isNot('number')); + expect(() => encoded.toJson(), returnsNormally); + } }); test('reads radio option identity and group selection', () { @@ -84,6 +170,121 @@ void main() { }); }); + testWidgets('rotates untrusted string hashes for each capture session', ( + tester, + ) async { + Future captureHash() async { + final targetKey = UniqueKey(); + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: Center( + child: Semantics( + value: 'Alice', + child: ElevatedButton( + key: targetKey, + onPressed: () {}, + child: const Text('Capture'), + ), + ), + ), + ), + ), + ); + await _waitForCaptures(tester); + await tester.tap(find.byKey(targetKey)); + await _waitForCaptures(tester); + final tap = TugboatReplay.controller!.session!.events.firstWhere( + (event) => event.type == 'tap', + ); + return ((_semanticAnnotationFrom(tap)!['value'] as Map)['value']); + } + + final first = await captureHash(); + await tester.pumpWidget(const SizedBox()); + TugboatReplay.resetForTest(); + final second = await captureHash(); + + expect(first, isNot(second)); + }); + + testWidgets('controller hash keys stay isolated across concurrent sessions', ( + tester, + ) async { + final firstKey = GlobalKey(); + final secondKey = GlobalKey(); + await tester.pumpWidget( + MaterialApp( + home: Row( + children: [ + Expanded( + child: RepaintBoundary( + key: firstKey, + child: Semantics( + button: true, + value: 'Alice', + child: const SizedBox.expand(), + ), + ), + ), + Expanded( + child: RepaintBoundary( + key: secondKey, + child: Semantics( + button: true, + value: 'Alice', + child: const SizedBox.expand(), + ), + ), + ), + ], + ), + ), + ); + + final firstController = TugboatReplayController( + config: _testConfig, + boundaryKey: firstKey, + ); + final secondController = TugboatReplayController( + config: _testConfig, + boundaryKey: secondKey, + ); + await firstController.initialize(); + await secondController.initialize(); + + firstController.start(const Size(400, 600), 'test'); + await tester.pump(); + firstController.recordPointerDown( + tester.getCenter(find.byKey(firstKey)), + pointer: 1, + ); + final firstTap = firstController.session!.events.lastWhere( + (event) => event.type == 'tap', + ); + final firstHash = + ((_semanticAnnotationFrom(firstTap)!['value'] as Map)['value']); + + secondController.start(const Size(400, 600), 'test'); + await tester.pump(); + firstController.recordPointerDown( + tester.getCenter(find.byKey(firstKey)), + pointer: 2, + ); + final secondTap = firstController.session!.events.lastWhere( + (event) => event.type == 'tap', + ); + final secondHash = + ((_semanticAnnotationFrom(secondTap)!['value'] as Map)['value']); + + expect(firstHash, secondHash); + firstController.dispose(); + secondController.dispose(); + await tester.pumpWidget(const SizedBox()); + }); + testWidgets('switch tap emits before/after control values', (tester) async { var enabled = false; await tester.pumpWidget( @@ -116,9 +317,22 @@ void main() { expect(tapValue?['role'], 'switch'); expect((tapValue?['value'] as Map)['value'], isFalse); - final settledValue = _controlValueFrom(settled); + final settledValue = _controlValueTransitionFrom(settled); expect(settledValue?['role'], 'switch'); + expect( + settledValue?['schemaVersion'], + tugboatControlValueTransitionSchemaVersion, + ); + expect(settled.data, isNot(contains('controlValue'))); expect((settledValue?['before'] as Map)['value'], isA()); + expect( + (settledValue?['before'] as Map)['schemaVersion'], + tugboatControlValueSchemaVersion, + ); + expect( + (settledValue?['after'] as Map)['schemaVersion'], + tugboatControlValueSchemaVersion, + ); expect( ((settledValue?['before'] as Map)['value'] as Map)['value'], isFalse, @@ -180,7 +394,7 @@ void main() { expect(tapValue['index'], 1); final settled = session.events.firstWhere((e) => e.type == 'tap_settled'); - final settledValue = _controlValueFrom(settled)!; + final settledValue = _controlValueTransitionFrom(settled)!; expect(((settledValue['after'] as Map)['value'] as Map)['value'], 2); expect(((settledValue['after'] as Map)['groupValue'] as Map)['value'], 2); expect((settledValue['after'] as Map)['selected'], isTrue); @@ -342,7 +556,7 @@ void main() { expect(json, contains('str:')); }); - test('semantic properties encode value and label tokens', () { + test('semantic properties hash arbitrary value and label strings', () { final snapshot = tugboatControlValueFromSemanticsProperties( const SemanticsProperties( button: true, @@ -353,9 +567,9 @@ void main() { ); expect(snapshot?.role, 'button'); expect(snapshot?.sources, ['semantics']); - expect(snapshot?.value?.kind, 'number'); - expect(snapshot?.value?.value, 15); - expect(snapshot?.semanticValue?.value, 15); + expect(snapshot?.value?.kind, 'token'); + expect(snapshot?.value?.value, startsWith('str:')); + expect(snapshot?.semanticValue?.value, startsWith('str:')); expect(snapshot?.semanticLabel?.value, startsWith('str:')); expect(snapshot?.selected, isTrue); }); @@ -375,7 +589,7 @@ void main() { children: [ Semantics( button: true, - identifier: 'duration-15', + identifier: 'tugboat:duration-15', value: '15', label: 'Duration 15 seconds', selected: selected == '15', @@ -387,7 +601,7 @@ void main() { ), Semantics( button: true, - identifier: 'duration-30', + identifier: 'tugboat:duration-30', value: '30', label: 'Duration 30 seconds', selected: selected == '30', @@ -413,15 +627,15 @@ void main() { final tap = session.events.firstWhere((e) => e.type == 'tap'); final tapValue = _controlValueFrom(tap)!; expect(tapValue['sources'], contains('semantics')); - expect((tapValue['semanticValue'] as Map)['value'], 30); - expect((tapValue['value'] as Map)['value'], 30); + expect((tapValue['semanticValue'] as Map)['value'], startsWith('str:')); + expect((tapValue['value'] as Map)['value'], startsWith('str:')); expect((tapValue['semanticLabel'] as Map)['value'], startsWith('str:')); expect(tapValue.toString(), isNot(contains('Duration 30 seconds'))); final annotation = _semanticAnnotationFrom(tap)!; expect(annotation['role'], 'button'); expect((annotation['identifier'] as Map)['value'], 'duration-30'); - expect((annotation['value'] as Map)['value'], 30); + expect((annotation['value'] as Map)['value'], startsWith('str:')); expect((annotation['label'] as Map)['value'], startsWith('str:')); expect(selected, '30'); }); @@ -453,9 +667,210 @@ void main() { expect(tapSemantic, isNotNull); expect(tapSemantic?['role'], 'button'); - expect((tapSemantic?['label'] as Map)['value'], 'Generate'); + expect((tapSemantic?['label'] as Map)['value'], startsWith('str:')); expect(settledSemantic, isNotNull); - expect((settledSemantic?['label'] as Map)['value'], 'Generate'); + expect((settledSemantic?['label'] as Map)['value'], startsWith('str:')); + }); + + testWidgets('rapid taps retain per-interaction after values', (tester) async { + var enabled = false; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => TugboatReplay.wrapApp( + config: const TugboatReplayConfig( + profile: TugboatCaptureProfile.exploration, + settleDelay: Duration(milliseconds: 120), + enableGlobalPointerCapture: false, + capturePixelRatio: 1.0, + ), + child: child!, + ), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return Switch( + key: const Key('rapid-switch'), + value: enabled, + onChanged: (next) => setState(() => enabled = next), + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('rapid-switch'))); + await tester.pump(); + await tester.tap(find.byKey(const Key('rapid-switch'))); + await tester.pump(); + await tester.pump(const Duration(milliseconds: 150)); + for (var i = 0; i < 3; i += 1) { + await _waitForCaptures(tester); + } + + final session = TugboatReplay.controller!.session!; + final taps = session.events.where((event) => event.type == 'tap').toList(); + final settles = session.events + .where((event) => event.type == 'tap_settled') + .toList(); + expect(taps, hasLength(2)); + expect(settles, hasLength(2)); + + final first = _controlValueTransitionFrom( + settles.firstWhere((event) => event.relatedEventId == taps[0].id), + )!; + final second = _controlValueTransitionFrom( + settles.firstWhere((event) => event.relatedEventId == taps[1].id), + )!; + + expect(((first['before'] as Map)['value'] as Map)['value'], isFalse); + expect(((first['after'] as Map)['value'] as Map)['value'], isTrue); + expect(((second['before'] as Map)['value'] as Map)['value'], isTrue); + expect(((second['after'] as Map)['value'] as Map)['value'], isFalse); + }); + + testWidgets('same-frame taps omit ambiguous after values', (tester) async { + var enabled = false; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + return Switch( + key: const Key('same-frame-switch'), + value: enabled, + onChanged: (next) => setState(() => enabled = next), + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('same-frame-switch'))); + await tester.tap(find.byKey(const Key('same-frame-switch'))); + await tester.pump(); + for (var i = 0; i < 3; i += 1) { + await _waitForCaptures(tester); + } + + final session = TugboatReplay.controller!.session!; + final settles = session.events + .where((event) => event.type == 'tap_settled') + .toList(); + expect(settles, hasLength(2)); + for (final settled in settles) { + final transition = _controlValueTransitionFrom(settled)!; + expect(transition, contains('before')); + expect(transition, isNot(contains('after'))); + } + }); + + testWidgets('semantics-only controls are resampled without accessibility', ( + tester, + ) async { + const renderKey = GlobalObjectKey('semantics-only-control'); + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => TugboatReplay.wrapApp( + config: const TugboatReplayConfig( + profile: TugboatCaptureProfile.productionLean, + settleDelay: Duration.zero, + enableGlobalPointerCapture: false, + capturePixelRatio: 1.0, + ), + child: child!, + ), + home: Scaffold( + body: GestureDetector( + onTap: () { + final renderObject = renderKey.currentContext!.findRenderObject(); + (renderObject! as _SemanticsOnlyRenderBox).value = 'on'; + }, + child: const _SemanticsOnlyControl(key: renderKey, value: 'off'), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(renderKey)); + await _waitForCaptures(tester); + + final settled = TugboatReplay.controller!.session!.events.firstWhere( + (event) => event.type == 'tap_settled', + ); + final transition = _controlValueTransitionFrom(settled)!; + expect( + ((transition['before'] as Map)['value'] as Map)['value'], + startsWith('str:'), + ); + expect( + ((transition['after'] as Map)['value'] as Map)['value'], + startsWith('str:'), + ); + expect( + ((transition['after'] as Map)['value'] as Map)['value'], + isNot(((transition['before'] as Map)['value'] as Map)['value']), + ); + }); + + testWidgets('settle never borrows a replacement at the old coordinate', ( + tester, + ) async { + var showOriginal = true; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) { + if (showOriginal) { + return Semantics( + identifier: 'tugboat:original-switch', + child: Switch( + key: const Key('original-switch'), + value: false, + onChanged: (_) => setState(() => showOriginal = false), + ), + ); + } + return Semantics( + identifier: 'tugboat:replacement-switch', + child: const Switch( + key: Key('replacement-switch'), + value: true, + onChanged: null, + ), + ); + }, + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('original-switch'))); + await _waitForCaptures(tester); + + final settled = TugboatReplay.controller!.session!.events.firstWhere( + (event) => event.type == 'tap_settled', + ); + final transition = _controlValueTransitionFrom(settled)!; + final annotation = _semanticAnnotationFrom(settled)!; + + expect(((transition['before'] as Map)['value'] as Map)['value'], isFalse); + expect(transition, isNot(contains('after'))); + expect((annotation['identifier'] as Map)['value'], 'original-switch'); + expect( + (annotation['identifier'] as Map)['value'], + isNot('replacement-switch'), + ); }); testWidgets('scroll events carry semanticAnnotation when present', ( @@ -467,7 +882,7 @@ void main() { TugboatReplay.wrapApp(config: _testConfig, child: child!), home: Scaffold( body: Semantics( - identifier: 'preset-list', + identifier: 'tugboat:preset-list', label: 'Preset options', child: ListView( key: const Key('preset-list'), @@ -491,18 +906,14 @@ void main() { final scrollEvents = session.events .where((e) => e.type == 'scroll_start' || e.type == 'scroll_end') .toList(); - expect(scrollEvents, isNotEmpty); - final annotated = scrollEvents - .map(_semanticAnnotationFrom) - .whereType>() - .toList(); - expect(annotated, isNotEmpty); - expect( - annotated.any((annotation) { - final identifier = annotation['identifier']; - return identifier is Map && identifier['value'] == 'preset-list'; - }), - isTrue, + final start = scrollEvents.firstWhere( + (event) => event.type == 'scroll_start', ); + final end = scrollEvents.firstWhere((event) => event.type == 'scroll_end'); + for (final event in [start, end]) { + final annotation = _semanticAnnotationFrom(event); + expect(annotation, isNotNull); + expect((annotation?['identifier'] as Map)['value'], 'preset-list'); + } }); } diff --git a/packages/tugboat/test/semantics_flags_compat_test.dart b/packages/tugboat/test/semantics_flags_compat_test.dart index 6a83ac0..3b519d8 100644 --- a/packages/tugboat/test/semantics_flags_compat_test.dart +++ b/packages/tugboat/test/semantics_flags_compat_test.dart @@ -1,5 +1,6 @@ import 'dart:ui'; +import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:tugboat/src/semantics_flags_compat.dart'; @@ -11,18 +12,66 @@ void main() { }, ); - test('semanticsEnabledFromFlags reads explicit enabled state', () { - expect( - semanticsEnabledFromFlags( - SemanticsFlags.none.copyWith(hasEnabledState: true, isEnabled: true), + testWidgets('semanticsEnabledFromFlags reads explicit enabled state', ( + tester, + ) async { + for (final value in [true, false]) { + await tester.pumpWidget( + Directionality( + textDirection: TextDirection.ltr, + child: Semantics( + enabled: value, + child: const SizedBox(width: 20, height: 20), + ), + ), + ); + final flags = tester + .getSemantics(find.byType(Semantics)) + .getSemanticsData() + .flagsCollection; + expect(semanticsEnabledFromFlags(flags), value); + } + }); + + testWidgets('semanticsCheckedFromFlags reads true, false, and none', ( + tester, + ) async { + for (final value in [true, false, null]) { + await tester.pumpWidget( + Directionality( + textDirection: TextDirection.ltr, + child: Semantics( + checked: value, + child: const SizedBox(width: 20, height: 20), + ), + ), + ); + final flags = tester + .getSemantics(find.byType(Semantics)) + .getSemanticsData() + .flagsCollection; + expect(semanticsCheckedFromFlags(flags), value); + } + }); + + testWidgets('semanticsCheckedFromFlags treats mixed as unavailable', ( + tester, + ) async { + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: Checkbox(tristate: true, value: null, onChanged: (_) {}), + ), ), - isTrue, ); + final flags = tester + .getSemantics(find.byType(Checkbox)) + .getSemanticsData() + .flagsCollection; + final dynamic runtimeFlags = flags; expect( - semanticsEnabledFromFlags( - SemanticsFlags.none.copyWith(hasEnabledState: true, isEnabled: false), - ), - isFalse, + semanticsCheckedFromFlags(flags), + runtimeFlags.isChecked is bool ? isFalse : isNull, ); }); } From 6423a60c5b7facd5206c019fc63da42537d0a7f5 Mon Sep 17 00:00:00 2001 From: Chinmay-KB Date: Wed, 29 Jul 2026 17:45:54 +0530 Subject: [PATCH 7/9] fix(replay): preserve canonical control metadata --- packages/tugboat/CHANGELOG.md | 3 + packages/tugboat/lib/src/controller.dart | 4 + .../lib/src/interaction_transaction.dart | 9 ++ packages/tugboat/test/control_value_test.dart | 108 ++++++++++++++++++ 4 files changed, 124 insertions(+) diff --git a/packages/tugboat/CHANGELOG.md b/packages/tugboat/CHANGELOG.md index 98b502f..bba6ad9 100644 --- a/packages/tugboat/CHANGELOG.md +++ b/packages/tugboat/CHANGELOG.md @@ -10,6 +10,9 @@ - **Causal control-value transitions** — settled control values are captured from the original interaction target and use a distinct transition payload. +- **Canonical interaction parity** — canonical-only tap and swipe results retain + their post-interaction control and semantic metadata without relying on + legacy projection events. - **Cross-SDK semantics flags** — checked-state capture compiles on the package's declared Flutter 3.35 minimum and newer enum-based SDKs. diff --git a/packages/tugboat/lib/src/controller.dart b/packages/tugboat/lib/src/controller.dart index 3efdbef..b386c6f 100644 --- a/packages/tugboat/lib/src/controller.dart +++ b/packages/tugboat/lib/src/controller.dart @@ -2818,6 +2818,8 @@ class TugboatReplayController extends ChangeNotifier { ? InteractionResultStatus.changed : InteractionResultStatus.unchanged; pending.resultObservedAtMs = atMs; + pending.resultControlValue = controlValue; + pending.resultSemanticAnnotation = semantic; if (scrollStartEventId != null) pending.addEvidence(scrollStartEventId); if (config.emitLegacyInteractionProjection) { _addEvent( @@ -3160,6 +3162,8 @@ class TugboatReplayController extends ChangeNotifier { pending.resultStateAnchor = afterState; pending.resultRoute = observation.route; pending.resultObservedAtMs = atMs; + pending.controlValueTransition = controlValueTransition; + pending.resultSemanticAnnotation = semanticAnnotation; if (observation.routeEventId != null) { pending.addEvidence(observation.routeEventId!); } diff --git a/packages/tugboat/lib/src/interaction_transaction.dart b/packages/tugboat/lib/src/interaction_transaction.dart index 4af1681..14f77bf 100644 --- a/packages/tugboat/lib/src/interaction_transaction.dart +++ b/packages/tugboat/lib/src/interaction_transaction.dart @@ -171,6 +171,9 @@ class InteractionTransaction { String? afterFrame; int? resultObservedAtMs; TugboatStateAnchor? resultStateAnchor; + TugboatControlValue? resultControlValue; + Map? controlValueTransition; + TugboatSemanticAnnotation? resultSemanticAnnotation; Completer? _successorSignal; @@ -223,6 +226,12 @@ class InteractionTransaction { if (resultStateAnchor != null) 'stateAnchor': resultStateAnchor!.toJson(), if (afterFrame != null) 'afterFrame': afterFrame, if (resultObservedAtMs != null) 'observedAtMs': resultObservedAtMs, + if (resultControlValue != null) + 'controlValue': resultControlValue!.toJson(), + if (controlValueTransition != null) + 'controlValueTransition': controlValueTransition, + if (resultSemanticAnnotation != null) + 'semanticAnnotation': resultSemanticAnnotation!.toJson(), }; Map attributionToJson({int? windowMs}) => { diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart index cd2d3c2..71f0aa7 100644 --- a/packages/tugboat/test/control_value_test.dart +++ b/packages/tugboat/test/control_value_test.dart @@ -7,6 +7,16 @@ import 'package:tugboat/tugboat.dart'; const _testConfig = TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, + enableGlobalPointerCapture: false, + capturePixelRatio: 1.0, +); + +const _canonicalTestConfig = TugboatReplayConfig( + profile: TugboatCaptureProfile.exploration, + settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, + interactionPublishMode: TugboatInteractionPublishMode.canonicalOnly, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ); @@ -261,6 +271,10 @@ void main() { tester.getCenter(find.byKey(firstKey)), pointer: 1, ); + firstController.recordPointerUp( + tester.getCenter(find.byKey(firstKey)), + pointer: 1, + ); final firstTap = firstController.session!.events.lastWhere( (event) => event.type == 'tap', ); @@ -273,6 +287,10 @@ void main() { tester.getCenter(find.byKey(firstKey)), pointer: 2, ); + firstController.recordPointerUp( + tester.getCenter(find.byKey(firstKey)), + pointer: 2, + ); final secondTap = firstController.session!.events.lastWhere( (event) => event.type == 'tap', ); @@ -341,6 +359,94 @@ void main() { expect(enabled, isTrue); }); + testWidgets('canonical-only tap retains the control transition', ( + tester, + ) async { + var enabled = false; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _canonicalTestConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) => Switch( + key: const Key('canonical-switch'), + value: enabled, + onChanged: (next) => setState(() => enabled = next), + ), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('canonical-switch'))); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + expect( + session.events.where((event) => event.type == 'tap_settled'), + isEmpty, + ); + final interaction = session.events.firstWhere( + (event) => event.type == 'interaction', + ); + final result = Map.from( + interaction.data['result']! as Map, + ); + final transition = Map.from( + result['controlValueTransition']! as Map, + ); + expect(transition['role'], 'switch'); + expect(((transition['before'] as Map)['value'] as Map)['value'], isFalse); + expect(((transition['after'] as Map)['value'] as Map)['value'], isTrue); + }); + + testWidgets('canonical-only swipe retains final control metadata', ( + tester, + ) async { + var value = 0.0; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _canonicalTestConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) => Slider( + key: const Key('canonical-slider'), + value: value, + onChanged: (next) => setState(() => value = next), + ), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.drag( + find.byKey(const Key('canonical-slider')), + const Offset(80, 0), + ); + await _waitForCaptures(tester); + + final session = TugboatReplay.controller!.session!; + expect(session.events.where((event) => event.type == 'swipe'), isEmpty); + final interaction = session.events.firstWhere( + (event) => + event.type == 'interaction' && + (event.data['gesture'] == 'swipe' || + event.data['gesture'] == 'scroll'), + ); + final result = Map.from( + interaction.data['result']! as Map, + ); + final controlValue = Map.from( + result['controlValue']! as Map, + ); + expect(controlValue['role'], 'slider'); + expect((controlValue['value'] as Map)['value'], isA()); + }); + testWidgets('radio tap records which option was selected', (tester) async { int? selected = 1; await tester.pumpWidget( @@ -680,6 +786,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.exploration, settleDelay: Duration(milliseconds: 120), + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), @@ -780,6 +887,7 @@ void main() { config: const TugboatReplayConfig( profile: TugboatCaptureProfile.productionLean, settleDelay: Duration.zero, + interactionClaimWindow: Duration.zero, enableGlobalPointerCapture: false, capturePixelRatio: 1.0, ), From 03b9ce0a5b18d4026660423ec47ec03423924946 Mon Sep 17 00:00:00 2001 From: Chinmay-KB Date: Wed, 29 Jul 2026 19:18:43 +0530 Subject: [PATCH 8/9] feat(replay): retain raw control analytics values --- docs/README.md | 2 +- docs/design/capture-and-fingerprint.md | 18 +- packages/tugboat/CHANGELOG.md | 13 + packages/tugboat/README.md | 44 ++- packages/tugboat/lib/src/anchor_resolver.dart | 2 +- packages/tugboat/lib/src/control_value.dart | 302 ++++++++++++++---- packages/tugboat/lib/src/sdk_version.dart | 2 +- packages/tugboat/lib/tugboat.dart | 5 +- packages/tugboat/pubspec.yaml | 2 +- packages/tugboat/test/control_value_test.dart | 173 +++++++--- 10 files changed, 433 insertions(+), 130 deletions(-) diff --git a/docs/README.md b/docs/README.md index ac68cda..933f394 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,7 +28,7 @@ verified in their own repositories. ## Current compatibility -- package version: `0.4.16`; +- package version: `0.4.17`; - session JSON schema: `8`; - fingerprint schema: `6`; - minimum Dart SDK: `3.9.2`; diff --git a/docs/design/capture-and-fingerprint.md b/docs/design/capture-and-fingerprint.md index 3c2f822..0142ebc 100644 --- a/docs/design/capture-and-fingerprint.md +++ b/docs/design/capture-and-fingerprint.md @@ -235,17 +235,17 @@ Developer-authored identity strings can still be emitted: - widget type names or configured `widgetNames` replacements; - canonical structural paths. -Interaction events may also carry a privacy-safe `controlValue` payload for +Interaction events may also carry a `controlValue` payload (schema version 4) for valued controls (checkbox, switch, radio, slider, dropdown / menu item, chip) and for hit targets that expose Flutter semantic annotations: - bools and numbers are emitted literally; -- enums are emitted as tokens; +- enums and developer identifiers are emitted literally; - arbitrary strings, including numeric strings and single-word values, are - hashed with a session-scoped secret (`str:`) and omit raw length; -- semantics identifiers are hashed by default. Apps may retain an explicitly - developer-authored short token by prefixing it with `tugboat:`; the prefix is - removed from the emitted value. + emitted literally; +- explicit custom-control values can be supplied with + `TugboatControlValueScope`, including a stable `controlKey`, optional unit, + and numeric `min`, `max`, and `step` metadata. `tap` includes a `controlValue` snapshot sampled at pointer-down. `tap_settled` uses the distinct `controlValueTransition` contract with @@ -256,13 +256,13 @@ carry a `controlValue` snapshot sampled at pointer-up. When a typed widget value is unavailable (custom GestureDetector rows, bottom sheets, etc.), the SDK still samples `SemanticsProperties` / live semantics -nodes under the pointer and records `semanticValue` / `semanticLabel` with the -same untrusted-string encoding rules. Standard controls may include both widget +nodes under the pointer and records raw `semanticValue` / `semanticLabel`. +Standard controls may include both widget state and semantic annotations under `sources: ["semantics","widget"]`. Independently, every interaction event (`tap`, `tap_settled`, `swipe`, `scroll_start`, `scroll_end`) may carry a top-level `semanticAnnotation` -payload whenever Flutter semantics expose an identifier, label, value, or +payload (schema version 2) whenever Flutter semantics expose an identifier, label, value, or selection flag on the target. This covers ordinary buttons and scrollables as well as valued controls. The field is named `semanticAnnotation` to avoid colliding with `tap_settled.data.settleObservation.semantic` (state-signature diff --git a/packages/tugboat/CHANGELOG.md b/packages/tugboat/CHANGELOG.md index bba6ad9..d76afa7 100644 --- a/packages/tugboat/CHANGELOG.md +++ b/packages/tugboat/CHANGELOG.md @@ -1,3 +1,16 @@ +## 0.4.17 + +### Changed + +- **Raw control and semantic values** — control values, semantic values, and + semantic labels are now sent verbatim instead of being tokenized. This makes + slider positions, durations, and template identifiers available for session + summaries and aggregate analysis. +- **Explicit custom-control values** — `TugboatControlValueScope` exposes a + stable `controlKey`, typed number/duration/enum value, optional unit, and + numeric range metadata for controls whose value is not readable from a + standard Flutter widget. + ## 0.4.16 ### Added diff --git a/packages/tugboat/README.md b/packages/tugboat/README.md index 256ffa6..00689d3 100644 --- a/packages/tugboat/README.md +++ b/packages/tugboat/README.md @@ -5,7 +5,7 @@ checkpoints around meaningful interactions, compact structural anchors, route transitions, scrolling evidence, and optional viewport semantic maps. Capture can be sent to the local exploration WebSocket, the HTTP collector, or both. -The current package version is `0.4.16`. Session JSON uses schema version `8` +The current package version is `0.4.17`. Session JSON uses schema version `8` (readers still accept `6`), and structural fingerprints use fingerprint schema version `6`. @@ -209,10 +209,11 @@ Available mask levels are `explicitOnly`, `allTextAndMedia`, `allText`, stay visible; other custom-painted or decorated image surfaces are not classified by this mode, so wrap them in `TugboatSensitive` when needed). -The structural telemetry does not retain arbitrary `Text`, accessibility, -tooltip, or icon label strings. Dynamic list discriminators are hashed before -they enter canonical paths. Telemetry does include developer-authored routing -and identity strings where applicable: +Control values and semantic strings are retained verbatim so session summaries +and aggregate analysis can use values such as slider positions, video duration, +and selected templates. Dynamic list discriminators remain hashed before they +enter canonical paths. Telemetry also includes developer-authored routing and +identity strings where applicable: - route names in `route_change.data` and anchor `routeKey` fields; - `TugboatSubView.label` in state/scroll context; @@ -222,10 +223,35 @@ and identity strings where applicable: - normalized bounds, pointer coordinates, scroll metrics, and screenshot pixels after the configured masking policy is applied. -Screenshots are the only captured surface that can contain rendered user -content. Choose an explicit production masking policy and test custom widgets, -platform views, and overlays in the target app before enabling production -capture. +Screenshots and telemetry can contain rendered or semantic user content. Choose +an explicit production masking policy and test custom widgets, platform views, +overlays, and semantic labels before enabling production capture. + +### Explicit custom-control values + +Standard Flutter controls expose their typed state automatically. Wrap custom +controls when the app knows a more useful stable key, unit, or range: + +```dart +TugboatControlValueScope( + controlKey: 'video_duration', + role: 'slider', + unit: 'milliseconds', + min: 1_000, + max: 60_000, + step: 1_000, + value: TugboatVisibleControlValue.duration( + const Duration(seconds: 15), + ), + child: MyDurationSlider(), +) +``` + +For template or preset selection, use a stable enum identifier: + +```dart +value: TugboatVisibleControlValue.enumId('modern_minimal'), +``` ## Event and frame model diff --git a/packages/tugboat/lib/src/anchor_resolver.dart b/packages/tugboat/lib/src/anchor_resolver.dart index 4b7bf14..dc81719 100644 --- a/packages/tugboat/lib/src/anchor_resolver.dart +++ b/packages/tugboat/lib/src/anchor_resolver.dart @@ -300,7 +300,7 @@ class AnchorResolver { ); } - /// Privacy-safe semantic annotation for an [element] already in the tree. + /// Semantic annotation for an [element] already in the tree. TugboatSemanticAnnotation? semanticAnnotationForElement(Element element) { final rootContext = rootKey.currentContext; final rootRender = rootContext?.findRenderObject(); diff --git a/packages/tugboat/lib/src/control_value.dart b/packages/tugboat/lib/src/control_value.dart index 2f91be0..96a97e6 100644 --- a/packages/tugboat/lib/src/control_value.dart +++ b/packages/tugboat/lib/src/control_value.dart @@ -1,18 +1,19 @@ part of 'anchors.dart'; -/// Schema version for privacy-safe control value payloads. -const int tugboatControlValueSchemaVersion = 2; +/// Schema version for raw control value payloads. +const int tugboatControlValueSchemaVersion = 4; /// Schema version for `tap_settled.controlValueTransition`. const int tugboatControlValueTransitionSchemaVersion = 1; /// Schema version for per-interaction semantic annotations. -const int tugboatSemanticAnnotationSchemaVersion = 1; +const int tugboatSemanticAnnotationSchemaVersion = 2; final RegExp _developerTokenPattern = RegExp(r'^[A-Za-z0-9_./:-]{1,64}$'); const String _developerTokenPrefix = 'tugboat:'; +// AnchorResolver owns a per-controller key and still uses this zone to isolate +// its capture work. Raw control values no longer depend on the key. const Symbol _controlValueHashKeyZoneKey = #tugboatControlValueHashKey; -final List _defaultControlValueHashKey = _newControlValueHashKey(); List _newControlValueHashKey() { final random = Random.secure(); @@ -23,24 +24,14 @@ T _withControlValueHashKey(List key, T Function() body) { return runZoned(body, zoneValues: {_controlValueHashKeyZoneKey: key}); } -String _controlValueHash(String value) { - final key = - Zone.current[_controlValueHashKeyZoneKey] as List? ?? - _defaultControlValueHashKey; - return Hmac( - sha256, - key, - ).convert(utf8.encode(value)).toString().substring(0, 16); -} - -/// Encodes a single control scalar without retaining free-text labels. +/// Encodes a single control scalar for analytics payloads. class TugboatEncodedControlScalar { const TugboatEncodedControlScalar._({required this.kind, this.value}); - /// `null`, `bool`, `number`, or `token`. + /// `null`, `bool`, `number`, `string`, `enum`, or a typed explicit value. final String kind; - /// Literal bool/num, or a privacy-safe token string. + /// Raw scalar value. final Object? value; factory TugboatEncodedControlScalar.encode(Object? raw) { @@ -54,45 +45,32 @@ class TugboatEncodedControlScalar { return TugboatEncodedControlScalar._(kind: 'number', value: raw); } if (raw is num) { - final text = raw.toString(); return TugboatEncodedControlScalar._( - kind: 'token', - value: 'num:${_controlValueHash(text)}', + kind: 'string', + value: raw.toString(), ); } if (raw is Enum) { return TugboatEncodedControlScalar._( - kind: 'token', + kind: 'enum', value: '${raw.runtimeType}.${raw.name}', ); } if (raw is String) { - final trimmed = raw.trim(); - if (trimmed.isEmpty) { - return const TugboatEncodedControlScalar._(kind: 'null'); - } - return TugboatEncodedControlScalar._( - kind: 'token', - value: 'str:${_controlValueHash(trimmed)}', - ); + return TugboatEncodedControlScalar._(kind: 'string', value: raw); } - final text = raw.toString(); - return TugboatEncodedControlScalar._( - kind: 'token', - value: '${raw.runtimeType}:${_controlValueHash(text)}', - ); + return TugboatEncodedControlScalar._(kind: 'string', value: raw.toString()); } - /// Retains only identifiers explicitly prefixed with `tugboat:`. + /// Retains developer identifiers without the `tugboat:` namespace prefix. static TugboatEncodedControlScalar encodeDeveloperToken(String raw) { - final trimmed = raw.trim(); - if (trimmed.startsWith(_developerTokenPrefix)) { - final token = trimmed.substring(_developerTokenPrefix.length); + if (raw.startsWith(_developerTokenPrefix)) { + final token = raw.substring(_developerTokenPrefix.length); if (_developerTokenPattern.hasMatch(token)) { - return TugboatEncodedControlScalar._(kind: 'token', value: token); + return TugboatEncodedControlScalar._(kind: 'enum', value: token); } } - return TugboatEncodedControlScalar.encode(trimmed); + return TugboatEncodedControlScalar.encode(raw); } Map toJson() => { @@ -110,7 +88,144 @@ class TugboatEncodedControlScalar { int get hashCode => Object.hash(kind, value); } -/// Privacy-safe semantic annotation for any interaction target. +/// A developer-declared typed control value. +class TugboatVisibleControlValue { + const TugboatVisibleControlValue._(this._encoded); + + final TugboatEncodedControlScalar _encoded; + + /// A finite numeric value, such as a slider position or percentage. + factory TugboatVisibleControlValue.number(num value) { + if (!value.isFinite) { + throw ArgumentError.value(value, 'value', 'must be finite'); + } + return TugboatVisibleControlValue._( + TugboatEncodedControlScalar._(kind: 'number', value: value), + ); + } + + /// A boolean control state. + TugboatVisibleControlValue.boolean(bool value) + : _encoded = TugboatEncodedControlScalar._(kind: 'bool', value: value); + + /// A duration represented as an exact non-negative number of milliseconds. + factory TugboatVisibleControlValue.duration(Duration value) { + if (value.isNegative) { + throw ArgumentError.value(value, 'value', 'must not be negative'); + } + return TugboatVisibleControlValue._( + TugboatEncodedControlScalar._( + kind: 'duration_ms', + value: value.inMilliseconds, + ), + ); + } + + /// A stable, developer-authored enum or template identifier. + factory TugboatVisibleControlValue.enumId(String value) { + final trimmed = value.trim(); + if (!_developerTokenPattern.hasMatch(trimmed)) { + throw ArgumentError.value( + value, + 'value', + 'must be 1-64 ASCII identifier characters', + ); + } + return TugboatVisibleControlValue._( + TugboatEncodedControlScalar._(kind: 'enum', value: trimmed), + ); + } + + Map toJson() => _encoded.toJson(); +} + +/// Declares a typed analytics value for a custom interactive control. +/// +/// Wrap controls whose actual state is not available from a standard Flutter +/// widget. [controlKey] should be a stable developer-owned identifier, for +/// example `video_duration`, `text_curve`, or `template`. +class TugboatControlValueScope extends StatelessWidget { + const TugboatControlValueScope({ + super.key, + required this.controlKey, + required this.value, + required this.child, + this.role, + this.unit, + this.min, + this.max, + this.step, + }); + + final String controlKey; + final TugboatVisibleControlValue value; + final Widget child; + + /// Optional role override, such as `slider`, `dropdown`, or `chip`. + final String? role; + + /// Optional stable unit, such as `ratio`, `percent`, or `milliseconds`. + final String? unit; + + /// Optional inclusive lower bound for [value]. + final num? min; + + /// Optional inclusive upper bound for [value]. + final num? max; + + /// Optional increment for [value]. + final num? step; + + @override + Widget build(BuildContext context) => child; + + TugboatControlValue? _toControlValue({required String fallbackRole}) { + if (!_hasValidNumericMetadata()) return null; + final normalizedKey = controlKey.trim(); + if (!_developerTokenPattern.hasMatch(normalizedKey)) return null; + final normalizedRole = role?.trim(); + final normalizedUnit = unit?.trim(); + return TugboatControlValue( + role: normalizedRole != null && normalizedRole.isNotEmpty + ? normalizedRole + : fallbackRole, + sources: const ['developer'], + controlKey: normalizedKey, + unit: + normalizedUnit != null && + _developerTokenPattern.hasMatch(normalizedUnit) + ? normalizedUnit + : null, + value: value._encoded, + min: _finiteNumber(min), + max: _finiteNumber(max), + step: _finiteNumber(step), + ); + } + + TugboatEncodedControlScalar? _finiteNumber(num? value) { + if (value == null || !value.isFinite) return null; + return TugboatEncodedControlScalar.encode(value); + } + + bool _hasValidNumericMetadata() { + if ((min != null && !min!.isFinite) || + (max != null && !max!.isFinite) || + (step != null && !step!.isFinite) || + (min != null && max != null && min! > max!) || + (step != null && step! <= 0)) { + return false; + } + if (min == null && max == null && step == null) return true; + final encoded = value._encoded; + if (encoded.kind != 'number' && encoded.kind != 'duration_ms') return false; + final numericValue = encoded.value as num; + return (min == null || numericValue >= min!) && + (max == null || numericValue <= max!); + } +} + +/// Semantic annotation for any interaction target. /// /// Attached to taps, settles, swipes, and scrolls whenever Flutter semantics /// expose an identifier, label, value, or selection flag under the target. @@ -132,10 +247,10 @@ class TugboatSemanticAnnotation { /// Developer-authored semantics identifier when set. final TugboatEncodedControlScalar? identifier; - /// Encoded semantics label (hashed when free-text). + /// Raw semantics label when present. final TugboatEncodedControlScalar? label; - /// Encoded semantics value (numbers/tokens retained). + /// Raw semantics value when present. final TugboatEncodedControlScalar? value; final bool? selected; @@ -348,16 +463,19 @@ TugboatSemanticAnnotation? tugboatSemanticAnnotationForElement( /// Prefer typed widget state for standard Material/Cupertino controls. When /// the hit target exposes Flutter semantics, [semanticValue] / [semanticLabel] /// are attached as well so custom rows (e.g. GestureDetector lists) can still -/// report developer-authored semantic tokens. +/// report developer-authored semantic values. /// -/// Bools, finite numbers, and enums are retained. Every ordinary string, -/// including numeric and short token-shaped strings, is hashed. Only explicit -/// developer identifiers use [TugboatEncodedControlScalar.encodeDeveloperToken]. +/// Bools, finite numbers, enums, and strings are retained as raw values. class TugboatControlValue { const TugboatControlValue({ required this.role, this.widgetType, this.sources = const ['widget'], + this.controlKey, + this.unit, + this.min, + this.max, + this.step, this.value, this.groupValue, this.selected, @@ -379,6 +497,21 @@ class TugboatControlValue { /// Provenance markers such as `widget` and/or `semantics`. final List sources; + /// Stable developer-owned key for an explicitly visible custom value. + final String? controlKey; + + /// Optional unit for [value], such as `ratio` or `milliseconds`. + final String? unit; + + /// Optional inclusive lower bound for an explicitly declared value. + final TugboatEncodedControlScalar? min; + + /// Optional inclusive upper bound for an explicitly declared value. + final TugboatEncodedControlScalar? max; + + /// Optional increment for an explicitly declared value. + final TugboatEncodedControlScalar? step; + /// Primary sampled value (option identity, toggle state, slider position, /// or best-effort semantic value when no typed widget value exists). final TugboatEncodedControlScalar? value; @@ -417,6 +550,11 @@ class TugboatControlValue { String? role, String? widgetType, List? sources, + String? controlKey, + String? unit, + TugboatEncodedControlScalar? min, + TugboatEncodedControlScalar? max, + TugboatEncodedControlScalar? step, TugboatEncodedControlScalar? value, TugboatEncodedControlScalar? groupValue, bool? selected, @@ -431,6 +569,11 @@ class TugboatControlValue { role: role ?? this.role, widgetType: widgetType ?? this.widgetType, sources: sources ?? this.sources, + controlKey: controlKey ?? this.controlKey, + unit: unit ?? this.unit, + min: min ?? this.min, + max: max ?? this.max, + step: step ?? this.step, value: value ?? this.value, groupValue: groupValue ?? this.groupValue, selected: selected ?? this.selected, @@ -447,6 +590,11 @@ class TugboatControlValue { 'role': role, if (widgetType != null && widgetType!.isNotEmpty) 'widgetType': widgetType, if (sources.isNotEmpty) 'sources': sources, + if (controlKey != null && controlKey!.isNotEmpty) 'controlKey': controlKey, + if (unit != null && unit!.isNotEmpty) 'unit': unit, + if (min != null) 'min': min!.toJson(), + if (max != null) 'max': max!.toJson(), + if (step != null) 'step': step!.toJson(), if (value != null) 'value': value!.toJson(), if (groupValue != null) 'groupValue': groupValue!.toJson(), if (selected != null) 'selected': selected, @@ -464,6 +612,11 @@ class TugboatControlValue { role == other.role && widgetType == other.widgetType && _listEquals(sources, other.sources) && + controlKey == other.controlKey && + unit == other.unit && + min == other.min && + max == other.max && + step == other.step && value == other.value && groupValue == other.groupValue && selected == other.selected && @@ -479,6 +632,11 @@ class TugboatControlValue { role, widgetType, Object.hashAll(sources), + controlKey, + unit, + min, + max, + step, value, groupValue, selected, @@ -490,7 +648,7 @@ class TugboatControlValue { ); } -/// Reads a privacy-safe control value from [widget], or null when unsupported. +/// Reads a control value from [widget], or null when unsupported. TugboatControlValue? tugboatControlValueForWidget(Widget widget, {int? index}) { final widgetType = widget.runtimeType.toString(); @@ -667,18 +825,18 @@ TugboatControlValue? tugboatControlValueFromSemanticsProperties( ? 'switch' : 'semantic'); + final semanticState = role == 'checkbox' || role == 'switch' + ? (checked ?? toggled ?? selected) + : null; + return TugboatControlValue( role: role, widgetType: widgetType, sources: const ['semantics'], value: semanticValue ?? - (checked != null - ? TugboatEncodedControlScalar.encode(checked) - : toggled != null - ? TugboatEncodedControlScalar.encode(toggled) - : selected != null - ? TugboatEncodedControlScalar.encode(selected) + (semanticState != null + ? TugboatEncodedControlScalar.encode(semanticState) : null), selected: selected ?? checked ?? toggled, index: index, @@ -725,17 +883,17 @@ TugboatControlValue? tugboatControlValueFromSemanticsNode( ? data.role.name : 'semantic'); + final semanticState = role == 'checkbox' || role == 'switch' + ? (checked ?? toggled ?? selected) + : null; + return TugboatControlValue( role: role, sources: const ['semantics'], value: semanticValue ?? - (checked != null - ? TugboatEncodedControlScalar.encode(checked) - : toggled != null - ? TugboatEncodedControlScalar.encode(toggled) - : selected != null - ? TugboatEncodedControlScalar.encode(selected) + (semanticState != null + ? TugboatEncodedControlScalar.encode(semanticState) : null), selected: selected ?? checked ?? toggled, semanticValue: semanticValue, @@ -749,21 +907,17 @@ TugboatControlValue? tugboatMergeControlValues( TugboatControlValue? semanticsValue, ) { if (widgetValue == null) return semanticsValue; - if (semanticsValue == null) { - return widgetValue.sources.contains('widget') - ? widgetValue - : widgetValue.copyWith(sources: const ['widget']); - } + if (semanticsValue == null) return widgetValue; final sources = { ...widgetValue.sources, ...semanticsValue.sources, - 'widget', - 'semantics', }.toList()..sort(); return widgetValue.copyWith( sources: sources, + controlKey: widgetValue.controlKey ?? semanticsValue.controlKey, + unit: widgetValue.unit ?? semanticsValue.unit, value: widgetValue.value ?? semanticsValue.value, selected: widgetValue.selected ?? semanticsValue.selected, semanticValue: semanticsValue.semanticValue ?? widgetValue.semanticValue, @@ -775,8 +929,12 @@ TugboatControlValue? tugboatMergeControlValues( TugboatControlValue? tugboatControlValueForElement(Element hitElement) { TugboatControlValue? widgetValue; TugboatControlValue? semanticsValue; + TugboatControlValueScope? developerScope; void consider(Element element) { + developerScope ??= element.widget is TugboatControlValueScope + ? element.widget as TugboatControlValueScope + : null; final index = widgetValue == null ? _optionIndexAmongSiblings(element) : null; @@ -799,12 +957,16 @@ TugboatControlValue? tugboatControlValueForElement(Element hitElement) { consider(hitElement); hitElement.visitAncestorElements((ancestor) { consider(ancestor); - return widgetValue == null || semanticsValue == null; + return true; }); final merged = tugboatMergeControlValues(widgetValue, semanticsValue); - if (merged == null || !merged.hasPayload) return null; - return merged; + final developerValue = developerScope?._toControlValue( + fallbackRole: merged?.role ?? 'semantic', + ); + final value = tugboatMergeControlValues(developerValue, merged); + if (value == null || !value.hasPayload) return null; + return value; } int? _optionIndexAmongSiblings(Element element) { diff --git a/packages/tugboat/lib/src/sdk_version.dart b/packages/tugboat/lib/src/sdk_version.dart index d124cc6..9f79b32 100644 --- a/packages/tugboat/lib/src/sdk_version.dart +++ b/packages/tugboat/lib/src/sdk_version.dart @@ -1,3 +1,3 @@ // Keep this in sync with packages/tugboat/pubspec.yaml. The SDK version test // reads pubspec.yaml directly so release bumps fail fast if this drifts. -const tugboatSdkVersion = '0.4.16'; +const tugboatSdkVersion = '0.4.17'; diff --git a/packages/tugboat/lib/tugboat.dart b/packages/tugboat/lib/tugboat.dart index ddb4b67..bfd7708 100644 --- a/packages/tugboat/lib/tugboat.dart +++ b/packages/tugboat/lib/tugboat.dart @@ -6,6 +6,8 @@ export 'src/anchors.dart' TugboatStateAnchor, TugboatTargetAnchor, TugboatEncodedControlScalar, + TugboatVisibleControlValue, + TugboatControlValueScope, TugboatControlValue, TugboatSemanticAnnotation, tugboatControlValueSchemaVersion, @@ -44,7 +46,8 @@ export 'src/health.dart' TugboatSanitizedFailure; export 'src/lifecycle.dart' show TugboatLifecycleState, TugboatLifecycleNotifier; -export 'src/interaction_transaction.dart' show tugboatDefaultReconciliationWindow; +export 'src/interaction_transaction.dart' + show tugboatDefaultReconciliationWindow; export 'src/models.dart'; export 'src/coordinate_space.dart' show diff --git a/packages/tugboat/pubspec.yaml b/packages/tugboat/pubspec.yaml index 94fc392..9d01bdd 100644 --- a/packages/tugboat/pubspec.yaml +++ b/packages/tugboat/pubspec.yaml @@ -1,7 +1,7 @@ name: tugboat description: >- Screenshot-based session replay with compact interaction anchors for Tugboat. -version: 0.4.16 +version: 0.4.17 repository: https://github.com/blendto/tugboat-flutter issue_tracker: https://github.com/blendto/tugboat-flutter/issues homepage: https://github.com/blendto/tugboat-flutter diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart index 71f0aa7..ecbcdc3 100644 --- a/packages/tugboat/test/control_value_test.dart +++ b/packages/tugboat/test/control_value_test.dart @@ -124,24 +124,25 @@ void main() { expect(slider?.value?.value, 0.4); }); - test('hashes every untrusted string scalar', () { + test('keeps raw string scalars visible', () { final freeText = TugboatEncodedControlScalar.encode('Secret Option Name'); - expect(freeText.kind, 'token'); - expect(freeText.value, startsWith('str:')); - expect(freeText.value, isNot(contains('Secret'))); + expect(freeText.kind, 'string'); + expect(freeText.value, 'Secret Option Name'); final oneWordName = TugboatEncodedControlScalar.encode('Alice'); - expect(oneWordName.value, startsWith('str:')); - expect(oneWordName.value, isNot('Alice')); - expect(oneWordName.value, isNot('str:${tugboatLabelHash('Alice')}')); + expect(oneWordName.value, 'Alice'); final numericPii = TugboatEncodedControlScalar.encode('123456'); - expect(numericPii.value, startsWith('str:')); - expect(numericPii.value, isNot(123456)); + expect(numericPii.value, '123456'); + + final whitespace = TugboatEncodedControlScalar.encode(' value '); + expect(whitespace.value, ' value '); + final empty = TugboatEncodedControlScalar.encode(''); + expect(empty.toJson(), {'kind': 'string', 'value': ''}); final implicitIdentifier = TugboatEncodedControlScalar.encodeDeveloperToken('123456'); - expect(implicitIdentifier.value, startsWith('str:')); + expect(implicitIdentifier.value, '123456'); final explicitIdentifier = TugboatEncodedControlScalar.encodeDeveloperToken( @@ -178,9 +179,107 @@ void main() { expect(radio?.groupValue?.value, 1); expect(radio?.selected, isFalse); }); + + test('keeps explicitly declared safe values visible', () { + final duration = TugboatVisibleControlValue.duration( + const Duration(seconds: 15), + ); + final template = TugboatVisibleControlValue.enumId('modern-minimal'); + + expect(duration.toJson(), {'kind': 'duration_ms', 'value': 15000}); + expect(template.toJson(), {'kind': 'enum', 'value': 'modern-minimal'}); + }); + }); + + testWidgets('developer value scope preserves an explicit slider value', ( + tester, + ) async { + var value = 0.25; + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: StatefulBuilder( + builder: (context, setState) => TugboatControlValueScope( + controlKey: 'text_curve', + value: TugboatVisibleControlValue.number(value), + role: 'slider', + unit: 'ratio', + min: 0, + max: 1, + step: 0.01, + child: Slider( + key: const Key('visible-slider'), + value: value, + onChanged: (next) => setState(() => value = next), + ), + ), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tapAt( + tester.getCenter(find.byKey(const Key('visible-slider'))) + + const Offset(70, 0), + ); + await _waitForCaptures(tester); + + final settled = TugboatReplay.controller!.session!.events.firstWhere( + (event) => event.type == 'tap_settled', + ); + final transition = _controlValueTransitionFrom(settled)!; + final before = transition['before'] as Map; + final after = transition['after'] as Map; + + expect(before['controlKey'], 'text_curve'); + expect(before['unit'], 'ratio'); + expect((before['min'] as Map)['value'], 0); + expect((before['max'] as Map)['value'], 1); + expect((before['step'] as Map)['value'], 0.01); + expect((before['value'] as Map)['kind'], 'number'); + expect((before['value'] as Map)['value'], 0.25); + expect((after['value'] as Map)['kind'], 'number'); + expect((after['value'] as Map)['value'], isNot(0.25)); }); - testWidgets('rotates untrusted string hashes for each capture session', ( + testWidgets('invalid developer range metadata is not emitted', (tester) async { + await tester.pumpWidget( + MaterialApp( + builder: (context, child) => + TugboatReplay.wrapApp(config: _testConfig, child: child!), + home: Scaffold( + body: TugboatControlValueScope( + controlKey: 'text_curve', + value: TugboatVisibleControlValue.number(0.5), + min: 1, + max: 0, + step: 0, + child: Slider( + key: const Key('invalid-range-slider'), + value: 0.5, + onChanged: (_) {}, + ), + ), + ), + ), + ); + await _waitForCaptures(tester); + + await tester.tap(find.byKey(const Key('invalid-range-slider'))); + await _waitForCaptures(tester); + + final tap = TugboatReplay.controller!.session!.events.firstWhere( + (event) => event.type == 'tap', + ); + final controlValue = _controlValueFrom(tap)!; + expect(controlValue, isNot(contains('controlKey'))); + expect(controlValue, isNot(contains('min'))); + }); + + testWidgets('retains raw semantic values across capture sessions', ( tester, ) async { Future captureHash() async { @@ -217,10 +316,11 @@ void main() { TugboatReplay.resetForTest(); final second = await captureHash(); - expect(first, isNot(second)); + expect(first, 'Alice'); + expect(second, 'Alice'); }); - testWidgets('controller hash keys stay isolated across concurrent sessions', ( + testWidgets('controllers retain equivalent raw semantic values', ( tester, ) async { final firstKey = GlobalKey(); @@ -621,7 +721,7 @@ void main() { expect((tapValue?['value'] as Map)['value'], isTrue); }); - testWidgets('free-text dropdown values stay hashed in session json', ( + testWidgets('free-text dropdown values stay visible in session json', ( tester, ) async { var selected = 'alpha-code'; @@ -658,11 +758,11 @@ void main() { await _waitForCaptures(tester); final json = TugboatReplay.controller!.session!.toJson().toString(); - expect(json, isNot(contains('Visible Secret City Name'))); - expect(json, contains('str:')); + expect(json, contains('Visible Secret City Name')); + expect(json, contains('alpha-code')); }); - test('semantic properties hash arbitrary value and label strings', () { + test('semantic properties retain raw value and label strings', () { final snapshot = tugboatControlValueFromSemanticsProperties( const SemanticsProperties( button: true, @@ -673,10 +773,11 @@ void main() { ); expect(snapshot?.role, 'button'); expect(snapshot?.sources, ['semantics']); - expect(snapshot?.value?.kind, 'token'); - expect(snapshot?.value?.value, startsWith('str:')); - expect(snapshot?.semanticValue?.value, startsWith('str:')); - expect(snapshot?.semanticLabel?.value, startsWith('str:')); + expect(snapshot?.value?.kind, 'string'); + expect(snapshot?.value?.value, '15'); + expect(snapshot?.semanticValue?.kind, 'string'); + expect(snapshot?.semanticValue?.value, '15'); + expect(snapshot?.semanticLabel?.value, 'Duration fifteen seconds'); expect(snapshot?.selected, isTrue); }); @@ -733,16 +834,20 @@ void main() { final tap = session.events.firstWhere((e) => e.type == 'tap'); final tapValue = _controlValueFrom(tap)!; expect(tapValue['sources'], contains('semantics')); - expect((tapValue['semanticValue'] as Map)['value'], startsWith('str:')); - expect((tapValue['value'] as Map)['value'], startsWith('str:')); - expect((tapValue['semanticLabel'] as Map)['value'], startsWith('str:')); - expect(tapValue.toString(), isNot(contains('Duration 30 seconds'))); + expect((tapValue['semanticValue'] as Map)['value'], '30'); + expect((tapValue['value'] as Map)['value'], '30'); + expect((tapValue['semanticLabel'] as Map)['value'], 'Duration 30 seconds'); + expect(tapValue.toString(), contains('Duration 30 seconds')); final annotation = _semanticAnnotationFrom(tap)!; + expect( + annotation['schemaVersion'], + tugboatSemanticAnnotationSchemaVersion, + ); expect(annotation['role'], 'button'); expect((annotation['identifier'] as Map)['value'], 'duration-30'); - expect((annotation['value'] as Map)['value'], startsWith('str:')); - expect((annotation['label'] as Map)['value'], startsWith('str:')); + expect((annotation['value'] as Map)['value'], '30'); + expect((annotation['label'] as Map)['value'], 'Duration 30 seconds'); expect(selected, '30'); }); @@ -773,9 +878,9 @@ void main() { expect(tapSemantic, isNotNull); expect(tapSemantic?['role'], 'button'); - expect((tapSemantic?['label'] as Map)['value'], startsWith('str:')); + expect((tapSemantic?['label'] as Map)['value'], 'Generate'); expect(settledSemantic, isNotNull); - expect((settledSemantic?['label'] as Map)['value'], startsWith('str:')); + expect((settledSemantic?['label'] as Map)['value'], 'Generate'); }); testWidgets('rapid taps retain per-interaction after values', (tester) async { @@ -913,14 +1018,8 @@ void main() { (event) => event.type == 'tap_settled', ); final transition = _controlValueTransitionFrom(settled)!; - expect( - ((transition['before'] as Map)['value'] as Map)['value'], - startsWith('str:'), - ); - expect( - ((transition['after'] as Map)['value'] as Map)['value'], - startsWith('str:'), - ); + expect(((transition['before'] as Map)['value'] as Map)['value'], 'off'); + expect(((transition['after'] as Map)['value'] as Map)['value'], 'on'); expect( ((transition['after'] as Map)['value'] as Map)['value'], isNot(((transition['before'] as Map)['value'] as Map)['value']), From 15c302e22b9c160035a7251a172f360c8d86f7be Mon Sep 17 00:00:00 2001 From: Chinmay-KB Date: Thu, 30 Jul 2026 12:26:23 +0530 Subject: [PATCH 9/9] test(replay): format control value coverage --- packages/tugboat/test/control_value_test.dart | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/tugboat/test/control_value_test.dart b/packages/tugboat/test/control_value_test.dart index ecbcdc3..d965219 100644 --- a/packages/tugboat/test/control_value_test.dart +++ b/packages/tugboat/test/control_value_test.dart @@ -245,7 +245,9 @@ void main() { expect((after['value'] as Map)['value'], isNot(0.25)); }); - testWidgets('invalid developer range metadata is not emitted', (tester) async { + testWidgets('invalid developer range metadata is not emitted', ( + tester, + ) async { await tester.pumpWidget( MaterialApp( builder: (context, child) => @@ -840,10 +842,7 @@ void main() { expect(tapValue.toString(), contains('Duration 30 seconds')); final annotation = _semanticAnnotationFrom(tap)!; - expect( - annotation['schemaVersion'], - tugboatSemanticAnnotationSchemaVersion, - ); + expect(annotation['schemaVersion'], tugboatSemanticAnnotationSchemaVersion); expect(annotation['role'], 'button'); expect((annotation['identifier'] as Map)['value'], 'duration-30'); expect((annotation['value'] as Map)['value'], '30');