fix: stop stamping recovered iOS captures truncated; confirm Android alert dismissal

.github/workflows/ios.yml

@@ -214,6 +214,7 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPrivateAXInteractiveFiltersLoginLikeHiddenDrawer \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSparsePayloadReasonMatrix \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Unreleased
 
+- Fixed: iOS snapshots no longer report `truncated: true` merely because a later backend produced
+  them. The runner stamped every recovered capture as truncated — including a complete private-AX
+  tree taken while the XCTest channel was penalized as slow — so a strict `is absent` / `wait absent`
+  refused it with "capture was truncated" on loaded CI hosts. `truncated` now tracks completeness
+  only: payload truncation, a depth-limited capture, or a sparse terminal payload.
+- Fixed: Android `alert accept` / `alert dismiss` return only once the dialog has left the
+  accessibility tree (a different alert taking its place counts as dismissed), matching the iOS
+  runner's re-check. Previously they returned right after the button press, so the next read could
+  still see only the dialog window. A dialog that stays visible past the action budget now fails with
+  `alert <action> did not dismiss the visible alert`.
 - Added strict `wait absent <selector> [timeoutMs]` polling for zero selector matches. Incomplete,
   sparse, truncated, scoped, depth-limited, and Android unreadable captures cannot prove absence;
   deadline diagnostics retain typed capture evidence and stable first-match details (#2236).

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -645,7 +645,11 @@ extension RunnerTests {
       // Legacy human text for older daemons that read message instead of snapshotQuality.
       message: Self.legacyQualityMessage(quality) ?? payload.message,
       nodes: payload.nodes,
-      truncated: payload.truncated == true || state != "healthy" || capture.effectiveDepth != nil,
+      // Completeness, never provenance: a whole tree that a later backend produced (state
+      // "recovered") stays untruncated, so strict absence reads can trust it. Only a real cap
+      // (payload truncation, a depth-limited private AX capture) or a sparse terminal payload
+      // is truncated.
+      truncated: payload.truncated == true || state == "sparse" || capture.effectiveDepth != nil,
       qualityPayload: capture.qualityPayload.flatMap { quality in
         guard let nodes = quality.nodes else { return nil }
         return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
@@ -886,6 +890,46 @@ extension RunnerTests {
     XCTAssertEqual(payload.nodes?.count, 1)
   }
 
+  func testStampedPayloadTruncationTracksCompletenessNotRecoveryProvenance() {
+    let complete = SnapshotBackendCapture(
+      payload: DataPayload(
+        nodes: [
+          planTestNode(index: 0, type: "Application", label: "App"),
+          planTestNode(index: 1, type: "Button", label: "Open", parentIndex: 0),
+        ],
+        truncated: false
+      ),
+      effectiveDepth: nil
+    )
+    let deferred: (reason: String, code: String) = (
+      "XCTest-backed snapshot tiers were deferred after recent slow accessibility work", "deferred"
+    )
+
+    // The CI signature behind `is absent ... capture was truncated`: a complete private AX
+    // tree selected while the XCTest channel is penalized is whole, and must say so.
+    let recovered = stampedSnapshotPayload(
+      complete, backend: .privateAX, state: "recovered", reason: deferred)
+    XCTAssertEqual(recovered.snapshotQuality?.state, "recovered")
+    XCTAssertEqual(recovered.truncated, false)
+
+    let depthLimited = stampedSnapshotPayload(
+      SnapshotBackendCapture(payload: complete.payload, effectiveDepth: 56),
+      backend: .privateAX, state: "recovered", reason: deferred)
+    XCTAssertEqual(depthLimited.truncated, true)
+
+    let cappedPayload = stampedSnapshotPayload(
+      SnapshotBackendCapture(
+        payload: DataPayload(nodes: complete.payload.nodes ?? [], truncated: true),
+        effectiveDepth: nil),
+      backend: .recursiveTree, state: "healthy", reason: nil)
+    XCTAssertEqual(cappedPayload.truncated, true)
+
+    let sparse = stampedSnapshotPayload(
+      complete, backend: .querySweep, state: "sparse",
+      reason: ("snapshot returned no semantic controls or content", "sparse-tree"))
+    XCTAssertEqual(sparse.truncated, true)
+  }
+
   func testSnapshotQualityCarriesUnscopedQualityPayload() {
     let quality = DataPayload(
       nodes: [planTestNode(index: 0, type: "Application", label: "App")],

packages/platform-android/src/__tests__/alert.test.ts

@@ -9,9 +9,22 @@ const runAndroidAdb = vi.fn(async (_device: DeviceInfo, _args: string[]) => ({
   stderr: '',
 }));
 vi.mock('../adb.ts', () => ({ runAndroidAdb }));
+// The dismissal re-check polls at the contract interval; the clock is the assertion, not the wait.
+vi.mock('@agent-device/host-kit/retry', () => ({ sleep: async () => {} }));
 
 const { handleAndroidAlert } = await import('../alert.ts');
 
+const dialog = [
+  node(0, 'android.app.AlertDialog'),
+  text(1, 'Automation confirmation', 'android:id/alertTitle'),
+  button(2, 'Cancel', 'android:id/button2', { x: 210, y: 612 }),
+];
+
+/** The dialog is in the tree until the button press lands, then gone — the real timeline. */
+function dialogUntilPressed(nodes = dialog) {
+  return async () => (runAndroidAdb.mock.calls.length === 0 ? nodes : []);
+}
+
 const device: DeviceInfo = {
   platform: 'android',
   id: 'emulator-5554',
@@ -23,11 +36,7 @@ const device: DeviceInfo = {
 test('dismissing a button alert records the tapped button and its coordinates', async () => {
   runAndroidAdb.mockClear();
   const result = await handleAndroidAlert(device, 'dismiss', {
-    captureNodes: async () => [
-      node(0, 'android.app.AlertDialog'),
-      text(1, 'Automation confirmation', 'android:id/alertTitle'),
-      button(2, 'Cancel', 'android:id/button2', { x: 210, y: 612 }),
-    ],
+    captureNodes: dialogUntilPressed(),
   });
 
   assert.deepEqual(result, {
@@ -52,11 +61,11 @@ test('dismissing a button alert records the tapped button and its coordinates',
 test('accepting a button alert records the tapped button and its coordinates', async () => {
   runAndroidAdb.mockClear();
   const result = await handleAndroidAlert(device, 'accept', {
-    captureNodes: async () => [
+    captureNodes: dialogUntilPressed([
       node(0, 'android.app.AlertDialog'),
       text(1, 'Automation confirmation', 'android:id/alertTitle'),
       button(2, 'OK', 'android:id/button1', { x: 52, y: 612 }),
-    ],
+    ]),
   });
 
   assert.equal(result.kind, 'alertHandled');
@@ -66,14 +75,74 @@ test('accepting a button alert records the tapped button and its coordinates', a
 test('a fallback Back dismissal (no matching button) carries no coordinates', async () => {
   runAndroidAdb.mockClear();
   const result = await handleAndroidAlert(device, 'dismiss', {
-    captureNodes: async () => [
+    captureNodes: dialogUntilPressed([
       node(0, 'android.app.AlertDialog'),
       text(1, 'Automation confirmation', 'android:id/alertTitle'),
-    ],
+    ]),
   });
 
   assert.equal(result.kind, 'alertHandled');
   assert.ok(result.kind === 'alertHandled' && !('coordinates' in result));
   assert.equal(result.kind === 'alertHandled' ? result.button : undefined, 'Back');
   assert.deepEqual(runAndroidAdb.mock.calls[0]?.[1], ['shell', 'input', 'keyevent', '4']);
 });
+
+test('dismiss returns only after the dialog has left the tree', async () => {
+  runAndroidAdb.mockClear();
+  // Pre-press lookup, then two captures that still show the closing dialog, then the app.
+  const captures = [dialog, dialog, dialog, []];
+  let reads = 0;
+  const result = await handleAndroidAlert(device, 'dismiss', {
+    captureNodes: async () => captures[Math.min(reads++, captures.length - 1)] ?? [],
+  });
+
+  assert.equal(result.kind, 'alertHandled');
+  assert.equal(reads, 4);
+  assert.equal(runAndroidAdb.mock.calls.length, 1);
+});
+
+test('a different alert replacing the pressed one counts as dismissed', async () => {
+  runAndroidAdb.mockClear();
+  const followUp = [
+    node(0, 'android.app.AlertDialog'),
+    text(1, 'Discard changes?', 'android:id/alertTitle'),
+    button(2, 'Keep', 'android:id/button2', { x: 210, y: 612 }),
+  ];
+  const result = await handleAndroidAlert(device, 'dismiss', {
+    captureNodes: async () => (runAndroidAdb.mock.calls.length === 0 ? dialog : followUp),
+  });
+
+  assert.equal(result.kind, 'alertHandled');
+  assert.equal(
+    result.kind === 'alertHandled' ? result.alert.title : undefined,
+    'Automation confirmation',
+  );
+});
+
+test('dismiss fails when the dialog is still visible after the action budget', async () => {
+  runAndroidAdb.mockClear();
+  vi.useFakeTimers({ now: 0, toFake: ['Date'] });
+  try {
+    let reads = 0;
+    await assert.rejects(
+      handleAndroidAlert(device, 'dismiss', {
+        captureNodes: async () => {
+          // Every post-press capture costs wall clock; the dialog never leaves.
+          if (reads++ > 0) vi.setSystemTime(Date.now() + 700);
+          return dialog;
+        },
+      }),
+      (error: unknown) =>
+        error instanceof Error &&
+        error.message === 'alert dismiss did not dismiss the visible alert' &&
+        (error as { code?: string }).code === 'COMMAND_FAILED',
+    );
+    assert.equal(runAndroidAdb.mock.calls.length, 1);
+    assert.ok(
+      reads >= 4,
+      `expected the re-check to poll until the budget expired, got ${reads} reads`,
+    );
+  } finally {
+    vi.useRealTimers();
+  }
+});

packages/platform-android/src/alert.ts

@@ -103,6 +103,7 @@ async function handleAndroidAlertAction(
   const button = chooseAndroidAlertButton(candidate.buttons, action);
   if (button) {
     await pressAndroid(device, button.x, button.y);
+    await confirmAndroidAlertDismissed(candidate.alert, action, captureNodes);
     return buildAndroidAlertHandledResponse(action, candidate.alert, button.label, {
       x: button.x,
       y: button.y,
@@ -111,6 +112,7 @@ async function handleAndroidAlertAction(
 
   if (action === 'dismiss') {
     await backAndroid(device);
+    await confirmAndroidAlertDismissed(candidate.alert, action, captureNodes);
     return buildAndroidAlertHandledResponse(action, candidate.alert, 'Back');
   }
 
@@ -120,6 +122,35 @@ async function handleAndroidAlertAction(
   });
 }
 
+/**
+ * `alert accept|dismiss` means the dialog is gone, not that a button was pressed: the next
+ * command reads the app, and a capture that lands while the dialog window is still up sees
+ * only the dialog. A different alert taking its place counts as dismissed. Bounded by the
+ * same budget the pre-press lookup uses; iOS's runner applies the same re-check.
+ */
+async function confirmAndroidAlertDismissed(
+  dismissed: AndroidAlertInfo,
+  action: 'accept' | 'dismiss',
+  captureNodes: AndroidAlertOptions['captureNodes'],
+): Promise<void> {
+  const start = Date.now();
+  for (;;) {
+    const current = await readAndroidAlertCandidate(captureNodes);
+    if (!current || !sameAndroidAlert(current.alert, dismissed)) return;
+    if (Date.now() - start >= ALERT_ACTION_RETRY_MS) {
+      throw new AppError('COMMAND_FAILED', `alert ${action} did not dismiss the visible alert`, {
+        alert: dismissed,
+        hint: 'The alert button was pressed but the dialog is still visible. Inspect alert get --json, then press the visible button by label/ref or retry.',
+      });
+    }
+    await sleep(ALERT_POLL_INTERVAL_MS);
+  }
+}
+
+function sameAndroidAlert(left: AndroidAlertInfo, right: AndroidAlertInfo): boolean {
+  return left.title === right.title && left.buttons.join('\u0000') === right.buttons.join('\u0000');
+}
+
 async function pollAndroidAlertCandidate(
   captureNodes: AndroidAlertOptions['captureNodes'],
   timeoutMs: number,

test/integration/android-emulator-e2e/live-automation-scenario.ts

@@ -164,9 +164,13 @@ export async function assertAutomationSystem(context: LiveContext): Promise<void
   const alert = await runStep(context, 'inspect Android native alert', ['alert', 'get']);
   assertJsonContains(alert, 'Automation confirmation', 'alert get should expose fixture dialog');
   await runStep(context, 'dismiss Android native alert', ['alert', 'dismiss']);
+  // The dismissal is confirmed, but the fixture's re-render after the button callback is the
+  // app's own timing: wait for the outcome, then pin it to the canary element.
+  await assertWaitText(context, 'Alert result: cancelled');
   await assertElementText(context, 'id="automation-alert-result"', 'Alert result: cancelled');
   await runStep(context, 'reopen Android native alert', ['click', 'id="automation-open-alert"']);
   await runStep(context, 'accept Android native alert', ['alert', 'accept']);
+  await assertWaitText(context, 'Alert result: accepted');
   await assertElementText(context, 'id="automation-alert-result"', 'Alert result: accepted');
   verifyCommand(context, C.alert, 'alert wait/get/dismiss/accept produce fixture-visible results');
 

test/integration/ios-simulator-e2e/snapshot-backend-conformance.ts

@@ -82,13 +82,14 @@ export function assertSnapshotBackendConformance(
     quality?.state === 'healthy' || quality?.state === 'recovered',
     `${backend} capture must have a non-sparse quality verdict: ${JSON.stringify(quality)}`,
   );
-  // The existing wire contract marks any recovered capture as truncated, including a complete
-  // private-AX payload selected after the XCTest channel was deferred. Assert that relationship
-  // instead of conflating recovery provenance with missing fixture controls.
+  // `truncated` is a completeness fact, never recovery provenance: a complete capture stays
+  // untruncated even when a later backend produced it after the XCTest channel was deferred
+  // (state "recovered"). The fixture screen fits every backend's budget, so truncation here is
+  // a wire regression, not missing fixture controls.
   assert.equal(
     snapshot.truncated,
-    quality.state !== 'healthy',
-    `${backend} quality/truncation flags disagree: ${JSON.stringify(quality)}`,
+    false,
+    `${backend} reported a truncated capture of the fixture screen: ${JSON.stringify(quality)}`,
   );
   assert.ok(
     snapshot.nodes.length >= fixture.minimumNodeCount,

test/integration/live-device-e2e/runtime.ts

@@ -169,13 +169,13 @@ export function createLiveDeviceHarness<
     const unexpectedFailure =
       result.status !== 0 && !failedAsExpected && stepOptions.allowFailure !== true;
     if (unexpectedFailure) {
-      const screenshotPath =
-        fullArgs[0] === 'wait' ? await captureWaitTimeoutScreenshot(context) : undefined;
+      const evidence = await captureFailedStepEvidence(context);
       const message = [
         formatResultDebug(step, fullArgs, result),
         `scenario: ${context.currentScenario}`,
         `artifacts: ${context.artifactDir}`,
-        `screenshot: ${screenshotPath ?? '(capture failed or not applicable)'}`,
+        `screenshot: ${evidence.screenshotPath ?? '(capture failed)'}`,
+        `snapshot: ${evidence.snapshotPath ?? '(capture failed)'}`,
       ].join('\n');
       fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message);
       assert.fail(message);
@@ -185,21 +185,37 @@ export function createLiveDeviceHarness<
     }
   }
 
-  /** Best-effort: never throws, returns undefined on a failed capture. */
-  async function captureWaitTimeoutScreenshot(context: Context): Promise<string | undefined> {
-    const screenshotPath = path.join(
-      context.artifactDir,
-      `wait-timeout-${context.stepHistory.length}.png`,
-    );
+  /**
+   * What the device showed when a step failed: the pixels and the accessibility tree the
+   * next capture would have read. Best-effort, never throws; a failed capture yields undefined.
+   */
+  async function captureFailedStepEvidence(
+    context: Context,
+  ): Promise<{ screenshotPath?: string; snapshotPath?: string }> {
+    const stem = path.join(context.artifactDir, `failed-step-${context.stepHistory.length}`);
+    const screenshotPath = `${stem}.png`;
+    const snapshotPath = `${stem}-snapshot.json`;
+    const runCli = options.runCli ?? runBuiltCliJson;
+    const evidence: { screenshotPath?: string; snapshotPath?: string } = {};
     try {
-      const capture = await (options.runCli ?? runBuiltCliJson)(
+      const screenshot = await runCli(
         options.commonFlags(context, ['screenshot', screenshotPath]),
         context.env,
       );
-      return capture.status === 0 ? screenshotPath : undefined;
+      if (screenshot.status === 0) evidence.screenshotPath = screenshotPath;
+    } catch {
+      // evidence only
+    }
+    try {
+      const snapshot = await runCli(options.commonFlags(context, ['snapshot']), context.env);
+      if (snapshot.status === 0 && snapshot.json !== undefined) {
+        fs.writeFileSync(snapshotPath, JSON.stringify(snapshot.json, null, 2));
+        evidence.snapshotPath = snapshotPath;
+      }
     } catch {
-      return undefined;
+      // evidence only
     }
+    return evidence;
   }
 
   function updateSessionState(context: Context, command: string | undefined, status: number): void {