Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions packages/capture-kit/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,14 @@
"types": "./src/ios-snapshot-acquisition.ts",
"default": "./src/ios-snapshot-acquisition.ts"
},
"./ios-snapshot-planning": {
"types": "./src/ios-snapshot-planning.ts",
"default": "./src/ios-snapshot-planning.ts"
},
"./ios-snapshot-engine": {
"types": "./src/ios-snapshot-engine/index.ts",
"default": "./src/ios-snapshot-engine/index.ts"
},
"./ios-snapshot-planning": {
"types": "./src/ios-snapshot-planning.ts",
"default": "./src/ios-snapshot-planning.ts"
},
"./mobile-snapshot-semantics": {
"types": "./src/mobile-snapshot-semantics.ts",
"default": "./src/mobile-snapshot-semantics.ts"
Expand Down Expand Up @@ -62,6 +62,14 @@
"types": "./src/png-worker-client.ts",
"default": "./src/png-worker-client.ts"
},
"./post-gesture-stability": {
"types": "./src/post-gesture-stability.ts",
"default": "./src/post-gesture-stability.ts"
},
"./react-native-overlay": {
"types": "./src/react-native-overlay.ts",
"default": "./src/react-native-overlay.ts"
},
"./screenshot-density": {
"types": "./src/screenshot-density.ts",
"default": "./src/screenshot-density.ts"
Expand All @@ -70,6 +78,10 @@
"types": "./src/screenshot-diff-pixels.ts",
"default": "./src/screenshot-diff-pixels.ts"
},
"./screenshot-overlay": {
"types": "./src/screenshot-overlay.ts",
"default": "./src/screenshot-overlay.ts"
},
"./snapshot-desktop-projection": {
"types": "./src/snapshot-desktop-projection.ts",
"default": "./src/snapshot-desktop-projection.ts"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import assert from 'node:assert/strict';
import { test } from 'vitest';
import type { SnapshotNode } from '@agent-device/kernel/snapshot';
import { makeSnapshotState } from '../../__tests__/test-utils/snapshot-builders.ts';
import { makeSnapshotState } from './snapshot-state.fixtures.ts';
import {
isAndroidUnlabeledClickableSource,
resolveAndroidOverlaySourceRect,
} from '../screenshot-overlay/android.ts';
} from './screenshot-overlay-android.ts';

const SCREEN = { x: 0, y: 0, width: 1080, height: 1920 };

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { Rect, SnapshotNode, SnapshotState } from '@agent-device/kernel/snapshot';
import { isViewportRootNode, normalizeType } from '@agent-device/contracts/snapshot';
import { hasPositiveRect, rectArea, rectContains, unionRects } from './rects.ts';
import { hasPositiveRect, rectArea, rectContains, unionRects } from './screenshot-overlay-rects.ts';

/**
* Android overlay policy (#1983): which Android nodes earn an overlay ref, and what rectangle
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { Rect, ScreenshotOverlayRef } from '@agent-device/kernel/snapshot';
import type { PNG } from '@agent-device/capture-kit/png';
import { clamp } from '../snapshot/screenshot-overlay/rects.ts';
import type { PNG } from './png.ts';
import { clamp } from './screenshot-overlay-rects.ts';

/**
* Rasterizing one overlay ref onto a decoded PNG: border, badge, and the bitmap glyphs the badge
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ import { test } from 'vitest';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { PNG } from '@agent-device/capture-kit/png';
import { annotateScreenshotWithRefs, buildScreenshotOverlayRefs } from '../screenshot-overlay.ts';
import { makeSnapshotState } from '../../__tests__/test-utils/snapshot-builders.ts';
import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts';
import { PNG } from './png.ts';
import { annotateScreenshotWithRefs, buildScreenshotOverlayRefs } from './screenshot-overlay.ts';
import { makeSnapshotState } from './snapshot-state.fixtures.ts';
import { mkdtempForTestSync } from './tmp-dir.fixtures.ts';

function writeSolidPng(filePath: string, width: number, height: number): void {
const png = new PNG({ width, height });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ import {
type SnapshotNode,
type SnapshotState,
} from '@agent-device/kernel/snapshot';
import { decodePngAsync, encodePngAsync } from '@agent-device/capture-kit/png-worker-client';
import { decodePngAsync, encodePngAsync } from './png-worker-client.ts';
import {
projectSnapshotRectToScreenshot,
resolveSnapshotBounds,
} from '@agent-device/capture-kit/snapshot-rect-projection';
import { analyzeReactNativeOverlay } from '../core/react-native-overlay.ts';
} from './snapshot-rect-projection.ts';
import { analyzeReactNativeOverlay } from './react-native-overlay.ts';
import {
findNearestAncestor,
isMeaningfulSignal,
Expand All @@ -21,14 +21,9 @@ import {
import {
isAndroidUnlabeledClickableSource,
resolveAndroidOverlaySourceRect,
} from '../snapshot/screenshot-overlay/android.ts';
} from './screenshot-overlay-android.ts';
import { drawOverlayRef } from './screenshot-overlay-draw.ts';
import {
clamp,
hasPositiveRect,
rectArea,
rectContains,
} from '../snapshot/screenshot-overlay/rects.ts';
import { clamp, hasPositiveRect, rectArea, rectContains } from './screenshot-overlay-rects.ts';

const MAX_OVERLAY_REFS = 24;
const ACTIONABLE_ROLE_TYPES = [
Expand Down
19 changes: 19 additions & 0 deletions packages/capture-kit/src/snapshot-state.fixtures.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import {
attachRefs,
type RawSnapshotNode,
type SnapshotState,
type SnapshotStateProvenance,
} from '@agent-device/kernel/snapshot';

export function makeSnapshotState(
raw: RawSnapshotNode[],
// The provenance pair stays correlated: overrides carry it as one value, never as two
// independently typed fields.
overrides?: Omit<Partial<SnapshotState>, 'backend' | 'producer'> & SnapshotStateProvenance,
): SnapshotState {
return {
nodes: attachRefs(raw),
createdAt: Date.now(),
...overrides,
};
}
8 changes: 8 additions & 0 deletions packages/host-kit/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@
"types": "./src/archive.ts",
"default": "./src/archive.ts"
},
"./code-signature": {
"types": "./src/code-signature.ts",
"default": "./src/code-signature.ts"
},
"./code-signature-cache": {
"types": "./src/code-signature-cache.ts",
"default": "./src/code-signature-cache.ts"
},
"./command": {
"types": "./src/command.ts",
"default": "./src/command.ts"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { afterEach, test, vi } from 'vitest';
import { computeDaemonCodeSignature } from '../code-signature.ts';
import { resolveCachedDaemonCodeSignature } from '../code-signature-cache.ts';
import { mkdtempForTestSync } from '../../__tests__/test-utils/tmp-dir.ts';
import { computeDaemonCodeSignature } from './code-signature.ts';
import { resolveCachedDaemonCodeSignature } from './code-signature-cache.ts';
import { mkdtempForTestSync } from './internal/tmp-dir.fixtures.ts';

afterEach(() => {
vi.restoreAllMocks();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import crypto from 'node:crypto';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { publishFileSync } from '@agent-device/host-kit/file';
import { publishFileSync } from './file.ts';
import {
buildDaemonCodeFileLabel,
formatDaemonCodeSignature,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import crypto from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import { findProjectRoot } from '@agent-device/host-kit/version';
import { findProjectRoot } from './version.ts';

// Any quoted, relative-path-shaped string literal is treated as a module
// specifier, rather than matching the `import`/`export`/`from` grammar
Expand Down
6 changes: 5 additions & 1 deletion packages/selectors/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"private": true,
"sideEffects": false,
"type": "module",
"description": "Shared selector matching, argument, and replay semantics for agent-device. `.` is string-only; `./ast` is the published parser surface behind `agent-device/selectors`; `./engine` is the resolve/list surface reserved for the selector-pipeline owner (R19).",
"description": "Shared selector matching, argument, and replay semantics for agent-device. `.` is string-only; `./ast` is the published parser surface behind `agent-device/selectors`; `./engine` is the resolve/list surface reserved for the selector-pipeline owner (R19); `./parameterized-recorded-fill` parameterizes recorded fill payloads against their selectors.",
"dependencies": {
"@agent-device/ad-script": "workspace:*",
"@agent-device/contracts": "workspace:*",
Expand All @@ -22,6 +22,10 @@
"./engine": {
"types": "./src/engine.ts",
"default": "./src/engine.ts"
},
"./parameterized-recorded-fill": {
"types": "./src/parameterized-recorded-fill.ts",
"default": "./src/parameterized-recorded-fill.ts"
}
},
"devDependencies": {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { TargetAnnotationV1 } from '@agent-device/contracts/replay';
import { selectorContainsValue } from '@agent-device/selectors';
import { selectorContainsValue } from './index.ts';

const STRUCTURAL_ROOT_OUTPUT_KEYS = new Set([
'action',
Expand Down
16 changes: 12 additions & 4 deletions scripts/layering/package-boundaries.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -404,8 +404,11 @@ test('the real tree parses, declares, and passes R11', () => {
'@agent-device/capture-kit/png-rgb-difference',
'@agent-device/capture-kit/png-size',
'@agent-device/capture-kit/png-worker-client',
'@agent-device/capture-kit/post-gesture-stability',
'@agent-device/capture-kit/react-native-overlay',
'@agent-device/capture-kit/screenshot-density',
'@agent-device/capture-kit/screenshot-diff-pixels',
'@agent-device/capture-kit/screenshot-overlay',
'@agent-device/capture-kit/snapshot-desktop-projection',
'@agent-device/capture-kit/snapshot-occlusion',
'@agent-device/capture-kit/snapshot-quality-backend-capabilities',
Expand Down Expand Up @@ -452,6 +455,8 @@ test('the real tree parses, declares, and passes R11', () => {
);
assert.deepEqual([...hostKitPackage.exportTargets.keys()].sort(), [
'@agent-device/host-kit/archive',
'@agent-device/host-kit/code-signature',
'@agent-device/host-kit/code-signature-cache',
'@agent-device/host-kit/command',
'@agent-device/host-kit/diagnostics',
'@agent-device/host-kit/file',
Expand Down Expand Up @@ -583,17 +588,20 @@ test('the real tree parses, declares, and passes R11', () => {
);
const selectorsPackage = packages.find((pkg) => pkg.name === '@agent-device/selectors');
assert.ok(selectorsPackage, 'selectors package must exist');
// Three subpaths, and each split is the point: `.` is the string-only façade
// Four subpaths, and each split is the point: `.` is the string-only façade
// every in-repo consumer uses, `./ast` is the published parser surface that
// `agent-device/selectors` has shipped since before the engine moved into
// this package, and `./engine` is the resolve/list surface reserved for the
// this package, `./engine` is the resolve/list surface reserved for the
// selector-pipeline owner (R19, #1656) — a route reaching it skips the
// structural stages its policy row declares. A fourth subpath, or the AST
// leaking into `.`, fails here.
// structural stages its policy row declares — and
// `./parameterized-recorded-fill` is the recorded-fill parameterization the
// daemon used to own (#2340). A fifth subpath, or the AST leaking into `.`,
// fails here.
assert.deepEqual([...selectorsPackage.exportTargets.keys()].sort(), [
'@agent-device/selectors',
'@agent-device/selectors/ast',
'@agent-device/selectors/engine',
'@agent-device/selectors/parameterized-recorded-fill',
]);
assert.deepEqual([...selectorsPackage.workspaceDependencies].sort(), [
'@agent-device/ad-script',
Expand Down
2 changes: 1 addition & 1 deletion src/commands/react-native/overlay.ts
Original file line number Diff line number Diff line change
@@ -1 +1 @@
export * from '../../core/react-native-overlay.ts';
export * from '@agent-device/capture-kit/react-native-overlay';
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {
classifyBaselineSurfaceEvidence,
type InteractionSurfaceSignature,
} from '../interaction-outcome-policy.ts';
import { decidePostGestureStabilityVerdict as decideWithHooks } from '../post-gesture-stability.ts';
import { decidePostGestureStabilityVerdict as decideWithHooks } from '@agent-device/capture-kit/post-gesture-stability';
import {
applicationRootNode,
keyboardWindowNodes,
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/android-snapshot-timeout-evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
} from '@agent-device/contracts/snapshot-timeout-evidence';
import { isAndroidSnapshotTimeoutError } from '../snapshot/snapshot-timeout-policy.ts';
import { contextFromFlags } from './context.ts';
import { annotateScreenshotWithRefs } from './screenshot-overlay.ts';
import { annotateScreenshotWithRefs } from '@agent-device/capture-kit/screenshot-overlay';
import { screenshotExecutionFromContext } from './screenshot-runtime.ts';
import {
resolveBoundScreenshotRuntime,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ vi.mock('@agent-device/host-kit/retry', async (importOriginal) => ({
import { resolveDaemonPaths, type DaemonPaths } from '../../config.ts';
import { sendToDaemon, type DaemonRequest, type DaemonResponse } from '../daemon-client.ts';
import { attachActiveSessionAddressHint } from '../daemon-client-lifecycle.ts';
import { computeDaemonCodeSignature } from '../../code-signature.ts';
import { computeDaemonCodeSignature } from '@agent-device/host-kit/code-signature';
import { sendRequest } from '../daemon-client-transport.ts';
import {
closeLoopbackServer,
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/client/__tests__/daemon-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import {
waitForProcessExit,
} from '@agent-device/host-kit/process';
import { sendToDaemon } from '../daemon-client.ts';
import { computeDaemonCodeSignature } from '../../code-signature.ts';
import { computeDaemonCodeSignature } from '@agent-device/host-kit/code-signature';
import { downloadRemoteArtifact } from '../../../remote/daemon-artifacts.ts';
import {
cleanupFailedDaemonStartupMetadata,
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/client/__tests__/daemon-launch-spec.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { afterEach, test, vi } from 'vitest';
import { computeDaemonCodeSignature } from '../../code-signature.ts';
import { computeDaemonCodeSignature } from '@agent-device/host-kit/code-signature';
import { resolveDaemonLaunchSpec, resolveLocalDaemonCodeSignature } from '../daemon-launch-spec.ts';
import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo';

Expand Down
5 changes: 3 additions & 2 deletions src/daemon/client/daemon-launch-spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { AppError } from '@agent-device/kernel/errors';
import { findProjectRoot } from '@agent-device/host-kit/version';
import { createTtlMemo } from '@agent-device/kernel/ttl-memo';

import { computeDaemonCodeSignature } from '../code-signature.ts';
import { computeDaemonCodeSignature } from '@agent-device/host-kit/code-signature';

export type DaemonLaunchSpec = {
root: string;
Expand Down Expand Up @@ -71,6 +71,7 @@ export async function resolveLocalDaemonCodeSignature(): Promise<string> {
if (!launchSpec.useSrc) {
return computeDaemonCodeSignature(launchSpec.distPath, launchSpec.root);
}
const { resolveCachedDaemonCodeSignature } = await import('../code-signature-cache.ts');
const { resolveCachedDaemonCodeSignature } =
await import('@agent-device/host-kit/code-signature-cache');
return resolveCachedDaemonCodeSignature(launchSpec.srcPath, launchSpec.root);
}
2 changes: 1 addition & 1 deletion src/daemon/deferred-interaction-outcome.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import {
retryPendingInteractionOutcome,
type InteractionRetryTap,
} from './interaction-outcome-policy.ts';
import { runPostGestureStabilityLoop } from './post-gesture-stability.ts';
import { runPostGestureStabilityLoop } from '@agent-device/capture-kit/post-gesture-stability';
import type { SessionState } from './types.ts';

/**
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/handlers/react-native.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { PUBLIC_COMMANDS } from '../../command-catalog.ts';
import {
analyzeReactNativeOverlay,
type ReactNativeOverlayDismissTarget,
} from '../../core/react-native-overlay.ts';
} from '@agent-device/capture-kit/react-native-overlay';
import { normalizeError } from '@agent-device/kernel/errors';
import { stripUndefined } from '@agent-device/kernel/record';
import { successText } from '@agent-device/kernel/success-text';
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/interaction/internal/interaction-common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
import type { MultiTargetAnnotationV1 } from '@agent-device/contracts/replay';
import { inferFillText } from '../../action-utils.ts';
import { recordedInputPlaceholder } from '@agent-device/ad-script';
import { parameterizeRecordedFillPayload } from '../../parameterized-recorded-fill.ts';
import { parameterizeRecordedFillPayload } from '@agent-device/selectors/parameterized-recorded-fill';
import type { InteractionFinalizationOperations } from './types.ts';

export function finalizeTouchInteraction(params: {
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/screenshot-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import type {
import { createDaemonRuntimeSessionStore } from './runtime-session.ts';
import { assertScreenshotCropPolicy } from './screenshot-crop-target.ts';
import { buildScreenshotCropWarnings, cropScreenshotToSelector } from './screenshot-crop.ts';
import { annotateScreenshotWithRefs } from './screenshot-overlay.ts';
import { annotateScreenshotWithRefs } from '@agent-device/capture-kit/screenshot-overlay';
import {
resolveBoundScreenshotRuntime,
type BoundScreenshotRuntime,
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/server/server-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { isAgentDeviceDaemonProcess } from '../daemon-process.ts';

export { readVersion } from '@agent-device/host-kit/version';
export { readProcessStartTime } from '@agent-device/host-kit/process';
export { resolveDaemonCodeSignature } from '../code-signature.ts';
export { resolveDaemonCodeSignature } from '@agent-device/host-kit/code-signature';

export type DaemonLockInfo = {
pid: number;
Expand Down
2 changes: 1 addition & 1 deletion src/daemon/session-action-recorder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import {
parameterizeRecordedResultEcho,
parameterizeTargetEvidenceEcho,
targetEvidenceCarriesAnyLiteral,
} from './parameterized-recorded-fill.ts';
} from '@agent-device/selectors/parameterized-recorded-fill';
import type { TargetEvidenceMode } from './session-target-evidence.ts';

export type RecordActionEntry = {
Expand Down
Loading