diff --git a/package.json b/package.json index 67d411b2..0459d945 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "test:afc": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/afc.spec.js\"", "test:all": "node --enable-source-maps --experimental-test-module-mocks --test --test-timeout=60000 \"build/test/unit/**/*.spec.js\" \"build/test/integration/**/*.spec.js\"", "test:app-service": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/app-service.spec.js\"", + "test:accessibility-audit": "node --enable-source-maps --test --test-timeout=120000 \"build/test/integration/accessibility-audit.spec.js\"", "test:coredevice-device-info": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/device-info-coredevice.spec.js\"", "test:device-control": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/device-control.spec.js\"", "test:configuration": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/configuration.spec.js\"", diff --git a/src/index.ts b/src/index.ts index 9529d86b..b3101e34 100644 --- a/src/index.ts +++ b/src/index.ts @@ -59,6 +59,25 @@ export type { ListAppsOptions, } from './services/ios/app-service/index.js'; export {PasteboardService} from './services/ios/pasteboard/index.js'; +export {AccessibilityAuditService} from './services/ios/accessibility-audit/index.js'; +export type {AxDeviceSetting} from './services/ios/accessibility-audit/index.js'; +export {AxAuditDtxTransport} from './services/ios/accessibility-audit/dtx-transport.js'; +export type {InvokeOptions as AxInvokeOptions} from './services/ios/accessibility-audit/dtx-transport.js'; +export {AX_OBJECT_TYPE, deserializeAxObject} from './services/ios/accessibility-audit/ax-deserialize.js'; +export {AxPoint} from './services/ios/accessibility-audit/ax-values.js'; +export { + serializeAxAttribute, + serializeAxElement, + toAxElement, + toInspectedElement, +} from './services/ios/accessibility-audit/ax-element.js'; +export type { + AxElement, + AxElementAttribute, + AxInspectedElement, + AxInspectorSection, +} from './services/ios/accessibility-audit/ax-element.js'; +export type {InspectOptions, RunAuditOptions, AxAuditIssue} from './services/ios/accessibility-audit/index.js'; export {CoreDeviceInfoService} from './services/ios/device-info/index.js'; export type { CoreDeviceAttributes, diff --git a/src/services.ts b/src/services.ts index 2c41b225..00bf4a88 100644 --- a/src/services.ts +++ b/src/services.ts @@ -4,6 +4,7 @@ import { resolveTunnelServicePorts, } from './lib/tunnel/tunnel-service-resolver.js'; import type {DVTInstruments, SyslogService as SyslogServiceType, XCTestServices} from './lib/types.js'; +import {AccessibilityAuditService} from './services/ios/accessibility-audit/index.js'; import AfcService from './services/ios/afc/index.js'; import {AppService} from './services/ios/app-service/index.js'; import {type Service} from './services/ios/base-service.js'; @@ -139,6 +140,16 @@ export async function startCoreDeviceInfoService(udid: string): Promise { + await requireCatalogService(udid, AccessibilityAuditService.RSD_SERVICE_NAME); + return AccessibilityAuditService.start(udid); +} + /** * Start the CoreDevice device-control service for the given device UDID. */ diff --git a/src/services/ios/accessibility-audit/ax-deserialize.ts b/src/services/ios/accessibility-audit/ax-deserialize.ts new file mode 100644 index 00000000..aa300bec --- /dev/null +++ b/src/services/ios/accessibility-audit/ax-deserialize.ts @@ -0,0 +1,53 @@ +/** + * The accessibility audit daemon wraps every value it returns in a recursive + * `{Value, ObjectType}` envelope. `ObjectType: "passthrough"` is a plain boxed + * value; any other `ObjectType` names a typed object (e.g. + * `AXAuditDeviceSetting_v1`) whose `Value` is a dictionary of fields. + */ +import {util} from '@appium/support'; + +/** Key under which a typed (non-passthrough) object records its `ObjectType`. */ +export const AX_OBJECT_TYPE = '__axObjectType'; + +/** A decoded typed object: its fields plus the {@link AX_OBJECT_TYPE} tag. */ +export type AxTypedObject = Record & {[AX_OBJECT_TYPE]: string}; + +function isEnvelope(value: unknown): value is {Value: unknown; ObjectType: string} { + return ( + util.isPlainObject(value) && + 'ObjectType' in value && + typeof (value as {ObjectType: unknown}).ObjectType === 'string' + ); +} + +/** + * Recursively unwraps the daemon's serialized-object envelopes. + * + * @param value A value decoded from an NSKeyedArchiver reply. + */ +export function deserializeAxObject(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(deserializeAxObject); + } + if (!isEnvelope(value)) { + if (util.isPlainObject(value)) { + // A plain dictionary with no ObjectType: deserialize each field. + const out: Record = {}; + for (const [key, inner] of Object.entries(value)) { + out[key] = deserializeAxObject(inner); + } + return out; + } + return value; + } + + const inner = deserializeAxObject(value.Value); + if (value.ObjectType === 'passthrough') { + return inner; + } + // A typed object. Spread its fields (when it has them) and tag the type. + if (util.isPlainObject(inner)) { + return {...(inner as Record), [AX_OBJECT_TYPE]: value.ObjectType}; + } + return {value: inner, [AX_OBJECT_TYPE]: value.ObjectType}; +} diff --git a/src/services/ios/accessibility-audit/ax-element.ts b/src/services/ios/accessibility-audit/ax-element.ts new file mode 100644 index 00000000..46f2108b --- /dev/null +++ b/src/services/ios/accessibility-audit/ax-element.ts @@ -0,0 +1,183 @@ +import {util} from '@appium/support'; + +import {AX_OBJECT_TYPE} from './ax-deserialize.js'; + +/** + * A handle to one element in the device's accessibility tree. + * + * `platformElement` is the daemon's opaque 20-byte identifier and is what makes + * the handle usable in later calls — it has to be sent back verbatim. + */ +export interface AxElement { + /** The daemon's opaque element identifier. */ + platformElement: Buffer; + /** The element's `accessibilityIdentifier`, when it has one. */ + accessibilityIdentifier?: string; +} + +/** + * One attribute the daemon exposes for an element, e.g. `Label` or `Traits`. + * + * These are descriptors only — they carry no value. Reading a value takes a + * second call (`deviceElement:valueForAttribute:`) passing the element and the + * descriptor back, which is exactly what Xcode's Inspector does to fill each row + * of its panel. + */ +export interface AxElementAttribute { + /** Wire name, e.g. `TraitsHumanReadable`. Pass this back to read a value. */ + name: string; + /** Display name, e.g. `Traits`. */ + humanReadableName: string; + /** Whether the value can be written back. */ + settable: boolean; + /** Whether reading it performs an action rather than returning data. */ + performsAction: boolean; + /** Whether the daemon considers this internal/debug-only. */ + isInternal: boolean; + /** The daemon's value-type discriminator. */ + valueType?: number; + /** The raw descriptor, needed verbatim when asking for the value. */ + raw: Record; +} + +/** A titled group of attributes — `Basic`, `Actions`, `Element`, `Hierarchy`. */ +export interface AxInspectorSection { + /** Stable identifier, e.g. `Basic_v1`. */ + identifier: string; + /** Display title, e.g. `Basic`. */ + title: string; + /** The attributes in this section. */ + attributes: AxElementAttribute[]; +} + +/** The inspector panel the device pushes when the focused element changes. */ +export interface AxInspectedElement { + /** What VoiceOver would announce, when the daemon provides it. */ + spokenDescription?: string; + /** The caption shown above the panel, when present. */ + caption?: string; + /** The panel's sections, in the order the device sent them. */ + sections: AxInspectorSection[]; +} + +/** Recovers a `Buffer` from a decoded `NS.data` blob. */ +function toBuffer(value: unknown): Buffer | undefined { + if (Buffer.isBuffer(value)) { + return value; + } + if (util.isPlainObject(value)) { + // The archiver decodes NSData into an index-keyed object. + const bytes = Object.values(value as Record).filter((b): b is number => typeof b === 'number'); + if (bytes.length > 0) { + return Buffer.from(bytes); + } + } + return undefined; +} + +/** + * Parses a deserialized `AXAuditElement_v1`. + * + * The `_v1` suffixes are the daemon's own wire keys, not our assumption. A + * future shape would carry different keys, so this returns `undefined` rather + * than misreading one. + */ +export function toAxElement(value: unknown): AxElement | undefined { + if (!util.isPlainObject(value)) { + return undefined; + } + const fields = value as Record; + const platformValue = fields.PlatformElementValue_v1; + const container = util.isPlainObject(platformValue) + ? ((platformValue as Record)['NS.data'] ?? platformValue) + : undefined; + const platformElement = toBuffer(container); + if (!platformElement) { + return undefined; + } + return { + platformElement, + accessibilityIdentifier: + typeof fields.AccessibilityIdentifier_v1 === 'string' ? fields.AccessibilityIdentifier_v1 : undefined, + }; +} + +/** + * Rebuilds the serialized form the daemon expects when an element is passed + * back, matching what Xcode's Inspector sends. + */ +export function serializeAxElement(element: AxElement): Record { + const value: Record = { + PlatformElementValue_v1: {ObjectType: 'passthrough', Value: element.platformElement}, + }; + if (element.accessibilityIdentifier !== undefined) { + value.AccessibilityIdentifier_v1 = {ObjectType: 'passthrough', Value: element.accessibilityIdentifier}; + } + return { + ObjectType: 'AXAuditElement_v1', + Value: {ObjectType: 'passthrough', Value: value}, + }; +} + +function toAttribute(value: unknown): AxElementAttribute | undefined { + if (!util.isPlainObject(value)) { + return undefined; + } + const fields = value as Record; + const name = fields.AttributeNameValue_v1; + if (typeof name !== 'string') { + return undefined; + } + return { + name, + humanReadableName: typeof fields.HumanReadableNameValue_v1 === 'string' ? fields.HumanReadableNameValue_v1 : name, + settable: fields.SettableValue_v1 === true, + performsAction: fields.PerformsActionValue_v1 === true, + isInternal: fields.IsInternal_v1 === true, + valueType: typeof fields.ValueTypeValue_v1 === 'number' ? fields.ValueTypeValue_v1 : undefined, + raw: stripTag(fields), + }; +} + +/** Drops the decoder's type tag so the object round-trips as the daemon sent it. */ +function stripTag(fields: Record): Record { + return Object.fromEntries(Object.entries(fields).filter(([key]) => key !== AX_OBJECT_TYPE)); +} + +/** Rebuilds an attribute descriptor for the wire. */ +export function serializeAxAttribute(attribute: AxElementAttribute): Record { + const value = Object.fromEntries( + Object.entries(attribute.raw).map(([key, inner]) => [key, {ObjectType: 'passthrough', Value: inner}]), + ); + return { + ObjectType: 'AXAuditElementAttribute_v1', + Value: {ObjectType: 'passthrough', Value: value}, + }; +} + +/** Parses the payload of an inbound `hostInspectorCurrentElementChanged:`. */ +export function toInspectedElement(value: unknown): AxInspectedElement { + const fields = (util.isPlainObject(value) ? value : {}) as Record; + const rawSections = Array.isArray(fields.InspectorSectionsValue_v1) ? fields.InspectorSectionsValue_v1 : []; + const sections: AxInspectorSection[] = []; + for (const rawSection of rawSections) { + if (!util.isPlainObject(rawSection)) { + continue; + } + const section = rawSection as Record; + const rawAttributes = Array.isArray(section.ElementAttributesValue_v1) ? section.ElementAttributesValue_v1 : []; + sections.push({ + identifier: typeof section.IdentifierValue_v1 === 'string' ? section.IdentifierValue_v1 : '', + title: typeof section.TitleValue_v1 === 'string' ? section.TitleValue_v1 : '', + attributes: rawAttributes + .map(toAttribute) + .filter((attribute): attribute is AxElementAttribute => attribute !== undefined), + }); + } + return { + spokenDescription: + typeof fields.SpokenDescriptionValue_v1 === 'string' ? fields.SpokenDescriptionValue_v1 : undefined, + caption: typeof fields.CaptionTextValue_v1 === 'string' ? fields.CaptionTextValue_v1 : undefined, + sections, + }; +} diff --git a/src/services/ios/accessibility-audit/ax-values.ts b/src/services/ios/accessibility-audit/ax-values.ts new file mode 100644 index 00000000..e2c0a85e --- /dev/null +++ b/src/services/ios/accessibility-audit/ax-values.ts @@ -0,0 +1,46 @@ +import {PlistUID} from '../../../lib/plist/index.js'; + +/** + * A point passed to the accessibility daemon, in normalized device coordinates + * (`0..1` across the screen's width and height). + * + * Carried as its own type because the daemon calls `CGPointValue` on the + * argument, so it has to arrive as an `NSValue` wrapping a `CGPoint` — an + * archived array, dictionary or bare double is rejected. Verified live on + * iOS 27.0: a double yields "Cannot get value with size 16. The type encoded as + * d is expected to be 8 bytes", and a dictionary yields + * "-[__NSDictionaryI CGPointValue]: unrecognized selector". + */ +export class AxPoint { + constructor( + readonly x: number, + readonly y: number, + ) {} +} + +/** + * Builds the NSKeyedArchiver graph for an `NSValue` holding a `CGPoint`. + * + * `NS.special` discriminates the wrapped struct — 1 for a point. The device's + * own replies use 3 for rects (seen on `ElementRectValue_v1`), which is what + * corroborates the numbering. + */ +export function archiveAxPoint(point: AxPoint): Record { + return { + $version: 100000, + $archiver: 'NSKeyedArchiver', + $top: {root: new PlistUID(1)}, + $objects: [ + '$null', + { + 'NS.special': 1, + 'NS.pointval': `{${point.x}, ${point.y}}`, + $class: new PlistUID(2), + }, + { + $classes: ['NSValue', 'NSObject'], + $classname: 'NSValue', + }, + ], + }; +} diff --git a/src/services/ios/accessibility-audit/dtx-transport.ts b/src/services/ios/accessibility-audit/dtx-transport.ts new file mode 100644 index 00000000..024c68bc --- /dev/null +++ b/src/services/ios/accessibility-audit/dtx-transport.ts @@ -0,0 +1,646 @@ +import net from 'node:net'; + +import {getLogger} from '../../../lib/logger.js'; +import {createBinaryPlist} from '../../../lib/plist/binary-plist-creator.js'; +import {parseBinaryPlist} from '../../../lib/plist/binary-plist-parser.js'; +import {createPlist} from '../../../lib/plist/plist-creator.js'; +import {BaseService, stripSSL} from '../base-service.js'; +import {ChannelFragmenter} from '../dvt/channel-fragmenter.js'; +import {DTXMessage, DTX_CONSTANTS, MessageAux} from '../dvt/dtx-message.js'; +import {decodeNSKeyedArchiver} from '../dvt/nskeyedarchiver-decoder.js'; +import {NSKeyedArchiverEncoder} from '../dvt/nskeyedarchiver-encoder.js'; +import {AxPoint, archiveAxPoint} from './ax-values.js'; + +const log = getLogger('AxAuditDtx'); + +/** First four bytes of a DTX message header (`0x1f3d5b79`, little-endian). */ +const DTX_MAGIC = Buffer.from([0x79, 0x5b, 0x3d, 0x1f]); + +/** The accessibility audit daemon speaks entirely on the DTX control channel. */ +const CONTROL_CHANNEL = 0; + +/** How long to wait for the TCP connection to the shim. */ +const CONNECT_TIMEOUT_MS = 30000; + +/** How long the checkin handshake may take before the socket is torn down. */ +const HANDSHAKE_TIMEOUT_MS = 30000; + +/** Safety valve for the checkin loop; iOS 27 sends two plists. */ +const MAX_CHECKIN_PLISTS = 8; + +/** Safety valve for fragment reassembly, far above any real message. */ +const MAX_FRAGMENTS_PER_MESSAGE = 65536; + +/** `DTXPrimitiveArray` header: a capacity word then the item-block length. */ +const AUX_HEADER_SIZE = 16; +/** Offset of the item-block length within that header. */ +const AUX_ITEMS_LENGTH_OFFSET = 8; +/** Width of the marker, type and 32-bit value words. */ +const AUX_WORD_SIZE = 4; +/** Width of a 64-bit auxiliary value. */ +const AUX_INT64_SIZE = 8; + +/** Options for {@link AxAuditDtxTransport.invoke}. */ +export interface InvokeOptions { + /** How long to wait for the reply, in milliseconds. Defaults to 15000. */ + timeoutMs?: number; +} + +/** + * DTX transport for the accessibility audit daemon + * (`com.apple.accessibility.axAuditDaemon.remoteserver.shim.remote`). + * + * This service is a DTX endpoint like the DVT instruments hub, but its RSD shim + * needs a different opening than {@link DVTSecureSocketProxyService}: a single + * `RSDCheckin` is answered by **two** framed plists — `{RSDCheckin}` then + * `{StartService}` — and only then does the raw DTX stream begin. Consuming both + * is the whole difference; reading just one leaves the second plist's bytes in + * front of the DTX stream and every later frame misparses. This was verified by + * packet-capturing Xcode's own Accessibility Inspector session. + * + * The exchange is also bidirectional: the device sends its own + * `_notifyOfPublishedCapabilities:` and a `hostApiVersion` request (neither must + * be answered for the read calls to work, verified live), and it delivers some + * results as calls back to the host rather than as replies — e.g. an audit's + * issues. Inbound calls are routed to {@link waitForInbound} waiters by selector + * and otherwise dropped. + * + * Reuses the DTX primitives from `../dvt/` unchanged — frame headers, the + * auxiliary `DTXPrimitiveArray` encoding, fragment reassembly, and + * NSKeyedArchiver coding — so this only owns the connection lifecycle and the + * request/reply dispatch. + */ +export class AxAuditDtxTransport extends BaseService { + static readonly RSD_SERVICE_NAME = 'com.apple.accessibility.axAuditDaemon.remoteserver.shim.remote'; + + private socket: net.Socket | null = null; + private readBuffer = Buffer.alloc(0); + private readWaiter: (() => void) | null = null; + private closed = false; + private closeError: Error | undefined; + + private nextMessageId = 1; + private readonly fragmenters = new Map(); + private readonly pendingReplies = new Map< + number, + {resolve: (value: unknown) => void; reject: (error: Error) => void} + >(); + private readonly inboundWaiters = new Map< + string, + Array<{resolve: (args: unknown[]) => void; reject: (error: Error) => void}> + >(); + private readonly inboundListeners = new Map void>>(); + + private constructor(udid: string) { + super(udid); + this.fragmenters.set(CONTROL_CHANNEL, new ChannelFragmenter()); + } + + /** + * Connects to the daemon, performs the checkin + StartService handshake, + * reaches the DTX phase, and publishes host capabilities. + * + * @param udid Target device UDID. + */ + static async connect(udid: string): Promise { + const transport = new AxAuditDtxTransport(udid); + await transport.open(); + return transport; + } + + private async open(): Promise { + // A raw socket on purpose: this protocol interleaves length-prefixed plists + // and raw DTX on one stream, so it cannot share ServiceConnection's plist + // pipeline (which would consume the checkin reply before the DTX reader sees + // it). Only the address resolution is reused. + const [host, port] = await this.resolveServiceAddress(AxAuditDtxTransport.RSD_SERVICE_NAME); + const socket = await new Promise((resolve, reject) => { + // The address family is left to Node: the tunnel hands out an IPv6 literal + // today, but nothing in this protocol is v6-specific. + const created = net.createConnection({host, port}, () => { + created.setTimeout(0); + created.setKeepAlive(true); + // Request/reply frames are small; Nagle would delay every one of them. + created.setNoDelay(true); + resolve(created); + }); + created.setTimeout(CONNECT_TIMEOUT_MS, () => { + created.destroy(); + reject(new Error(`Timed out after ${CONNECT_TIMEOUT_MS}ms connecting to [${host}]:${port}`)); + }); + created.once('error', reject); + }); + stripSSL(socket); + socket.on('data', (chunk: Buffer) => this.onData(chunk)); + socket.on('close', () => this.onClose(new Error('Accessibility audit connection closed'))); + socket.on('error', (error: Error) => this.onClose(error)); + this.socket = socket; + + try { + await withDeadline(this.performCheckin(), HANDSHAKE_TIMEOUT_MS, 'RSDCheckin handshake'); + } catch (error) { + this.close(); + throw error; + } + await this.publishCapabilities(); + } + + private onData(chunk: Buffer): void { + this.readBuffer = Buffer.concat([this.readBuffer, chunk]); + const waiter = this.readWaiter; + if (waiter) { + this.readWaiter = null; + waiter(); + } + } + + private onClose(error: Error): void { + if (this.closed) { + return; + } + this.closed = true; + this.closeError = error; + const waiter = this.readWaiter; + if (waiter) { + this.readWaiter = null; + waiter(); + } + for (const pending of this.pendingReplies.values()) { + pending.reject(error); + } + this.pendingReplies.clear(); + // Inbound waiters have to be failed too, or they hang until their own + // timeout even though nothing can arrive any more. + for (const waiters of this.inboundWaiters.values()) { + for (const waiter of waiters.splice(0)) { + waiter.reject(error); + } + } + this.inboundWaiters.clear(); + this.inboundListeners.clear(); + } + + /** Waits until at least `length` bytes are buffered, then returns them without consuming. */ + private async peek(length: number): Promise { + while (this.readBuffer.length < length) { + if (this.closed) { + throw this.closeError ?? new Error('Connection closed'); + } + await new Promise((resolve) => { + this.readWaiter = resolve; + }); + } + return this.readBuffer.subarray(0, length); + } + + /** Reads and consumes exactly `length` bytes. */ + private async readExact(length: number): Promise { + const data = await this.peek(length); + const copy = Buffer.from(data); + this.readBuffer = this.readBuffer.subarray(length); + return copy; + } + + private write(buffer: Buffer): void { + if (!this.socket || this.closed) { + throw this.closeError ?? new Error('Accessibility audit connection is not open'); + } + this.socket.write(buffer); + } + + /** + * Sends `RSDCheckin` and consumes every framed plist the daemon returns until + * the DTX stream begins. On iOS 27 that is `{RSDCheckin}` then `{StartService}`. + */ + private async performCheckin(): Promise { + const requestXml = createPlist({ + Label: 'appium-internal', + ProtocolVersion: '2', + Request: 'RSDCheckin', + }); + const body = Buffer.from(String(requestXml), 'utf8'); + const framed = Buffer.alloc(4 + body.length); + framed.writeUInt32BE(body.length, 0); + body.copy(framed, 4); + this.write(framed); + + // Plists are length-prefixed (4-byte big-endian); the DTX stream is not, so + // the magic marks the boundary. + const consumed: string[] = []; + for (;;) { + const head = await this.peek(4); + if (head.equals(DTX_MAGIC)) { + break; + } + if (consumed.length >= MAX_CHECKIN_PLISTS) { + throw new Error(`No DTX stream after ${MAX_CHECKIN_PLISTS} checkin plists: ${consumed.join(', ')}`); + } + const length = head.readUInt32BE(0); + await this.readExact(4); + const plist = await this.readExact(length); + const request = plist.toString('utf8').match(/Request<\/key>\s*([^<]*) { + for (let fragments = 0; ; fragments += 1) { + if (fragments >= MAX_FRAGMENTS_PER_MESSAGE) { + throw new Error(`DTX message exceeded ${MAX_FRAGMENTS_PER_MESSAGE} fragments`); + } + const header = DTXMessage.parseMessageHeader(await this.readExact(DTX_CONSTANTS.MESSAGE_HEADER_SIZE)); + const channel = Math.abs(header.channelCode); + let fragmenter = this.fragmenters.get(channel); + if (!fragmenter) { + fragmenter = new ChannelFragmenter(); + this.fragmenters.set(channel, fragmenter); + } + // The first fragment of a multi-fragment message is a header-only marker. + if (header.fragmentCount > 1 && header.fragmentId === 0) { + continue; + } + const fragment = await this.readExact(header.length); + fragmenter.addFragment(header, fragment); + const message = fragmenter.get(); + if (message) { + return { + channel, + identifier: header.identifier, + conversationIndex: header.conversationIndex, + payload: message, + }; + } + } + } + + /** + * Splits a reassembled payload into its object (the return value on a reply, + * or the selector on an inbound call) and its decoded auxiliary arguments. + */ + private decodePayload(payload: Buffer): {object: unknown; args: unknown[]} { + const payloadHeader = DTXMessage.parsePayloadHeader(payload); + const compression = (payloadHeader.flags & 0xff000) >> 12; + if (compression !== 0) { + throw new Error(`Unsupported DTX compression type ${compression}`); + } + const body = payload.subarray(DTX_CONSTANTS.PAYLOAD_HEADER_SIZE); + const auxBuffer = body.subarray(0, payloadHeader.auxiliaryLength); + const args = parseAuxiliary(auxBuffer); + + const objectSize = Number(payloadHeader.totalLength) - payloadHeader.auxiliaryLength; + const objectData = + objectSize > 0 ? body.subarray(payloadHeader.auxiliaryLength, payloadHeader.auxiliaryLength + objectSize) : null; + const object = objectData ? decodeNSKeyedArchiver(parseBinaryPlist(objectData)) : null; + return {object, args}; + } + + /** + * The single read loop; dispatches replies to callers and inbound calls to + * waiters. It runs until {@link recvMessage} throws, which happens when the + * socket closes — that is the only exit. + */ + private async readLoop(): Promise { + try { + for (;;) { + const message = await this.recvMessage(); + try { + this.dispatch(message); + } catch (error) { + // A frame this transport cannot decode (e.g. a compressed payload) + // must not take the whole session down with it. + log.debug(`Dropped undecodable DTX frame: ${error instanceof Error ? error.message : String(error)}`); + } + } + } catch (error) { + this.onClose(error instanceof Error ? error : new Error(String(error))); + } + } + + /** Routes one decoded message to its reply promise, listeners, or waiters. */ + private dispatch(message: {identifier: number; conversationIndex: number; payload: Buffer}): void { + const {object, args} = this.decodePayload(message.payload); + if (message.conversationIndex === 1) { + const pending = this.pendingReplies.get(message.identifier); + if (pending) { + this.pendingReplies.delete(message.identifier); + pending.resolve(object); + } else { + log.debug(`Reply for unknown id ${message.identifier} ignored`); + } + return; + } + // Inbound call from the device. The object is the selector; the args carry + // the payload (e.g. audit issues). Deliver it to a waiter if one is + // registered, otherwise drop it — the device does not require a reply for + // the read flows this transport supports. + const selector = typeof object === 'string' ? object : ''; + const listeners = selector ? this.inboundListeners.get(selector) : undefined; + for (const listener of [...(listeners ?? [])]) { + // A throwing listener is the caller's bug, not a transport failure. + try { + listener(args); + } catch (error) { + log.warn(`Listener for ${selector} threw: ${error instanceof Error ? error.message : String(error)}`); + } + } + const waiters = selector ? this.inboundWaiters.get(selector) : undefined; + if (waiters && waiters.length > 0) { + for (const waiter of waiters.splice(0)) { + waiter.resolve(args); + } + } else if (!listeners || listeners.length === 0) { + log.debug(`Inbound ${selector || 'message'} dropped (${args.length} arg(s))`); + } + } + + /** + * Subscribes to every inbound call with the given selector. + * + * Unlike {@link waitForInbound}, which resolves once, this stays registered — + * the device streams some results as a sequence of calls (an audit reports each + * issue via its own `hostFoundAuditIssue:`). + * + * @param selector The inbound selector to listen for. + * @param listener Receives the call's decoded arguments. + * @returns A function that removes the listener. + */ + onInbound(selector: string, listener: (args: unknown[]) => void): () => void { + const listeners = this.inboundListeners.get(selector) ?? []; + listeners.push(listener); + this.inboundListeners.set(selector, listeners); + return () => { + const index = listeners.indexOf(listener); + if (index >= 0) { + listeners.splice(index, 1); + } + }; + } + + /** + * Resolves with the arguments of the next inbound call whose selector matches. + * + * The device delivers some results as calls back to the host rather than as + * replies — e.g. an audit's issues arrive as + * `hostDeviceDidCompleteAuditCategoriesWithAuditIssues:`. + * + * @param selector The inbound selector to wait for. + * @param timeoutMs How long to wait before rejecting. Defaults to 60000. + */ + waitForInbound(selector: string, timeoutMs = 60000): Promise { + if (this.closed) { + return Promise.reject(this.closeError ?? new Error('Connection closed')); + } + return new Promise((resolve, reject) => { + const waiters = this.inboundWaiters.get(selector) ?? []; + const timer = setTimeout(() => { + const index = waiters.indexOf(waiter); + if (index >= 0) { + waiters.splice(index, 1); + } + reject(new Error(`Timed out after ${timeoutMs}ms waiting for inbound ${selector}`)); + }, timeoutMs); + const waiter = { + resolve: (args: unknown[]): void => { + clearTimeout(timer); + resolve(args); + }, + reject: (error: Error): void => { + clearTimeout(timer); + reject(error); + }, + }; + waiters.push(waiter); + this.inboundWaiters.set(selector, waiters); + }); + } + + private buildFrame( + selector: string | null, + aux: MessageAux | null, + expectsReply: boolean, + identifier: number, + ): Buffer { + const auxBuffer = aux ? buildAuxiliaryData(aux) : Buffer.alloc(0); + const selectorBuffer = + selector === null ? Buffer.alloc(0) : createBinaryPlist(new NSKeyedArchiverEncoder().encode(selector)); + let flags = DTX_CONSTANTS.INSTRUMENTS_MESSAGE_TYPE; + if (expectsReply) { + flags |= DTX_CONSTANTS.EXPECTS_REPLY_MASK; + } + const payloadHeader = DTXMessage.buildPayloadHeader({ + flags, + auxiliaryLength: auxBuffer.length, + totalLength: BigInt(auxBuffer.length + selectorBuffer.length), + }); + const messageHeader = DTXMessage.buildMessageHeader({ + magic: DTX_CONSTANTS.MESSAGE_HEADER_MAGIC, + cb: DTX_CONSTANTS.MESSAGE_HEADER_SIZE, + fragmentId: 0, + fragmentCount: 1, + length: DTX_CONSTANTS.PAYLOAD_HEADER_SIZE + auxBuffer.length + selectorBuffer.length, + identifier, + conversationIndex: 0, + channelCode: CONTROL_CHANNEL, + expectsReply: expectsReply ? 1 : 0, + }); + return Buffer.concat([messageHeader, payloadHeader, auxBuffer, selectorBuffer]); + } + + /** Publishes host capabilities and starts the read loop, matching Xcode's opening. */ + private async publishCapabilities(): Promise { + const aux = new MessageAux(); + aux.appendObj({ + 'com.apple.private.DTXBlockCompression': 0, + 'com.apple.private.DTXConnection': 1, + }); + this.write(this.buildFrame('_notifyOfPublishedCapabilities:', aux, false, this.nextMessageId++)); + // Only start consuming inbound frames after the handshake is on the wire, so + // the checkin reader above is the sole consumer until this point. + void this.readLoop(); + } + + /** + * Invokes a daemon selector and resolves with its decoded return value. + * + * @param selector The Objective-C selector, e.g. `deviceCapabilities`. + * @param aux Encoded arguments, or `null` for a no-argument selector. + * @param options Reply timeout. + */ + invoke(selector: string, aux: MessageAux | null = null, options: InvokeOptions = {}): Promise { + if (this.closed) { + return Promise.reject(this.closeError ?? new Error('Connection closed')); + } + const identifier = this.nextMessageId++; + const {timeoutMs = 15000} = options; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pendingReplies.delete(identifier); + reject(new Error(`Timed out after ${timeoutMs}ms waiting for reply to ${selector}`)); + }, timeoutMs); + this.pendingReplies.set(identifier, { + resolve: (value) => { + clearTimeout(timer); + resolve(value); + }, + reject: (error) => { + clearTimeout(timer); + reject(error); + }, + }); + try { + this.write(this.buildFrame(selector, aux, true, identifier)); + } catch (error) { + clearTimeout(timer); + this.pendingReplies.delete(identifier); + reject(error instanceof Error ? error : new Error(String(error))); + } + }); + } + + /** Sends a selector without waiting for a reply. */ + invokeOneway(selector: string, aux: MessageAux | null = null): void { + this.write(this.buildFrame(selector, aux, false, this.nextMessageId++)); + } + + /** Closes the connection. */ + close(): void { + this.onClose(new Error('Accessibility audit connection closed by caller')); + this.socket?.destroy(); + this.socket = null; + } +} + +/** + * Decodes a DTX `DTXPrimitiveArray` back into its values — the inverse of + * {@link buildAuxiliaryData}. Handles the three types this protocol uses: + * archived objects, 32-bit and 64-bit integers. + */ +function parseAuxiliary(buffer: Buffer): unknown[] { + if (buffer.length < AUX_HEADER_SIZE) { + return []; + } + // The leading word is NOT a fixed constant. `DTX_CONSTANTS.MESSAGE_AUX_MAGIC` + // (0x1f0) is only the value this codebase writes; the device also sends 0x3f0 + // and 0x7f0 — observed live, larger values on larger payloads, so it encodes a + // capacity rather than identifying the format. Requiring an exact match + // silently discarded every audit issue. The declared item length is validated + // instead, and the item loop bails on anything it does not recognise. + const itemsLength = Number(buffer.readBigUInt64LE(AUX_ITEMS_LENGTH_OFFSET)); + if (itemsLength <= 0 || itemsLength > buffer.length) { + return []; + } + const end = Math.min(AUX_HEADER_SIZE + itemsLength, buffer.length); + const values: unknown[] = []; + let offset = AUX_HEADER_SIZE; + while (offset + AUX_INT64_SIZE <= end) { + // Each item is prefixed by an empty-dictionary marker then a type word. + if (buffer.readUInt32LE(offset) === DTX_CONSTANTS.EMPTY_DICTIONARY) { + offset += AUX_WORD_SIZE; + } + const type = buffer.readUInt32LE(offset); + offset += AUX_WORD_SIZE; + if (type === DTX_CONSTANTS.AUX_TYPE_OBJECT) { + const length = buffer.readUInt32LE(offset); + offset += AUX_WORD_SIZE; + const objectData = buffer.subarray(offset, offset + length); + offset += length; + values.push(decodeNSKeyedArchiver(parseBinaryPlist(objectData))); + continue; + } + if (type === DTX_CONSTANTS.AUX_TYPE_INT32) { + values.push(buffer.readInt32LE(offset)); + offset += AUX_WORD_SIZE; + continue; + } + if (type === DTX_CONSTANTS.AUX_TYPE_INT64) { + values.push(Number(buffer.readBigInt64LE(offset))); + offset += AUX_INT64_SIZE; + continue; + } + // Unknown type — stop rather than misread the rest of the array. + break; + } + return values; +} + +/** + * Rejects if `promise` has not settled within `timeoutMs`. + */ +async function withDeadline(promise: Promise, timeoutMs: number, what: string): Promise { + let timer: NodeJS.Timeout | undefined; + let deadlineExpired = false; + promise.catch((error) => { + if (deadlineExpired) { + log.debug(`${what} failed after its deadline: ${error instanceof Error ? error.message : String(error)}`); + } + }); + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => { + deadlineExpired = true; + reject(new Error(`Timed out after ${timeoutMs}ms during ${what}`)); + }, timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +/** + * Encodes a {@link MessageAux} as a DTX `DTXPrimitiveArray`, byte-identical to + * {@link DVTSecureSocketProxyService}'s private encoder. + */ +function buildAuxiliaryData(aux: MessageAux): Buffer { + const values = aux.getValues(); + if (values.length === 0) { + return Buffer.alloc(0); + } + const parts: Buffer[] = []; + for (const {type, value} of values) { + const marker = Buffer.alloc(8); + marker.writeUInt32LE(DTX_CONSTANTS.EMPTY_DICTIONARY, 0); + marker.writeUInt32LE(type, 4); + parts.push(marker); + switch (type) { + case DTX_CONSTANTS.AUX_TYPE_OBJECT: { + // An AxPoint has to be archived as an NSValue, which the generic encoder + // cannot express — see `ax-values.ts`. + const archive = value instanceof AxPoint ? archiveAxPoint(value) : new NSKeyedArchiverEncoder().encode(value); + const encoded = createBinaryPlist(archive); + const length = Buffer.alloc(AUX_WORD_SIZE); + length.writeUInt32LE(encoded.length, 0); + parts.push(length, encoded); + break; + } + case DTX_CONSTANTS.AUX_TYPE_INT32: { + const encoded = Buffer.alloc(AUX_WORD_SIZE); + encoded.writeUInt32LE(value as number, 0); + parts.push(encoded); + break; + } + default: { + const encoded = Buffer.alloc(AUX_INT64_SIZE); + encoded.writeBigUInt64LE(BigInt(value as number | bigint), 0); + parts.push(encoded); + } + } + } + const itemsData = Buffer.concat(parts); + const header = Buffer.alloc(AUX_HEADER_SIZE); + header.writeBigUInt64LE(BigInt(DTX_CONSTANTS.MESSAGE_AUX_MAGIC), 0); + header.writeBigUInt64LE(BigInt(itemsData.length), AUX_ITEMS_LENGTH_OFFSET); + return Buffer.concat([header, itemsData]); +} diff --git a/src/services/ios/accessibility-audit/index.ts b/src/services/ios/accessibility-audit/index.ts new file mode 100644 index 00000000..e5787c62 --- /dev/null +++ b/src/services/ios/accessibility-audit/index.ts @@ -0,0 +1,400 @@ +import {util} from '@appium/support'; + +import {getLogger} from '../../../lib/logger.js'; +import {MessageAux} from '../dvt/dtx-message.js'; +import {AX_OBJECT_TYPE, deserializeAxObject} from './ax-deserialize.js'; +import { + type AxElement, + type AxElementAttribute, + type AxInspectedElement, + serializeAxAttribute, + serializeAxElement, + toAxElement, + toInspectedElement, +} from './ax-element.js'; +import {AxAuditDtxTransport, type InvokeOptions} from './dtx-transport.js'; + +const log = getLogger('AccessibilityAudit'); + +/** `deviceInspectorSetMonitoredEventType:` value that reports focus changes. */ +const MONITORED_EVENT_FOCUS = 2; +/** Value that disarms monitoring. */ +const MONITORED_EVENT_OFF = 0; + +/** + * One accessibility setting reported by + * {@link AccessibilityAuditService.getAccessibilitySettings}. + * + * The daemon names fields with a `_v1` suffix (and ships the historical typo + * `IdentiifierValue_v1`); this is the cleaned form. Unknown fields are preserved + * via the index signature so nothing is silently dropped. + */ +export interface AxDeviceSetting { + /** Stable identifier, e.g. `INVERT_COLORS`, `REDUCE_MOTION`. */ + identifier: string; + /** The daemon's setting-type discriminator. */ + settingType?: number; + /** Current value — a boolean toggle, a number, or a string depending on type. */ + currentValue?: unknown; + /** Whether the setting is currently enabled/available. */ + enabled?: boolean; + /** Tick-mark count for slider-style settings. */ + sliderTickMarks?: number; + [key: string]: unknown; +} + +/** + * CoreDevice accessibility audit service + * (`com.apple.accessibility.axAuditDaemon.remoteserver`) — the backend behind + * Xcode's Accessibility Inspector. + * + * Exposes the device's accessibility model over DTX: the audit catalogue, + * accessibility settings, and (later) the element tree and on-device audits. + * See {@link AxAuditDtxTransport} for the connection details that make this + * reachable over the RemoteXPC tunnel. + * + * @example + * ```ts + * const audit = await Services.startAccessibilityAuditService(udid); + * try { + * const settings = await audit.getAccessibilitySettings(); + * } finally { + * audit.close(); + * } + * ``` + */ +export class AccessibilityAuditService { + static readonly RSD_SERVICE_NAME = AxAuditDtxTransport.RSD_SERVICE_NAME; + + /** Live {@link observeFocusedElement} subscriptions; monitoring stays armed while > 0. */ + private observerCount = 0; + + /** Guards {@link runAudit} against overlapping calls on one instance. */ + private auditInFlight = false; + + private constructor(private readonly transport: AxAuditDtxTransport) {} + + /** + * Connects to the daemon and completes the DTX handshake. + * + * @param udid Target device UDID. + */ + static async start(udid: string): Promise { + return new AccessibilityAuditService(await AxAuditDtxTransport.connect(udid)); + } + + /** The daemon's API version (26 on iOS 27.0). */ + async getApiVersion(options?: InvokeOptions): Promise { + const value = await this.transport.invoke('deviceApiVersion', null, options); + if (typeof value !== 'number') { + throw new Error(`Expected a numeric API version, got ${JSON.stringify(value)}`); + } + return value; + } + + /** The selectors the device's daemon implements. */ + async getCapabilities(options?: InvokeOptions): Promise { + return asStringArray(await this.transport.invoke('deviceCapabilities', null, options), 'deviceCapabilities'); + } + + /** The audit types the device supports, e.g. `testTypeContrast`. */ + async getSupportedAuditTypes(options?: InvokeOptions): Promise { + return asStringArray( + await this.transport.invoke('deviceAllSupportedAuditTypes', null, options), + 'deviceAllSupportedAuditTypes', + ); + } + + /** + * The device's accessibility settings and their current values. + */ + async getAccessibilitySettings(options?: InvokeOptions): Promise { + const raw = deserializeAxObject(await this.transport.invoke('deviceAccessibilitySettings', null, options)); + if (!Array.isArray(raw)) { + throw new Error(`Expected an array of settings, got ${JSON.stringify(raw)?.slice(0, 120)}`); + } + return raw.map(toDeviceSetting); + } + + /** + * Runs the given accessibility audits on whatever the device is currently + * showing and resolves with the issues found (empty when everything passes). + * + * The audit begins with a one-way `deviceBeginAuditTypes:` and completes when + * the device calls back with + * `hostDeviceDidCompleteAuditCategoriesWithAuditIssues:`. + * + * Only one audit may run per service instance — issues arrive on a shared + * inbound stream, so overlapping calls would each collect the other's. A + * concurrent call is rejected rather than allowed to mix results; use a second + * service instance to audit in parallel. + * + * @param auditTypes Audit types to run, from {@link getSupportedAuditTypes}. + * @param options Timeout for the completion callback. + */ + async runAudit(auditTypes: string[], options: RunAuditOptions = {}): Promise { + if (this.auditInFlight) { + throw new Error('An audit is already running on this service instance; await it or use a second instance'); + } + this.auditInFlight = true; + // Issues are streamed one per `hostFoundAuditIssue:` call and the + // completion callback carries no arguments. Older releases are reported to + // return them in the completion instead, so both are collected and the + // streamed set wins when present. + const issues: AxAuditIssue[] = []; + const stopIssues = this.transport.onInbound('hostFoundAuditIssue:', (args) => { + const issue = deserializeAxObject(args[0]); + if (util.isPlainObject(issue)) { + issues.push(issue as AxAuditIssue); + } + }); + const stopLog = options.onLog + ? this.transport.onInbound('hostAppendAuditLog:', (args) => { + if (typeof args[0] === 'string') { + options.onLog?.(args[0]); + } + }) + : undefined; + + try { + if (options.targetPid !== undefined) { + // Narrows the audit to one process; omitted, the daemon uses the + // foreground app. + const pidAux = new MessageAux(); + pidAux.appendObj(options.targetPid); + this.transport.invokeOneway('deviceSetAuditTargetPid:', pidAux); + } + const completion = this.transport.waitForInbound( + 'hostDeviceDidCompleteAuditCategoriesWithAuditIssues:', + options.timeoutMs, + ); + const aux = new MessageAux(); + aux.appendObj(auditTypes); + this.transport.invokeOneway('deviceBeginAuditTypes:', aux); + const completionArgs = await completion; + return issues.length > 0 ? issues : issuesFromCompletion(completionArgs); + } finally { + this.auditInFlight = false; + stopIssues(); + stopLog?.(); + } + } + + /** + * Returns the element the device's accessibility focus is currently on. + * + * The daemon does not answer a query for this. Xcode's Inspector arms a + * monitoring session and the device *pushes* the element back as an inbound + * `hostInspectorCurrentElementChanged:` call, so that is what this reproduces: + * arm, ask focus to report, wait for the push, disarm. Captured from a live + * Inspector session — `deviceFetchElementAtNormalizedDeviceCoordinate:` + * returns `null` on iOS 27 no matter how it is called. + * + * @param options Timeout, and whether to draw the on-device highlight. + */ + async getFocusedElement(options: InspectOptions = {}): Promise { + const {timeoutMs = 15000, showVisuals = false} = options; + const pushed = this.transport.waitForInbound('hostInspectorCurrentElementChanged:', timeoutMs); + this.setMonitoredEventType(MONITORED_EVENT_FOCUS); + if (showVisuals) { + this.setShowVisuals(true); + } + try { + const empty = new MessageAux(); + // An empty dictionary means "whatever is focused now" — this is exactly + // what the Inspector sends. + empty.appendObj({}); + this.transport.invokeOneway('deviceInspectorFocusOnElement:', empty); + // `waitForInbound` resolves with the call's whole argument list; the panel + // is the first argument. + const [payload] = await pushed; + return toInspectedElement(deserializeAxObject(payload)); + } finally { + if (showVisuals) { + this.setShowVisuals(false); + } + // Leave monitoring armed if an observer is relying on it. + if (this.observerCount === 0) { + this.setMonitoredEventType(MONITORED_EVENT_OFF); + } + } + } + + /** + * Subscribes to focus changes, delivering an inspector panel each time the + * device's accessibility focus moves. + * + * Monitoring stays armed until every observer has unsubscribed. The listener + * is called synchronously; a rejected promise returned from an async one is + * not awaited, so handle errors inside it. + * + * @param listener Receives each pushed element. + * @param options Whether to draw the on-device highlight. + */ + observeFocusedElement( + listener: (element: AxInspectedElement) => void, + options: {showVisuals?: boolean} = {}, + ): () => void { + const stop = this.transport.onInbound('hostInspectorCurrentElementChanged:', (args) => { + listener(toInspectedElement(deserializeAxObject(args[0]))); + }); + this.observerCount += 1; + this.setMonitoredEventType(MONITORED_EVENT_FOCUS); + if (options.showVisuals) { + this.setShowVisuals(true); + } + let stopped = false; + return () => { + if (stopped) { + return; + } + stopped = true; + stop(); + if (options.showVisuals) { + this.setShowVisuals(false); + } + // Only the last observer may disarm; otherwise it would stop the others. + this.observerCount -= 1; + if (this.observerCount === 0) { + this.setMonitoredEventType(MONITORED_EVENT_OFF); + } + }; + } + + /** + * Reads one attribute's value for an element. + * + * Attribute descriptors carry no values, so each row of the inspector panel + * costs one call — the element handle and the descriptor both go back as they + * arrived. + * + * @param element The element handle. + * @param attribute A descriptor from {@link AxInspectedElement}'s sections. + * @param options Reply timeout. + */ + async getElementAttributeValue( + element: AxElement, + attribute: AxElementAttribute, + options?: InvokeOptions, + ): Promise { + const aux = new MessageAux(); + aux.appendObj(serializeAxElement(element)); + aux.appendObj(serializeAxAttribute(attribute)); + return deserializeAxObject(await this.transport.invoke('deviceElement:valueForAttribute:', aux, options)); + } + + /** + * Returns one of the daemon's well-known elements. + * + * Index `0` and `1` resolve on iOS 27; higher indices return `undefined`. + * + * @param index Which special element to fetch. + * @param options Reply timeout. + */ + async getSpecialElement(index: number, options?: InvokeOptions): Promise { + const aux = new MessageAux(); + aux.appendObj(index); + const value = deserializeAxObject(await this.transport.invoke('deviceFetchSpecialElement:', aux, options)); + return toAxElement(value); + } + + /** Arms or disarms the daemon's focus monitoring. */ + private setMonitoredEventType(type: number): void { + const aux = new MessageAux(); + aux.appendObj(type); + this.transport.invokeOneway('deviceInspectorSetMonitoredEventType:', aux); + } + + /** Toggles the on-device highlight the Inspector draws around the element. */ + private setShowVisuals(enabled: boolean): void { + const aux = new MessageAux(); + aux.appendObj(enabled); + this.transport.invokeOneway('deviceInspectorShowVisuals:', aux); + } + + /** Closes the underlying connection. */ + close(): void { + this.transport.close(); + } +} + +/** Options for {@link AccessibilityAuditService.getFocusedElement}. */ +export interface InspectOptions { + /** How long to wait for the device to push the element. Defaults to 15000. */ + timeoutMs?: number; + /** Draw the Inspector's highlight around the element on the device. */ + showVisuals?: boolean; +} + +/** + * One accessibility issue from {@link AccessibilityAuditService.runAudit}. + * + * The daemon's fields carry `_v1` suffixes and vary by audit type, so this is + * an open shape — the tag under {@link AX_OBJECT_TYPE} identifies the concrete + * type (`AXAuditIssue_v1`). + */ +export type AxAuditIssue = Record; + +/** Options for {@link AccessibilityAuditService.runAudit}. */ +export interface RunAuditOptions { + /** + * PID of the app to audit. Optional — the daemon audits the foreground app + * when this is omitted. + * + * Set it to audit a specific process regardless of what is frontmost. + */ + targetPid?: number; + /** How long to wait for the audit to complete, in milliseconds. */ + timeoutMs?: number; + /** Receives the device's own audit log lines as they stream in. */ + onLog?: (line: string) => void; +} + +/** + * Recovers audit issues carried in the completion callback's arguments. + * + * The device sends no arguments there and streams each issue separately, so this is + * the fallback path for releases that report them in the completion instead. + */ +function issuesFromCompletion(args: unknown[]): AxAuditIssue[] { + if (args.length === 0) { + return []; + } + const payload = deserializeAxObject(args[0]); + if (payload === null || payload === undefined) { + return []; + } + const list = Array.isArray(payload) ? payload : [payload]; + return list.filter((issue): issue is AxAuditIssue => util.isPlainObject(issue)); +} + +/** Narrows an unknown reply to `string[]`. */ +function asStringArray(value: unknown, selector: string): string[] { + if (!Array.isArray(value) || !value.every((entry) => typeof entry === 'string')) { + throw new Error(`Expected ${selector} to return an array of strings, got ${JSON.stringify(value)?.slice(0, 120)}`); + } + return value as string[]; +} + +/** Maps one deserialized `AXAuditDeviceSetting_v1` to the cleaned shape. */ +function toDeviceSetting(raw: unknown): AxDeviceSetting { + if (typeof raw !== 'object' || raw === null) { + throw new Error(`Malformed accessibility setting: ${JSON.stringify(raw)?.slice(0, 120)}`); + } + const fields = raw as Record; + const identifier = fields.IdentiifierValue_v1; + if (typeof identifier !== 'string') { + log.debug(`Setting without a string identifier: ${JSON.stringify(fields)?.slice(0, 120)}`); + } + return { + ...fields, + identifier: typeof identifier === 'string' ? identifier : String(identifier), + settingType: typeof fields.SettingTypeValue_v1 === 'number' ? fields.SettingTypeValue_v1 : undefined, + currentValue: fields.CurrentValueNumber_v1, + enabled: typeof fields.EnabledValue_v1 === 'boolean' ? fields.EnabledValue_v1 : undefined, + sliderTickMarks: typeof fields.SliderTickMarksValue_v1 === 'number' ? fields.SliderTickMarksValue_v1 : undefined, + }; +} + +export {AxAuditDtxTransport, AX_OBJECT_TYPE, MessageAux}; +export default AccessibilityAuditService; diff --git a/test/integration/accessibility-audit.spec.ts b/test/integration/accessibility-audit.spec.ts new file mode 100644 index 00000000..31ab2490 --- /dev/null +++ b/test/integration/accessibility-audit.spec.ts @@ -0,0 +1,157 @@ +import assert from 'node:assert/strict'; +import {after, before, describe, it} from 'node:test'; + +import {type AccessibilityAuditService} from '../../src/index.js'; +import * as Services from '../../src/services.js'; +import {requireDeviceUdid} from './helpers/device.js'; + +/** + * Integration tests for the accessibility audit service + * (`com.apple.accessibility.axAuditDaemon.remoteserver`), the DTX backend behind + * Xcode's Accessibility Inspector. + * + * Requires a physical iOS device with a running tunnel registry, and the shim + * present in the RSD catalog (needs a Developer Disk Image mounted). Set the + * UDID env var to the target device. + * + * The audit result depends on what is on the device's screen, so + * {@link AccessibilityAuditService.runAudit} is asserted only to complete and + * return an array — the count is whatever the current screen yields. + */ +describe('AccessibilityAuditService', {timeout: 90000}, function () { + let service: AccessibilityAuditService | null = null; + + before(async function () { + const udid = requireDeviceUdid(); + service = await Services.startAccessibilityAuditService(udid); + }); + + after(function () { + service?.close(); + }); + + it('reports the daemon API version', async function () { + const version = await service!.getApiVersion(); + + assert.strictEqual(typeof version, 'number'); + // The daemon advertises 26 on iOS 27.0; any positive integer is acceptable. + assert.ok(version > 0); + }); + + it('lists the selectors the daemon implements', async function () { + const capabilities = await service!.getCapabilities(); + + assert.ok(Array.isArray(capabilities)); + assert.ok(capabilities.length > 0); + // A stable, always-present selector. + assert.ok(capabilities.includes('deviceAccessibilitySettings')); + }); + + it('lists the supported audit types', async function () { + const types = await service!.getSupportedAuditTypes(); + + assert.ok(Array.isArray(types)); + assert.ok(types.length > 0); + assert.ok(types.every((type) => typeof type === 'string' && type.startsWith('testType'))); + }); + + it('reads the accessibility settings with their current values', async function () { + const settings = await service!.getAccessibilitySettings(); + + assert.ok(Array.isArray(settings)); + assert.ok(settings.length > 0); + for (const setting of settings) { + assert.strictEqual(typeof setting.identifier, 'string'); + assert.ok(setting.identifier.length > 0); + } + // Reduce Motion is a stock toggle present on every device. + assert.ok(settings.some((setting) => setting.identifier === 'REDUCE_MOTION')); + }); + + it('runs an audit over the current screen and returns its issues', async function () { + const types = await service!.getSupportedAuditTypes(); + + const issues = await service!.runAudit(types, {timeoutMs: 60000}); + + // Whatever is on screen: the flow must complete and yield an array. + assert.ok(Array.isArray(issues)); + }); + + it('streams the device audit log while auditing', async function () { + const lines: string[] = []; + + await service!.runAudit(['testTypeContrast'], {timeoutMs: 60000, onLog: (line) => lines.push(line)}); + + // The daemon narrates every audit; the run above must have produced some. + assert.ok(lines.length > 0); + assert.ok(lines.join('').includes('Test Starting')); + }); + + it('accepts an empty audit-type list without hanging', async function () { + const issues = await service!.runAudit([], {timeoutMs: 60000}); + + assert.ok(Array.isArray(issues)); + }); + + it('fetches a special element with a usable handle', async function (t) { + const element = await service!.getSpecialElement(0); + + assert.ok(element, 'index 0 should resolve on iOS 27'); + // The handle is opaque but must round-trip, so it has to be real bytes. + assert.ok(Buffer.isBuffer(element.platformElement)); + assert.ok(element.platformElement.length > 0); + t.diagnostic(`element identifier: ${element.accessibilityIdentifier ?? '(none)'}`); + }); + + it('returns the focused element inspector panel', async function () { + const panel = await service!.getFocusedElement({timeoutMs: 30000}); + + // The device pushes the whole panel; Basic is always present. + assert.ok(Array.isArray(panel.sections)); + assert.ok(panel.sections.length > 0); + const basic = panel.sections.find((section) => section.title === 'Basic'); + assert.ok(basic, 'a Basic section should be present'); + assert.ok(basic.attributes.length > 0); + // Attribute descriptors carry no values — only names and flags. + for (const attribute of basic.attributes) { + assert.strictEqual(typeof attribute.name, 'string'); + assert.strictEqual(typeof attribute.humanReadableName, 'string'); + assert.strictEqual(typeof attribute.settable, 'boolean'); + } + assert.ok(basic.attributes.some((attribute) => attribute.name === 'Label')); + }); + + it('reads attribute values for an element', async function (t) { + const element = await service!.getSpecialElement(0); + assert.ok(element); + const panel = await service!.getFocusedElement({timeoutMs: 30000}); + const basic = panel.sections.find((section) => section.title === 'Basic'); + assert.ok(basic); + + const values: Record = {}; + for (const attribute of basic.attributes) { + values[attribute.name] = await service!.getElementAttributeValue(element, attribute); + } + + // Which values are populated depends on the element in focus, so this + // asserts the call succeeds and returns the attributes asked for rather + // than pinning values that legitimately vary. + assert.deepStrictEqual(Object.keys(values).sort(), basic.attributes.map((attribute) => attribute.name).sort()); + t.diagnostic(`values: ${JSON.stringify(values).slice(0, 200)}`); + }); + + it('reports issues against a targeted app when one has them', async function (t) { + // Only meaningful with the deliberately-broken test app installed and in the + // foreground; without it there is nothing to find, so this records what it + // saw rather than asserting a count it cannot guarantee. + const types = await service!.getSupportedAuditTypes(); + const issues = await service!.runAudit(types, {timeoutMs: 60000}); + + for (const issue of issues) { + // Every issue must name the audit type that produced it. + assert.strictEqual(typeof issue.auditTestTypeValue_v1, 'string'); + assert.ok(types.includes(issue.auditTestTypeValue_v1 as string)); + } + t.diagnostic(`audit produced ${issues.length} issue(s) on the current screen`); + }); +}); diff --git a/test/unit/accessibility-audit/ax-deserialize.spec.ts b/test/unit/accessibility-audit/ax-deserialize.spec.ts new file mode 100644 index 00000000..bd7dc316 --- /dev/null +++ b/test/unit/accessibility-audit/ax-deserialize.spec.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import {describe, it} from 'node:test'; + +import {AX_OBJECT_TYPE, deserializeAxObject} from '../../../src/services/ios/accessibility-audit/ax-deserialize.js'; + +describe('deserializeAxObject', function () { + it('returns primitives unchanged', function () { + assert.strictEqual(deserializeAxObject(26), 26); + assert.strictEqual(deserializeAxObject('INVERT_COLORS'), 'INVERT_COLORS'); + assert.strictEqual(deserializeAxObject(true), true); + assert.strictEqual(deserializeAxObject(null), null); + }); + + it('unwraps a passthrough envelope to its inner value', function () { + assert.strictEqual(deserializeAxObject({ObjectType: 'passthrough', Value: 'REDUCE_MOTION'}), 'REDUCE_MOTION'); + assert.strictEqual(deserializeAxObject({ObjectType: 'passthrough', Value: 3}), 3); + }); + + it('unwraps nested passthrough envelopes', function () { + const nested = { + ObjectType: 'passthrough', + Value: {ObjectType: 'passthrough', Value: false}, + }; + assert.strictEqual(deserializeAxObject(nested), false); + }); + + it('tags a typed object and flattens its fields', function () { + // The real shape of one accessibility setting from the device. + const setting = { + ObjectType: 'AXAuditDeviceSetting_v1', + Value: { + ObjectType: 'passthrough', + Value: { + IdentiifierValue_v1: {ObjectType: 'passthrough', Value: 'INVERT_COLORS'}, + SettingTypeValue_v1: {ObjectType: 'passthrough', Value: 3}, + CurrentValueNumber_v1: {ObjectType: 'passthrough', Value: false}, + EnabledValue_v1: {ObjectType: 'passthrough', Value: true}, + }, + }, + }; + + assert.deepStrictEqual(deserializeAxObject(setting), { + IdentiifierValue_v1: 'INVERT_COLORS', + SettingTypeValue_v1: 3, + CurrentValueNumber_v1: false, + EnabledValue_v1: true, + [AX_OBJECT_TYPE]: 'AXAuditDeviceSetting_v1', + }); + }); + + it('deserializes each element of an array', function () { + const list = [ + {ObjectType: 'passthrough', Value: 'a'}, + {ObjectType: 'passthrough', Value: 'b'}, + ]; + assert.deepStrictEqual(deserializeAxObject(list), ['a', 'b']); + }); + + it('recurses into a plain dictionary that has no ObjectType', function () { + const plain = { + count: 2, + first: {ObjectType: 'passthrough', Value: 'x'}, + }; + assert.deepStrictEqual(deserializeAxObject(plain), {count: 2, first: 'x'}); + }); + + it('wraps a typed object whose value is not a dictionary under `value`', function () { + const typed = {ObjectType: 'AXAuditElement_v1', Value: {ObjectType: 'passthrough', Value: 'raw'}}; + assert.deepStrictEqual(deserializeAxObject(typed), {value: 'raw', [AX_OBJECT_TYPE]: 'AXAuditElement_v1'}); + }); + + it('handles the empty settings list', function () { + assert.deepStrictEqual(deserializeAxObject([]), []); + }); +});