feat(accessibility): add AccessibilityAuditService for on-device audits and element inspection

package.json

@@ -46,6 +46,7 @@
     "test:afc": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/afc.spec.js\"",
     "test:all": "node --enable-source-maps --experimental-test-module-mocks --test --test-timeout=60000 \"build/test/unit/**/*.spec.js\" \"build/test/integration/**/*.spec.js\"",
     "test:app-service": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/app-service.spec.js\"",
+    "test:accessibility-audit": "node --enable-source-maps --test --test-timeout=120000 \"build/test/integration/accessibility-audit.spec.js\"",
     "test:coredevice-device-info": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/device-info-coredevice.spec.js\"",
     "test:device-control": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/device-control.spec.js\"",
     "test:configuration": "node --enable-source-maps --test --test-timeout=60000 \"build/test/integration/configuration.spec.js\"",

src/index.ts

@@ -59,6 +59,25 @@ export type {
   ListAppsOptions,
 } from './services/ios/app-service/index.js';
 export {PasteboardService} from './services/ios/pasteboard/index.js';
+export {AccessibilityAuditService} from './services/ios/accessibility-audit/index.js';
+export type {AxDeviceSetting} from './services/ios/accessibility-audit/index.js';
+export {AxAuditDtxTransport} from './services/ios/accessibility-audit/dtx-transport.js';
+export type {InvokeOptions as AxInvokeOptions} from './services/ios/accessibility-audit/dtx-transport.js';
+export {AX_OBJECT_TYPE, deserializeAxObject} from './services/ios/accessibility-audit/ax-deserialize.js';
+export {AxPoint} from './services/ios/accessibility-audit/ax-values.js';
+export {
+  serializeAxAttribute,
+  serializeAxElement,
+  toAxElement,
+  toInspectedElement,
+} from './services/ios/accessibility-audit/ax-element.js';
+export type {
+  AxElement,
+  AxElementAttribute,
+  AxInspectedElement,
+  AxInspectorSection,
+} from './services/ios/accessibility-audit/ax-element.js';
+export type {InspectOptions, RunAuditOptions, AxAuditIssue} from './services/ios/accessibility-audit/index.js';
 export {CoreDeviceInfoService} from './services/ios/device-info/index.js';
 export type {
   CoreDeviceAttributes,

src/services.ts

@@ -4,6 +4,7 @@ import {
   resolveTunnelServicePorts,
 } from './lib/tunnel/tunnel-service-resolver.js';
 import type {DVTInstruments, SyslogService as SyslogServiceType, XCTestServices} from './lib/types.js';
+import {AccessibilityAuditService} from './services/ios/accessibility-audit/index.js';
 import AfcService from './services/ios/afc/index.js';
 import {AppService} from './services/ios/app-service/index.js';
 import {type Service} from './services/ios/base-service.js';
@@ -139,6 +140,16 @@ export async function startCoreDeviceInfoService(udid: string): Promise<CoreDevi
   return new CoreDeviceInfoService(udid);
 }
 
+/**
+ * Start the accessibility audit service for the given device UDID — the DTX
+ * backend behind Xcode's Accessibility Inspector. Exposes the device's
+ * accessibility settings and audit catalogue.
+ */
+export async function startAccessibilityAuditService(udid: string): Promise<AccessibilityAuditService> {
+  await requireCatalogService(udid, AccessibilityAuditService.RSD_SERVICE_NAME);
+  return AccessibilityAuditService.start(udid);
+}
+
 /**
  * Start the CoreDevice device-control service for the given device UDID.
  */

src/services/ios/accessibility-audit/ax-deserialize.ts

@@ -0,0 +1,54 @@
+/**
+ * The accessibility audit daemon wraps every value it returns in a recursive
+ * `{Value, ObjectType}` envelope. `ObjectType: "passthrough"` is a plain boxed
+ * value; any other `ObjectType` names a typed object (e.g.
+ * `AXAuditDeviceSetting_v1`) whose `Value` is a dictionary of fields.
+ */
+
+/** Key under which a typed (non-passthrough) object records its `ObjectType`. */
+export const AX_OBJECT_TYPE = '__axObjectType';
+
+/** A decoded typed object: its fields plus the {@link AX_OBJECT_TYPE} tag. */
+export type AxTypedObject = Record<string, unknown> & {[AX_OBJECT_TYPE]: string};
+
+function isEnvelope(value: unknown): value is {Value: unknown; ObjectType: string} {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    !Array.isArray(value) &&
+    'ObjectType' in value &&
+    typeof (value as {ObjectType: unknown}).ObjectType === 'string'
+  );
+}
+
+/**
+ * Recursively unwraps the daemon's serialized-object envelopes.
+ *
+ * @param value A value decoded from an NSKeyedArchiver reply.
+ */
+export function deserializeAxObject(value: unknown): unknown {
+  if (Array.isArray(value)) {
+    return value.map(deserializeAxObject);
+  }
+  if (!isEnvelope(value)) {
+    if (typeof value === 'object' && value !== null) {
+      // A plain dictionary with no ObjectType: deserialize each field.
+      const out: Record<string, unknown> = {};
+      for (const [key, inner] of Object.entries(value)) {
+        out[key] = deserializeAxObject(inner);
+      }
+      return out;
+    }
+    return value;
+  }
+
+  const inner = deserializeAxObject(value.Value);
+  if (value.ObjectType === 'passthrough') {
+    return inner;
+  }
+  // A typed object. Spread its fields (when it has them) and tag the type.
+  if (typeof inner === 'object' && inner !== null && !Array.isArray(inner)) {
+    return {...(inner as Record<string, unknown>), [AX_OBJECT_TYPE]: value.ObjectType};
+  }
+  return {value: inner, [AX_OBJECT_TYPE]: value.ObjectType};
+}

src/services/ios/accessibility-audit/ax-element.ts

@@ -0,0 +1,183 @@
+import {util} from '@appium/support';
+
+import {AX_OBJECT_TYPE} from './ax-deserialize.js';
+
+/**
+ * A handle to one element in the device's accessibility tree.
+ *
+ * `platformElement` is the daemon's opaque 20-byte identifier and is what makes
+ * the handle usable in later calls — it has to be sent back verbatim.
+ */
+export interface AxElement {
+  /** The daemon's opaque element identifier. */
+  platformElement: Buffer;
+  /** The element's `accessibilityIdentifier`, when it has one. */
+  accessibilityIdentifier?: string;
+}
+
+/**
+ * One attribute the daemon exposes for an element, e.g. `Label` or `Traits`.
+ *
+ * These are descriptors only — they carry no value. Reading a value takes a
+ * second call (`deviceElement:valueForAttribute:`) passing the element and the
+ * descriptor back, which is exactly what Xcode's Inspector does to fill each row
+ * of its panel.
+ */
+export interface AxElementAttribute {
+  /** Wire name, e.g. `TraitsHumanReadable`. Pass this back to read a value. */
+  name: string;
+  /** Display name, e.g. `Traits`. */
+  humanReadableName: string;
+  /** Whether the value can be written back. */
+  settable: boolean;
+  /** Whether reading it performs an action rather than returning data. */
+  performsAction: boolean;
+  /** Whether the daemon considers this internal/debug-only. */
+  isInternal: boolean;
+  /** The daemon's value-type discriminator. */
+  valueType?: number;
+  /** The raw descriptor, needed verbatim when asking for the value. */
+  raw: Record<string, unknown>;
+}
+
+/** A titled group of attributes — `Basic`, `Actions`, `Element`, `Hierarchy`. */
+export interface AxInspectorSection {
+  /** Stable identifier, e.g. `Basic_v1`. */
+  identifier: string;
+  /** Display title, e.g. `Basic`. */
+  title: string;
+  /** The attributes in this section. */
+  attributes: AxElementAttribute[];
+}
+
+/** The inspector panel the device pushes when the focused element changes. */
+export interface AxInspectedElement {
+  /** What VoiceOver would announce, when the daemon provides it. */
+  spokenDescription?: string;
+  /** The caption shown above the panel, when present. */
+  caption?: string;
+  /** The panel's sections, in the order the device sent them. */
+  sections: AxInspectorSection[];
+}
+
+/** Recovers a `Buffer` from a decoded `NS.data` blob. */
+function toBuffer(value: unknown): Buffer | undefined {
+  if (Buffer.isBuffer(value)) {
+    return value;
+  }
+  if (typeof value === 'object' && value !== null) {
+    // The archiver decodes NSData into an index-keyed object.
+    const bytes = Object.values(value as Record<string, unknown>).filter((b): b is number => typeof b === 'number');
+    if (bytes.length > 0) {
+      return Buffer.from(bytes);
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Parses a deserialized `AXAuditElement_v1`.
+ *
+ * The `_v1` suffixes are the daemon's own wire keys, not our assumption. A
+ * future shape would carry different keys, so this returns `undefined` rather
+ * than misreading one.
+ */
+export function toAxElement(value: unknown): AxElement | undefined {
+  if (!util.isPlainObject(value)) {
+    return undefined;
+  }
+  const fields = value as Record<string, unknown>;
+  const platformValue = fields.PlatformElementValue_v1;
+  const container = util.isPlainObject(platformValue)
+    ? ((platformValue as Record<string, unknown>)['NS.data'] ?? platformValue)
+    : undefined;
+  const platformElement = toBuffer(container);
+  if (!platformElement) {
+    return undefined;
+  }
+  return {
+    platformElement,
+    accessibilityIdentifier:
+      typeof fields.AccessibilityIdentifier_v1 === 'string' ? fields.AccessibilityIdentifier_v1 : undefined,
+  };
+}
+
+/**
+ * Rebuilds the serialized form the daemon expects when an element is passed
+ * back, matching what Xcode's Inspector sends.
+ */
+export function serializeAxElement(element: AxElement): Record<string, unknown> {
+  const value: Record<string, unknown> = {
+    PlatformElementValue_v1: {ObjectType: 'passthrough', Value: element.platformElement},
+  };
+  if (element.accessibilityIdentifier !== undefined) {
+    value.AccessibilityIdentifier_v1 = {ObjectType: 'passthrough', Value: element.accessibilityIdentifier};
+  }
+  return {
+    ObjectType: 'AXAuditElement_v1',
+    Value: {ObjectType: 'passthrough', Value: value},
+  };
+}
+
+function toAttribute(value: unknown): AxElementAttribute | undefined {
+  if (!util.isPlainObject(value)) {
+    return undefined;
+  }
+  const fields = value as Record<string, unknown>;
+  const name = fields.AttributeNameValue_v1;
+  if (typeof name !== 'string') {
+    return undefined;
+  }
+  return {
+    name,
+    humanReadableName: typeof fields.HumanReadableNameValue_v1 === 'string' ? fields.HumanReadableNameValue_v1 : name,
+    settable: fields.SettableValue_v1 === true,
+    performsAction: fields.PerformsActionValue_v1 === true,
+    isInternal: fields.IsInternal_v1 === true,
+    valueType: typeof fields.ValueTypeValue_v1 === 'number' ? fields.ValueTypeValue_v1 : undefined,
+    raw: stripTag(fields),
+  };
+}
+
+/** Drops the decoder's type tag so the object round-trips as the daemon sent it. */
+function stripTag(fields: Record<string, unknown>): Record<string, unknown> {
+  return Object.fromEntries(Object.entries(fields).filter(([key]) => key !== AX_OBJECT_TYPE));
+}
+
+/** Rebuilds an attribute descriptor for the wire. */
+export function serializeAxAttribute(attribute: AxElementAttribute): Record<string, unknown> {
+  const value = Object.fromEntries(
+    Object.entries(attribute.raw).map(([key, inner]) => [key, {ObjectType: 'passthrough', Value: inner}]),
+  );
+  return {
+    ObjectType: 'AXAuditElementAttribute_v1',
+    Value: {ObjectType: 'passthrough', Value: value},
+  };
+}
+
+/** Parses the payload of an inbound `hostInspectorCurrentElementChanged:`. */
+export function toInspectedElement(value: unknown): AxInspectedElement {
+  const fields = (util.isPlainObject(value) ? value : {}) as Record<string, unknown>;
+  const rawSections = Array.isArray(fields.InspectorSectionsValue_v1) ? fields.InspectorSectionsValue_v1 : [];
+  const sections: AxInspectorSection[] = [];
+  for (const rawSection of rawSections) {
+    if (!util.isPlainObject(rawSection)) {
+      continue;
+    }
+    const section = rawSection as Record<string, unknown>;
+    const rawAttributes = Array.isArray(section.ElementAttributesValue_v1) ? section.ElementAttributesValue_v1 : [];
+    sections.push({
+      identifier: typeof section.IdentifierValue_v1 === 'string' ? section.IdentifierValue_v1 : '',
+      title: typeof section.TitleValue_v1 === 'string' ? section.TitleValue_v1 : '',
+      attributes: rawAttributes
+        .map(toAttribute)
+        .filter((attribute): attribute is AxElementAttribute => attribute !== undefined),
+    });
+  }
+  return {
+    spokenDescription:
+      typeof fields.SpokenDescriptionValue_v1 === 'string' ? fields.SpokenDescriptionValue_v1 : undefined,
+    caption: typeof fields.CaptionTextValue_v1 === 'string' ? fields.CaptionTextValue_v1 : undefined,
+    sections,
+  };
+}

src/services/ios/accessibility-audit/ax-values.ts

@@ -0,0 +1,46 @@
+import {PlistUID} from '../../../lib/plist/index.js';
+
+/**
+ * A point passed to the accessibility daemon, in normalized device coordinates
+ * (`0..1` across the screen's width and height).
+ *
+ * Carried as its own type because the daemon calls `CGPointValue` on the
+ * argument, so it has to arrive as an `NSValue` wrapping a `CGPoint` — an
+ * archived array, dictionary or bare double is rejected. Verified live on
+ * iOS 27.0: a double yields "Cannot get value with size 16. The type encoded as
+ * d is expected to be 8 bytes", and a dictionary yields
+ * "-[__NSDictionaryI CGPointValue]: unrecognized selector".
+ */
+export class AxPoint {
+  constructor(
+    readonly x: number,
+    readonly y: number,
+  ) {}
+}
+
+/**
+ * Builds the NSKeyedArchiver graph for an `NSValue` holding a `CGPoint`.
+ *
+ * `NS.special` discriminates the wrapped struct — 1 for a point. The device's
+ * own replies use 3 for rects (seen on `ElementRectValue_v1`), which is what
+ * corroborates the numbering.
+ */
+export function archiveAxPoint(point: AxPoint): Record<string, unknown> {
+  return {
+    $version: 100000,
+    $archiver: 'NSKeyedArchiver',
+    $top: {root: new PlistUID(1)},
+    $objects: [
+      '$null',
+      {
+        'NS.special': 1,
+        'NS.pointval': `{${point.x}, ${point.y}}`,
+        $class: new PlistUID(2),
+      },
+      {
+        $classes: ['NSValue', 'NSObject'],
+        $classname: 'NSValue',
+      },
+    ],
+  };
+}