test: add additional tools

Author: pnarayanaswamy

README.md

@@ -108,18 +108,21 @@ The `.device-session` file is typically written by the test runner when it creat
 | ------------------- | ---------------------------------------------------------------- |
 | `device_snapshot`   | Capture the UI accessibility hierarchy. Call before interacting. |
 | `device_screenshot` | Capture a screenshot as base64 PNG. Optionally save to file.     |
+| `device_info`       | Get device platform, name, OS version, and device ID.            |
 | `device_app_state`  | Check if an app is running, installed, or absent.                |
 | `device_logs`       | Capture recent device logs (syslog/logcat) with optional filter. |
 
 ### Interaction
 
-| Tool                  | Description                                                        |
-| --------------------- | ------------------------------------------------------------------ |
-| `device_tap_element`  | Find an element by label/identifier/text/type and tap its center.  |
-| `device_type`         | Type text into the currently focused input field.                  |
-| `device_swipe`        | Swipe in a direction with optional start coordinates and distance. |
-| `device_wait_for`     | Poll until an element matching a query appears.                    |
-| `device_press_button` | Press a device button (home/back/enter/lock).                      |
+| Tool                     | Description                                                        |
+| ------------------------ | ------------------------------------------------------------------ |
+| `device_tap_element`     | Find an element by label/identifier/text/type and tap its center.  |
+| `device_tap_coordinates` | Tap at exact screen coordinates. Last resort when queries fail.    |
+| `device_type`            | Type text into the currently focused input field.                  |
+| `device_swipe`           | Swipe in a direction with optional start coordinates and distance. |
+| `device_long_press`      | Long press an element for context menus or drag initiation.        |
+| `device_wait_for`        | Poll until an element matching a query appears.                    |
+| `device_press_button`    | Press a device button (home/back/enter/lock).                      |
 
 ### App & Device Control
 
@@ -132,28 +135,31 @@ The `.device-session` file is typically written by the test runner when it creat
 
 ### Element Identification
 
-Elements are identified by accessibility attributes — not internal refs:
+Elements are identified by accessibility attributes — not internal refs. Matching is **fuzzy**: partial text and case-insensitive matches work. For example, querying `{ label: "Confirm" }` matches an element with label `"Confirm Transaction"`.
 
 - **iOS**: accessibility label, accessibility identifier
 - **Android**: content-description, resource-id, text
 
 ### Backend Implementation
 
-| Tool                      | iOS (IDB)             | Android (ADB)        | Appium (W3C WebDriver)        |
-| ------------------------- | --------------------- | -------------------- | ----------------------------- |
-| `device_snapshot`         | `idb ui describe-all` | `uiautomator dump`   | `mobile: source`              |
-| `device_screenshot`       | `idb screenshot`      | `screencap` + `pull` | `mobile: getScreenshot`       |
-| `device_tap_element`      | find + `idb ui tap`   | find + `input tap`   | find + W3C Actions            |
-| `device_type`             | `idb ui text`         | `input text`         | `findElement` + `sendKeys`    |
-| `device_swipe`            | `idb ui swipe`        | `input swipe`        | W3C Actions                   |
-| `device_wait_for`         | poll snapshot         | poll snapshot        | poll snapshot                 |
-| `device_app_state`        | `idb list-apps`       | `dumpsys activity`   | `mobile: queryAppState`       |
-| `device_open_app`         | `idb launch`          | `monkey -p`          | `mobile: activateApp`         |
-| `device_close_app`        | `idb terminate`       | `am force-stop`      | `mobile: terminateApp`        |
-| `device_press_button`     | `idb ui key`          | `input keyevent`     | `mobile: pressButton/Key`     |
-| `device_dismiss_keyboard` | `idb ui key RETURN`   | `input keyevent 111` | `mobile: hideKeyboard`        |
-| `device_dismiss_alert`    | find button + tap     | find button + tap    | `mobile: accept/dismissAlert` |
-| `device_logs`             | `idb log`             | `logcat`             | `mobile: getLog`              |
+| Tool                      | iOS (IDB)               | Android (ADB)        | Appium (W3C WebDriver)        |
+| ------------------------- | ----------------------- | -------------------- | ----------------------------- |
+| `device_snapshot`         | `idb ui describe-all`   | `uiautomator dump`   | `mobile: source`              |
+| `device_screenshot`       | `idb screenshot`        | `screencap` + `pull` | `mobile: getScreenshot`       |
+| `device_info`             | `idb describe`          | `getprop`            | session capabilities          |
+| `device_tap_element`      | find + `idb ui tap`     | find + `input tap`   | find + W3C Actions            |
+| `device_tap_coordinates`  | `idb ui tap x y`        | `input tap x y`      | W3C Actions                   |
+| `device_type`             | `idb ui text`           | `input text`         | `findElement` + `sendKeys`    |
+| `device_swipe`            | `idb ui swipe`          | `input swipe`        | W3C Actions                   |
+| `device_long_press`       | `idb ui tap --duration` | `input swipe` (hold) | W3C Actions (pause)           |
+| `device_wait_for`         | poll snapshot           | poll snapshot        | poll snapshot                 |
+| `device_app_state`        | `idb list-apps`         | `dumpsys activity`   | `mobile: queryAppState`       |
+| `device_open_app`         | `idb launch`            | `monkey -p`          | `mobile: activateApp`         |
+| `device_close_app`        | `idb terminate`         | `am force-stop`      | `mobile: terminateApp`        |
+| `device_press_button`     | `idb ui key`            | `input keyevent`     | `mobile: pressButton/Key`     |
+| `device_dismiss_keyboard` | `idb ui key RETURN`     | `input keyevent 111` | `mobile: hideKeyboard`        |
+| `device_dismiss_alert`    | find button + tap       | find button + tap    | `mobile: accept/dismissAlert` |
+| `device_logs`             | `idb log`               | `logcat`             | `mobile: getLog`              |
 
 ## MCP Client Configuration
 
@@ -229,16 +235,16 @@ Add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
 @metamask/device-mcp
 ├── src/
 │   ├── index.ts                # Entry point — lazy backend, stdio MCP server
-│   ├── server.ts               # MCP server — registers 13 tools
+│   ├── server.ts               # MCP server — registers 16 tools
 │   ├── backends/
-│   │   ├── types.ts            # DeviceBackend interface (13 operations)
+│   │   ├── types.ts            # DeviceBackend interface (16 operations)
 │   │   ├── idb-backend.ts      # iOS local — IDB commands + JSON parser
 │   │   ├── adb-backend.ts      # Android local — ADB commands + XML parser
 │   │   ├── appium-backend.ts   # Remote — Appium/BrowserStack via W3C WebDriver
 │   │   ├── webdriver-client.ts # Minimal W3C WebDriver HTTP client (fetch)
 │   │   ├── session-file.ts     # .device-session file reader
 │   │   └── index.ts            # createBackend() + createLazyBackend() factory
-│   ├── tools/                  # One file per MCP tool (13 tools)
+│   ├── tools/                  # One file per MCP tool (16 tools)
 │   └── utils/
 │       ├── exec.ts             # Shell execution wrapper
 │       ├── platform.ts         # Platform auto-detection
@@ -249,7 +255,7 @@ Add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
 
 ```bash
 yarn build        # Compile TypeScript
-yarn test         # Run tests (49 tests)
+yarn test         # Run tests (69 tests)
 yarn lint         # Lint everything (ESLint + Prettier + changelog)
 yarn lint:fix     # Auto-fix lint issues
 yarn dev          # Watch mode compilation

SKILL.md

@@ -1,6 +1,6 @@
 # @metamask/device-mcp — Agent Reference
 
-You interact with a mobile device (iOS simulator or Android emulator) through the `device-mcp` MCP server. It exposes 6 tools for inspecting and controlling the device UI.
+You interact with a mobile device (iOS simulator, Android emulator, or BrowserStack remote device) through the `device-mcp` MCP server. It exposes 16 tools for inspecting UI state, interacting with elements, capturing evidence, and controlling app lifecycle.
 
 ## When to Use This
 
@@ -23,6 +23,9 @@ device_snapshot             # 3. See the result — NEVER assume screen state
 - **ALWAYS call `device_snapshot` before interacting.** You cannot guess what's on screen.
 - **NEVER call `device_snapshot` twice without acting in between.** If the screen hasn't changed, the hierarchy won't either.
 - **ALWAYS call `device_snapshot` after interacting.** Confirm the action had the expected effect.
+- **Call `device_dismiss_keyboard` after typing.** The keyboard obscures elements below the input.
+- **Call `device_dismiss_alert` when a system dialog appears.** Permission prompts block all other interaction.
+- **Use `device_screenshot` for visual evidence.** Snapshots show accessibility data; screenshots show what the user sees.
 
 ## Tools
 
@@ -56,6 +59,8 @@ Find an element by query and tap its center. Specify at least one of: `label`, `
 3. `text` — visible text content (matches both `value` and `label`)
 4. `type` — element class (least specific, combine with other fields)
 
+**Matching is fuzzy:** partial text and case-insensitive matches work. `{ label: "Confirm" }` matches `"Confirm Transaction"`.
+
 **Examples:**
 
 ```json
@@ -64,6 +69,22 @@ Find an element by query and tap its center. Specify at least one of: `label`, `
 { "text": "Send", "type": "Button" }
 ```
 
+### device_tap_coordinates
+
+Tap at exact screen coordinates. Use as a **last resort** when element queries fail. Get coordinates from `device_snapshot` frame data.
+
+```json
+{ "x": 195, "y": 722 }
+```
+
+### device_long_press
+
+Long press an element. Used for context menus, drag initiation, and other long-press interactions.
+
+```json
+{ "label": "Transaction", "durationMs": 2000 }
+```
+
 ### device_type
 
 Type text into the currently focused input. **Tap an input field first** to focus it, then call this.
@@ -104,6 +125,12 @@ Check if an app is running, installed, or absent.
 { "bundleId": "io.metamask" }
 ```
 
+### device_info (read-only)
+
+Get device platform, name, OS version, and device ID.
+
+**When:** Understanding which device/platform you're connected to. Including platform-specific instructions in your workflow.
+
 ### device_screenshot (read-only)
 
 Capture a screenshot of the current screen. Returns base64 PNG image data.
@@ -192,8 +219,10 @@ device_snapshot                                          # verify we're there
 device_snapshot                                          # see available inputs
 device_tap_element { "identifier": "email-input" }      # focus the field
 device_type { "text": "user@example.com" }              # type into it
+device_dismiss_keyboard                                  # hide keyboard
 device_tap_element { "identifier": "password-input" }   # focus next field
 device_type { "text": "secret123" }                     # type password
+device_dismiss_keyboard                                  # hide keyboard
 device_tap_element { "identifier": "submit-btn" }       # submit
 device_snapshot                                          # verify result
 ```
@@ -211,10 +240,27 @@ device_snapshot                                          # check again
 
 ```
 device_snapshot                                          # see dialog
-device_tap_element { "label": "Allow" }                 # accept permission
+device_dismiss_alert { "accept": true }                  # accept permission
 device_snapshot                                          # verify dismissed
 ```
 
+### Launch app, debug, collect evidence
+
+```
+device_open_app { "bundleId": "io.metamask" }            # launch app
+device_snapshot                                          # see initial screen
+device_screenshot { "outputPath": "./debug/home.png" }   # visual evidence
+device_logs { "filter": "MetaMask", "durationSeconds": 60 } # check for errors
+device_close_app { "bundleId": "io.metamask" }           # stop app
+```
+
+### Navigate with device buttons
+
+```
+device_press_button { "button": "home" }                 # go to home screen
+device_press_button { "button": "back" }                 # go back (Android)
+```
+
 ## Error Handling
 
 All tools return `isError: true` on failure with a descriptive message. Common errors:

eslint.config.mjs

@@ -60,6 +60,7 @@ const config = createConfig([
       '@typescript-eslint/unbound-method': 'off',
       '@typescript-eslint/no-non-null-assertion': 'off',
       'no-restricted-globals': 'off',
+      'n/no-sync': 'off',
     },
   },
 ]);

src/backends/adb-backend.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from 'vitest';
 
 import { parseAndroidHierarchy, parseNodeAttributes } from './adb-backend.js';
+import { findElement } from '../utils/element.js';
 
 const SAMPLE_UIAUTOMATOR_XML = `<?xml version="1.0" encoding="UTF-8"?>
 <hierarchy rotation="0">
@@ -12,71 +13,70 @@ const SAMPLE_UIAUTOMATOR_XML = `<?xml version="1.0" encoding="UTF-8"?>
 </hierarchy>`;
 
 describe('parseAndroidHierarchy', () => {
-  it('parses uiautomator XML into flat element list', () => {
+  it('builds a tree with parent-child relationships', () => {
     const elements = parseAndroidHierarchy(SAMPLE_UIAUTOMATOR_XML);
-    expect(elements).toHaveLength(4);
+    expect(elements).toHaveLength(1);
+    expect(elements[0].type).toBe('android.widget.FrameLayout');
+    expect(elements[0].children).toHaveLength(3);
   });
 
-  it('extracts text content', () => {
+  it('nests children under the correct parent', () => {
     const elements = parseAndroidHierarchy(SAMPLE_UIAUTOMATOR_XML);
-    const title = elements.find((element) => element.value === 'MetaMask');
-    expect(title).toBeDefined();
-    expect(title!.identifier).toBe('io.metamask:id/title');
-    expect(title!.type).toBe('android.widget.TextView');
+    const frame = elements[0];
+    expect(frame.children![0].value).toBe('MetaMask');
+    expect(frame.children![1].label).toBe('Account avatar');
+    expect(frame.children![2].value).toBe('$0.00');
   });
 
-  it('extracts content-desc as label', () => {
+  it('self-closing nodes have no children', () => {
     const elements = parseAndroidHierarchy(SAMPLE_UIAUTOMATOR_XML);
-    const avatar = elements.find(
-      (element) => element.label === 'Account avatar',
-    );
-    expect(avatar).toBeDefined();
-    expect(avatar!.identifier).toBe('io.metamask:id/identicon');
+    const title = elements[0].children![0];
+    expect(title.children).toBeUndefined();
   });
 
-  it('parses bounds into frame coordinates', () => {
+  it('findElement still finds nested elements', () => {
     const elements = parseAndroidHierarchy(SAMPLE_UIAUTOMATOR_XML);
-    const title = elements.find((element) => element.value === 'MetaMask');
-    expect(title!.frame).toStrictEqual({
-      x: 100,
-      y: 200,
-      width: 400,
-      height: 60,
-    });
+    const title = findElement(elements, { identifier: 'io.metamask:id/title' });
+    expect(title).toBeDefined();
+    expect(title!.value).toBe('MetaMask');
   });
 
-  it('detects disabled elements', () => {
-    const elements = parseAndroidHierarchy(SAMPLE_UIAUTOMATOR_XML);
-    const balance = elements.find((element) => element.value === '$0.00');
-    expect(balance).toBeDefined();
-    expect(balance!.enabled).toBe(false);
+  it('handles empty XML gracefully', () => {
+    expect(parseAndroidHierarchy('')).toStrictEqual([]);
   });
 
-  it('handles empty XML gracefully', () => {
-    const elements = parseAndroidHierarchy('');
-    expect(elements).toStrictEqual([]);
+  it('handles deeply nested XML', () => {
+    const deepXml = `<hierarchy>
+      <node class="Root" bounds="[0,0][100,100]">
+        <node class="Mid" bounds="[10,10][90,90]">
+          <node class="Leaf" text="deep" bounds="[20,20][80,80]" />
+        </node>
+      </node>
+    </hierarchy>`;
+    const elements = parseAndroidHierarchy(deepXml);
+    expect(elements).toHaveLength(1);
+    expect(elements[0].children).toHaveLength(1);
+    expect(elements[0].children![0].children).toHaveLength(1);
+    expect(elements[0].children![0].children![0].value).toBe('deep');
   });
 });
 
 describe('parseNodeAttributes', () => {
   it('returns null for missing bounds', () => {
-    const result = parseNodeAttributes('text="hello" class="View"');
-    expect(result).toBeNull();
+    expect(parseNodeAttributes('text="hello" class="View"')).toBeNull();
   });
 
   it('returns null for malformed bounds', () => {
-    const result = parseNodeAttributes(
-      'text="hello" class="View" bounds="invalid"',
-    );
-    expect(result).toBeNull();
+    expect(
+      parseNodeAttributes('text="hello" class="View" bounds="invalid"'),
+    ).toBeNull();
   });
 
   it('parses a complete attribute string', () => {
     const attrs =
       'text="Send" resource-id="btn_send" class="Button" ' +
       'content-desc="Send button" enabled="true" bounds="[10,20][110,70]"';
-    const result = parseNodeAttributes(attrs);
-    expect(result).toStrictEqual({
+    expect(parseNodeAttributes(attrs)).toStrictEqual({
       type: 'Button',
       label: 'Send button',
       value: 'Send',