fix(ios): paste text through WDA clipboard

Author: quanru

packages/core/src/device/device-options.ts

@@ -113,12 +113,18 @@ export type IOSDeviceInputOpt = {
   /** Automatically dismiss the keyboard after input is completed */
   autoDismissKeyboard?: boolean;
   /**
-   * Per-character delay (ms) used when typing into iOS input fields. When set
-   * to a positive number, characters are dispatched to WebDriverAgent one at a
-   * time with this gap between them, which prevents app-side reaction windows
-   * (re-render, predictive bar, autocorrect) from swallowing a contiguous
-   * block of keystrokes. Set to 0 to send the whole string in a single
-   * `/wda/keys` request (faster, but lossy on slow-reacting inputs).
+   * Strategy used to enter text into iOS input fields. `paste` writes the text
+   * to the iOS pasteboard and triggers a paste shortcut, avoiding occasional
+   * character loss from WebDriverAgent key typing on longer inputs. `type`
+   * keeps the old `/wda/keys` delivery path.
+   * @default 'paste'
+   */
+  keyboardInputStrategy?: 'paste' | 'type';
+  /**
+   * Per-character delay (ms) used by the `type` strategy and by the fallback
+   * path when paste is not supported by the active WebDriverAgent/iOS runtime.
+   * When set to a positive number, characters are dispatched to
+   * WebDriverAgent one at a time with this gap between them.
    * @default 80
    */
   keyboardTypeDelay?: number;

packages/core/tests/unit-test/device-options.test.ts

@@ -60,6 +60,7 @@ describe('Device Options Type Definitions', () => {
         sessionId: 'external-session-id',
         useWDA: true,
         autoDismissKeyboard: true,
+        keyboardInputStrategy: 'paste',
       };
 
       // Type check - this will fail at compile time if types are incorrect
@@ -85,6 +86,7 @@ describe('Device Options Type Definitions', () => {
     test('IOSDeviceInputOpt should include keyboard options', () => {
       const inputOptions: IOSDeviceInputOpt = {
         autoDismissKeyboard: true,
+        keyboardInputStrategy: 'type',
         keyboardTypeDelay: 80,
       };
 
@@ -137,6 +139,7 @@ describe('Device Options Type Definitions', () => {
         sessionId: 'external-session-id',
         useWDA: true,
         autoDismissKeyboard: true,
+        keyboardInputStrategy: 'paste',
 
         // YAML-specific
         launch: 'com.example.app',

packages/ios/src/device.ts

@@ -470,16 +470,33 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
 
     const delayMs =
       options?.keyboardTypeDelay ?? this.options?.keyboardTypeDelay ?? 80;
+    const keyboardInputStrategy =
+      options?.keyboardInputStrategy ??
+      this.options?.keyboardInputStrategy ??
+      'paste';
 
-    debugDevice(`Typing text: "${text}" (delayMs=${delayMs})`);
+    debugDevice(
+      `Input text: "${text}" (strategy=${keyboardInputStrategy}, delayMs=${delayMs})`,
+    );
 
     try {
       // Wait a bit to ensure keyboard is ready
       await sleep(200);
-      await this.wdaBackend.typeText(text, { delayMs });
+      if (keyboardInputStrategy === 'type') {
+        await this.wdaBackend.typeText(text, { delayMs });
+      } else {
+        try {
+          await this.wdaBackend.pasteText(text);
+        } catch (pasteError) {
+          debugDevice(
+            `Failed to paste text with WDA, falling back to typing: ${pasteError}`,
+          );
+          await this.wdaBackend.typeText(text, { delayMs });
+        }
+      }
       await sleep(300); // Give more time for text to appear
     } catch (error) {
-      debugDevice(`Failed to type text with WDA: ${error}`);
+      debugDevice(`Failed to input text with WDA: ${error}`);
       throw error;
     }
 

packages/ios/src/ios-webdriver-client.ts

@@ -8,6 +8,7 @@ const debugIOS = getDebug('webdriver:ios');
 const WDA_MJPEG_SCREENSHOT_QUALITY = 50;
 const WDA_MJPEG_FRAMERATE = 30;
 const WDA_MJPEG_SCALING_FACTOR = 50;
+const XCUI_KEY_MODIFIER_COMMAND = 1 << 4;
 
 export class IOSWebDriverClient extends WebDriverClient {
   async launchApp(bundleId: string): Promise<void> {
@@ -305,6 +306,56 @@ export class IOSWebDriverClient extends WebDriverClient {
     }
   }
 
+  async pasteText(text: string): Promise<void> {
+    this.ensureSession();
+
+    // Keep the same surrounding-whitespace behavior as typeText.
+    const cleanText = text.trim();
+    if (!cleanText) {
+      return;
+    }
+
+    try {
+      await this.setPasteboardText(cleanText);
+      await this.sendPasteShortcut();
+      debugIOS(`Pasted text: "${text}"`);
+    } catch (error) {
+      debugIOS(`Failed to paste text "${text}": ${error}`);
+      throw new Error(`Failed to paste text: ${error}`);
+    }
+  }
+
+  private async setPasteboardText(text: string): Promise<void> {
+    this.ensureSession();
+
+    await this.makeRequest(
+      'POST',
+      `/session/${this.sessionId}/wda/setPasteboard`,
+      {
+        content: Buffer.from(text, 'utf8').toString('base64'),
+        contentType: 'plaintext',
+      },
+    );
+  }
+
+  private async sendPasteShortcut(): Promise<void> {
+    this.ensureSession();
+
+    const elementId = (await this.getActiveElement()) ?? '0';
+    await this.makeRequest(
+      'POST',
+      `/session/${this.sessionId}/wda/element/${elementId}/keyboardInput`,
+      {
+        keys: [
+          {
+            key: 'v',
+            modifierFlags: XCUI_KEY_MODIFIER_COMMAND,
+          },
+        ],
+      },
+    );
+  }
+
   async typeText(text: string, options?: { delayMs?: number }): Promise<void> {
     this.ensureSession();
 

packages/ios/tests/unit-test/device.test.ts

@@ -32,6 +32,7 @@ describe('IOSDevice', () => {
       longPress: vi.fn().mockResolvedValue(undefined),
       swipe: vi.fn().mockResolvedValue(undefined),
       pinch: vi.fn().mockResolvedValue(undefined),
+      pasteText: vi.fn().mockResolvedValue(undefined),
       typeText: vi.fn().mockResolvedValue(undefined),
       clearActiveElement: vi.fn().mockResolvedValue(true),
       pressKey: vi.fn().mockResolvedValue(undefined),
@@ -236,31 +237,42 @@ describe('IOSDevice', () => {
       expect(mockWdaClient.tap).toHaveBeenNthCalledWith(1, 10, 20);
       expect(mockWdaClient.tap).toHaveBeenNthCalledWith(2, 30, 40);
       expect(mockWdaClient.clearActiveElement).toHaveBeenCalledTimes(2);
-      expect(mockWdaClient.typeText).toHaveBeenNthCalledWith(1, 'from action', {
-        delayMs: 80,
-      });
-      expect(mockWdaClient.typeText).toHaveBeenNthCalledWith(
+      expect(mockWdaClient.pasteText).toHaveBeenNthCalledWith(1, 'from action');
+      expect(mockWdaClient.pasteText).toHaveBeenNthCalledWith(
         2,
         'from pointer',
-        { delayMs: 80 },
       );
+      expect(mockWdaClient.typeText).not.toHaveBeenCalled();
     });
 
-    it('forwards keyboardTypeDelay from per-call options to the WDA backend', async () => {
-      await device.inputPrimitives.keyboard.typeText('per call', {
-        autoDismissKeyboard: false,
-        keyboardTypeDelay: 25,
-      } as any);
+    it('forwards keyboardTypeDelay from per-call options when using type strategy', async () => {
+      const deviceWithTypeStrategy = new IOSDevice({
+        wdaPort: DEFAULT_WDA_PORT,
+        wdaHost: 'localhost',
+        keyboardInputStrategy: 'type',
+      });
+
+      await deviceWithTypeStrategy.inputPrimitives.keyboard.typeText(
+        'per call',
+        {
+          autoDismissKeyboard: false,
+          keyboardTypeDelay: 25,
+        } as any,
+      );
 
       expect(mockWdaClient.typeText).toHaveBeenLastCalledWith('per call', {
         delayMs: 25,
       });
+      expect(mockWdaClient.pasteText).not.toHaveBeenCalled();
+
+      await deviceWithTypeStrategy.destroy();
     });
 
-    it('falls back to device-level keyboardTypeDelay when the call omits it', async () => {
+    it('falls back to device-level keyboardTypeDelay when using type strategy', async () => {
       const deviceWithDelay = new IOSDevice({
         wdaPort: DEFAULT_WDA_PORT,
         wdaHost: 'localhost',
+        keyboardInputStrategy: 'type',
         keyboardTypeDelay: 0,
       });
 
@@ -274,6 +286,21 @@ describe('IOSDevice', () => {
 
       await deviceWithDelay.destroy();
     });
+
+    it('falls back to WDA typing when paste input is unavailable', async () => {
+      mockWdaClient.pasteText = vi
+        .fn()
+        .mockRejectedValue(new Error('Paste not supported'));
+
+      await device.inputPrimitives.keyboard.typeText('fallback text', {
+        autoDismissKeyboard: false,
+      } as any);
+
+      expect(mockWdaClient.pasteText).toHaveBeenCalledWith('fallback text');
+      expect(mockWdaClient.typeText).toHaveBeenLastCalledWith('fallback text', {
+        delayMs: 80,
+      });
+    });
   });
 
   describe('Device Operations', () => {
@@ -415,13 +442,12 @@ describe('IOSDevice', () => {
       expect(mockWdaClient.swipe).toHaveBeenCalledWith(100, 200, 300, 400, 500);
     });
 
-    it('should type text', async () => {
+    it('should paste text by default', async () => {
       await device.connect();
 
       await (device as any).typeText('Hello World');
-      expect(mockWdaClient.typeText).toHaveBeenCalledWith('Hello World', {
-        delayMs: 80,
-      });
+      expect(mockWdaClient.pasteText).toHaveBeenCalledWith('Hello World');
+      expect(mockWdaClient.typeText).not.toHaveBeenCalled();
     });
 
     it('should press home button', async () => {
@@ -554,6 +580,9 @@ describe('IOSDevice', () => {
 
     it('should handle text input failure', async () => {
       await device.connect();
+      mockWdaClient.pasteText = vi
+        .fn()
+        .mockRejectedValue(new Error('Paste text failed'));
       mockWdaClient.typeText = vi
         .fn()
         .mockRejectedValue(new Error('Type text failed'));
@@ -607,6 +636,7 @@ describe('IOSDevice', () => {
       const mockBackend = {
         ...mockWdaClient,
         createSession: vi.fn().mockResolvedValue({ sessionId: 'test-session' }),
+        pasteText: vi.fn().mockResolvedValue(undefined),
         typeText: vi.fn().mockResolvedValue(undefined),
         dismissKeyboard: vi
           .fn()
@@ -625,10 +655,9 @@ describe('IOSDevice', () => {
       await deviceWithAutoDismiss.connect();
       await (deviceWithAutoDismiss as any).typeText('test text');
 
-      // Should call typeText and swipe (for keyboard dismiss)
-      expect(mockBackend.typeText).toHaveBeenCalledWith('test text', {
-        delayMs: 80,
-      });
+      // Should call pasteText and swipe (for keyboard dismiss)
+      expect(mockBackend.pasteText).toHaveBeenCalledWith('test text');
+      expect(mockBackend.typeText).not.toHaveBeenCalled();
       expect(mockBackend.swipe).toHaveBeenCalled();
     });
   });

packages/ios/tests/unit-test/wda-backend.test.ts

@@ -80,6 +80,66 @@ describe('IOSWebDriverClient.typeText delivery modes', () => {
   });
 });
 
+describe('IOSWebDriverClient.pasteText', () => {
+  it('sets the iOS pasteboard and sends Command+V to the active element', async () => {
+    const { client, makeRequest } = createClientWithSession();
+    makeRequest.mockImplementation(async (_method, path) => {
+      if (path === '/session/session-under-test/element/active') {
+        return { value: { ELEMENT: 'active-element-id' } };
+      }
+      return undefined;
+    });
+
+    await client.pasteText('Hello 世界');
+
+    expect(makeRequest).toHaveBeenCalledTimes(3);
+    expect(makeRequest).toHaveBeenNthCalledWith(
+      1,
+      'POST',
+      '/session/session-under-test/wda/setPasteboard',
+      {
+        content: Buffer.from('Hello 世界', 'utf8').toString('base64'),
+        contentType: 'plaintext',
+      },
+    );
+    expect(makeRequest).toHaveBeenNthCalledWith(
+      2,
+      'GET',
+      '/session/session-under-test/element/active',
+    );
+    expect(makeRequest).toHaveBeenNthCalledWith(
+      3,
+      'POST',
+      '/session/session-under-test/wda/element/active-element-id/keyboardInput',
+      {
+        keys: [{ key: 'v', modifierFlags: 16 }],
+      },
+    );
+  });
+
+  it('falls back to the active application when no active element is available', async () => {
+    const { client, makeRequest } = createClientWithSession();
+
+    await client.pasteText('Hello');
+
+    expect(makeRequest).toHaveBeenLastCalledWith(
+      'POST',
+      '/session/session-under-test/wda/element/0/keyboardInput',
+      {
+        keys: [{ key: 'v', modifierFlags: 16 }],
+      },
+    );
+  });
+
+  it('trims surrounding whitespace and skips empty input', async () => {
+    const { client, makeRequest } = createClientWithSession();
+
+    await client.pasteText('   ');
+
+    expect(makeRequest).not.toHaveBeenCalled();
+  });
+});
+
 describe('IOSWebDriverClient - Simple Tests', () => {
   describe('Module Structure', () => {
     it('should export IOSWebDriverClient class', async () => {
@@ -116,6 +176,7 @@ describe('IOSWebDriverClient - Simple Tests', () => {
         'tap',
         'swipe',
         'typeText',
+        'pasteText',
         'pressKey',
         'launchApp',
         'openUrl',