Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/core/src/device/device-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,16 @@ export type AndroidDeviceOpt = {
export type IOSDeviceInputOpt = {
/** Automatically dismiss the keyboard after input is completed */
autoDismissKeyboard?: boolean;
/**
* Per-character delay (ms) used when typing into iOS input fields. When set
* to a positive number, characters are dispatched to WebDriverAgent one at a
* time with this gap between them, which prevents app-side reaction windows
* (re-render, predictive bar, autocorrect) from swallowing a contiguous
* block of keystrokes. Set to 0 to send the whole string in a single
* `/wda/keys` request (faster, but lossy on slow-reacting inputs).
* @default 80
*/
keyboardTypeDelay?: number;
};

/**
Expand Down
1 change: 1 addition & 0 deletions packages/core/tests/unit-test/device-options.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ describe('Device Options Type Definitions', () => {
test('IOSDeviceInputOpt should include keyboard options', () => {
const inputOptions: IOSDeviceInputOpt = {
autoDismissKeyboard: true,
keyboardTypeDelay: 80,
};

expect(inputOptions).toBeDefined();
Expand Down
7 changes: 5 additions & 2 deletions packages/ios/src/device.ts
Original file line number Diff line number Diff line change
Expand Up @@ -468,12 +468,15 @@ ScreenSize: ${size.width}x${size.height} (DPR: ${size.scale})
const shouldAutoDismissKeyboard =
options?.autoDismissKeyboard ?? this.options?.autoDismissKeyboard ?? true;

debugDevice(`Typing text: "${text}"`);
const delayMs =
options?.keyboardTypeDelay ?? this.options?.keyboardTypeDelay ?? 80;

debugDevice(`Typing text: "${text}" (delayMs=${delayMs})`);

try {
// Wait a bit to ensure keyboard is ready
await sleep(200);
await this.wdaBackend.typeText(text);
await this.wdaBackend.typeText(text, { delayMs });
await sleep(300); // Give more time for text to appear
} catch (error) {
debugDevice(`Failed to type text with WDA: ${error}`);
Expand Down
42 changes: 34 additions & 8 deletions packages/ios/src/ios-webdriver-client.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { sleep } from '@midscene/core/utils';
import { getDebug } from '@midscene/shared/logger';
import { WebDriverClient } from '@midscene/webdriver';

Expand Down Expand Up @@ -304,17 +305,42 @@ export class IOSWebDriverClient extends WebDriverClient {
}
}

async typeText(text: string): Promise<void> {
async typeText(text: string, options?: { delayMs?: number }): Promise<void> {
this.ensureSession();

// Clean the text to avoid unwanted trailing spaces
const cleanText = text.trim();
if (!cleanText) {
return;
}

const chars = Array.from(cleanText);
const delayMs = options?.delayMs ?? 0;

try {
// Clean the text to avoid unwanted trailing spaces
const cleanText = text.trim();
// Use WebDriverAgent's keys endpoint with array value
await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
value: cleanText.split(''), // Must be an array of characters
});
debugIOS(`Typed text: "${text}"`);
if (delayMs > 0) {
// Per-character sends with an inter-key gap prevent a single app-side
// reaction window (re-render, predictive bar, autocorrect) from
// swallowing a contiguous block of keystrokes.
for (let i = 0; i < chars.length; i++) {
await this.makeRequest(
'POST',
`/session/${this.sessionId}/wda/keys`,
{
value: [chars[i]],
},
);
if (i < chars.length - 1) {
await sleep(delayMs);
}
}
} else {
// Use WebDriverAgent's keys endpoint with array value
await this.makeRequest('POST', `/session/${this.sessionId}/wda/keys`, {
value: chars, // Must be an array of characters
});
}
debugIOS(`Typed text: "${text}" (delayMs=${delayMs})`);
} catch (error) {
debugIOS(`Failed to type text "${text}": ${error}`);
throw new Error(`Failed to type text: ${error}`);
Expand Down
55 changes: 48 additions & 7 deletions packages/ios/tests/unit-test/device.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,43 @@ describe('IOSDevice', () => {
expect(mockWdaClient.tap).toHaveBeenNthCalledWith(1, 10, 20);
expect(mockWdaClient.tap).toHaveBeenNthCalledWith(2, 30, 40);
expect(mockWdaClient.clearActiveElement).toHaveBeenCalledTimes(2);
expect(mockWdaClient.typeText).toHaveBeenNthCalledWith(1, 'from action');
expect(mockWdaClient.typeText).toHaveBeenNthCalledWith(2, 'from pointer');
expect(mockWdaClient.typeText).toHaveBeenNthCalledWith(1, 'from action', {
delayMs: 80,
});
expect(mockWdaClient.typeText).toHaveBeenNthCalledWith(
2,
'from pointer',
{ delayMs: 80 },
);
});

it('forwards keyboardTypeDelay from per-call options to the WDA backend', async () => {
await device.inputPrimitives.keyboard.typeText('per call', {
autoDismissKeyboard: false,
keyboardTypeDelay: 25,
} as any);

expect(mockWdaClient.typeText).toHaveBeenLastCalledWith('per call', {
delayMs: 25,
});
});

it('falls back to device-level keyboardTypeDelay when the call omits it', async () => {
const deviceWithDelay = new IOSDevice({
wdaPort: DEFAULT_WDA_PORT,
wdaHost: 'localhost',
keyboardTypeDelay: 0,
});

await deviceWithDelay.inputPrimitives.keyboard.typeText('default delay', {
autoDismissKeyboard: false,
} as any);

expect(mockWdaClient.typeText).toHaveBeenLastCalledWith('default delay', {
delayMs: 0,
});

await deviceWithDelay.destroy();
});
});

Expand Down Expand Up @@ -383,8 +418,10 @@ describe('IOSDevice', () => {
it('should type text', async () => {
await device.connect();

await device.typeText('Hello World');
expect(mockWdaClient.typeText).toHaveBeenCalledWith('Hello World');
await (device as any).typeText('Hello World');
expect(mockWdaClient.typeText).toHaveBeenCalledWith('Hello World', {
delayMs: 80,
});
});

it('should press home button', async () => {
Expand Down Expand Up @@ -521,7 +558,9 @@ describe('IOSDevice', () => {
.fn()
.mockRejectedValue(new Error('Type text failed'));

await expect(device.typeText('test')).rejects.toThrow('Type text failed');
await expect((device as any).typeText('test')).rejects.toThrow(
'Type text failed',
);
});
});

Expand Down Expand Up @@ -584,10 +623,12 @@ describe('IOSDevice', () => {
});

await deviceWithAutoDismiss.connect();
await deviceWithAutoDismiss.typeText('test text');
await (deviceWithAutoDismiss as any).typeText('test text');

// Should call typeText and swipe (for keyboard dismiss)
expect(mockBackend.typeText).toHaveBeenCalledWith('test text');
expect(mockBackend.typeText).toHaveBeenCalledWith('test text', {
delayMs: 80,
});
expect(mockBackend.swipe).toHaveBeenCalled();
});
});
Expand Down
81 changes: 80 additions & 1 deletion packages/ios/tests/unit-test/wda-backend.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,84 @@
import { DEFAULT_WDA_PORT } from '@midscene/shared/constants';
import { describe, expect, it } from 'vitest';
import { describe, expect, it, vi } from 'vitest';
import { IOSWebDriverClient } from '../../src/ios-webdriver-client';

function createClientWithSession() {
const client = new IOSWebDriverClient({
port: DEFAULT_WDA_PORT,
host: 'localhost',
});
// Bypass createSession() — we only want to observe outbound HTTP calls.
(client as any).sessionId = 'session-under-test';
const makeRequest = vi
.spyOn(client as any, 'makeRequest')
.mockResolvedValue(undefined);
return { client, makeRequest };
}

describe('IOSWebDriverClient.typeText delivery modes', () => {
it('sends the whole string in one /wda/keys request when delayMs is 0 (default)', async () => {
const { client, makeRequest } = createClientWithSession();

await client.typeText('Al is amazing');

expect(makeRequest).toHaveBeenCalledTimes(1);
expect(makeRequest).toHaveBeenCalledWith(
'POST',
'/session/session-under-test/wda/keys',
{
value: [
'A',
'l',
' ',
'i',
's',
' ',
'a',
'm',
'a',
'z',
'i',
'n',
'g',
],
},
);
});

it('emits one /wda/keys request per character when delayMs > 0', async () => {
const { client, makeRequest } = createClientWithSession();

await client.typeText('Hi!', { delayMs: 1 });

expect(makeRequest).toHaveBeenCalledTimes(3);
expect(makeRequest).toHaveBeenNthCalledWith(
1,
'POST',
'/session/session-under-test/wda/keys',
{ value: ['H'] },
);
expect(makeRequest).toHaveBeenNthCalledWith(
2,
'POST',
'/session/session-under-test/wda/keys',
{ value: ['i'] },
);
expect(makeRequest).toHaveBeenNthCalledWith(
3,
'POST',
'/session/session-under-test/wda/keys',
{ value: ['!'] },
);
});

it('trims surrounding whitespace and skips empty input', async () => {
const { client, makeRequest } = createClientWithSession();

await client.typeText(' ');

expect(makeRequest).not.toHaveBeenCalled();
});
});

describe('IOSWebDriverClient - Simple Tests', () => {
describe('Module Structure', () => {
Expand Down
36 changes: 36 additions & 0 deletions packages/web-integration/src/puppeteer/base-page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,42 @@ export class Page<
return (await page.context().newCDPSession(page)) as ScreencastCdpSession;
}

async waitForDomQuiet(opts?: {
quietMs?: number;
timeoutMs?: number;
}): Promise<void> {
const quietMs = opts?.quietMs ?? 100;
const timeoutMs = opts?.timeoutMs ?? 500;
try {
await this.evaluate(
([q, total]: [number, number]) =>
new Promise<void>((resolve) => {
let settleTimer: ReturnType<typeof setTimeout> | undefined;
const done = () => {
obs.disconnect();
clearTimeout(hardTimer);
if (settleTimer) clearTimeout(settleTimer);
resolve();
};
const obs = new MutationObserver(() => {
if (settleTimer) clearTimeout(settleTimer);
settleTimer = setTimeout(done, q);
});
obs.observe(document.body, {
childList: true,
subtree: true,
attributes: true,
characterData: true,
Comment on lines +480 to +484
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Scope DOM-quiet wait to the input subtree

waitForDomQuiet observes document.body with subtree/attributes/characterData, so any unrelated mutations anywhere on the page keep resetting the quiet timer until the 500ms hard timeout. Because typeText now calls this on every replace-mode input (web-page.ts), dynamic pages with ticking counters/spinners can incur an extra ~500ms per field and may hit higher-level action timeouts even when the target input is stable. Limit observation to the target element (or a narrow ancestor) so unrelated DOM churn does not throttle typing.

Useful? React with 👍 / 👎.

});
const hardTimer = setTimeout(done, total);
}),
[quietMs, timeoutMs],
);
} catch (error) {
debugPage('waitForDomQuiet failed: %s', error);
}
}

async flushPendingVisualUpdate(): Promise<void> {
const activeStream = this.activeMjpegStream;
if (!activeStream) return;
Expand Down
10 changes: 10 additions & 0 deletions packages/web-integration/src/web-page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,10 @@ export abstract class AbstractWebPage extends AbstractInterface {
stopLoading?(): Promise<void>;
navigationState?(): Promise<{ isLoading: boolean }>;
flushPendingVisualUpdate?(): Promise<void>;
waitForDomQuiet?(opts?: {
quietMs?: number;
timeoutMs?: number;
}): Promise<void>;

get mouse(): MouseAction {
return {
Expand Down Expand Up @@ -457,6 +461,12 @@ export function createWebInputPrimitives(
const element = opts?.target;
if (element && opts?.replace !== false) {
await page.clearInput(element as ElementInfo);
// Frameworks (React/Vue/etc.) often re-render in response to
// the `input` event fired by clearing. If that re-render lands
// between clearInput returning and the first typed character,
// the keypresses can be dropped. Wait for the DOM to settle
// before starting to type.
await page.waitForDomQuiet?.();
} else if (element) {
const target = element as ElementInfo;
await page.mouse.click(target.center[0], target.center[1], {
Expand Down
Loading
Loading