feat(cli): add auto-learn feature for Android app UI mapping

Add --auto-learn flag to automatically discover and map Android app UI
structure (navigation tabs, interactive elements, page layouts, jump
targets) using VLM-guided agents. Includes code review fixes for
security (argument sanitization), error handling, type safety, and
timeout protection.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: ZevGit

packages/ui-tars/cli/src/auto-learn/__tests__/app-launcher.test.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect } from 'vitest';
+import { launchApp, adbClick, adbBack, adbHome, adbWake } from '../app-launcher';
+
+describe('app-launcher argument sanitization', () => {
+  describe('launchApp', () => {
+    it('should reject deviceId with shell injection characters', () => {
+      expect(() => launchApp('device1; rm -rf /', 'com.example.app')).toThrow();
+    });
+
+    it('should reject pkg with shell injection characters', () => {
+      expect(() => launchApp('device1', 'com.example.app; rm -rf /')).toThrow();
+    });
+
+    it('should accept valid package names', () => {
+      // This will fail at ADB level (no device), but should NOT throw sanitization error
+      try {
+        launchApp('emulator-5554', 'com.example.app');
+      } catch (err: any) {
+        // ADB error is expected, but NOT sanitization error
+        expect(err.message).not.toContain('Invalid argument');
+      }
+    });
+
+    it('should accept valid device IDs with dots and hyphens', () => {
+      try {
+        launchApp('192.168.1.1:5555', 'com.app');
+      } catch (err: any) {
+        expect(err.message).not.toContain('Invalid argument');
+      }
+    });
+
+    it('should reject pkg with spaces', () => {
+      expect(() => launchApp('device1', 'com.example app')).toThrow('Invalid argument');
+    });
+  });
+
+  describe('adbClick', () => {
+    it('should reject deviceId with injection characters', () => {
+      expect(() => adbClick('device; evil', 100, 200)).toThrow('Invalid argument');
+    });
+  });
+
+  describe('adbBack', () => {
+    it('should reject deviceId with injection characters', () => {
+      expect(() => adbBack('device; evil')).toThrow('Invalid argument');
+    });
+  });
+
+  describe('adbHome', () => {
+    it('should reject deviceId with injection characters', () => {
+      expect(() => adbHome('device; evil')).toThrow('Invalid argument');
+    });
+  });
+
+  describe('adbWake', () => {
+    it('should reject deviceId with injection characters', () => {
+      expect(() => adbWake('device; evil')).toThrow('Invalid argument');
+    });
+  });
+});

packages/ui-tars/cli/src/auto-learn/__tests__/page-signatures.test.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect } from 'vitest';
+import {
+  matchPageSignature,
+  getLayoutType,
+  isSecondaryPage,
+} from '../page-signatures';
+
+describe('matchPageSignature', () => {
+  it('should match home page features', () => {
+    expect(matchPageSignature('推荐卡片和banner轮播')).toBe('首页');
+    expect(matchPageSignature('banner推荐内容')).toBe('首页');
+  });
+
+  it('should match theater page features', () => {
+    expect(matchPageSignature('左侧分类筛选列表')).toBe('剧场');
+    expect(matchPageSignature('剧目卡片和分类筛选')).toBe('剧场');
+  });
+
+  it('should match messages page features', () => {
+    expect(matchPageSignature('消息列表聊天记录')).toBe('消息');
+  });
+
+  it('should match profile page features', () => {
+    expect(matchPageSignature('用户头像和钻石余额')).toBe('我的');
+    expect(matchPageSignature('个人中心设置')).toBe('我的');
+  });
+
+  it('should match secondary pages based on parent context', () => {
+    expect(matchPageSignature('剧目封面播放按钮', '剧场')).toBe('剧目详情');
+    expect(matchPageSignature('搜索框搜索结果', '首页')).toBe('搜索页');
+    expect(matchPageSignature('播放器全屏', '剧目详情')).toBe('播放页');
+  });
+
+  it('should return Unknown for unrecognized pages', () => {
+    expect(matchPageSignature('随机未知页面')).toBe('Unknown');
+  });
+
+  it('should use notFeatures to reduce false positives', () => {
+    // "推荐" would match 首页, but "左侧筛选" is a notFeature for 首页
+    // So this should match 剧场 instead
+    const result = matchPageSignature('左侧筛选和剧目列表');
+    expect(result).toBe('剧场');
+  });
+});
+
+describe('getLayoutType', () => {
+  it('should return correct layout for known pages', () => {
+    expect(getLayoutType('首页')).toBe('scrollable-feed');
+    expect(getLayoutType('剧场')).toBe('split-view');
+    expect(getLayoutType('消息')).toBe('list');
+    expect(getLayoutType('我的')).toBe('form');
+  });
+
+  it('should return unknown for pages without layout definition', () => {
+    expect(getLayoutType('AI伴侣')).toBe('unknown');
+    expect(getLayoutType('Unknown')).toBe('unknown');
+  });
+});
+
+describe('isSecondaryPage', () => {
+  it('should return true for secondary pages', () => {
+    expect(isSecondaryPage('剧目详情')).toBe(true);
+    expect(isSecondaryPage('搜索页')).toBe(true);
+    expect(isSecondaryPage('播放页')).toBe(true);
+    expect(isSecondaryPage('设置页')).toBe(true);
+  });
+
+  it('should return false for primary pages', () => {
+    expect(isSecondaryPage('首页')).toBe(false);
+    expect(isSecondaryPage('剧场')).toBe(false);
+    expect(isSecondaryPage('AI伴侣')).toBe(false);
+    expect(isSecondaryPage('消息')).toBe(false);
+    expect(isSecondaryPage('我的')).toBe(false);
+  });
+
+  it('should return false for unknown pages', () => {
+    expect(isSecondaryPage('Unknown')).toBe(false);
+  });
+});

packages/ui-tars/cli/src/auto-learn/__tests__/recording-operator.test.ts

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { RecordingOperator } from '../recording-operator';
+import type { Operator, ScreenshotOutput, ExecuteParams, ExecuteOutput } from '@ui-tars/sdk/core';
+
+describe('RecordingOperator', () => {
+  let mockBaseOp: Operator;
+  let recordingOp: RecordingOperator;
+
+  const createScreenshotOutput = (base64 = 'mock-screenshot'): ScreenshotOutput => ({
+    base64,
+    scaleFactor: 2,
+  });
+
+  const createExecuteParams = (
+    actionType = 'click',
+    coords = [100, 200],
+    thought = '我要点击【按钮】',
+  ): ExecuteParams =>
+    ({
+      parsedPrediction: {
+        action_type: actionType,
+        action_inputs: { start_coords: coords },
+        thought,
+      },
+    }) as unknown as ExecuteParams;
+
+  beforeEach(() => {
+    mockBaseOp = {
+      screenshot: vi.fn().mockResolvedValue(createScreenshotOutput()),
+      execute: vi.fn().mockResolvedValue({} as ExecuteOutput),
+    } as unknown as Operator;
+
+    recordingOp = new RecordingOperator(mockBaseOp);
+  });
+
+  describe('screenshot', () => {
+    it('should delegate to base operator and cache last screenshot', async () => {
+      const result = await recordingOp.screenshot();
+
+      expect(mockBaseOp.screenshot).toHaveBeenCalled();
+      expect(result.base64).toBe('mock-screenshot');
+      expect(recordingOp.getLastScreenshot()).toBe('mock-screenshot');
+    });
+  });
+
+  describe('execute', () => {
+    it('should delegate to base operator', async () => {
+      const params = createExecuteParams();
+      await recordingOp.execute(params);
+
+      expect(mockBaseOp.execute).toHaveBeenCalledWith(params);
+    });
+
+    it('should record click actions with screenshot before', async () => {
+      await recordingOp.screenshot(); // Set last screenshot
+      await recordingOp.execute(createExecuteParams('click', [100, 200]));
+
+      const actions = recordingOp.getActions();
+      expect(actions).toHaveLength(1);
+      expect(actions[0].type).toBe('click');
+      expect(actions[0].inputs).toEqual({ start_coords: [100, 200] });
+      expect(actions[0].thought).toBe('我要点击【按钮】');
+      expect(actions[0].screenshotBefore).toBe('mock-screenshot');
+    });
+
+    it('should record non-click actions', async () => {
+      await recordingOp.execute(createExecuteParams('scroll', [200, 300], '向上滚动'));
+
+      const actions = recordingOp.getActions();
+      expect(actions[0].type).toBe('scroll');
+    });
+
+    it('should auto-finish after repeated clicks at same position', async () => {
+      // Click same position 4 times
+      for (let i = 0; i < 4; i++) {
+        await recordingOp.execute(createExecuteParams('click', [100, 200]));
+      }
+
+      const actions = recordingOp.getActions();
+      const finishedAction = actions.find((a) => a.type === 'finished');
+      expect(finishedAction).toBeDefined();
+      expect(finishedAction?.thought).toBe(
+        'Auto-finished due to repeated clicks at same position',
+      );
+    });
+
+    it('should not auto-finish for different click positions', async () => {
+      const positions = [
+        [100, 200],
+        [200, 300],
+        [300, 400],
+        [400, 500],
+      ];
+      for (const pos of positions) {
+        await recordingOp.execute(createExecuteParams('click', pos));
+      }
+
+      const actions = recordingOp.getActions();
+      expect(actions.find((a) => a.type === 'finished')).toBeUndefined();
+    });
+
+    it('should take after-screenshot for click actions', async () => {
+      await recordingOp.execute(createExecuteParams('click', [100, 200]));
+
+      // Should have called screenshot again for after-screenshot
+      expect(mockBaseOp.screenshot).toHaveBeenCalledTimes(1);
+      const immediateScreenshots = recordingOp.getImmediateScreenshots();
+      expect(immediateScreenshots).toHaveLength(1);
+      expect(immediateScreenshots[0].actionIndex).toBe(0);
+    });
+  });
+
+  describe('getActions', () => {
+    it('should return all recorded actions', async () => {
+      await recordingOp.execute(createExecuteParams('click', [100, 200]));
+      await recordingOp.execute(createExecuteParams('scroll', [200, 300]));
+      await recordingOp.execute(createExecuteParams('type'));
+
+      expect(recordingOp.getActions()).toHaveLength(3);
+    });
+
+    it('should return empty array initially', () => {
+      expect(recordingOp.getActions()).toEqual([]);
+    });
+  });
+});