feat: use different color for annotations (#366)

.github/workflows/ai-evaluation.yml

@@ -0,0 +1,54 @@
+name: AI unit test
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to checkout'
+        required: false
+        default: 'main'
+        type: string
+
+jobs:
+  main:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        node-version: [18.19.0]
+
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
+      MIDSCENE_MODEL_NAME: gpt-4o-2024-11-20
+      CI: 1
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.inputs.branch || 'main' }}
+
+    - name: Setup pnpm
+      uses: pnpm/action-setup@v2
+      with:
+        version: 9.3.0
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+        cache: 'pnpm'
+
+    - name: Install dependencies
+      run: pnpm install --frozen-lockfile
+
+    - name: Build project
+      run: pnpm run build
+
+    - name: Run evaluation
+      run: |
+        cd packages/evaluation
+        pnpm run evaluate:inspect
+        pnpm run evaluate:assertion

biome.json

@@ -21,7 +21,8 @@
       "**/playwright-report/**",
       "**/todo-report.spec.ts-snapshots/**",
       "**/visualizer/scripts/fixture/*",
-      "**/unpacked-extension/*"
+      "**/unpacked-extension/*",
+      "**/page-data/**"
     ]
   },
   "javascript": {

packages/evaluation/.gitignore

@@ -0,0 +1,4 @@
+
+# Midscene.js dump files
+midscene_run/report
+midscene_run/tmp

packages/evaluation/data-generator/fixture.ts

@@ -0,0 +1,5 @@
+import type { PlayWrightAiFixtureType } from '@midscene/web';
+import { PlaywrightAiFixture } from '@midscene/web/playwright';
+import { test as base } from '@playwright/test';
+
+export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());

packages/evaluation/data-generator/generator-headed.spec.ts

@@ -0,0 +1,23 @@
+import { PlaywrightWebPage } from '@midscene/web/playwright';
+import { test } from './fixture';
+import { generateExtractData, generateTestDataPath } from './utils';
+
+function sleep(time: number) {
+  return new Promise((resolve) => {
+    setTimeout(() => {
+      resolve(0);
+    }, time);
+  });
+}
+
+test('taobao', async ({ page, ai }) => {
+  const playwrightPage = new PlaywrightWebPage(page);
+  page.setViewportSize({ width: 1280, height: 800 });
+
+  await page.goto('https://www.taobao.com/');
+
+  // for --ui
+  await sleep(5000);
+
+  await generateExtractData(playwrightPage, generateTestDataPath('taobao'));
+});

packages/web-integration/tests/ai/web/playwright/generate-test-data.spec.ts → packages/evaluation/data-generator/generator-headless.spec.ts

@@ -1,6 +1,6 @@
-import { generateExtractData, generateTestDataPath } from '@/debug';
-import { PlaywrightWebPage } from '@/playwright';
+import { PlaywrightWebPage } from '@midscene/web/playwright';
 import { test } from './fixture';
+import { generateExtractData, generateTestDataPath } from './utils';
 
 function sleep(time: number) {
   return new Promise((resolve) => {
@@ -10,7 +10,7 @@ function sleep(time: number) {
   });
 }
 
-test('generate todo test data', async ({ page }) => {
+test('todo app', async ({ page }) => {
   const playwrightPage = new PlaywrightWebPage(page);
   await page.goto('https://todomvc.com/examples/react/dist/');
   // Add data
@@ -33,7 +33,7 @@ test('generate todo test data', async ({ page }) => {
   );
 });
 
-test('generate visualstudio test data', async ({ page }) => {
+test('visualstudio code', async ({ page }) => {
   const playwrightPage = new PlaywrightWebPage(page);
 
   await page.goto('https://code.visualstudio.com/');
@@ -45,7 +45,7 @@ test('generate visualstudio test data', async ({ page }) => {
   );
 });
 
-test('generate githubstatus test data', async ({ page }) => {
+test('github status page', async ({ page }) => {
   const playwrightPage = new PlaywrightWebPage(page);
 
   await page.setViewportSize({ width: 1920, height: 1080 });
@@ -95,7 +95,7 @@ test('antd widget - carousel', async ({ page }) => {
   );
 });
 
-test('generate online order test data', async ({ page, ai }) => {
+test('heytea online order', async ({ page, ai }) => {
   const playwrightPage = new PlaywrightWebPage(page);
 
   page.setViewportSize({ width: 400, height: 905 });
@@ -109,7 +109,7 @@ test('generate online order test data', async ({ page, ai }) => {
   );
 });
 
-test('generate online order list test data (zh-cn)', async ({ page, ai }) => {
+test('heytea online order list (zh-cn)', async ({ page, ai }) => {
   const playwrightPage = new PlaywrightWebPage(page);
 
   page.setViewportSize({ width: 400, height: 905 });
@@ -127,19 +127,7 @@ test('generate online order list test data (zh-cn)', async ({ page, ai }) => {
   );
 });
 
-test('generate taobao test data', async ({ page, ai }) => {
-  const playwrightPage = new PlaywrightWebPage(page);
-  page.setViewportSize({ width: 1280, height: 800 });
-
-  await page.goto('https://www.taobao.com/');
-
-  // for --ui
-  await sleep(5000);
-
-  await generateExtractData(playwrightPage, generateTestDataPath('taobao'));
-});
-
-test('generate douyin test data', async ({ page, ai }) => {
+test('douyin', async ({ page, ai }) => {
   const playwrightPage = new PlaywrightWebPage(page);
 
   page.setViewportSize({ width: 1280, height: 800 });

packages/web-integration/src/debug/index.ts → packages/evaluation/data-generator/utils.ts

@@ -1,14 +1,17 @@
+import assert from 'node:assert';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { NodeType } from '@midscene/shared/constants';
+
 import path from 'node:path';
-import type { WebPage } from '@/common/page';
 import { descriptionOfTree } from '@midscene/core/tree';
-import { NodeType } from '@midscene/shared/constants';
-import type { ElementInfo } from '@midscene/shared/extractor';
 import {
   processImageElementInfo,
   resizeImgBase64,
   saveBase64Image,
 } from '@midscene/shared/img';
+import type { WebPage } from '@midscene/web';
+
+import type { ElementInfo } from '@midscene/shared/extractor';
 
 export async function generateExtractData(
   page: WebPage,
@@ -109,13 +112,10 @@ export async function generateExtractData(
 }
 
 export function generateTestDataPath(testDataName: string) {
-  // `dist/lib/index.js` Is the default export path
-  const modulePath = require
-    .resolve('@midscene/core')
-    .replace('dist/lib/index.js', '');
+  assert(testDataName, 'testDataName is required');
   const midsceneTestDataPath = path.join(
-    modulePath,
-    `tests/ai/evaluate/test-data/${testDataName}`,
+    __dirname,
+    `../page-data/${testDataName}`,
   );
 
   return midsceneTestDataPath;

packages/evaluation/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "evaluation",
+  "private": true,
+  "scripts": {
+    "update-page-data:headless": "playwright test ./data-generator/generator-headless.spec.ts",
+    "update-page-data:headed": "playwright test ./data-generator/generator-headed.spec.ts --headed",
+    "evaluate:inspect": "npx vitest --run tests/llm-inspect.test.ts",
+    "evaluate:inspect:update": "UPDATE_AI_DATA=true npm run evaluate:inspect",
+    "evaluate:assertion": "npx vitest --run tests/assertion.test.ts",
+    "evaluate:assertion:update": "UPDATE_AI_DATA=true npm run evaluate:assertion"
+  },
+  "dependencies": {
+    "@midscene/core": "workspace:*",
+    "@midscene/shared": "workspace:*",
+    "@midscene/web": "workspace:*"
+  },
+  "devDependencies": {
+    "dotenv": "16.4.5",
+    "playwright": "1.44.1",
+    "@playwright/test": "^1.44.1",
+    "typescript": "~5.0.4",
+    "vitest": "^1.6.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "license": "MIT"
+}

packages/midscene/tests/ai/evaluate/ai-data/assertion/online_order.json → packages/evaluation/page-cases/assertion/online_order.json

@@ -1,5 +1,5 @@
 {
-  "testDataPath": "test-data/online_order",
+  "testDataPath": "online_order",
   "testCases": [
     {
       "prompt": "there are three tabs in the page, named 'Menu', 'Reviews', 'Merchant'",
@@ -26,4 +26,4 @@
       "expected": false
     }
   ]
-}
+}

packages/midscene/tests/ai/evaluate/ai-data/inspect/aweme_login.json → packages/evaluation/page-cases/inspect/aweme_login.json

@@ -1,5 +1,5 @@
 {
-  "testDataPath": "test-data/aweme-login",
+  "testDataPath": "aweme-login",
   "testCases": [
     {
       "prompt": "密码登录",
@@ -72,4 +72,4 @@
       ]
     }
   ]
-}
+}

packages/evaluation/page-cases/inspect/aweme_play.json

@@ -0,0 +1,58 @@
+{
+  "testDataPath": "aweme-play",
+  "testCases": [
+    {
+      "prompt": "左下角暂停按钮",
+      "response": [
+        {
+          "id": "3"
+        }
+      ]
+    },
+    {
+      "prompt": "点赞（爱心）按钮",
+      "response": [
+        {
+          "id": "afifi",
+          "indexId": 22
+        }
+      ]
+    },
+    {
+      "prompt": "评论按钮",
+      "response": [
+        {
+          "id": "jldma",
+          "indexId": 24
+        }
+      ]
+    },
+    {
+      "prompt": "书签收藏按钮",
+      "response": [
+        {
+          "id": "nmgcl",
+          "indexId": 26
+        }
+      ]
+    },
+    {
+      "prompt": "分享按钮",
+      "response": [
+        {
+          "id": "eabap",
+          "indexId": 28
+        }
+      ]
+    },
+    {
+      "prompt": "右下角区域声音按钮",
+      "response": [
+        {
+          "id": "djknm",
+          "indexId": 9
+        }
+      ]
+    }
+  ]
+}

packages/midscene/tests/ai/evaluate/ai-data/inspect/online_order.json → packages/evaluation/page-cases/inspect/online_order.json

@@ -1,5 +1,5 @@
 {
-  "testDataPath": "test-data/online_order",
+  "testDataPath": "online_order",
   "testCases": [
     {
       "prompt": "Top left menu bar icon",
@@ -62,4 +62,4 @@
       ]
     }
   ]
-}
+}

packages/midscene/tests/ai/evaluate/ai-data/inspect/online_order_list.json → packages/evaluation/page-cases/inspect/online_order_list.json

@@ -1,5 +1,5 @@
 {
-  "testDataPath": "test-data/online_order_list",
+  "testDataPath": "online_order_list",
   "testCases": [
     {
       "prompt": "'清爽不喝腻'下面第二个饮品的名称",
@@ -42,4 +42,4 @@
       ]
     }
   ]
-}
+}