Merge branch 'main' into feat/extension-bridge-mode

Author: yuyutaotao

packages/midscene/src/ai-model/coze/index.ts

@@ -124,7 +124,6 @@ export async function AiInspectElement<
   multi: boolean;
   targetElementDescription: string;
   callAI?: typeof callAiFn<AIElementResponse | [number, number]>;
-  useModel?: 'coze' | 'openAI';
   quickAnswer?: AISingleElementResponse;
 }): Promise<{
   parseResult: AIElementResponse;
@@ -195,9 +194,8 @@ export async function AiExtractElementInfo<
 >(options: {
   dataQuery: string | Record<string, string>;
   context: UIContext<ElementType>;
-  useModel?: 'coze' | 'openAI';
 }) {
-  const { dataQuery, context, useModel } = options;
+  const { dataQuery, context } = options;
   const systemPrompt = systemPromptToExtract();
 
   const { screenshotBase64 } = context;
@@ -256,9 +254,8 @@ export async function AiAssert<
 >(options: {
   assertion: string;
   context: UIContext<ElementType>;
-  useModel?: 'coze' | 'openAI';
 }) {
-  const { assertion, context, useModel } = options;
+  const { assertion, context } = options;
 
   assert(assertion, 'assertion should be a string');
 

packages/midscene/src/ai-model/inspect.ts

@@ -41,7 +41,7 @@ import { findElementSchema } from '../prompt/element_inspector';
 import { planSchema } from '../prompt/planning';
 import { assertSchema } from '../prompt/util';
 
-export function checkAIConfig(preferVendor?: 'coze' | 'openAI') {
+export function checkAIConfig(preferVendor?: 'openAI') {
   if (preferVendor && preferVendor !== 'openAI') return false;
   if (getAIConfig(OPENAI_API_KEY)) return true;
   if (getAIConfig(MIDSCENE_USE_AZURE_OPENAI)) return true;

packages/midscene/src/ai-model/openai/index.ts

@@ -1,5 +1,4 @@
 import { AiAssert } from '@/ai-model';
-import { preferCozeModel } from '@/ai-model/coze';
 /* eslint-disable max-lines-per-function */
 import { describe, expect, it, vi } from 'vitest';
 import { getPageDataOfTestName } from '../evaluate/test-suite/util';
@@ -9,38 +8,28 @@ vi.setConfig({
   hookTimeout: 30 * 1000,
 });
 
-const modelList: Array<'openAI' | 'coze'> = ['openAI'];
+describe('assert', () => {
+  it('todo pass', async () => {
+    const { context } = await getPageDataOfTestName('todo');
 
-if (preferCozeModel('coze')) {
-  modelList.push('coze');
-}
-
-modelList.forEach((model) => {
-  describe('assert', () => {
-    it('todo pass', async () => {
-      const { context } = await getPageDataOfTestName('todo');
-
-      const {
-        content: { pass },
-      } = await AiAssert({
-        assertion: 'Three tasks have been added',
-        context,
-        useModel: model,
-      });
-      expect(pass).toBe(true);
+    const {
+      content: { pass },
+    } = await AiAssert({
+      assertion: 'Three tasks have been added',
+      context,
     });
+    expect(pass).toBe(true);
+  });
 
-    it('todo error', async () => {
-      const { context } = await getPageDataOfTestName('todo');
+  it('todo error', async () => {
+    const { context } = await getPageDataOfTestName('todo');
 
-      const {
-        content: { pass, thought },
-      } = await AiAssert({
-        assertion: 'There are four tasks in the task list',
-        context,
-        useModel: model,
-      });
-      expect(pass).toBe(false);
+    const {
+      content: { pass, thought },
+    } = await AiAssert({
+      assertion: 'There are four tasks in the task list',
+      context,
     });
+    expect(pass).toBe(false);
   });
 });

packages/midscene/tests/ai/assert/assert.test.ts

@@ -0,0 +1,45 @@
+import { plan } from '@/ai-model';
+/* eslint-disable max-lines-per-function */
+import { describe, expect, it, vi } from 'vitest';
+import { makePlanResultStable } from '../../util';
+import { getPageDataOfTestName, repeat } from './../test-suite/util';
+
+vi.setConfig({
+  testTimeout: 180 * 1000,
+  hookTimeout: 30 * 1000,
+});
+
+describe('automation - planning input', () => {
+  repeat(5, () =>
+    it('input value', async () => {
+      const { context } = await getPageDataOfTestName('todo');
+      const instructions = [
+        'In the taskbar, type learning english',
+        'In the taskbar, type learning english and hit Enter key',
+      ];
+
+      for (const instruction of instructions) {
+        const { actions } = await plan(instruction, { context });
+        const res = makePlanResultStable(actions);
+        expect(res).toMatchSnapshot();
+      }
+    }),
+  );
+
+  repeat(5, () =>
+    it('input value Add, delete, correct and check', async () => {
+      const { context } = await getPageDataOfTestName('todo-input-with-value');
+      const instructions = [
+        'Append "tomorrow" to the existing content in the task input box',
+        'Replace "English" with "Skiing" in the existing content of the task input box',
+        'Delete "English" from the existing content in the task input box',
+      ];
+
+      for (const instruction of instructions) {
+        const { actions } = await plan(instruction, { context });
+        const res = makePlanResultStable(actions);
+        expect(res).toMatchSnapshot();
+      }
+    }),
+  );
+});

packages/midscene/tests/ai/evaluate/plan/planning-input.test.ts

@@ -1,5 +1,4 @@
 import { AiExtractElementInfo } from '@/ai-model';
-import { preferCozeModel } from '@/ai-model/coze';
 import { describe, expect, it, vi } from 'vitest';
 import { getPageDataOfTestName } from '../evaluate/test-suite/util';
 
@@ -8,46 +7,35 @@ vi.setConfig({
   hookTimeout: 30 * 1000,
 });
 
-const modelList: Array<'openAI' | 'coze'> = ['openAI'];
+describe('extract', () => {
+  it('todo', async () => {
+    const { context } = await getPageDataOfTestName('todo');
 
-if (preferCozeModel('coze')) {
-  modelList.push('coze');
-}
-
-modelList.forEach((model) => {
-  describe(`assert ${model}`, () => {
-    it('todo', async () => {
-      const { context } = await getPageDataOfTestName('todo');
-
-      const { parseResult } = await AiExtractElementInfo({
-        dataQuery: 'Array<string>, Complete task list, string is the task',
-        context,
-        useModel: model,
-      });
-      expect(parseResult).toMatchSnapshot();
+    const { parseResult } = await AiExtractElementInfo({
+      dataQuery: 'Array<string>, Complete task list, string is the task',
+      context,
     });
+    expect(parseResult).toMatchSnapshot();
+  });
 
-    it('online order', async () => {
-      const { context } = await getPageDataOfTestName('online_order');
+  it('online order', async () => {
+    const { context } = await getPageDataOfTestName('online_order');
 
-      const { parseResult } = await AiExtractElementInfo({
-        dataQuery: '{name: string, price: string}[], 饮品名称和价格',
-        context,
-        useModel: model,
-      });
-      expect(parseResult).toMatchSnapshot();
+    const { parseResult } = await AiExtractElementInfo({
+      dataQuery: '{name: string, price: string}[], 饮品名称和价格',
+      context,
     });
+    expect(parseResult).toMatchSnapshot();
+  });
 
-    it('todo obj', async () => {
-      const { context } = await getPageDataOfTestName('todo');
+  it('todo obj', async () => {
+    const { context } = await getPageDataOfTestName('todo');
 
-      const { parseResult } = await AiExtractElementInfo({
-        dataQuery:
-          '{checked: boolean; text: string}[],Complete task list, string is the task',
-        context,
-        useModel: model,
-      });
-      expect(parseResult).toMatchSnapshot();
+    const { parseResult } = await AiExtractElementInfo({
+      dataQuery:
+        '{checked: boolean; text: string}[],Complete task list, string is the task',
+      context,
     });
+    expect(parseResult).toMatchSnapshot();
   });
 });

packages/midscene/tests/ai/evaluate/planning.test.ts renamed to packages/midscene/tests/ai/evaluate/plan/planning.test.ts

@@ -1,4 +1,3 @@
-import { preferCozeModel } from '@/ai-model/coze';
 import type { PlanningAction } from '@/types';
 
 export function makePlanResultStable(plans: PlanningAction[]) {
@@ -15,7 +14,3 @@ export function makePlanResultStable(plans: PlanningAction[]) {
     return plan;
   });
 }
-
-export const modelList: Array<'openAI' | 'coze'> = preferCozeModel('coze')
-  ? ['openAI', 'coze']
-  : ['openAI'];