feat(web): added ai cache capability

.gitignore

@@ -97,8 +97,6 @@ blob-report/
 playwright/.cache/
 
 # MidScene.js dump files
-midscene_run/
-midscene-report/
 __ai_responses__/
 
 

packages/midscene/src/ai-model/inspect.ts

@@ -8,7 +8,7 @@ export async function AiInspectElement<ElementType extends BaseElement = BaseEle
   context: UIContext<ElementType>;
   multi: boolean;
   findElementDescription: string;
-  callAI?: typeof callToGetJSONObject;
+  callAI?: typeof callToGetJSONObject<AIElementParseResponse>;
 }) {
   const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options;
   const { screenshotBase64 } = context;
@@ -35,7 +35,7 @@ export async function AiInspectElement<ElementType extends BaseElement = BaseEle
       ],
     },
   ];
-  const parseResult = await callAI<AIElementParseResponse>(msgs);
+  const parseResult = await callAI(msgs);
   return {
     parseResult,
     elementById,

packages/midscene/src/ai-model/prompt/util.ts

@@ -199,7 +199,15 @@ export async function describeUserPage<ElementType extends BaseElement = BaseEle
   context: Omit<UIContext<ElementType>, 'describer'>,
 ) {
   const { screenshotBase64 } = context;
-  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  let width: number;
+  let height: number;
+
+  if (context.size) {
+    ({ width, height } = context.size);
+  } else {
+    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    ({ width, height } = imgSize);
+  }
 
   const elementsInfo = context.content;
   const idElementMap: Record<string, ElementType> = {};

packages/midscene/src/automation/planning.ts

@@ -1,6 +1,6 @@
 import { ChatCompletionMessageParam } from 'openai/resources';
 import { PlanningAction, PlanningAIResponse, UIContext } from '@/types';
-import { callToGetJSONObject as callAI } from '@/ai-model/openai';
+import { callToGetJSONObject } from '@/ai-model/openai';
 import { describeUserPage } from '@/ai-model';
 
 const characteristic =
@@ -60,7 +60,14 @@ export function systemPromptToTaskPlanning(query: string) {
   `;
 }
 
-export async function plan(context: UIContext, userPrompt: string): Promise<{ plans: PlanningAction[] }> {
+export async function plan(
+  userPrompt: string,
+  opts: {
+    context: UIContext;
+    callAI?: typeof callToGetJSONObject<PlanningAIResponse>;
+  },
+): Promise<{ plans: PlanningAction[] }> {
+  const { callAI = callToGetJSONObject<PlanningAIResponse>, context } = opts || {};
   const { screenshotBase64 } = context;
   const { description } = await describeUserPage(context);
   const systemPrompt = systemPromptToTaskPlanning(userPrompt);
@@ -84,7 +91,7 @@ export async function plan(context: UIContext, userPrompt: string): Promise<{ pl
     },
   ];
 
-  const planFromAI = await callAI<PlanningAIResponse>(msgs);
+  const planFromAI = await callAI(msgs);
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }

packages/midscene/src/insight/index.ts

@@ -16,6 +16,7 @@ import {
   BaseElement,
   DumpSubscriber,
   InsightExtractParam,
+  AIElementParseResponse,
 } from '@/types';
 
 const sortByOrder = (a: UISection, b: UISection) => {
@@ -26,8 +27,9 @@ const sortByOrder = (a: UISection, b: UISection) => {
   }
 };
 
-export interface FindElementOptions {
+export interface LocateOpts {
   multi?: boolean;
+  callAI?: typeof callAI<AIElementParseResponse>;
 }
 
 // export type UnwrapDataShape<T> = T extends EnhancedQuery<infer DataShape> ? DataShape : {};
@@ -36,19 +38,19 @@ export type AnyValue<T> = {
   [K in keyof T]: unknown extends T[K] ? any : T[K];
 };
 
-export default class Insight<ElementType extends BaseElement = BaseElement> {
-  contextRetrieverFn: () => Promise<UIContext<ElementType>> | UIContext<ElementType>;
+export default class Insight<
+  ElementType extends BaseElement = BaseElement,
+  ContextType extends UIContext<ElementType> = UIContext<ElementType>,
+> {
+  contextRetrieverFn: () => Promise<ContextType> | ContextType;
 
   aiVendorFn: typeof callAI = callAI;
 
   onceDumpUpdatedFn?: DumpSubscriber;
 
   taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
 
-  constructor(
-    context: UIContext<ElementType> | (() => Promise<UIContext<ElementType>> | UIContext<ElementType>),
-    opt?: InsightOptions,
-  ) {
+  constructor(context: ContextType | (() => Promise<ContextType> | ContextType), opt?: InsightOptions) {
     assert(context, 'context is required for Insight');
     if (typeof context === 'function') {
       this.contextRetrieverFn = context;
@@ -64,19 +66,20 @@ export default class Insight<ElementType extends BaseElement = BaseElement> {
     }
   }
 
-  async locate(queryPrompt: string): Promise<ElementType | null>;
+  async locate(queryPrompt: string, opt?: { callAI: LocateOpts['callAI'] }): Promise<ElementType | null>;
   async locate(queryPrompt: string, opt: { multi: true }): Promise<ElementType[]>;
-  async locate(queryPrompt: string, opt?: FindElementOptions) {
+  async locate(queryPrompt: string, opt?: LocateOpts) {
+    const { callAI = this.aiVendorFn, multi = false } = opt || {};
     assert(queryPrompt, 'query is required for located');
     const dumpSubscriber = this.onceDumpUpdatedFn;
     this.onceDumpUpdatedFn = undefined;
     const context = await this.contextRetrieverFn();
 
     const startTime = Date.now();
     const { parseResult, systemPrompt, elementById } = await AiInspectElement({
-      callAI: this.aiVendorFn,
+      callAI,
       context,
-      multi: Boolean(opt?.multi),
+      multi: Boolean(multi),
       findElementDescription: queryPrompt,
     });
     // const parseResult = await this.aiVendorFn<AIElementParseResponse>(msgs);
@@ -282,4 +285,12 @@ export default class Insight<ElementType extends BaseElement = BaseElement> {
 
     return mergedData;
   }
+
+  setAiVendorFn<T>(aiVendorFn: typeof callAI<T>) {
+    const origin = this.aiVendorFn;
+    this.aiVendorFn<T> = aiVendorFn;
+    return () => {
+      this.aiVendorFn = origin;
+    };
+  }
 }

packages/midscene/src/insight/utils.ts

@@ -58,7 +58,11 @@ export function writeInsightDump(
     const length = logContent.push(dataString);
     logIdIndexMap[id] = length - 1;
   }
-  writeDumpFile(logFileName, logFileExt, `[\n${logContent.join(',\n')}\n]`);
+  writeDumpFile({
+    fileName: logFileName,
+    fileExt: logFileExt,
+    fileContent: `[\n${logContent.join(',\n')}\n]`,
+  });
 
   return id;
 }

packages/midscene/src/utils.ts

@@ -43,20 +43,32 @@ export const groupedActionDumpFileExt = 'web-dump.json';
 export function getDumpDir() {
   return logDir;
 }
+
 export function setDumpDir(dir: string) {
   logDir = dir;
 }
 
-export function writeDumpFile(fileName: string, fileExt: string, fileContent: string) {
+export function getDumpDirPath(type: 'dump' | 'cache') {
+  return join(getDumpDir(), type);
+}
+
+export function writeDumpFile(opts: {
+  fileName: string;
+  fileExt: string;
+  fileContent: string;
+  type?: 'dump' | 'cache';
+}) {
+  const { fileName, fileExt, fileContent, type = 'dump' } = opts;
+  const targetDir = getDumpDirPath(type);
+  if (!existsSync(targetDir)) {
+    mkdirSync(targetDir, { recursive: true });
+  }
   // Ensure directory exists
   if (!logEnvReady) {
-    assert(logDir, 'logDir should be set before writing dump file');
-    if (!existsSync(logDir)) {
-      mkdirSync(logDir, { recursive: true });
-    }
+    assert(targetDir, 'logDir should be set before writing dump file');
 
     // gitIgnore in the parent directory
-    const gitIgnorePath = join(logDir, '../.gitignore');
+    const gitIgnorePath = join(targetDir, '../../.gitignore');
     let gitIgnoreContent = '';
     if (existsSync(gitIgnorePath)) {
       gitIgnoreContent = readFileSync(gitIgnorePath, 'utf-8');
@@ -67,16 +79,19 @@ export function writeDumpFile(fileName: string, fileExt: string, fileContent: st
     if (!gitIgnoreContent.includes(`${logDirName}/`)) {
       writeFileSync(
         gitIgnorePath,
-        `${gitIgnoreContent}\n# MidScene.js dump files\n${logDirName}/\n`,
+        `${gitIgnoreContent}\n# MidScene.js dump files\n${logDirName}/midscene-report\n${logDirName}/dump-logger\n`,
         'utf-8',
       );
     }
     logEnvReady = true;
   }
 
-  const filePath = join(getDumpDir(), `${fileName}.${fileExt}`);
+  const filePath = join(targetDir, `${fileName}.${fileExt}`);
   writeFileSync(filePath, fileContent);
-  copyFileSync(filePath, join(getDumpDir(), `latest.${fileExt}`));
+
+  if (type === 'dump') {
+    copyFileSync(filePath, join(targetDir, `latest.${fileExt}`));
+  }
 
   return filePath;
 }

packages/midscene/tests/ai-model/inspector/__snapshots__/todo_inspector.test.ts.snap

@@ -2,7 +2,7 @@
   {
     "elements": [
       {
-        "id": "2",
+        "id": "b0ca2e8c69",
       },
     ],
     "error": [],
@@ -11,7 +11,7 @@
   {
     "elements": [
       {
-        "id": "8",
+        "id": "b9807d7de6",
       },
     ],
     "error": [],
@@ -20,7 +20,7 @@
   {
     "elements": [
       {
-        "id": "9",
+        "id": "c5a7702fed",
       },
     ],
     "error": [],
@@ -29,7 +29,7 @@
   {
     "elements": [
       {
-        "id": "10",
+        "id": "c84a3afdac",
       },
     ],
     "error": [],
@@ -38,7 +38,7 @@
   {
     "elements": [
       {
-        "id": "15",
+        "id": "defa24dedd",
       },
     ],
     "error": [],

packages/midscene/tests/ai-model/inspector/__snapshots__/xicha_inspector.test.ts.snap

@@ -2,7 +2,7 @@
   {
     "elements": [
       {
-        "id": "1",
+        "id": "922e98a196",
       },
     ],
     "error": [],
@@ -11,16 +11,16 @@
   {
     "elements": [
       {
-        "id": "2",
+        "id": "83ffa89342",
       },
     ],
     "error": [],
-    "prompt": "Switch language(include：中文、english text)",
+    "prompt": "Toggle language text button(Could be：中文、english text)",
   },
   {
     "elements": [
       {
-        "id": "4",
+        "id": "a525985342",
       },
     ],
     "error": [],
@@ -29,10 +29,10 @@
   {
     "elements": [
       {
-        "id": "22",
+        "id": "3fb89d359f",
       },
       {
-        "id": "28",
+        "id": "c4300a7c45",
       },
     ],
     "error": [],
@@ -41,10 +41,10 @@
   {
     "elements": [
       {
-        "id": "23",
+        "id": "ae0ba24c99",
       },
       {
-        "id": "29",
+        "id": "a50d88f84c",
       },
     ],
     "error": [],
@@ -53,7 +53,7 @@
   {
     "elements": [
       {
-        "id": "30",
+        "id": "df4f252aab",
       },
     ],
     "error": [],

packages/midscene/tests/ai-model/inspector/xicha_inspector.test.ts → packages/midscene/tests/ai-model/inspector/online_order_inspector.test.ts

@@ -1,5 +1,5 @@
-import { test, expect } from 'vitest';
 import path from 'path';
+import { test, expect } from 'vitest';
 import { getPageTestData, repeat, runTestCases, writeFileSyncWithDir } from './util';
 import { AiInspectElement } from '@/ai-model';
 
@@ -9,7 +9,7 @@ const testCases = [
     multi: false,
   },
   {
-    description: 'Switch language(include：中文、english text)',
+    description: 'Toggle language text button(Could be：中文、english text)',
     multi: false,
   },
   {
@@ -31,21 +31,28 @@ const testCases = [
 ];
 
 repeat(5, (repeatIndex) => {
-  test('xicha: inspect element', async () => {
-    const { context } = await getPageTestData(path.join(__dirname, './test-data/xicha'));
+  test(
+    'xicha: inspect element',
+    async () => {
+      const { context } = await getPageTestData(path.join(__dirname, './test-data/online_order'));
 
-    const { aiResponse, filterUnStableinf } = await runTestCases(testCases, async (testCase)=>{
+      const { aiResponse, filterUnStableinf } = await runTestCases(testCases, async (testCase) => {
         const { parseResult } = await AiInspectElement({
           context,
           multi: testCase.multi,
           findElementDescription: testCase.description,
         });
         return parseResult;
-    });
-    writeFileSyncWithDir(path.join(__dirname, `__ai_responses__/xicha-inspector-element-${repeatIndex}.json`), JSON.stringify(aiResponse, null, 2), { encoding: 'utf-8'});
-    expect(filterUnStableinf).toMatchFileSnapshot('./__snapshots__/xicha_inspector.test.ts.snap');
-  }, {
-    timeout: 99999,
-  });
+      });
+      writeFileSyncWithDir(
+        path.join(__dirname, `__ai_responses__/online_order-inspector-element-${repeatIndex}.json`),
+        JSON.stringify(aiResponse, null, 2),
+        { encoding: 'utf-8' },
+      );
+      expect(filterUnStableinf).toMatchFileSnapshot('./__snapshots__/online_order_inspector.test.ts.snap');
+    },
+    {
+      timeout: 99999,
+    },
+  );
 });
-