diff --git a/.gitignore b/.gitignore index bbdcdbe97..213be5775 100644 --- a/.gitignore +++ b/.gitignore @@ -97,8 +97,6 @@ blob-report/ playwright/.cache/ # MidScene.js dump files -midscene_run/ -midscene-report/ __ai_responses__/ diff --git a/packages/midscene/src/ai-model/inspect.ts b/packages/midscene/src/ai-model/inspect.ts index ff7abae73..69984ab7c 100644 --- a/packages/midscene/src/ai-model/inspect.ts +++ b/packages/midscene/src/ai-model/inspect.ts @@ -8,7 +8,7 @@ export async function AiInspectElement; multi: boolean; findElementDescription: string; - callAI?: typeof callToGetJSONObject; + callAI?: typeof callToGetJSONObject; }) { const { context, multi, findElementDescription, callAI = callToGetJSONObject } = options; const { screenshotBase64 } = context; @@ -35,7 +35,7 @@ export async function AiInspectElement(msgs); + const parseResult = await callAI(msgs); return { parseResult, elementById, diff --git a/packages/midscene/src/ai-model/prompt/util.ts b/packages/midscene/src/ai-model/prompt/util.ts index 7a4e9806c..bfaba849a 100644 --- a/packages/midscene/src/ai-model/prompt/util.ts +++ b/packages/midscene/src/ai-model/prompt/util.ts @@ -199,7 +199,15 @@ export async function describeUserPage, 'describer'>, ) { const { screenshotBase64 } = context; - const { width, height } = await imageInfoOfBase64(screenshotBase64); + let width: number; + let height: number; + + if (context.size) { + ({ width, height } = context.size); + } else { + const imgSize = await imageInfoOfBase64(screenshotBase64); + ({ width, height } = imgSize); + } const elementsInfo = context.content; const idElementMap: Record = {}; diff --git a/packages/midscene/src/automation/planning.ts b/packages/midscene/src/automation/planning.ts index 9015acd97..c8ab48ab9 100644 --- a/packages/midscene/src/automation/planning.ts +++ b/packages/midscene/src/automation/planning.ts @@ -1,6 +1,6 @@ import { ChatCompletionMessageParam } from 'openai/resources'; import { PlanningAction, PlanningAIResponse, UIContext } from '@/types'; -import { callToGetJSONObject as callAI } from '@/ai-model/openai'; +import { callToGetJSONObject } from '@/ai-model/openai'; import { describeUserPage } from '@/ai-model'; const characteristic = @@ -60,7 +60,14 @@ export function systemPromptToTaskPlanning(query: string) { `; } -export async function plan(context: UIContext, userPrompt: string): Promise<{ plans: PlanningAction[] }> { +export async function plan( + userPrompt: string, + opts: { + context: UIContext; + callAI?: typeof callToGetJSONObject; + }, +): Promise<{ plans: PlanningAction[] }> { + const { callAI = callToGetJSONObject, context } = opts || {}; const { screenshotBase64 } = context; const { description } = await describeUserPage(context); const systemPrompt = systemPromptToTaskPlanning(userPrompt); @@ -84,7 +91,7 @@ export async function plan(context: UIContext, userPrompt: string): Promise<{ pl }, ]; - const planFromAI = await callAI(msgs); + const planFromAI = await callAI(msgs); if (planFromAI.error) { throw new Error(planFromAI.error); } diff --git a/packages/midscene/src/insight/index.ts b/packages/midscene/src/insight/index.ts index e95053269..7366ec498 100644 --- a/packages/midscene/src/insight/index.ts +++ b/packages/midscene/src/insight/index.ts @@ -16,6 +16,7 @@ import { BaseElement, DumpSubscriber, InsightExtractParam, + AIElementParseResponse, } from '@/types'; const sortByOrder = (a: UISection, b: UISection) => { @@ -26,8 +27,9 @@ const sortByOrder = (a: UISection, b: UISection) => { } }; -export interface FindElementOptions { +export interface LocateOpts { multi?: boolean; + callAI?: typeof callAI; } // export type UnwrapDataShape = T extends EnhancedQuery ? DataShape : {}; @@ -36,8 +38,11 @@ export type AnyValue = { [K in keyof T]: unknown extends T[K] ? any : T[K]; }; -export default class Insight { - contextRetrieverFn: () => Promise> | UIContext; +export default class Insight< + ElementType extends BaseElement = BaseElement, + ContextType extends UIContext = UIContext, +> { + contextRetrieverFn: () => Promise | ContextType; aiVendorFn: typeof callAI = callAI; @@ -45,10 +50,7 @@ export default class Insight { taskInfo?: Omit; - constructor( - context: UIContext | (() => Promise> | UIContext), - opt?: InsightOptions, - ) { + constructor(context: ContextType | (() => Promise | ContextType), opt?: InsightOptions) { assert(context, 'context is required for Insight'); if (typeof context === 'function') { this.contextRetrieverFn = context; @@ -64,9 +66,10 @@ export default class Insight { } } - async locate(queryPrompt: string): Promise; + async locate(queryPrompt: string, opt?: { callAI: LocateOpts['callAI'] }): Promise; async locate(queryPrompt: string, opt: { multi: true }): Promise; - async locate(queryPrompt: string, opt?: FindElementOptions) { + async locate(queryPrompt: string, opt?: LocateOpts) { + const { callAI = this.aiVendorFn, multi = false } = opt || {}; assert(queryPrompt, 'query is required for located'); const dumpSubscriber = this.onceDumpUpdatedFn; this.onceDumpUpdatedFn = undefined; @@ -74,9 +77,9 @@ export default class Insight { const startTime = Date.now(); const { parseResult, systemPrompt, elementById } = await AiInspectElement({ - callAI: this.aiVendorFn, + callAI, context, - multi: Boolean(opt?.multi), + multi: Boolean(multi), findElementDescription: queryPrompt, }); // const parseResult = await this.aiVendorFn(msgs); @@ -282,4 +285,12 @@ export default class Insight { return mergedData; } + + setAiVendorFn(aiVendorFn: typeof callAI) { + const origin = this.aiVendorFn; + this.aiVendorFn = aiVendorFn; + return () => { + this.aiVendorFn = origin; + }; + } } diff --git a/packages/midscene/src/insight/utils.ts b/packages/midscene/src/insight/utils.ts index 7ce89eea6..95b959c84 100644 --- a/packages/midscene/src/insight/utils.ts +++ b/packages/midscene/src/insight/utils.ts @@ -58,7 +58,11 @@ export function writeInsightDump( const length = logContent.push(dataString); logIdIndexMap[id] = length - 1; } - writeDumpFile(logFileName, logFileExt, `[\n${logContent.join(',\n')}\n]`); + writeDumpFile({ + fileName: logFileName, + fileExt: logFileExt, + fileContent: `[\n${logContent.join(',\n')}\n]`, + }); return id; } diff --git a/packages/midscene/src/utils.ts b/packages/midscene/src/utils.ts index 2fcbbbc0b..dded24b0c 100644 --- a/packages/midscene/src/utils.ts +++ b/packages/midscene/src/utils.ts @@ -43,20 +43,32 @@ export const groupedActionDumpFileExt = 'web-dump.json'; export function getDumpDir() { return logDir; } + export function setDumpDir(dir: string) { logDir = dir; } -export function writeDumpFile(fileName: string, fileExt: string, fileContent: string) { +export function getDumpDirPath(type: 'dump' | 'cache') { + return join(getDumpDir(), type); +} + +export function writeDumpFile(opts: { + fileName: string; + fileExt: string; + fileContent: string; + type?: 'dump' | 'cache'; +}) { + const { fileName, fileExt, fileContent, type = 'dump' } = opts; + const targetDir = getDumpDirPath(type); + if (!existsSync(targetDir)) { + mkdirSync(targetDir, { recursive: true }); + } // Ensure directory exists if (!logEnvReady) { - assert(logDir, 'logDir should be set before writing dump file'); - if (!existsSync(logDir)) { - mkdirSync(logDir, { recursive: true }); - } + assert(targetDir, 'logDir should be set before writing dump file'); // gitIgnore in the parent directory - const gitIgnorePath = join(logDir, '../.gitignore'); + const gitIgnorePath = join(targetDir, '../../.gitignore'); let gitIgnoreContent = ''; if (existsSync(gitIgnorePath)) { gitIgnoreContent = readFileSync(gitIgnorePath, 'utf-8'); @@ -67,16 +79,19 @@ export function writeDumpFile(fileName: string, fileExt: string, fileContent: st if (!gitIgnoreContent.includes(`${logDirName}/`)) { writeFileSync( gitIgnorePath, - `${gitIgnoreContent}\n# MidScene.js dump files\n${logDirName}/\n`, + `${gitIgnoreContent}\n# MidScene.js dump files\n${logDirName}/midscene-report\n${logDirName}/dump-logger\n`, 'utf-8', ); } logEnvReady = true; } - const filePath = join(getDumpDir(), `${fileName}.${fileExt}`); + const filePath = join(targetDir, `${fileName}.${fileExt}`); writeFileSync(filePath, fileContent); - copyFileSync(filePath, join(getDumpDir(), `latest.${fileExt}`)); + + if (type === 'dump') { + copyFileSync(filePath, join(targetDir, `latest.${fileExt}`)); + } return filePath; } diff --git a/packages/midscene/tests/ai-model/inspector/__snapshots__/todo_inspector.test.ts.snap b/packages/midscene/tests/ai-model/inspector/__snapshots__/todo_inspector.test.ts.snap index a68142724..d3003acdb 100644 --- a/packages/midscene/tests/ai-model/inspector/__snapshots__/todo_inspector.test.ts.snap +++ b/packages/midscene/tests/ai-model/inspector/__snapshots__/todo_inspector.test.ts.snap @@ -2,7 +2,7 @@ { "elements": [ { - "id": "2", + "id": "b0ca2e8c69", }, ], "error": [], @@ -11,7 +11,7 @@ { "elements": [ { - "id": "8", + "id": "b9807d7de6", }, ], "error": [], @@ -20,7 +20,7 @@ { "elements": [ { - "id": "9", + "id": "c5a7702fed", }, ], "error": [], @@ -29,7 +29,7 @@ { "elements": [ { - "id": "10", + "id": "c84a3afdac", }, ], "error": [], @@ -38,7 +38,7 @@ { "elements": [ { - "id": "15", + "id": "defa24dedd", }, ], "error": [], diff --git a/packages/midscene/tests/ai-model/inspector/__snapshots__/xicha_inspector.test.ts.snap b/packages/midscene/tests/ai-model/inspector/__snapshots__/xicha_inspector.test.ts.snap index 22a124355..090cc4101 100644 --- a/packages/midscene/tests/ai-model/inspector/__snapshots__/xicha_inspector.test.ts.snap +++ b/packages/midscene/tests/ai-model/inspector/__snapshots__/xicha_inspector.test.ts.snap @@ -2,7 +2,7 @@ { "elements": [ { - "id": "1", + "id": "922e98a196", }, ], "error": [], @@ -11,16 +11,16 @@ { "elements": [ { - "id": "2", + "id": "83ffa89342", }, ], "error": [], - "prompt": "Switch language(include:中文、english text)", + "prompt": "Toggle language text button(Could be:中文、english text)", }, { "elements": [ { - "id": "4", + "id": "a525985342", }, ], "error": [], @@ -29,10 +29,10 @@ { "elements": [ { - "id": "22", + "id": "3fb89d359f", }, { - "id": "28", + "id": "c4300a7c45", }, ], "error": [], @@ -41,10 +41,10 @@ { "elements": [ { - "id": "23", + "id": "ae0ba24c99", }, { - "id": "29", + "id": "a50d88f84c", }, ], "error": [], @@ -53,7 +53,7 @@ { "elements": [ { - "id": "30", + "id": "df4f252aab", }, ], "error": [], diff --git a/packages/midscene/tests/ai-model/inspector/xicha_inspector.test.ts b/packages/midscene/tests/ai-model/inspector/online_order_inspector.test.ts similarity index 54% rename from packages/midscene/tests/ai-model/inspector/xicha_inspector.test.ts rename to packages/midscene/tests/ai-model/inspector/online_order_inspector.test.ts index 63aeff9c7..09183d768 100644 --- a/packages/midscene/tests/ai-model/inspector/xicha_inspector.test.ts +++ b/packages/midscene/tests/ai-model/inspector/online_order_inspector.test.ts @@ -1,5 +1,5 @@ -import { test, expect } from 'vitest'; import path from 'path'; +import { test, expect } from 'vitest'; import { getPageTestData, repeat, runTestCases, writeFileSyncWithDir } from './util'; import { AiInspectElement } from '@/ai-model'; @@ -9,7 +9,7 @@ const testCases = [ multi: false, }, { - description: 'Switch language(include:中文、english text)', + description: 'Toggle language text button(Could be:中文、english text)', multi: false, }, { @@ -31,21 +31,28 @@ const testCases = [ ]; repeat(5, (repeatIndex) => { - test('xicha: inspect element', async () => { - const { context } = await getPageTestData(path.join(__dirname, './test-data/xicha')); + test( + 'xicha: inspect element', + async () => { + const { context } = await getPageTestData(path.join(__dirname, './test-data/online_order')); - const { aiResponse, filterUnStableinf } = await runTestCases(testCases, async (testCase)=>{ + const { aiResponse, filterUnStableinf } = await runTestCases(testCases, async (testCase) => { const { parseResult } = await AiInspectElement({ context, multi: testCase.multi, findElementDescription: testCase.description, }); return parseResult; - }); - writeFileSyncWithDir(path.join(__dirname, `__ai_responses__/xicha-inspector-element-${repeatIndex}.json`), JSON.stringify(aiResponse, null, 2), { encoding: 'utf-8'}); - expect(filterUnStableinf).toMatchFileSnapshot('./__snapshots__/xicha_inspector.test.ts.snap'); - }, { - timeout: 99999, - }); + }); + writeFileSyncWithDir( + path.join(__dirname, `__ai_responses__/online_order-inspector-element-${repeatIndex}.json`), + JSON.stringify(aiResponse, null, 2), + { encoding: 'utf-8' }, + ); + expect(filterUnStableinf).toMatchFileSnapshot('./__snapshots__/online_order_inspector.test.ts.snap'); + }, + { + timeout: 99999, + }, + ); }); - diff --git a/packages/midscene/tests/ai-model/inspector/test-data/githubstatus/element-snapshot.json b/packages/midscene/tests/ai-model/inspector/test-data/githubstatus/element-snapshot.json index 93c3ecac6..8620329f7 100644 --- a/packages/midscene/tests/ai-model/inspector/test-data/githubstatus/element-snapshot.json +++ b/packages/midscene/tests/ai-model/inspector/test-data/githubstatus/element-snapshot.json @@ -1,11 +1,13 @@ [ { - "id": "1", + "id": "af212c0aa4", + "indexId": "1", + "nodeHashId": "af212c0aa4", "attributes": { "class": ".custom-header-container", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='1']", + "locator": "[_midscene_retrieve_task_id='af212c0aa4']", "center": [ 960, 248 @@ -19,12 +21,14 @@ } }, { - "id": "2", + "id": "2e61aa0fa8", + "indexId": "2", + "nodeHashId": "2e61aa0fa8", "attributes": { "class": ".header.d-md-flex.flex-md-justify-between.flex-md-items-center.px-4.py-3.text-center.text-md-left.bg-white.box-shadow-large", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='2']", + "locator": "[_midscene_retrieve_task_id='2e61aa0fa8']", "center": [ 960, 37 @@ -38,12 +42,14 @@ } }, { - "id": "3", + "id": "5c2d0fc07e", + "indexId": "3", + "nodeHashId": "5c2d0fc07e", "attributes": { "class": ".f4.list-style-none.py-2.mt-2.mt-md-0", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='3']", + "locator": "[_midscene_retrieve_task_id='5c2d0fc07e']", "center": [ 399, 36 @@ -57,13 +63,15 @@ } }, { - "id": "4", + "id": "7480b80cb3", + "indexId": "4", + "nodeHashId": "7480b80cb3", "attributes": { "class": ".mr-3.mr-lg-4.py-2", "href": "https://help.github.com", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='4']", + "locator": "[_midscene_retrieve_task_id='7480b80cb3']", "center": [ 41, 37 @@ -77,13 +85,15 @@ } }, { - "id": "5", + "id": "f3a0bad871", + "indexId": "5", + "nodeHashId": "f3a0bad871", "attributes": { "class": ".mr-3.mr-lg-4.py-2", "href": "https://github.community", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='5']", + "locator": "[_midscene_retrieve_task_id='f3a0bad871']", "center": [ 128, 37 @@ -97,13 +107,15 @@ } }, { - "id": "6", + "id": "2b27b9d4f4", + "indexId": "6", + "nodeHashId": "2b27b9d4f4", "attributes": { "class": ".py-2.text-gray", "href": "/", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='6']", + "locator": "[_midscene_retrieve_task_id='2b27b9d4f4']", "center": [ 220, 37 @@ -117,12 +129,14 @@ } }, { - "id": "7", + "id": "40bf454063", + "indexId": "7", + "nodeHashId": "40bf454063", "attributes": { "class": ".f4.list-style-none.py-2.text-md-right", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='7']", + "locator": "[_midscene_retrieve_task_id='40bf454063']", "center": [ 1522, 36 @@ -136,13 +150,15 @@ } }, { - "id": "8", + "id": "44c2ed902a", + "indexId": "8", + "nodeHashId": "44c2ed902a", "attributes": { "class": ".py-2", "href": "https://github.com", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='8']", + "locator": "[_midscene_retrieve_task_id='44c2ed902a']", "center": [ 1586, 37 @@ -156,13 +172,15 @@ } }, { - "id": "9", + "id": "7d552ded8a", + "indexId": "9", + "nodeHashId": "7d552ded8a", "attributes": { "class": ".py-2.ml-3.ml-lg-4", "href": "https://twitter.com/githubstatus", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='9']", + "locator": "[_midscene_retrieve_task_id='7d552ded8a']", "center": [ 1683, 37 @@ -176,12 +194,14 @@ } }, { - "id": "10", + "id": "c6687feb44", + "indexId": "10", + "nodeHashId": "c6687feb44", "attributes": { "class": ".d-inline.py-2.ml-3.ml-lg-4", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='10']", + "locator": "[_midscene_retrieve_task_id='c6687feb44']", "center": [ 1816, 37 @@ -195,14 +215,16 @@ } }, { - "id": "11", + "id": "9bf8cb5fec", + "indexId": "11", + "nodeHashId": "9bf8cb5fec", "attributes": { "class": ".updates-dropdown-container", "data-js-hook": "updates-dropdown-container", "style": "display: inline-block;", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='11']", + "locator": "[_midscene_retrieve_task_id='9bf8cb5fec']", "center": [ 1816, 36 @@ -216,8 +238,10 @@ } }, { - "id": "12", - "locator": "[_midscene_retrieve_task_id='12']", + "id": "4e12022b70", + "indexId": "12", + "nodeHashId": "4e12022b70", + "locator": "[_midscene_retrieve_task_id='4e12022b70']", "attributes": { "src": "https://user-images.githubusercontent.com/19292210/60553863-044dd200-9cea-11e9-987e-7db84449f215.png", "class": ".illo-desktop-header", @@ -238,12 +262,14 @@ ] }, { - "id": "13", + "id": "33bccce59f", + "indexId": "13", + "nodeHashId": "33bccce59f", "attributes": { "class": ".page-status.status-none", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='13']", + "locator": "[_midscene_retrieve_task_id='33bccce59f']", "center": [ 960, 428 @@ -257,12 +283,14 @@ } }, { - "id": "14", + "id": "221b0092a0", + "indexId": "14", + "nodeHashId": "221b0092a0", "attributes": { "class": ".status.font-large", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='14']", + "locator": "[_midscene_retrieve_task_id='221b0092a0']", "center": [ 632, 427 @@ -276,12 +304,14 @@ } }, { - "id": "15", + "id": "9cfcb84f47", + "indexId": "15", + "nodeHashId": "9cfcb84f47", "attributes": { "class": ".components-section.font-regular", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='15']", + "locator": "[_midscene_retrieve_task_id='9cfcb84f47']", "center": [ 960, 756 @@ -295,12 +325,14 @@ } }, { - "id": "16", + "id": "1d2f8b067e", + "indexId": "16", + "nodeHashId": "1d2f8b067e", "attributes": { "class": ".components-container.two-columns", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='16']", + "locator": "[_midscene_retrieve_task_id='1d2f8b067e']", "center": [ 960, 783 @@ -314,12 +346,14 @@ } }, { - "id": "17", + "id": "1775a8e5ac", + "indexId": "17", + "nodeHashId": "1775a8e5ac", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='17']", + "locator": "[_midscene_retrieve_task_id='1775a8e5ac']", "center": [ 717, 605 @@ -333,14 +367,16 @@ } }, { - "id": "18", + "id": "9b8133c565", + "indexId": "18", + "nodeHashId": "9b8133c565", "attributes": { "data-component-id": "8l4ygp009s5s", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='18']", + "locator": "[_midscene_retrieve_task_id='9b8133c565']", "center": [ 716, 606 @@ -354,12 +390,14 @@ } }, { - "id": "19", + "id": "786dafff55", + "indexId": "19", + "nodeHashId": "786dafff55", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='19']", + "locator": "[_midscene_retrieve_task_id='786dafff55']", "center": [ 546, 593 @@ -373,12 +411,14 @@ } }, { - "id": "20", + "id": "9fd504c355", + "indexId": "20", + "nodeHashId": "9fd504c355", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='20']", + "locator": "[_midscene_retrieve_task_id='9fd504c355']", "center": [ 618, 591 @@ -392,12 +432,14 @@ } }, { - "id": "21", + "id": "15865efd31", + "indexId": "21", + "nodeHashId": "15865efd31", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='21']", + "locator": "[_midscene_retrieve_task_id='15865efd31']", "center": [ 716, 619 @@ -411,12 +453,14 @@ } }, { - "id": "22", + "id": "cac86a3a9a", + "indexId": "22", + "nodeHashId": "cac86a3a9a", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='22']", + "locator": "[_midscene_retrieve_task_id='cac86a3a9a']", "center": [ 1206, 605 @@ -430,14 +474,16 @@ } }, { - "id": "23", + "id": "bfd2275c7b", + "indexId": "23", + "nodeHashId": "bfd2275c7b", "attributes": { "data-component-id": "brv1bkgrwx7q", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='23']", + "locator": "[_midscene_retrieve_task_id='bfd2275c7b']", "center": [ 1205, 606 @@ -451,12 +497,14 @@ } }, { - "id": "24", + "id": "244ab8c4c3", + "indexId": "24", + "nodeHashId": "244ab8c4c3", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='24']", + "locator": "[_midscene_retrieve_task_id='244ab8c4c3']", "center": [ 1030, 593 @@ -470,12 +518,14 @@ } }, { - "id": "25", + "id": "b0bdb14a2c", + "indexId": "25", + "nodeHashId": "b0bdb14a2c", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='25']", + "locator": "[_midscene_retrieve_task_id='b0bdb14a2c']", "center": [ 1097, 591 @@ -489,12 +539,14 @@ } }, { - "id": "26", + "id": "e4932f3f94", + "indexId": "26", + "nodeHashId": "e4932f3f94", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='26']", + "locator": "[_midscene_retrieve_task_id='e4932f3f94']", "center": [ 1205, 619 @@ -508,12 +560,14 @@ } }, { - "id": "27", + "id": "6ec8080ca6", + "indexId": "27", + "nodeHashId": "6ec8080ca6", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='27']", + "locator": "[_midscene_retrieve_task_id='6ec8080ca6']", "center": [ 717, 695 @@ -527,14 +581,16 @@ } }, { - "id": "28", + "id": "5d616d114f", + "indexId": "28", + "nodeHashId": "5d616d114f", "attributes": { "data-component-id": "4230lsnqdsld", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='28']", + "locator": "[_midscene_retrieve_task_id='5d616d114f']", "center": [ 716, 695 @@ -548,12 +604,14 @@ } }, { - "id": "29", + "id": "675456f129", + "indexId": "29", + "nodeHashId": "675456f129", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='29']", + "locator": "[_midscene_retrieve_task_id='675456f129']", "center": [ 531, 682 @@ -567,12 +625,14 @@ } }, { - "id": "30", + "id": "cd6731b4d0", + "indexId": "30", + "nodeHashId": "cd6731b4d0", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='30']", + "locator": "[_midscene_retrieve_task_id='cd6731b4d0']", "center": [ 588, 681 @@ -586,12 +646,14 @@ } }, { - "id": "31", + "id": "4abfbbde8a", + "indexId": "31", + "nodeHashId": "4abfbbde8a", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='31']", + "locator": "[_midscene_retrieve_task_id='4abfbbde8a']", "center": [ 716, 708 @@ -605,12 +667,14 @@ } }, { - "id": "32", + "id": "646e1d48fb", + "indexId": "32", + "nodeHashId": "646e1d48fb", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='32']", + "locator": "[_midscene_retrieve_task_id='646e1d48fb']", "center": [ 1206, 695 @@ -624,14 +688,16 @@ } }, { - "id": "33", + "id": "8b54cbe825", + "indexId": "33", + "nodeHashId": "8b54cbe825", "attributes": { "data-component-id": "kr09ddfgbfsf", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='33']", + "locator": "[_midscene_retrieve_task_id='8b54cbe825']", "center": [ 1205, 695 @@ -645,12 +711,14 @@ } }, { - "id": "34", + "id": "49b1a5e3e1", + "indexId": "34", + "nodeHashId": "49b1a5e3e1", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='34']", + "locator": "[_midscene_retrieve_task_id='49b1a5e3e1']", "center": [ 1005, 682 @@ -664,12 +732,14 @@ } }, { - "id": "35", + "id": "ce72a7bb29", + "indexId": "35", + "nodeHashId": "ce72a7bb29", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='35']", + "locator": "[_midscene_retrieve_task_id='ce72a7bb29']", "center": [ 1046, 681 @@ -683,12 +753,14 @@ } }, { - "id": "36", + "id": "44c6cea361", + "indexId": "36", + "nodeHashId": "44c6cea361", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='36']", + "locator": "[_midscene_retrieve_task_id='44c6cea361']", "center": [ 1205, 708 @@ -702,12 +774,14 @@ } }, { - "id": "37", + "id": "1a7bbd910b", + "indexId": "37", + "nodeHashId": "1a7bbd910b", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='37']", + "locator": "[_midscene_retrieve_task_id='1a7bbd910b']", "center": [ 717, 784 @@ -721,14 +795,16 @@ } }, { - "id": "38", + "id": "d04547b004", + "indexId": "38", + "nodeHashId": "d04547b004", "attributes": { "data-component-id": "hhtssxt0f5v2", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='38']", + "locator": "[_midscene_retrieve_task_id='d04547b004']", "center": [ 716, 785 @@ -742,12 +818,14 @@ } }, { - "id": "39", + "id": "0ef54614d8", + "indexId": "39", + "nodeHashId": "0ef54614d8", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='39']", + "locator": "[_midscene_retrieve_task_id='0ef54614d8']", "center": [ 542, 772 @@ -761,12 +839,14 @@ } }, { - "id": "40", + "id": "00228c1c08", + "indexId": "40", + "nodeHashId": "00228c1c08", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='40']", + "locator": "[_midscene_retrieve_task_id='00228c1c08']", "center": [ 611, 770 @@ -780,12 +860,14 @@ } }, { - "id": "41", + "id": "206fe45dc0", + "indexId": "41", + "nodeHashId": "206fe45dc0", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='41']", + "locator": "[_midscene_retrieve_task_id='206fe45dc0']", "center": [ 716, 798 @@ -799,12 +881,14 @@ } }, { - "id": "42", + "id": "075070b70a", + "indexId": "42", + "nodeHashId": "075070b70a", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='42']", + "locator": "[_midscene_retrieve_task_id='075070b70a']", "center": [ 1206, 784 @@ -818,14 +902,16 @@ } }, { - "id": "43", + "id": "ebbf5e2217", + "indexId": "43", + "nodeHashId": "ebbf5e2217", "attributes": { "data-component-id": "br0l2tvcx85d", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='43']", + "locator": "[_midscene_retrieve_task_id='ebbf5e2217']", "center": [ 1205, 785 @@ -839,12 +925,14 @@ } }, { - "id": "44", + "id": "89c422b218", + "indexId": "44", + "nodeHashId": "89c422b218", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='44']", + "locator": "[_midscene_retrieve_task_id='89c422b218']", "center": [ 1009, 772 @@ -858,12 +946,14 @@ } }, { - "id": "45", + "id": "e923998486", + "indexId": "45", + "nodeHashId": "e923998486", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='45']", + "locator": "[_midscene_retrieve_task_id='e923998486']", "center": [ 1055, 770 @@ -877,12 +967,14 @@ } }, { - "id": "46", + "id": "967447a1c6", + "indexId": "46", + "nodeHashId": "967447a1c6", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='46']", + "locator": "[_midscene_retrieve_task_id='967447a1c6']", "center": [ 1205, 798 @@ -896,12 +988,14 @@ } }, { - "id": "47", + "id": "11a7c1ed98", + "indexId": "47", + "nodeHashId": "11a7c1ed98", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='47']", + "locator": "[_midscene_retrieve_task_id='11a7c1ed98']", "center": [ 717, 874 @@ -915,14 +1009,16 @@ } }, { - "id": "48", + "id": "1c9544f01a", + "indexId": "48", + "nodeHashId": "1c9544f01a", "attributes": { "data-component-id": "st3j38cctv9l", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='48']", + "locator": "[_midscene_retrieve_task_id='1c9544f01a']", "center": [ 716, 874 @@ -936,12 +1032,14 @@ } }, { - "id": "49", + "id": "41a6c2ed19", + "indexId": "49", + "nodeHashId": "41a6c2ed19", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='49']", + "locator": "[_midscene_retrieve_task_id='41a6c2ed19']", "center": [ 528, 861 @@ -955,12 +1053,14 @@ } }, { - "id": "50", + "id": "b8f8f86b4c", + "indexId": "50", + "nodeHashId": "b8f8f86b4c", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='50']", + "locator": "[_midscene_retrieve_task_id='b8f8f86b4c']", "center": [ 581, 860 @@ -974,12 +1074,14 @@ } }, { - "id": "51", + "id": "4d10b3b116", + "indexId": "51", + "nodeHashId": "4d10b3b116", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='51']", + "locator": "[_midscene_retrieve_task_id='4d10b3b116']", "center": [ 716, 887 @@ -993,12 +1095,14 @@ } }, { - "id": "52", + "id": "7d79505914", + "indexId": "52", + "nodeHashId": "7d79505914", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='52']", + "locator": "[_midscene_retrieve_task_id='7d79505914']", "center": [ 1206, 874 @@ -1012,14 +1116,16 @@ } }, { - "id": "53", + "id": "e502c0d9c5", + "indexId": "53", + "nodeHashId": "e502c0d9c5", "attributes": { "data-component-id": "vg70hn9s2tyj", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='53']", + "locator": "[_midscene_retrieve_task_id='e502c0d9c5']", "center": [ 1205, 874 @@ -1033,12 +1139,14 @@ } }, { - "id": "54", + "id": "cae85d0269", + "indexId": "54", + "nodeHashId": "cae85d0269", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='54']", + "locator": "[_midscene_retrieve_task_id='cae85d0269']", "center": [ 1004, 861 @@ -1052,12 +1160,14 @@ } }, { - "id": "55", + "id": "85a6924c98", + "indexId": "55", + "nodeHashId": "85a6924c98", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='55']", + "locator": "[_midscene_retrieve_task_id='85a6924c98']", "center": [ 1045, 860 @@ -1071,12 +1181,14 @@ } }, { - "id": "56", + "id": "352bb8a0c7", + "indexId": "56", + "nodeHashId": "352bb8a0c7", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='56']", + "locator": "[_midscene_retrieve_task_id='352bb8a0c7']", "center": [ 1205, 887 @@ -1090,12 +1202,14 @@ } }, { - "id": "57", + "id": "7ebaccd865", + "indexId": "57", + "nodeHashId": "7ebaccd865", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='57']", + "locator": "[_midscene_retrieve_task_id='7ebaccd865']", "center": [ 717, 963 @@ -1109,14 +1223,16 @@ } }, { - "id": "58", + "id": "318eb23523", + "indexId": "58", + "nodeHashId": "318eb23523", "attributes": { "data-component-id": "h2ftsgbw7kmk", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='58']", + "locator": "[_midscene_retrieve_task_id='318eb23523']", "center": [ 716, 963 @@ -1130,12 +1246,14 @@ } }, { - "id": "59", + "id": "f397787172", + "indexId": "59", + "nodeHashId": "f397787172", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='59']", + "locator": "[_midscene_retrieve_task_id='f397787172']", "center": [ 538, 950 @@ -1149,12 +1267,14 @@ } }, { - "id": "60", + "id": "d7f5fbb4d0", + "indexId": "60", + "nodeHashId": "d7f5fbb4d0", "attributes": { "class": ".tooltip-base.tool.tooltipstered", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='60']", + "locator": "[_midscene_retrieve_task_id='d7f5fbb4d0']", "center": [ 602, 949 @@ -1168,12 +1288,14 @@ } }, { - "id": "61", + "id": "aafa155f98", + "indexId": "61", + "nodeHashId": "aafa155f98", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='61']", + "locator": "[_midscene_retrieve_task_id='aafa155f98']", "center": [ 716, 976 @@ -1187,12 +1309,14 @@ } }, { - "id": "62", + "id": "aa396a5b1d", + "indexId": "62", + "nodeHashId": "aa396a5b1d", "attributes": { "class": ".component-container.border-color", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='62']", + "locator": "[_midscene_retrieve_task_id='aa396a5b1d']", "center": [ 1206, 963 @@ -1206,14 +1330,16 @@ } }, { - "id": "63", + "id": "ecfafeca6d", + "indexId": "63", + "nodeHashId": "ecfafeca6d", "attributes": { "data-component-id": "pjmpxvq2cmr2", "class": ".component-inner-container.status-green.", "data-component-status": "operational", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='63']", + "locator": "[_midscene_retrieve_task_id='ecfafeca6d']", "center": [ 1205, 963 @@ -1227,12 +1353,14 @@ } }, { - "id": "64", + "id": "18dce47ff0", + "indexId": "64", + "nodeHashId": "18dce47ff0", "attributes": { "class": ".name", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='64']", + "locator": "[_midscene_retrieve_task_id='18dce47ff0']", "center": [ 1008, 950 @@ -1246,12 +1374,14 @@ } }, { - "id": "65", + "id": "3ac8478628", + "indexId": "65", + "nodeHashId": "3ac8478628", "attributes": { "class": ".status-msg", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='65']", + "locator": "[_midscene_retrieve_task_id='3ac8478628']", "center": [ 1205, 976 diff --git a/packages/midscene/tests/ai-model/inspector/test-data/xicha/element-snapshot.json b/packages/midscene/tests/ai-model/inspector/test-data/online_order/element-snapshot.json similarity index 71% rename from packages/midscene/tests/ai-model/inspector/test-data/xicha/element-snapshot.json rename to packages/midscene/tests/ai-model/inspector/test-data/online_order/element-snapshot.json index eaed8aded..75c108450 100644 --- a/packages/midscene/tests/ai-model/inspector/test-data/xicha/element-snapshot.json +++ b/packages/midscene/tests/ai-model/inspector/test-data/online_order/element-snapshot.json @@ -1,7 +1,9 @@ [ { - "id": "1", - "locator": "[_midscene_retrieve_task_id='1']", + "id": "6ad26dfdca", + "indexId": "1", + "nodeHashId": "6ad26dfdca", + "locator": "[_midscene_retrieve_task_id='6ad26dfdca']", "attributes": { "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAsBAMAAADsqkcyAAAABGdBTUEAALGPC/xhBQAAAAFzUkdCAK7OHOkAAAAhUExURUxpcfOtAPOtAPSuAPSuAPStAPOvAPKsAO+vAO+vAPSuADVGh/QAAAAKdFJOUwA/gN/FmG9QECBSnvm3AAAAUElEQVQoz2NgGDSAURALEGCIWoUFLGVYhRUweGETXcIwTQkLyGAYZqDYGAtwZ7DCHlRYRRczlGAzxHy4BVUbtgSRiSv54EhsOJImjoQ8aAAAnI6GWHtsGtQAAAAASUVORK5CYII=", "class": ".icon", @@ -20,12 +22,14 @@ ] }, { - "id": "2", + "id": "ba59909699", + "indexId": "2", + "nodeHashId": "ba59909699", "attributes": { "class": ".lang-item-text.lang-show-text", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='2']", + "locator": "[_midscene_retrieve_task_id='ba59909699']", "center": [ 81, 28 @@ -39,8 +43,10 @@ } }, { - "id": "3", - "locator": "[_midscene_retrieve_task_id='3']", + "id": "f05ef98d09", + "indexId": "3", + "nodeHashId": "f05ef98d09", + "locator": "[_midscene_retrieve_task_id='f05ef98d09']", "attributes": { "src": "https://dec-pub-img.obs.ap-southeast-3.myhuaweicloud.com/rfkle/16509415998546650.jpg", "class": ".img", @@ -59,8 +65,10 @@ ] }, { - "id": "4", - "locator": "[_midscene_retrieve_task_id='4']", + "id": "f775c69cb4", + "indexId": "4", + "nodeHashId": "f775c69cb4", + "locator": "[_midscene_retrieve_task_id='f775c69cb4']", "attributes": { "src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADIAAAAyCAYAAAAeP4ixAAAACXBIWXMAABYlAAAWJQFJUiTwAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAPeSURBVHgB7VlNTttAFH7PSauqqCqV+rPE7FCSSrlBHS5QeoLCCQonIJyA9ASkN+AE4GV3BDWQqBvCupUadqhK5vXN2E6m4BnbUWtbbT6JxMTjsb95733zzRhgiSWW+C+AaRtubHguVMGFHDHs+37atolEJIFKFY8IwIO8QTACxO5l/+QgqWklqcHLV+tnTLcJRQBhlT+95y/cm+/fRp9tTau2k7Wat82dufKYI/KRgI4hLwhwEXEfMfjmXzq25lYi4MwiMR70T3chZ9Tr3ioBHvLhqkzx4dAfmdo6to6IVGhlOMZQAMhJf18rES60teCLi64ACNLum6CYdiIFYyrmEXEEPLW1tRLBqNCJRlAAKuK31Hpma2snEtYIId5AAXj0aE6EMKxXA6yqxZKrLkage0XHivKeQ9ZhQRgLoPbwwv8EGaFPtiyxnYsvJ3v6+V7PH9caLXWMCUSMEWk2vdmFFKNaLIttRZTTz+GHgAXgVGE3cgycvrsbDc+71ygSGoI1a1+mE7e38xHAGCL6CFFB8qzDXCOa3AnnvvxOBc3SAJEyp5XqF2DuFIi6cSaRQgnm1HMtXS0uv8NLvxuFHQHfwAJgSXWjY4FgGAxU0Z5Nzqa+jCdQm4AmhgmRgkjIPI/Nbwtcl2sw8FBq1I2WnUSkmIsRQWG/UOJhFTq8DlAj5hAeqTVLSqw8cfbnhpTapnZci0H9Lapaum6bzJqUR0F0EN7IdSp4mkRGRqL+evNQqpS6D5CfIN2RkCxGRLvQqkiDC5+lVyNTxataffOIlwDNuwQaDe/D4xU8m5HglKIJ7Nj6B83n2QbJOCHK4kK5fkwhrZd9v11rePKu+8HNidcxuF1vtMYUpQaBK4IHC/8ln6awY7PmWWCOSEbnK8mICa1z4fvRb2rCZALqb44xR2Rv0PdbaUikdcD2hVVGhA/WUingsO1gg8ADoVwrR6bHxM6zbChISAfshMNtc8BGIsr5krIOmQ1jSKgLfwDKAc/zxuiAzfI7n4B+QIHQ08/mgM3yG6kWwjWUBJiVSJLzzR0pHHAskSTnW0bEp1aC880baRxwqTcfZiC6Dr4y1ojufKeiVKmVjYjufCslIDJzF1lVK43zzROa4GSeR1I537ygb52aHHB8RAre870LSjGg8REJnS+UJSL6gBoccNKWaTkUS9szMDlgA5Fw5wIKelN1B2zjveiYJ+hYNx5r4xHEOX9u8eFqrd66Yko+FAie0bd4UNUMf/sAerFt4n6UpvHnBE+hJBGJMJnSu68D/zjunPGtbkAG5CbBWx6SxK2hvwn5WoMjcpB1dbnEEkv8g/gF7BF6kDDXbdkAAAAASUVORK5CYII=", "class": ".icon", @@ -79,11 +87,13 @@ ] }, { - "id": "5", + "id": "67bd3fba57", + "indexId": "5", + "nodeHashId": "67bd3fba57", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='5']", + "locator": "[_midscene_retrieve_task_id='67bd3fba57']", "center": [ 200, 80 @@ -97,11 +107,13 @@ } }, { - "id": "6", + "id": "c900f4b633", + "indexId": "6", + "nodeHashId": "c900f4b633", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='6']", + "locator": "[_midscene_retrieve_task_id='c900f4b633']", "center": [ 209, 98 @@ -115,11 +127,13 @@ } }, { - "id": "7", + "id": "6f761c984e", + "indexId": "7", + "nodeHashId": "6f761c984e", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='7']", + "locator": "[_midscene_retrieve_task_id='6f761c984e']", "center": [ 209, 133 @@ -133,8 +147,10 @@ } }, { - "id": "8", - "locator": "[_midscene_retrieve_task_id='8']", + "id": "49118bbc89", + "indexId": "8", + "nodeHashId": "49118bbc89", + "locator": "[_midscene_retrieve_task_id='49118bbc89']", "attributes": { "src": "https://dec-pub-img.obs.ap-southeast-3.myhuaweicloud.com/rfkle/16558697767831080.jpeg", "nodeType": "IMG Node" @@ -152,11 +168,13 @@ ] }, { - "id": "9", + "id": "31c3a4bba6", + "indexId": "9", + "nodeHashId": "31c3a4bba6", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='9']", + "locator": "[_midscene_retrieve_task_id='31c3a4bba6']", "center": [ 200, 482 @@ -170,11 +188,13 @@ } }, { - "id": "10", + "id": "079f969064", + "indexId": "10", + "nodeHashId": "079f969064", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='10']", + "locator": "[_midscene_retrieve_task_id='079f969064']", "center": [ 128, 569 @@ -188,11 +208,13 @@ } }, { - "id": "11", + "id": "b0931353bd", + "indexId": "11", + "nodeHashId": "b0931353bd", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='11']", + "locator": "[_midscene_retrieve_task_id='b0931353bd']", "center": [ 200, 569 @@ -206,11 +228,13 @@ } }, { - "id": "12", + "id": "afab9cd02f", + "indexId": "12", + "nodeHashId": "afab9cd02f", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='12']", + "locator": "[_midscene_retrieve_task_id='afab9cd02f']", "center": [ 272, 569 @@ -224,11 +248,13 @@ } }, { - "id": "13", + "id": "88e7fb5aed", + "indexId": "13", + "nodeHashId": "88e7fb5aed", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='13']", + "locator": "[_midscene_retrieve_task_id='88e7fb5aed']", "center": [ 29, 623 @@ -242,11 +268,13 @@ } }, { - "id": "14", + "id": "cee3c6676f", + "indexId": "14", + "nodeHashId": "cee3c6676f", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='14']", + "locator": "[_midscene_retrieve_task_id='cee3c6676f']", "center": [ 35, 683 @@ -260,11 +288,13 @@ } }, { - "id": "15", + "id": "512ab699a2", + "indexId": "15", + "nodeHashId": "512ab699a2", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='15']", + "locator": "[_midscene_retrieve_task_id='512ab699a2']", "center": [ 35, 743 @@ -278,11 +308,13 @@ } }, { - "id": "16", + "id": "3948acb8ad", + "indexId": "16", + "nodeHashId": "3948acb8ad", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='16']", + "locator": "[_midscene_retrieve_task_id='3948acb8ad']", "center": [ 23, 803 @@ -296,11 +328,13 @@ } }, { - "id": "17", + "id": "985ed5922f", + "indexId": "17", + "nodeHashId": "985ed5922f", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='17']", + "locator": "[_midscene_retrieve_task_id='985ed5922f']", "center": [ 23, 863 @@ -314,11 +348,13 @@ } }, { - "id": "18", + "id": "e54bc06de4", + "indexId": "18", + "nodeHashId": "e54bc06de4", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='18']", + "locator": "[_midscene_retrieve_task_id='e54bc06de4']", "center": [ 126, 633 @@ -332,8 +368,10 @@ } }, { - "id": "19", - "locator": "[_midscene_retrieve_task_id='19']", + "id": "02bf1e20e5", + "indexId": "19", + "nodeHashId": "02bf1e20e5", + "locator": "[_midscene_retrieve_task_id='02bf1e20e5']", "attributes": { "src": "https://dec-pub-img.obs.ap-southeast-3.myhuaweicloud.com/djlxq/16509428649460869.jpeg", "class": ".img", @@ -352,11 +390,13 @@ ] }, { - "id": "20", + "id": "c93bedf4d3", + "indexId": "20", + "nodeHashId": "c93bedf4d3", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='20']", + "locator": "[_midscene_retrieve_task_id='c93bedf4d3']", "center": [ 288, 670 @@ -370,12 +410,14 @@ } }, { - "id": "21", + "id": "1b673db570", + "indexId": "21", + "nodeHashId": "1b673db570", "attributes": { "class": ".van-ellipsis", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='21']", + "locator": "[_midscene_retrieve_task_id='1b673db570']", "center": [ 288, 691 @@ -389,12 +431,14 @@ } }, { - "id": "22", + "id": "14103376fb", + "indexId": "22", + "nodeHashId": "14103376fb", "attributes": { "class": ".price", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='22']", + "locator": "[_midscene_retrieve_task_id='14103376fb']", "center": [ 211, 736 @@ -408,12 +452,14 @@ } }, { - "id": "23", + "id": "580cfae23c", + "indexId": "23", + "nodeHashId": "580cfae23c", "attributes": { "class": ".handle-btn", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='23']", + "locator": "[_midscene_retrieve_task_id='580cfae23c']", "center": [ 355, 732 @@ -427,11 +473,13 @@ } }, { - "id": "24", + "id": "7f10fa6626", + "indexId": "24", + "nodeHashId": "7f10fa6626", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='24']", + "locator": "[_midscene_retrieve_task_id='7f10fa6626']", "center": [ 134, 776 @@ -445,8 +493,10 @@ } }, { - "id": "25", - "locator": "[_midscene_retrieve_task_id='25']", + "id": "eb294263b8", + "indexId": "25", + "nodeHashId": "eb294263b8", + "locator": "[_midscene_retrieve_task_id='eb294263b8']", "attributes": { "src": "https://dec-pub-img.obs.ap-southeast-3.myhuaweicloud.com/djlxq/16509428062990437.jpeg", "class": ".img", @@ -465,11 +515,13 @@ ] }, { - "id": "26", + "id": "2e297d0b4a", + "indexId": "26", + "nodeHashId": "2e297d0b4a", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='26']", + "locator": "[_midscene_retrieve_task_id='2e297d0b4a']", "center": [ 288, 813 @@ -483,12 +535,14 @@ } }, { - "id": "27", + "id": "9f0e2a5328", + "indexId": "27", + "nodeHashId": "9f0e2a5328", "attributes": { "class": ".van-ellipsis", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='27']", + "locator": "[_midscene_retrieve_task_id='9f0e2a5328']", "center": [ 288, 834 @@ -502,12 +556,14 @@ } }, { - "id": "28", + "id": "0250e12e67", + "indexId": "28", + "nodeHashId": "0250e12e67", "attributes": { "class": ".price", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='28']", + "locator": "[_midscene_retrieve_task_id='0250e12e67']", "center": [ 211, 879 @@ -521,12 +577,14 @@ } }, { - "id": "29", + "id": "925c254744", + "indexId": "29", + "nodeHashId": "925c254744", "attributes": { "class": ".handle-btn", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='29']", + "locator": "[_midscene_retrieve_task_id='925c254744']", "center": [ 355, 875 @@ -540,8 +598,10 @@ } }, { - "id": "30", - "locator": "[_midscene_retrieve_task_id='30']", + "id": "cad3004a2d", + "indexId": "30", + "nodeHashId": "cad3004a2d", + "locator": "[_midscene_retrieve_task_id='cad3004a2d']", "attributes": { "src": "/img/customer.8a8d9dc9.png", "nodeType": "IMG Node" diff --git a/packages/midscene/tests/ai-model/inspector/test-data/xicha/input.png b/packages/midscene/tests/ai-model/inspector/test-data/online_order/input.png similarity index 100% rename from packages/midscene/tests/ai-model/inspector/test-data/xicha/input.png rename to packages/midscene/tests/ai-model/inspector/test-data/online_order/input.png diff --git a/packages/midscene/tests/ai-model/inspector/test-data/xicha/output.png b/packages/midscene/tests/ai-model/inspector/test-data/online_order/output.png similarity index 100% rename from packages/midscene/tests/ai-model/inspector/test-data/xicha/output.png rename to packages/midscene/tests/ai-model/inspector/test-data/online_order/output.png diff --git a/packages/midscene/tests/ai-model/inspector/test-data/xicha/output_without_text.png b/packages/midscene/tests/ai-model/inspector/test-data/online_order/output_without_text.png similarity index 100% rename from packages/midscene/tests/ai-model/inspector/test-data/xicha/output_without_text.png rename to packages/midscene/tests/ai-model/inspector/test-data/online_order/output_without_text.png diff --git a/packages/midscene/tests/ai-model/inspector/test-data/xicha/resize-output.png b/packages/midscene/tests/ai-model/inspector/test-data/online_order/resize-output.png similarity index 100% rename from packages/midscene/tests/ai-model/inspector/test-data/xicha/resize-output.png rename to packages/midscene/tests/ai-model/inspector/test-data/online_order/resize-output.png diff --git a/packages/midscene/tests/ai-model/inspector/test-data/todo/element-snapshot.json b/packages/midscene/tests/ai-model/inspector/test-data/todo/element-snapshot.json index 09b5e2a7e..503a15545 100644 --- a/packages/midscene/tests/ai-model/inspector/test-data/todo/element-snapshot.json +++ b/packages/midscene/tests/ai-model/inspector/test-data/todo/element-snapshot.json @@ -1,10 +1,12 @@ [ { - "id": "1", + "id": "ab22d01377", + "indexId": "1", + "nodeHashId": "ab22d01377", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='1']", + "locator": "[_midscene_retrieve_task_id='ab22d01377']", "center": [ 640, 66 @@ -18,8 +20,10 @@ } }, { - "id": "2", - "locator": "[_midscene_retrieve_task_id='2']", + "id": "3530a9c1eb", + "indexId": "2", + "nodeHashId": "3530a9c1eb", + "locator": "[_midscene_retrieve_task_id='3530a9c1eb']", "attributes": { "class": ".new-todo", "id": "todo-input", @@ -41,8 +45,10 @@ ] }, { - "id": "3", - "locator": "[_midscene_retrieve_task_id='3']", + "id": "eb02ad0e19", + "indexId": "3", + "nodeHashId": "eb02ad0e19", + "locator": "[_midscene_retrieve_task_id='eb02ad0e19']", "attributes": { "class": ".toggle-all", "type": "checkbox", @@ -62,13 +68,15 @@ ] }, { - "id": "4", + "id": "22625b5e51", + "indexId": "4", + "nodeHashId": "22625b5e51", "attributes": { "class": ".toggle-all-label", "for": "toggle-all", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='4']", + "locator": "[_midscene_retrieve_task_id='22625b5e51']", "center": [ 388, 164 @@ -82,8 +90,10 @@ } }, { - "id": "5", - "locator": "[_midscene_retrieve_task_id='5']", + "id": "e0a509c9a3", + "indexId": "5", + "nodeHashId": "e0a509c9a3", + "locator": "[_midscene_retrieve_task_id='e0a509c9a3']", "attributes": { "class": ".toggle", "type": "checkbox", @@ -103,12 +113,14 @@ ] }, { - "id": "6", + "id": "46449cb0ef", + "indexId": "6", + "nodeHashId": "46449cb0ef", "attributes": { "data-testid": "todo-item-label", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='6']", + "locator": "[_midscene_retrieve_task_id='46449cb0ef']", "center": [ 640, 226 @@ -122,8 +134,10 @@ } }, { - "id": "7", - "locator": "[_midscene_retrieve_task_id='7']", + "id": "c0751f3b26", + "indexId": "7", + "nodeHashId": "c0751f3b26", + "locator": "[_midscene_retrieve_task_id='c0751f3b26']", "attributes": { "class": ".toggle", "type": "checkbox", @@ -143,12 +157,14 @@ ] }, { - "id": "8", + "id": "b5bacc879a", + "indexId": "8", + "nodeHashId": "b5bacc879a", "attributes": { "data-testid": "todo-item-label", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='8']", + "locator": "[_midscene_retrieve_task_id='b5bacc879a']", "center": [ 640, 286 @@ -162,8 +178,10 @@ } }, { - "id": "9", - "locator": "[_midscene_retrieve_task_id='9']", + "id": "7ccd467339", + "indexId": "9", + "nodeHashId": "7ccd467339", + "locator": "[_midscene_retrieve_task_id='7ccd467339']", "attributes": { "class": ".destroy", "data-testid": "todo-item-button", @@ -182,8 +200,10 @@ ] }, { - "id": "10", - "locator": "[_midscene_retrieve_task_id='10']", + "id": "eb987bf616", + "indexId": "10", + "nodeHashId": "eb987bf616", + "locator": "[_midscene_retrieve_task_id='eb987bf616']", "attributes": { "class": ".toggle", "type": "checkbox", @@ -203,12 +223,14 @@ ] }, { - "id": "11", + "id": "6c30e37d29", + "indexId": "11", + "nodeHashId": "6c30e37d29", "attributes": { "data-testid": "todo-item-label", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='11']", + "locator": "[_midscene_retrieve_task_id='6c30e37d29']", "center": [ 640, 346 @@ -222,12 +244,14 @@ } }, { - "id": "12", + "id": "12fb207e82", + "indexId": "12", + "nodeHashId": "12fb207e82", "attributes": { "class": ".todo-count", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='12']", + "locator": "[_midscene_retrieve_task_id='12fb207e82']", "center": [ 416, 395 @@ -241,13 +265,15 @@ } }, { - "id": "13", + "id": "944c1fae15", + "indexId": "13", + "nodeHashId": "944c1fae15", "attributes": { "class": ".selected", "href": "#/", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='13']", + "locator": "[_midscene_retrieve_task_id='944c1fae15']", "center": [ 565, 395 @@ -261,13 +287,15 @@ } }, { - "id": "14", + "id": "fc1a3e34a0", + "indexId": "14", + "nodeHashId": "fc1a3e34a0", "attributes": { "class": ".", "href": "#/active", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='14']", + "locator": "[_midscene_retrieve_task_id='fc1a3e34a0']", "center": [ 613, 395 @@ -281,13 +309,15 @@ } }, { - "id": "15", + "id": "0f8f471e06", + "indexId": "15", + "nodeHashId": "0f8f471e06", "attributes": { "class": ".", "href": "#/completed", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='15']", + "locator": "[_midscene_retrieve_task_id='0f8f471e06']", "center": [ 688, 395 @@ -301,8 +331,10 @@ } }, { - "id": "16", - "locator": "[_midscene_retrieve_task_id='16']", + "id": "84b6988e83", + "indexId": "16", + "nodeHashId": "84b6988e83", + "locator": "[_midscene_retrieve_task_id='84b6988e83']", "attributes": { "class": ".clear-completed", "nodeType": "BUTTON Node" @@ -320,11 +352,13 @@ ] }, { - "id": "17", + "id": "586415981c", + "indexId": "17", + "nodeHashId": "586415981c", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='17']", + "locator": "[_midscene_retrieve_task_id='586415981c']", "center": [ 640, 486 @@ -338,11 +372,13 @@ } }, { - "id": "18", + "id": "d501ec8b0c", + "indexId": "18", + "nodeHashId": "d501ec8b0c", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='18']", + "locator": "[_midscene_retrieve_task_id='d501ec8b0c']", "center": [ 640, 508 @@ -356,11 +392,13 @@ } }, { - "id": "19", + "id": "332bc0052f", + "indexId": "19", + "nodeHashId": "332bc0052f", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='19']", + "locator": "[_midscene_retrieve_task_id='332bc0052f']", "center": [ 640, 530 @@ -374,12 +412,14 @@ } }, { - "id": "20", + "id": "70ba39c5c6", + "indexId": "20", + "nodeHashId": "70ba39c5c6", "attributes": { "href": "http://todomvc.com", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='20']", + "locator": "[_midscene_retrieve_task_id='70ba39c5c6']", "center": [ 656, 530 diff --git a/packages/midscene/tests/ai-model/inspector/test-data/visualstudio/element-snapshot.json b/packages/midscene/tests/ai-model/inspector/test-data/visualstudio/element-snapshot.json index 0d7eea97a..826ee887b 100644 --- a/packages/midscene/tests/ai-model/inspector/test-data/visualstudio/element-snapshot.json +++ b/packages/midscene/tests/ai-model/inspector/test-data/visualstudio/element-snapshot.json @@ -1,11 +1,13 @@ [ { - "id": "1", + "id": "bd82c77920", + "indexId": "1", + "nodeHashId": "bd82c77920", "attributes": { "class": ".navbar-fixed-container", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='1']", + "locator": "[_midscene_retrieve_task_id='bd82c77920']", "center": [ 640, 29 @@ -19,14 +21,16 @@ } }, { - "id": "2", + "id": "bd82c77920", + "indexId": "2", + "nodeHashId": "bd82c77920", "attributes": { "class": ".navbar.navbar-inverse.navbar-fixed-top.affix-top", "data-spy": "affix", "data-offset-top": "1", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='2']", + "locator": "[_midscene_retrieve_task_id='bd82c77920']", "center": [ 640, 29 @@ -40,13 +44,15 @@ } }, { - "id": "3", + "id": "bd82c77920", + "indexId": "3", + "nodeHashId": "bd82c77920", "attributes": { "role": "navigation", "aria-label": "Top Level", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='3']", + "locator": "[_midscene_retrieve_task_id='bd82c77920']", "center": [ 640, 29 @@ -60,12 +66,14 @@ } }, { - "id": "4", + "id": "4a1d2260db", + "indexId": "4", + "nodeHashId": "4a1d2260db", "attributes": { "class": ".container", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='4']", + "locator": "[_midscene_retrieve_task_id='4a1d2260db']", "center": [ 640, 29 @@ -79,12 +87,14 @@ } }, { - "id": "5", + "id": "ae5debb6cf", + "indexId": "5", + "nodeHashId": "ae5debb6cf", "attributes": { "class": ".nav.navbar-header", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='5']", + "locator": "[_midscene_retrieve_task_id='ae5debb6cf']", "center": [ 191, 29 @@ -98,11 +108,13 @@ } }, { - "id": "6", + "id": "a482926fdd", + "indexId": "6", + "nodeHashId": "a482926fdd", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='6']", + "locator": "[_midscene_retrieve_task_id='a482926fdd']", "center": [ 201, 29 @@ -116,13 +128,15 @@ } }, { - "id": "7", + "id": "894271f682", + "indexId": "7", + "nodeHashId": "894271f682", "attributes": { "class": ".navbar-collapse.collapse", "style": "max-height: min(60vh, 400px);", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='7']", + "locator": "[_midscene_retrieve_task_id='894271f682']", "center": [ 640, 29 @@ -136,12 +150,14 @@ } }, { - "id": "8", + "id": "38057e67de", + "indexId": "8", + "nodeHashId": "38057e67de", "attributes": { "class": ".nav.navbar-nav.navbar-left", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='8']", + "locator": "[_midscene_retrieve_task_id='38057e67de']", "center": [ 541, 29 @@ -155,13 +171,15 @@ } }, { - "id": "9", + "id": "8b9d1a6179", + "indexId": "9", + "nodeHashId": "8b9d1a6179", "attributes": { "id": "nav-docs", "href": "/docs", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='9']", + "locator": "[_midscene_retrieve_task_id='8b9d1a6179']", "center": [ 332, 29 @@ -175,13 +193,15 @@ } }, { - "id": "10", + "id": "42af1c2281", + "indexId": "10", + "nodeHashId": "42af1c2281", "attributes": { "id": "nav-updates", "href": "/updates", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='10']", + "locator": "[_midscene_retrieve_task_id='42af1c2281']", "center": [ 407, 29 @@ -195,13 +215,15 @@ } }, { - "id": "11", + "id": "2b762b1507", + "indexId": "11", + "nodeHashId": "2b762b1507", "attributes": { "id": "nav-blogs", "href": "/blogs", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='11']", + "locator": "[_midscene_retrieve_task_id='2b762b1507']", "center": [ 478, 29 @@ -215,13 +237,15 @@ } }, { - "id": "12", + "id": "fbd4626b77", + "indexId": "12", + "nodeHashId": "fbd4626b77", "attributes": { "id": "nav-extend", "href": "/api", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='12']", + "locator": "[_midscene_retrieve_task_id='fbd4626b77']", "center": [ 532, 29 @@ -235,7 +259,9 @@ } }, { - "id": "13", + "id": "839a7fdd30", + "indexId": "13", + "nodeHashId": "839a7fdd30", "attributes": { "href": "https://marketplace.visualstudio.com/VSCode", "target": "_blank", @@ -243,7 +269,7 @@ "id": "nav-extensions", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='13']", + "locator": "[_midscene_retrieve_task_id='839a7fdd30']", "center": [ 609, 29 @@ -257,13 +283,15 @@ } }, { - "id": "14", + "id": "c292124789", + "indexId": "14", + "nodeHashId": "c292124789", "attributes": { "id": "nav-faqs", "href": "/docs/supporting/faq", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='14']", + "locator": "[_midscene_retrieve_task_id='c292124789']", "center": [ 688, 29 @@ -277,13 +305,15 @@ } }, { - "id": "15", + "id": "606008ee28", + "indexId": "15", + "nodeHashId": "606008ee28", "attributes": { "id": "nav-learn", "href": "/learn", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='15']", + "locator": "[_midscene_retrieve_task_id='606008ee28']", "center": [ 748, 29 @@ -297,13 +327,15 @@ } }, { - "id": "16", + "id": "226cfbfa70", + "indexId": "16", + "nodeHashId": "226cfbfa70", "attributes": { "class": ".nav.navbar-nav.navbar-right", "role": "presentation", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='16']", + "locator": "[_midscene_retrieve_task_id='226cfbfa70']", "center": [ 1025, 29 @@ -317,11 +349,14 @@ } }, { - "id": "17", - "locator": "[_midscene_retrieve_task_id='17']", + "id": "e72ee416b2", + "indexId": "17", + "nodeHashId": "e72ee416b2", + "locator": "[_midscene_retrieve_task_id='e72ee416b2']", "attributes": { "type": "button", "class": ".theme-switch", + "id": "theme-toggle", "nodeType": "BUTTON Node" }, "content": "", @@ -337,8 +372,10 @@ ] }, { - "id": "18", - "locator": "[_midscene_retrieve_task_id='18']", + "id": "ffbf775768", + "indexId": "18", + "nodeHashId": "ffbf775768", + "locator": "[_midscene_retrieve_task_id='ffbf775768']", "attributes": { "class": ".theme-icon-dark", "src": "/assets/icons/theme-dark.svg", @@ -358,8 +395,10 @@ ] }, { - "id": "19", - "locator": "[_midscene_retrieve_task_id='19']", + "id": "0c513c2cab", + "indexId": "19", + "nodeHashId": "0c513c2cab", + "locator": "[_midscene_retrieve_task_id='0c513c2cab']", "attributes": { "type": "text", "name": "q", @@ -381,8 +420,10 @@ ] }, { - "id": "20", - "locator": "[_midscene_retrieve_task_id='20']", + "id": "d76390b753", + "indexId": "20", + "nodeHashId": "d76390b753", + "locator": "[_midscene_retrieve_task_id='d76390b753']", "attributes": { "tabindex": "0", "class": ".btn", @@ -403,8 +444,10 @@ ] }, { - "id": "21", - "locator": "[_midscene_retrieve_task_id='21']", + "id": "eec7cedece", + "indexId": "21", + "nodeHashId": "eec7cedece", + "locator": "[_midscene_retrieve_task_id='eec7cedece']", "attributes": { "class": ".search-icon-light", "src": "/assets/icons/search.svg", @@ -424,14 +467,16 @@ ] }, { - "id": "22", + "id": "23914e93fd", + "indexId": "22", + "nodeHashId": "23914e93fd", "attributes": { "class": ".link-button", "href": "/Download", "id": "nav-download", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='22']", + "locator": "[_midscene_retrieve_task_id='23914e93fd']", "center": [ 1147, 29 @@ -445,11 +490,13 @@ } }, { - "id": "23", + "id": "7a1a2d8f0e", + "indexId": "23", + "nodeHashId": "7a1a2d8f0e", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='23']", + "locator": "[_midscene_retrieve_task_id='7a1a2d8f0e']", "center": [ 1147, 30 @@ -463,12 +510,14 @@ } }, { - "id": "24", + "id": "398138147f", + "indexId": "24", + "nodeHashId": "398138147f", "attributes": { "class": ".updates-banner.home.", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='24']", + "locator": "[_midscene_retrieve_task_id='398138147f']", "center": [ 640, 85 @@ -482,12 +531,14 @@ } }, { - "id": "25", + "id": "c82d77bcb3", + "indexId": "25", + "nodeHashId": "c82d77bcb3", "attributes": { "class": ".container", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='25']", + "locator": "[_midscene_retrieve_task_id='c82d77bcb3']", "center": [ 624, 85 @@ -501,12 +552,14 @@ } }, { - "id": "26", + "id": "553bf30386", + "indexId": "26", + "nodeHashId": "553bf30386", "attributes": { "class": ".message", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='26']", + "locator": "[_midscene_retrieve_task_id='553bf30386']", "center": [ 624, 85 @@ -520,13 +573,15 @@ } }, { - "id": "27", + "id": "c4a6236bc9", + "indexId": "27", + "nodeHashId": "c4a6236bc9", "attributes": { "href": "/updates", "id": "banner-link-updates", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='27']", + "locator": "[_midscene_retrieve_task_id='c4a6236bc9']", "center": [ 404, 86 @@ -540,12 +595,14 @@ } }, { - "id": "28", + "id": "79cb132d74", + "indexId": "28", + "nodeHashId": "79cb132d74", "attributes": { "class": ".container.hero-content", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='28']", + "locator": "[_midscene_retrieve_task_id='79cb132d74']", "center": [ 640, 394 @@ -559,12 +616,14 @@ } }, { - "id": "29", + "id": "40e6911724", + "indexId": "29", + "nodeHashId": "40e6911724", "attributes": { "class": ".hero-text.col-sm-12.col-md-4", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='29']", + "locator": "[_midscene_retrieve_task_id='40e6911724']", "center": [ 240, 394 @@ -578,12 +637,14 @@ } }, { - "id": "30", + "id": "aed37e3385", + "indexId": "30", + "nodeHashId": "aed37e3385", "attributes": { "class": ".hero-tag", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='30']", + "locator": "[_midscene_retrieve_task_id='aed37e3385']", "center": [ 246, 195 @@ -597,11 +658,13 @@ } }, { - "id": "31", + "id": "fad8ef3f4d", + "indexId": "31", + "nodeHashId": "fad8ef3f4d", "attributes": { "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='31']", + "locator": "[_midscene_retrieve_task_id='fad8ef3f4d']", "center": [ 240, 337 @@ -615,12 +678,14 @@ } }, { - "id": "32", + "id": "2f9412a521", + "indexId": "32", + "nodeHashId": "2f9412a521", "attributes": { "class": ".download-content-wrapper", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='32']", + "locator": "[_midscene_retrieve_task_id='2f9412a521']", "center": [ 240, 543 @@ -634,13 +699,15 @@ } }, { - "id": "33", + "id": "d79e1a9930", + "indexId": "33", + "nodeHashId": "d79e1a9930", "attributes": { "id": "download-buttons", "class": ".download-hero.alt-downloads.win", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='33']", + "locator": "[_midscene_retrieve_task_id='d79e1a9930']", "center": [ 240, 499 @@ -654,12 +721,14 @@ } }, { - "id": "34", + "id": "d79e1a9930", + "indexId": "34", + "nodeHashId": "d79e1a9930", "attributes": { "class": ".primary-buttons", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='34']", + "locator": "[_midscene_retrieve_task_id='d79e1a9930']", "center": [ 240, 499 @@ -673,8 +742,10 @@ } }, { - "id": "35", - "locator": "[_midscene_retrieve_task_id='35']", + "id": "a115c043a2", + "indexId": "35", + "nodeHashId": "a115c043a2", + "locator": "[_midscene_retrieve_task_id='a115c043a2']", "attributes": { "class": ".link-button.dlink", "data-os": "win", @@ -694,13 +765,15 @@ ] }, { - "id": "36", + "id": "c755a574e2", + "indexId": "36", + "nodeHashId": "c755a574e2", "attributes": { "id": "download-matrix-label", "class": ".subtext", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='36']", + "locator": "[_midscene_retrieve_task_id='c755a574e2']", "center": [ 240, 551 @@ -714,7 +787,9 @@ } }, { - "id": "37", + "id": "ea2585add6", + "indexId": "37", + "nodeHashId": "ea2585add6", "attributes": { "href": "https://vscode.dev/", "target": "_blank", @@ -723,7 +798,7 @@ "id": "download-buttons-web", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='37']", + "locator": "[_midscene_retrieve_task_id='ea2585add6']", "center": [ 95, 552 @@ -737,13 +812,15 @@ } }, { - "id": "38", + "id": "34a6664272", + "indexId": "38", + "nodeHashId": "34a6664272", "attributes": { "href": "/insiders", "id": "download-buttons-insiders", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='38']", + "locator": "[_midscene_retrieve_task_id='34a6664272']", "center": [ 168, 552 @@ -757,12 +834,14 @@ } }, { - "id": "39", + "id": "1ef8370ece", + "indexId": "39", + "nodeHashId": "1ef8370ece", "attributes": { "href": "/Download", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='39']", + "locator": "[_midscene_retrieve_task_id='1ef8370ece']", "center": [ 295, 552 @@ -776,12 +855,14 @@ } }, { - "id": "40", + "id": "fc5a82d77a", + "indexId": "40", + "nodeHashId": "fc5a82d77a", "attributes": { "class": ".subtext.terms", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='40']", + "locator": "[_midscene_retrieve_task_id='fc5a82d77a']", "center": [ 240, 596 @@ -795,12 +876,14 @@ } }, { - "id": "41", + "id": "62187837c8", + "indexId": "41", + "nodeHashId": "62187837c8", "attributes": { "class": ".wrap-together", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='41']", + "locator": "[_midscene_retrieve_task_id='62187837c8']", "center": [ 160, 604 @@ -814,7 +897,9 @@ } }, { - "id": "42", + "id": "deba00f856", + "indexId": "42", + "nodeHashId": "deba00f856", "attributes": { "href": "https://code.visualstudio.com/license", "target": "_blank", @@ -822,7 +907,7 @@ "title": "View the Visual Studio Code license.", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='42']", + "locator": "[_midscene_retrieve_task_id='deba00f856']", "center": [ 99, 604 @@ -836,7 +921,9 @@ } }, { - "id": "43", + "id": "324717c5ff", + "indexId": "43", + "nodeHashId": "324717c5ff", "attributes": { "href": "https://go.microsoft.com/fwlink/?LinkId=521839", "target": "_blank", @@ -844,7 +931,7 @@ "title": "View the Microsoft privacy statement.", "nodeType": "TEXT Node" }, - "locator": "[_midscene_retrieve_task_id='43']", + "locator": "[_midscene_retrieve_task_id='324717c5ff']", "center": [ 190, 604 @@ -858,8 +945,10 @@ } }, { - "id": "44", - "locator": "[_midscene_retrieve_task_id='44']", + "id": "7eb7a9b4da", + "indexId": "44", + "nodeHashId": "7eb7a9b4da", + "locator": "[_midscene_retrieve_task_id='7eb7a9b4da']", "attributes": { "src": "/assets/home/home-screenshot-mac-2x-v2-light.png", "alt": "Visual Studio Code in action with AI-powered suggestions from GitHub Copilot, built-in terminal and powerful extensions for all languages and tools", diff --git a/packages/midscene/tests/ai-model/inspector/todo_inspector.test.ts b/packages/midscene/tests/ai-model/inspector/todo_inspector.test.ts index 357a66f48..fde5a6ab6 100644 --- a/packages/midscene/tests/ai-model/inspector/todo_inspector.test.ts +++ b/packages/midscene/tests/ai-model/inspector/todo_inspector.test.ts @@ -1,9 +1,8 @@ -import { it, expect } from 'vitest'; import path from 'path'; +import { it, expect } from 'vitest'; import { getPageTestData, repeat, runTestCases, writeFileSyncWithDir } from './util'; import { AiInspectElement } from '@/ai-model'; - const testTodoCases = [ { description: '任务输入框', @@ -27,23 +26,29 @@ const testTodoCases = [ }, ]; - repeat(2, (repeatIndex) => { - it('todo: inspect element', async () => { - const { context } = await getPageTestData(path.join(__dirname, './test-data/todo')); - - const { aiResponse, filterUnStableinf } = await runTestCases(testTodoCases, async (testCase)=>{ + it( + 'todo: inspect element', + async () => { + const { context } = await getPageTestData(path.join(__dirname, './test-data/todo')); + + const { aiResponse, filterUnStableinf } = await runTestCases(testTodoCases, async (testCase) => { const { parseResult } = await AiInspectElement({ - context, - multi: testCase.multi, - findElementDescription: testCase.description, + context, + multi: testCase.multi, + findElementDescription: testCase.description, }); return parseResult; - }); - writeFileSyncWithDir(path.join(__dirname, `__ai_responses__/todo-inspector-element-${repeatIndex}.json`), JSON.stringify(aiResponse, null, 2), { encoding: 'utf-8'}); - expect(filterUnStableinf).toMatchFileSnapshot('./__snapshots__/todo_inspector.test.ts.snap'); - }, { - timeout: 99999, - }); + }); + writeFileSyncWithDir( + path.join(__dirname, `__ai_responses__/todo-inspector-element-${repeatIndex}.json`), + JSON.stringify(aiResponse, null, 2), + { encoding: 'utf-8' }, + ); + expect(filterUnStableinf).toMatchFileSnapshot('./__snapshots__/todo_inspector.test.ts.snap'); + }, + { + timeout: 99999, + }, + ); }); - diff --git a/packages/midscene/tsconfig.json b/packages/midscene/tsconfig.json index 6430f7031..76513fe64 100644 --- a/packages/midscene/tsconfig.json +++ b/packages/midscene/tsconfig.json @@ -13,10 +13,10 @@ "@/*": ["./src/*"] }, "resolveJsonModule": true, - "rootDir": "./src", + "rootDir": ".", "skipLibCheck": true, "strict": true }, "exclude": ["**/node_modules"], - "include": ["src"] + "include": ["src", "tests"] } diff --git a/packages/web-integration/.gitignore b/packages/web-integration/.gitignore index e716e450b..73b38a431 100644 --- a/packages/web-integration/.gitignore +++ b/packages/web-integration/.gitignore @@ -1,3 +1,4 @@ # MidScene.js dump files -midscene_run/ +midscene_run/midscene-report +midscene_run/dump diff --git a/packages/web-integration/midscene-reporter.ts b/packages/web-integration/midscene-reporter.ts deleted file mode 100644 index f2c131570..000000000 --- a/packages/web-integration/midscene-reporter.ts +++ /dev/null @@ -1,150 +0,0 @@ -import path from 'path'; -import fs from 'fs'; -import assert from 'assert'; -import os from 'os'; -import type { - FullConfig, - FullResult, - Reporter, - Suite, - TestCase, - TestResult, - Location, -} from '@playwright/test/reporter'; -// @ts-expect-error -import fsExtra from 'fs-extra'; - -type TestData = { - testId: string; - title: string; - status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; - /** - * Running time in milliseconds. - */ - duration: number; - /** - * Optional location in the source where the step is defined. - */ - location?: Location; - dumpPath?: string; -}; - -const testDataList: Array = []; - -class MyReporter implements Reporter { - onBegin(config: FullConfig, suite: Suite) { - const suites = suite.allTests(); - console.log(`Starting the run with ${suites.length} tests`); - } - - onTestBegin(test: TestCase, _result: TestResult) { - console.log(`Starting test ${test.title}`); - } - - onTestEnd(test: TestCase, result: TestResult) { - const aiActionTestData = test.annotations.filter((annotation) => { - if (annotation.type === 'PLAYWRIGHT_AI_ACTION') { - return true; - } - return false; - }); - aiActionTestData.forEach((testData) => { - const parseData = JSON.parse(testData.description!); - if (parseData.testId === test.id) { - testDataList.push({ - testId: test.id, - title: test.title, - status: result.status, - duration: result.duration, - location: test.location, - dumpPath: parseData.dumpPath, - }); - } - }); - console.log(`Finished test ${test.title}: ${result.status}`); - } - - onEnd(result: FullResult) { - console.log('testDataList', JSON.stringify(testDataList)); - console.log(`Finished the run: ${result.status}`); - generateTestData(testDataList); - } -} - -function generateTestData(testDataList: Array) { - const filterDataList = testDataList.reduce((res, testData) => { - if (res.find((item) => item.testId === testData.testId)) { - return res; - } else { - return [...res, testData]; - } - }, [] as Array); - const projectDir = process.cwd(); - const reportDir = path.join(projectDir, 'midscene-report'); - - // Create a report folder - if (!fs.existsSync(reportDir)) { - fs.mkdirSync(reportDir); - } - - // Copy the contents of the report html folder to the report folder - const reportHtmlDir = path.join(projectDir, `node_modules/@midscene/visualizer-report/.output`); - const tempDir = path.join(os.tmpdir(), 'temp-folder'); - try { - // First copy to the temporary directory - fsExtra.copySync(reportHtmlDir, tempDir); - // Then move the contents of the temporary directory to the destination directory - fsExtra.moveSync(tempDir, reportDir, { overwrite: true }); - console.log('Copy completed!'); - } catch (err) { - console.error('An error occurred while copying the folder.', err); - } - - try { - fsExtra.removeSync(path.join(reportDir, 'public')); - console.log('Public Folder deleted successfully!'); - } catch (err) { - console.error('An error occurred while deleting the folder.', err); - } - - for (const testData of filterDataList) { - const { dumpPath } = testData; - if (dumpPath) { - const srcFile = dumpPath.split('/').pop(); - assert(srcFile, `Failed to get source file name from ${dumpPath}`); - const destFile = path.join(reportDir, 'public', srcFile); - fsExtra.copySync(dumpPath, destFile); - } - } - - try { - fsExtra.outputFileSync( - path.join(reportDir, 'public', 'test-data-list.json'), - JSON.stringify({ 'test-list': filterDataList }), - ); - console.log('File written successfully!'); - } catch (err) { - console.error('An error occurred while writing to the file.', err); - } - - const filePath = path.join(reportDir, 'index.js'); // File path - const searchValue = 'Server is listening on http://[::]:'; // The content to be replaced can be a string or a regular expression - const replaceValue = 'The report has been generated on http://127.0.0.1:'; // The replaced content - - try { - // Read file contents - let fileContent = fs.readFileSync(filePath, 'utf8'); - - // Replace file contents - fileContent = fileContent.replace(searchValue, replaceValue); - - // Writes the modified content to the file - fsExtra.outputFileSync(filePath, fileContent); - - console.log('File content replaced and written successfully!'); - } catch (err) { - console.error('An error occurred:', err); - } -} - -export default MyReporter; diff --git a/packages/web-integration/midscene_run/cache/ai-auto-todo.spec.ts:8(ai todo).json b/packages/web-integration/midscene_run/cache/ai-auto-todo.spec.ts:8(ai todo).json new file mode 100644 index 000000000..a3b91962c --- /dev/null +++ b/packages/web-integration/midscene_run/cache/ai-auto-todo.spec.ts:8(ai todo).json @@ -0,0 +1,356 @@ +{ + "pkgName": "@midscene/web", + "pkgVersion": "0.0.1", + "taskFile": "ai-auto-todo.spec.ts:8", + "taskTitle": "ai todo", + "aiTasks": [ + { + "type": "plan", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Enter \"Learn JS today\" in the task box, then press Enter to create", + "response": { + "plans": [ + { + "thought": "The task box is the input field with the placeholder 'What needs to be done?'.", + "type": "Locate", + "param": { + "prompt": "The input field with the placeholder text 'What needs to be done?'" + } + }, + { + "thought": "Enter the given text 'Learn JS today' into the input field.", + "type": "Input", + "param": { + "value": "Learn JS today" + } + }, + { + "thought": "Press Enter to create the task after entering the text.", + "type": "KeyboardPress", + "param": { + "value": "Enter" + } + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "The task box is the input field with the placeholder 'What needs to be done?'.", + "response": { + "elements": [ + { + "reason": "The element has the placeholder 'What needs to be done?' and matches the description as an input field.", + "text": "What needs to be done?", + "id": "3530a9c1eb" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create", + "response": { + "plans": [ + { + "thought": "Locate the element to input the task.", + "type": "Locate", + "param": { + "prompt": "The task input box with the placeholder 'What needs to be done?'" + } + }, + { + "thought": "Input the task description 'Learn Rust tomorrow'.", + "type": "Input", + "param": { + "value": "Learn Rust tomorrow" + } + }, + { + "thought": "Press Enter to create the task.", + "type": "KeyboardPress", + "param": { + "value": "Enter" + } + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Locate the element to input the task.", + "response": { + "elements": [ + { + "reason": "The element matches the description 'Locate the element to input the task.' as it is an input node with a placeholder indicating it is for inputting tasks.", + "text": "What needs to be done?", + "id": "3530a9c1eb" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create", + "response": { + "plans": [ + { + "thought": "首先定位到任务输入框,页面描述中包含了 'What needs to be done?' 作为其 placeholder。", + "type": "Locate", + "param": { + "prompt": "包含 'What needs to be done?' placeholder 的输入框" + } + }, + { + "thought": "在输入框中输入 'Learning AI the day after tomorrow'。", + "type": "Input", + "param": { + "value": "Learning AI the day after tomorrow" + } + }, + { + "thought": "按下 Enter 键以创建任务。", + "type": "KeyboardPress", + "param": { + "value": "Enter" + } + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "首先定位到任务输入框,页面描述中包含了 'What needs to be done?' 作为其 placeholder。", + "response": { + "elements": [ + { + "reason": "The element contains the 'What needs to be done?' as its placeholder in the INPUT Node, matching the user description.", + "text": "What needs to be done?", + "id": "3530a9c1eb" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Move your mouse over the second item in the task list and click the Delete button to the right of the second task", + "response": { + "plans": [ + { + "thought": "I need to locate the second item in the task list first.", + "type": "Locate", + "param": { + "prompt": "The text node with content 'Learn Rust tomorrow'." + } + }, + { + "thought": "Hover over the second task item after locating it.", + "type": "Hover", + "param": null + }, + { + "thought": "Locate the delete button to the right of the second task item.", + "type": "Locate", + "param": { + "prompt": "The button to the right of the text 'Learn Rust tomorrow'." + } + }, + { + "thought": "Click on the delete button after locating it.", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "I need to locate the second item in the task list first.", + "response": { + "elements": [ + { + "reason": "The second item in the task list is 'Learn Rust tomorrow', which is located below the first item 'Learn JS today'.", + "text": "Learn Rust tomorrow", + "id": "10bc260820" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Locate the delete button to the right of the second task item.", + "response": { + "elements": [ + { + "reason": "The element ID 7ccd467339 is the delete button located to the right of the second task item 'Learn Rust tomorrow', which matches the user's description.", + "text": "×", + "id": "7ccd467339" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Click the check button to the left of the second task", + "response": { + "plans": [ + { + "thought": "I need to locate the checkbox to the left of the second task to mark it complete.", + "type": "Locate", + "param": { + "prompt": "The checkbox to the left of 'Learning AI the day after tomorrow'" + } + }, + { + "thought": "Click the located checkbox to mark the second task as complete.", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "I need to locate the checkbox to the left of the second task to mark it complete.", + "response": { + "elements": [ + { + "reason": "The element is a checkbox and is located to the left of the second task in the list, as described in the request.", + "text": "", + "id": "c0751f3b26" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "Click the completed Status button below the task list", + "response": { + "plans": [ + { + "thought": "The task is asking to click the completed Status button. From the image and the json description, the 'Completed' button appears just below the task list.", + "type": "Locate", + "param": { + "prompt": "The Completed button below the task list" + } + }, + { + "thought": "After locating the Completed button, the next action is to click it.", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://todomvc.com/examples/react/dist/", + "size": { + "width": 1280, + "height": 720 + } + }, + "prompt": "The task is asking to click the completed Status button. From the image and the json description, the 'Completed' button appears just below the task list.", + "response": { + "elements": [ + { + "reason": "The element with ID '663a34de3b' is identified as the 'Completed' status button as it matches the content 'Completed' and is located in the correct region consistently with the provided image and task description.", + "text": "Completed", + "id": "663a34de3b" + } + ], + "errors": [] + } + } + ] +} \ No newline at end of file diff --git a/packages/web-integration/midscene_run/cache/ai-online-order.spec.ts:10(ai online order).json b/packages/web-integration/midscene_run/cache/ai-online-order.spec.ts:10(ai online order).json new file mode 100644 index 000000000..a2c9b7132 --- /dev/null +++ b/packages/web-integration/midscene_run/cache/ai-online-order.spec.ts:10(ai online order).json @@ -0,0 +1,527 @@ +{ + "pkgName": "@midscene/web", + "pkgVersion": "0.0.1", + "taskFile": "ai-online-order.spec.ts:10", + "taskTitle": "ai online order", + "aiTasks": [ + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击左上角语言切换按钮(英文、中文),在弹出的下拉列表中点击中文", + "response": { + "plans": [ + { + "thought": "先定位页面左上角的语言切换按钮。", + "type": "Locate", + "param": { + "prompt": "页面顶部左侧显示 'English' 的按钮。" + } + }, + { + "thought": "点击该语言切换按钮。", + "type": "Tap", + "param": null + }, + { + "thought": "定位下拉列表中的中文选项。", + "type": "Locate", + "param": { + "prompt": "下拉菜单里显示 '中文' 的选项。" + } + }, + { + "thought": "点击中文选项。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "先定位页面左上角的语言切换按钮。", + "response": { + "elements": [ + { + "reason": "The requested element is the language switch button located at the top left of the page. The text element with the content 'English' is in the top left section, matching the location and description.", + "text": "English", + "id": "57c026f0b8" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "定位下拉列表中的中文选项。", + "response": { + "elements": [ + { + "reason": "理由是找到该元素,节点类型为TEXT,内容是 “中文”,并且在页面中位于下拉列表中。", + "text": "中文", + "id": "efc0b2ff38" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "在向下滚动一屏", + "response": { + "plans": [ + { + "thought": "页面当前显示了一部分内容,需要向下滚动以查看更多内容。", + "type": "Scroll", + "param": { + "scrollType": "ScrollDown" + } + } + ] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "直接点击多肉葡萄的规格按钮", + "response": { + "plans": [ + { + "thought": "用户想要点击页面中的'多肉葡萄'的规格按钮。图像中可以看到这个按钮。", + "type": "Locate", + "param": { + "prompt": "在页面中部位置,‘多肉葡萄’项目下,文本内容为‘选规格’的黄色按钮" + } + }, + { + "thought": "找到符合条件的按钮,进行点击操作。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "用户想要点击页面中的'多肉葡萄'的规格按钮。图像中可以看到这个按钮。", + "response": { + "elements": [ + { + "reason": "The description indicates the user intends to click on the '规格' button labeled '选规格' next to the '多肉葡萄' item. Found '多肉葡萄(首创)' with content ID '65b6fcae72' in the JSON, and matched '选规格' button next to it.", + "text": "选规格", + "id": "427d3179c2" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击不使用吸管、点击冰沙推荐、点击正常冰推荐", + "response": { + "plans": [ + { + "thought": "首先找到“不使用吸管”的复选框,然后点击它。", + "type": "Locate", + "param": { + "prompt": "标有'不使用吸管'的复选框" + } + }, + { + "thought": "点击'不使用吸管'的复选框。", + "type": "Tap", + "param": null + }, + { + "thought": "接着找到“冰沙(推荐)”的复选框,然后点击它。", + "type": "Locate", + "param": { + "prompt": "标有'冰沙(推荐)'的复选框" + } + }, + { + "thought": "点击'冰沙(推荐)'的复选框。", + "type": "Tap", + "param": null + }, + { + "thought": "最后找到“正常冰(推荐)”的复选框,并点击它。", + "type": "Locate", + "param": { + "prompt": "标有'正常冰(推荐)'的复选框" + } + }, + { + "thought": "点击'正常冰(推荐)'的复选框。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "首先找到“不使用吸管”的复选框,然后点击它。", + "response": { + "elements": [ + { + "reason": "The element is closest to the description '首先找到“不使用吸管”的复选框,然后点击它。'. It is a checkbox input located at the coordinates (23, 524) with associated text '不使用吸管'.", + "text": "", + "id": "6384a887d9" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "接着找到“冰沙(推荐)”的复选框,然后点击它。", + "response": { + "elements": [ + { + "reason": "Based on user's instruction to find and click on the checkbox labeled with \"冰沙(推荐)\", I located an INPUT element with the type 'checkbox' and an adjacent TEXT Node with content '冰沙(推荐)'.", + "text": "冰沙(推荐)", + "id": "04f0538285" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "最后找到“正常冰(推荐)”的复选框,并点击它。", + "response": { + "elements": [ + { + "reason": "The element matches the description '正常冰(推荐)' and is a checkbox type input node located at the specified position in the image.", + "text": "正常冰(推荐)", + "id": "0b385ab4dc" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "向下滚动一屏", + "response": { + "plans": [ + { + "thought": "用户要求向下滚动一屏。页面展示的信息很多,支持滚动操作。", + "type": "Scroll", + "param": { + "scrollType": "ScrollDown" + } + } + ] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击标准甜、点击绿妍(推荐)、点击标准口味", + "response": { + "plans": [ + { + "thought": "找到并点击标准甜(推荐)的复选框", + "type": "Locate", + "param": { + "prompt": "标准甜(推荐)前面的复选框" + } + }, + { + "thought": "点击所找到的复选框", + "type": "Tap", + "param": null + }, + { + "thought": "找到并点击绿妍(推荐)的复选框", + "type": "Locate", + "param": { + "prompt": "绿妍(推荐)前面的复选框" + } + }, + { + "thought": "点击所找到的复选框", + "type": "Tap", + "param": null + }, + { + "thought": "找到并点击标准口味(推荐)的复选框", + "type": "Locate", + "param": { + "prompt": "标准口味(推荐)前面的复选框" + } + }, + { + "thought": "点击所找到的复选框", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到并点击标准甜(推荐)的复选框", + "response": { + "elements": [ + { + "reason": "Found the '标准甜(推荐)' checkbox which matches the user's description in the 'elementInfos'. The element type is 'checkbox' and it has the text '标准甜(推荐)'.", + "text": "标准甜(推荐)", + "id": "5280c060a7" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到并点击绿妍(推荐)的复选框", + "response": { + "elements": [ + { + "reason": "The element has the text '绿妍(推荐)' which matches the user's description. The element next to this text is a checkbox, as per the request.", + "text": "绿妍(推荐)", + "id": "e72c1a6a97" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到并点击标准口味(推荐)的复选框", + "response": { + "elements": [ + { + "reason": "The checkbox '标准口味(推荐)' closely matches the description '找到并点击标准口味(推荐)'. It is the '标准口味(推荐)' checkbox shown on the user interface.", + "text": "标准口味(推荐)", + "id": "7946eb054f" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "滚动到最下面", + "response": { + "plans": [ + { + "thought": "用户希望滚动到页面的最底部,该页面包含多个可选择的茶和甜度选项,因此滚动至底部是可行的。", + "type": "Scroll", + "param": { + "scrollType": "ScrollUntilBottom" + } + } + ] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击选好了按钮", + "response": { + "plans": [ + { + "thought": "点击按钮可进行下一步操作。", + "type": "Locate", + "param": { + "prompt": "页面底部黄色按钮,文本内容为'选好了'" + } + }, + { + "thought": "点击按钮以确认选择。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击按钮可进行下一步操作。", + "response": { + "elements": [ + { + "reason": "The button that can be clicked for the next step has the text content '选好了' and is of 'BUTTON Node' type.", + "text": "选好了", + "id": "c9de916ef0" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击右上角商品图标按钮", + "response": { + "plans": [ + { + "thought": "用户要求点击右上角的商品图标按钮。通过页面信息,看到右上角的商品图标按钮。", + "type": "Locate", + "param": { + "prompt": "右上角的商品图标按钮" + } + }, + { + "thought": "找到右上角的商品图标按钮并点击。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "用户要求点击右上角的商品图标按钮。通过页面信息,看到右上角的商品图标按钮。", + "response": { + "elements": [ + { + "reason": "The element is an image node located in the upper right corner of the page. According to the description, it looks like the shopping cart icon with a notification badge.", + "text": "", + "id": "f775c69cb4" + } + ], + "errors": [] + } + } + ] +} \ No newline at end of file diff --git a/packages/web-integration/midscene_run/cache/ai-xicha.spec.ts:10(ai order).json b/packages/web-integration/midscene_run/cache/ai-xicha.spec.ts:10(ai order).json new file mode 100644 index 000000000..f3fd78cfb --- /dev/null +++ b/packages/web-integration/midscene_run/cache/ai-xicha.spec.ts:10(ai order).json @@ -0,0 +1,527 @@ +{ + "pkgName": "@midscene/web", + "pkgVersion": "0.0.1", + "taskFile": "ai-xicha.spec.ts:10", + "taskTitle": "ai order", + "aiTasks": [ + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击左上角语言切换按钮(英文、中文),在弹出的下拉列表中点击中文", + "response": { + "plans": [ + { + "thought": "用户想要切换到中文,需要首先找到位于左上角的语言切换按钮。", + "type": "Locate", + "param": { + "prompt": "左上角显示‘English’的按钮" + } + }, + { + "thought": "点击语言切换按钮以显示语言下拉列表。", + "type": "Tap", + "param": null + }, + { + "thought": "在弹出的下拉列表中找到‘中文’选项。", + "type": "Locate", + "param": { + "prompt": "下拉列表中的‘中文’选项" + } + }, + { + "thought": "点击‘中文’选项以切换语言。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "用户想要切换到中文,需要首先找到位于左上角的语言切换按钮。", + "response": { + "elements": [ + { + "reason": "This element is the language switch button located in the top left corner. The text 'English' matches the description of the element for switching languages.", + "text": "English", + "id": "57c026f0b8" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "在弹出的下拉列表中找到‘中文’选项。", + "response": { + "elements": [ + { + "reason": "The element with the text '中文' matches the description of the '中文' option in the drop-down list.", + "text": "中文", + "id": "efc0b2ff38" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "在向下滚动一屏", + "response": { + "plans": [ + { + "thought": "必须向下滚动才能展示更多的内容。", + "type": "Scroll", + "param": { + "scrollType": "ScrollDown" + } + } + ] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "直接点击多肉葡萄的规格按钮", + "response": { + "plans": [ + { + "thought": "根据用户请求,需要点击多肉葡萄的“选规格”按钮。页面中多肉葡萄的规格按钮位于屏幕下部。", + "type": "Locate", + "param": { + "prompt": "带有‘多肉葡萄(首创)’的那个产品条目下的‘选规格’按钮" + } + }, + { + "thought": "找到‘选规格’按钮后,进行点击。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "根据用户请求,需要点击多肉葡萄的“选规格”按钮。页面中多肉葡萄的规格按钮位于屏幕下部。", + "response": { + "elements": [ + { + "reason": "The element corresponds to the '选规格' button for '多肉葡萄(首创)', which is located near the bottom portion of the page as specified in the request.", + "text": "选规格", + "id": "427d3179c2" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击不使用吸管、点击冰沙推荐、点击正常冰推荐", + "response": { + "plans": [ + { + "thought": "用户希望选择“多肉葡萄(首创)”的属性,包括不使用吸管、冰沙推荐、正常冰推荐。因此,首先需要找到并点击“不使用吸管”的选择框。", + "type": "Locate", + "param": { + "prompt": "带有“多肉葡萄(首创)”描述的页面上,标记为“不使用吸管”的勾选框,位于“绿色喜茶(必选, 最多选1)”的下面,该勾选框旁边是文字“可降解吸管上一行“" + } + }, + { + "thought": "现在点击“不使用吸管”选择框。", + "type": "Tap", + "param": null + }, + { + "thought": "接下来需要找到并点击“冰沙(推荐)”的选择框。", + "type": "Locate", + "param": { + "prompt": "带有“多肉葡萄(首创)”描述的页面上,标记为“冰沙(推荐)”的勾选框,位于“状态(必选, 最多选1)”的下面,该勾选框旁边是文字“冰沙(推荐)”" + } + }, + { + "thought": "现在点击“冰沙(推荐)选择框。", + "type": "Tap", + "param": null + }, + { + "thought": "最后,需要找到并点击“正常冰(推荐)”的选择框。", + "type": "Locate", + "param": { + "prompt": "带有“多肉葡萄(首创)”描述的页面上,标记为“正常冰(推荐)”的勾选框,位于“冰量(必选, 最多选1)”的下面,该勾选框旁边是文字“正常冰(推荐)”" + } + }, + { + "thought": "现在点击“正常冰(推荐)选择框。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "用户希望选择“多肉葡萄(首创)”的属性,包括不使用吸管、冰沙推荐、正常冰推荐。因此,首先需要找到并点击“不使用吸管”的选择框。", + "response": { + "elements": [ + { + "reason": "找到“不使用吸管”的选框,通过其旁边的文本“不可降解吸管”进行确认。", + "text": "不使用吸管", + "id": "6384a887d9" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "接下来需要找到并点击“冰沙(推荐)”的选择框。", + "response": { + "elements": [ + { + "reason": "The element has the text '冰沙(推荐)' which matches the description '冰沙(推荐)'.", + "text": "冰沙(推荐)", + "id": "e673260398" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "最后,需要找到并点击“正常冰(推荐)”的选择框。", + "response": { + "elements": [ + { + "reason": "The element was selected because it matches the description '正常冰(推荐)' and is a checkbox type.", + "text": "正常冰(推荐)", + "id": "f59967b279" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "向下滚动一屏", + "response": { + "plans": [ + { + "thought": "为了满足用户滚动页面的要求", + "type": "Scroll", + "param": { + "scrollType": "ScrollDown" + } + } + ] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击标准甜、点击绿妍(推荐)、点击标准口味", + "response": { + "plans": [ + { + "thought": "找到标记为标准甜推荐的复选框", + "type": "Locate", + "param": { + "prompt": "标有'标准甜(推荐)'旁边的复选框" + } + }, + { + "thought": "点击复选框来选择标准甜", + "type": "Tap", + "param": null + }, + { + "thought": "找到标记为绿妍推荐的复选框", + "type": "Locate", + "param": { + "prompt": "标有'绿妍(推荐)'旁边的复选框" + } + }, + { + "thought": "点击复选框来选择绿妍", + "type": "Tap", + "param": null + }, + { + "thought": "找到标记为标准口味推荐的复选框", + "type": "Locate", + "param": { + "prompt": "标有'标准口味(推荐)'旁边的复选框" + } + }, + { + "thought": "点击复选框来选择标准口味", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到标记为标准甜推荐的复选框", + "response": { + "elements": [ + { + "reason": "The element with the content '标准甜(推荐)' matches the description 标记为标准甜推荐的复选框. The checkbox input associated with this text has the ID 5280c060a7.", + "text": "标准甜(推荐)", + "id": "5280c060a7" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到标记为绿妍推荐的复选框", + "response": { + "elements": [ + { + "reason": "The element matches the description \"绿妍(推荐)\" for the checkbox indicating a recommendation for the tea base.", + "text": "绿妍(推荐)", + "id": "e72c1a6a97" + } + ], + "errors": [] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到标记为标准口味推荐的复选框", + "response": { + "elements": [ + { + "reason": "The checkbox labeled as '标准口味(推荐)' is identified by its text content and node type 'INPUT Node', which matches the user's description.", + "text": "标准口味(推荐)", + "id": "0d204274f0" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "滚动到最下面", + "response": { + "plans": [ + { + "thought": "用户要求滚动到页面的最底部,根据页面上下文确定整个页面包含可滚动的内容。", + "type": "Scroll", + "param": { + "scrollType": "ScrollUntilBottom" + } + } + ] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击选好了按钮", + "response": { + "plans": [ + { + "thought": "找到 '选好了' 按钮。", + "type": "Locate", + "param": { + "prompt": "页面底部黄色的“选好了”按钮" + } + }, + { + "thought": "点击 '选好了' 按钮。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "找到 '选好了' 按钮。", + "response": { + "elements": [ + { + "reason": "The element was identified as the button labeled '选好了' which translates to 'finished selection'. The button is used for proceeding after making selections, matching the user's request.", + "text": "选好了", + "id": "c9de916ef0" + } + ], + "errors": [] + } + }, + { + "type": "plan", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "点击右上角商品图标按钮", + "response": { + "plans": [ + { + "thought": "用户要求点击右上角的商品图标按钮,页面右上角有一个商品图标。", + "type": "Locate", + "param": { + "prompt": "右上角的商品图标按钮" + } + }, + { + "thought": "找到商品图标按钮后,点击它。", + "type": "Tap", + "param": null + } + ] + } + }, + { + "type": "locate", + "pageContext": { + "url": "https://heyteavivocity.meuu.online/home", + "size": { + "width": 400, + "height": 905 + } + }, + "prompt": "用户要求点击右上角的商品图标按钮,页面右上角有一个商品图标。", + "response": { + "elements": [ + { + "reason": "The element is located in the upper right corner, is an image type (IMG Node), and corresponds to the shopping cart icon button in the provided screenshot.", + "text": "", + "id": "f775c69cb4" + } + ], + "errors": [] + } + } + ] +} \ No newline at end of file diff --git a/packages/web-integration/package.json b/packages/web-integration/package.json index 2bf24965d..b82db62cd 100644 --- a/packages/web-integration/package.json +++ b/packages/web-integration/package.json @@ -50,11 +50,15 @@ "build:pkg": "modern build -c ./modern.config.ts", "build:script": "modern build -c ./modern.inspect.config.ts", "build:watch": "modern build -w -c ./modern.config.ts & modern build -w -c ./modern.inspect.config.ts", + "test": "vitest --run", + "test:all": "AITEST=true vitest --run", "new": "modern new", "upgrade": "modern upgrade", "prepublishOnly": "npm run build", "e2e": "playwright test --config=playwright.config.ts", - "e2e:ui": "playwright test --config=playwright.config.ts --ui" + "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts", + "e2e:ui": "playwright test --config=playwright.config.ts --ui", + "e2e:ui-cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts --ui" }, "files": [ "dist", @@ -63,11 +67,13 @@ "dependencies": { "openai": "4.47.1", "sharp": "0.33.3", + "inquirer": "10.1.5", "@midscene/core": "workspace:*", "@midscene/visualizer-report": "workspace:*" }, "devDependencies": { "@modern-js/module-tools": "^2.54.2", + "js-sha256": "0.11.0", "@types/node": "^18.0.0", "typescript": "~5.0.4", "vitest": "^1.6.0", diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts index dcb17cee7..7842a0646 100644 --- a/packages/web-integration/src/common/agent.ts +++ b/packages/web-integration/src/common/agent.ts @@ -1,6 +1,7 @@ import { ExecutionDump, GroupedActionDump } from '@midscene/core'; import { groupedActionDumpFileExt, writeDumpFile } from '@midscene/core/utils'; import { PageTaskExecutor } from '../common/tasks'; +import { AiTaskCache } from './task-cache'; import { WebPage } from '@/common/page'; export class PageAgent { @@ -12,42 +13,46 @@ export class PageAgent { dumpFile?: string; - constructor(page: WebPage, testId?: string) { + actionAgent: PageTaskExecutor; + + constructor(page: WebPage, opts: { testId?: string; taskFile?: string; cache?: AiTaskCache }) { this.page = page; - this.dumps = []; - this.testId = testId || String(process.pid); + this.dumps = [ + { + groupName: opts?.taskFile || 'unnamed', + executions: [], + }, + ]; + this.testId = opts?.testId || String(process.pid); + this.actionAgent = new PageTaskExecutor(this.page, { + cache: opts?.cache || { aiTasks: [] }, + }); } - appendDump(groupName: string, execution: ExecutionDump) { - let currentDump = this.dumps.find((dump) => dump.groupName === groupName); - if (!currentDump) { - currentDump = { - groupName, - executions: [], - }; - this.dumps.push(currentDump); - } + appendDump(execution: ExecutionDump) { + const currentDump = this.dumps[0]; currentDump.executions.push(execution); } writeOutActionDumps() { - this.dumpFile = writeDumpFile( - `playwright-${this.testId}`, - groupedActionDumpFileExt, - JSON.stringify(this.dumps), - ); + this.dumpFile = writeDumpFile({ + fileName: `playwright-${this.testId}`, + fileExt: groupedActionDumpFileExt, + fileContent: JSON.stringify(this.dumps), + }); } - async aiAction(taskPrompt: string, dumpCaseName = 'AI Action', dumpGroupName = 'MidScene / Web') { - const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName }); + async aiAction(taskPrompt: string) { let error: Error | undefined; try { - await actionAgent.action(taskPrompt); + await this.actionAgent.action(taskPrompt); } catch (e: any) { error = e; } - if (actionAgent.executionDump) { - this.appendDump(dumpGroupName, actionAgent.executionDump); + // console.log('cache logic', actionAgent.taskCache.generateTaskCache()); + if (this.actionAgent.executionDump) { + this.appendDump(this.actionAgent.executionDump); + // this.appendDump(dumpGroupName, actionAgent.executionDump); this.writeOutActionDumps(); } if (error) { @@ -57,17 +62,16 @@ export class PageAgent { } } - async aiQuery(demand: any, dumpCaseName = 'AI Query', dumpGroupName = 'MidScene / Web') { - const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName }); + async aiQuery(demand: any) { let error: Error | undefined; let result: any; try { - result = await actionAgent.query(demand); + result = await this.actionAgent.query(demand); } catch (e: any) { error = e; } - if (actionAgent.executionDump) { - this.appendDump(dumpGroupName, actionAgent.executionDump); + if (this.actionAgent.executionDump) { + this.appendDump(this.actionAgent.executionDump); this.writeOutActionDumps(); } if (error) { @@ -78,11 +82,11 @@ export class PageAgent { return result; } - async ai(taskPrompt: string, type = 'action', dumpCaseName = 'AI', dumpGroupName = 'MidScene / Web') { + async ai(taskPrompt: string, type = 'action') { if (type === 'action') { - return this.aiAction(taskPrompt, dumpCaseName, dumpGroupName); + return this.aiAction(taskPrompt); } else if (type === 'query') { - return this.aiQuery(taskPrompt, dumpCaseName, dumpGroupName); + return this.aiQuery(taskPrompt); } throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`); } diff --git a/packages/web-integration/src/common/task-cache.ts b/packages/web-integration/src/common/task-cache.ts new file mode 100644 index 000000000..aded39a24 --- /dev/null +++ b/packages/web-integration/src/common/task-cache.ts @@ -0,0 +1,131 @@ +import { AIElementParseResponse, PlanningAction } from '@midscene/core'; +import { WebUIContext } from './utils'; + +export type PlanTask = { + type: 'plan'; + prompt: string; + pageContext: { + url: string; + size: { + width: number; + height: number; + }; + }; + response: { plans: PlanningAction[] }; +}; + +export type LocateTask = { + type: 'locate'; + prompt: string; + pageContext: { + url: string; + size: { + width: number; + height: number; + }; + }; + response: AIElementParseResponse; +}; + +export type AiTasks = Array; + +export type AiTaskCache = { + aiTasks: AiTasks; +}; + +export class TaskCache { + cache: AiTaskCache | undefined; + + newCache: AiTaskCache; + + constructor(opts?: { cache: AiTaskCache }) { + this.cache = opts?.cache; + this.newCache = { + aiTasks: [], + }; + } + + /** + * Read and return cached responses asynchronously based on specific criteria + * This function is mainly used to read cached responses from a certain storage medium. + * It accepts three parameters: the page context information, the task type, and the user's prompt information. + * In the function, it first checks whether there is cached data. If there is, it retrieves the first task response from the cache. + * It then checks whether the task type is 'locate' and whether the corresponding element can be found in the new context. + * If the element cannot be found, it returns false, indicating that the cache is invalid. + * If the task type is correct and the user prompt matches, it checks whether the page context is the same. + * If the page context is the same, it returns the cached response, indicating that the cache hit is successful. + * If there is no cached data or the conditions are not met, the function returns false, indicating that no cache is available or the cache is not hit. + * + * @param pageContext UIContext type, representing the context information of the current page + * @param type String type, specifying the task type, can be 'plan' or 'locate' + * @param userPrompt String type, representing user prompt information + * @return Returns a Promise object that resolves to a boolean or object + */ + readCache(pageContext: WebUIContext, type: 'plan', userPrompt: string): PlanTask['response']; + readCache(pageContext: WebUIContext, type: 'locate', userPrompt: string): LocateTask['response']; + readCache( + pageContext: WebUIContext, + type: 'plan' | 'locate', + userPrompt: string, + ): PlanTask['response'] | LocateTask['response'] | false { + if (this.cache) { + const { aiTasks } = this.cache; + const index = aiTasks.findIndex((item) => item.prompt === userPrompt); + + if (index === -1) { + return false; + } + + const taskRes = aiTasks.splice(index, 1)[0]; + + // The corresponding element cannot be found in the new context + if ( + taskRes?.type === 'locate' && + !taskRes.response?.elements.every((element) => { + const findIndex = pageContext.content.findIndex( + (contentElement) => contentElement.id === element.id, + ); + if (findIndex === -1) { + return false; + } + return true; + }) + ) { + return false; + } + if ( + taskRes && + taskRes.type === type && + taskRes.prompt === userPrompt && + this.pageContextEqual(taskRes.pageContext, pageContext) + ) { + return taskRes.response; + } + } + return false; + } + + saveCache(cache: PlanTask | LocateTask) { + if (cache) { + this.newCache?.aiTasks.push(cache); + } + } + + pageContextEqual(taskPageContext: LocateTask['pageContext'], pageContext: WebUIContext) { + return ( + taskPageContext.size.width === pageContext.size.width && + taskPageContext.size.height === pageContext.size.height + ); + } + + /** + * Generate task cache data. + * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object. + * In the cache object, it may contain task-related information, states, or other necessary data. + * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache. + * @returns {Object} Returns a new cache object, which may include task cache data. + */ + generateTaskCache() { + return this.newCache; + } +} diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index 0c266977b..40345ec29 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -1,5 +1,6 @@ import assert from 'assert'; import Insight, { + AIElementParseResponse, DumpSubscriber, ExecutionDump, ExecutionRecorderItem, @@ -11,35 +12,37 @@ import Insight, { Executor, InsightDump, InsightExtractParam, + plan, PlanningAction, PlanningActionParamHover, PlanningActionParamInputOrKeyPress, PlanningActionParamScroll, PlanningActionParamTap, - plan, } from '@midscene/core'; import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils'; import { base64Encoded } from '@midscene/core/image'; import type { KeyInput, Page as PuppeteerPage } from 'puppeteer'; +import { ChatCompletionMessageParam } from 'openai/resources'; import { WebElementInfo } from '../web-element'; -import { parseContextFromWebPage } from './utils'; +import { parseContextFromWebPage, WebUIContext } from './utils'; +import { AiTaskCache, TaskCache } from './task-cache'; import { WebPage } from '@/common/page'; export class PageTaskExecutor { page: WebPage; - insight: Insight; - - taskExecutor: Executor; + insight: Insight; executionDump?: ExecutionDump; - constructor(page: WebPage, opt?: { taskName?: string }) { + taskCache: TaskCache; + + constructor(page: WebPage, opts: { cache: AiTaskCache }) { this.page = page; - this.insight = new Insight(async () => { + this.insight = new Insight(async () => { return await parseContextFromWebPage(page); }); - this.taskExecutor = new Executor(opt?.taskName || 'MidScene - PlayWrightAI'); + this.taskCache = new TaskCache(opts); } private async recordScreenshot(timing: ExecutionRecorderItem['timing']) { @@ -95,8 +98,33 @@ export class PageTaskExecutor { insightDump = dump; }; this.insight.onceDumpUpdatedFn = dumpCollector; - const element = await this.insight.locate(param.prompt); + const pageContext = await this.insight.contextRetrieverFn(); + const locateCache = this.taskCache.readCache(pageContext, 'locate', param.prompt); + let locateResult: AIElementParseResponse | undefined; + const callAI = this.insight.aiVendorFn; + const element = await this.insight.locate(param.prompt, { + callAI: async (message: ChatCompletionMessageParam[]) => { + if (locateCache) { + locateResult = locateCache; + return Promise.resolve(locateCache); + } + locateResult = await callAI(message); + return locateResult; + }, + }); + assert(element, `Element not found: ${param.prompt}`); + if (locateResult) { + this.taskCache.saveCache({ + type: 'locate', + pageContext: { + url: pageContext.url, + size: pageContext.size, + }, + prompt: param.prompt, + response: locateResult, + }); + } return { output: { element, @@ -104,6 +132,9 @@ export class PageTaskExecutor { log: { dump: insightDump, }, + cache: { + hit: Boolean(locateResult), + }, }; }, }; @@ -193,8 +224,8 @@ export class PageTaskExecutor { } async action(userPrompt: string /* , actionInfo?: { actionType?: EventActions[number]['action'] } */) { - this.taskExecutor.description = userPrompt; - const pageContext = await this.insight.contextRetrieverFn(); + const taskExecutor = new Executor(userPrompt); + taskExecutor.description = userPrompt; let plans: PlanningAction[] = []; const planningTask: ExecutionTaskPlanningApply = { @@ -202,45 +233,70 @@ export class PageTaskExecutor { param: { userPrompt, }, - async executor(param) { - const planResult = await plan(pageContext, param.userPrompt); + executor: async (param) => { + const pageContext = await this.insight.contextRetrieverFn(); + let planResult: { plans: PlanningAction[] }; + const planCache = this.taskCache.readCache(pageContext, 'plan', userPrompt); + if (planCache) { + planResult = planCache; + } else { + planResult = await plan(param.userPrompt, { + context: pageContext, + }); + } + assert(planResult.plans.length > 0, 'No plans found'); // eslint-disable-next-line prefer-destructuring plans = planResult.plans; + + this.taskCache.saveCache({ + type: 'plan', + pageContext: { + url: pageContext.url, + size: pageContext.size, + }, + prompt: userPrompt, + response: planResult, + }); return { output: planResult, + cache: { + hint: Boolean(planCache), + }, }; }, }; try { // plan - await this.taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask)); - await this.taskExecutor.flush(); - this.executionDump = this.taskExecutor.dump(); + await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask)); + await taskExecutor.flush(); + this.executionDump = taskExecutor.dump(); // append tasks const executables = await this.convertPlanToExecutable(plans); - await this.taskExecutor.append(executables); + await taskExecutor.append(executables); // flush actions - await this.taskExecutor.flush(); - this.executionDump = this.taskExecutor.dump(); + await taskExecutor.flush(); + this.executionDump = taskExecutor.dump(); assert( - this.taskExecutor.status !== 'error', - `failed to execute tasks: ${this.taskExecutor.status}, msg: ${this.taskExecutor.errorMsg || ''}`, + taskExecutor.status !== 'error', + `failed to execute tasks: ${taskExecutor.status}, msg: ${taskExecutor.errorMsg || ''}`, ); } catch (e: any) { // keep the dump before throwing - this.executionDump = this.taskExecutor.dump(); + this.executionDump = taskExecutor.dump(); const err = new Error(e.message, { cause: e }); throw err; } } async query(demand: InsightExtractParam) { - this.taskExecutor.description = JSON.stringify(demand); + const description = JSON.stringify(demand); + const taskExecutor = new Executor(description); + taskExecutor.description = description; let data: any; const queryTask: ExecutionTaskInsightQueryApply = { type: 'Insight', @@ -262,12 +318,12 @@ export class PageTaskExecutor { }, }; try { - await this.taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask)); - await this.taskExecutor.flush(); - this.executionDump = this.taskExecutor.dump(); + await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask)); + await taskExecutor.flush(); + this.executionDump = taskExecutor.dump(); } catch (e: any) { // keep the dump before throwing - this.executionDump = this.taskExecutor.dump(); + this.executionDump = taskExecutor.dump(); const err = new Error(e.message, { cause: e }); throw err; } diff --git a/packages/web-integration/src/common/utils.ts b/packages/web-integration/src/common/utils.ts index 1c8a5e820..2ac4a021c 100644 --- a/packages/web-integration/src/common/utils.ts +++ b/packages/web-integration/src/common/utils.ts @@ -8,12 +8,17 @@ import { getTmpFile } from '@midscene/core/utils'; import { WebElementInfo, WebElementInfoType } from '../web-element'; import { WebPage } from './page'; +export type WebUIContext = UIContext & { + url: string; +}; + export async function parseContextFromWebPage( page: WebPage, _opt?: PlaywrightParserOpt, -): Promise> { +): Promise { assert(page, 'page is required'); + const url = page.url(); const file = getTmpFile('jpeg'); await page.screenshot({ path: file, type: 'jpeg', quality: 75 }); const screenshotBuffer = readFileSync(file); @@ -27,6 +32,7 @@ export async function parseContextFromWebPage( content: elementsInfo, size, screenshotBase64, + url, }; } @@ -70,7 +76,7 @@ async function alignElements( * @param {string} dir - Home directory * @returns {string|null} - The most recent package.json file path or null */ -function findNearestPackageJson(dir: string) { +export function findNearestPackageJson(dir: string): string | null { const packageJsonPath = path.join(dir, 'package.json'); if (fs.existsSync(packageJsonPath)) { diff --git a/packages/web-integration/src/extractor/extractor.ts b/packages/web-integration/src/extractor/extractor.ts index 5f5082a7d..824dcb77d 100644 --- a/packages/web-integration/src/extractor/extractor.ts +++ b/packages/web-integration/src/extractor/extractor.ts @@ -1,4 +1,5 @@ import { + generateHash, getNodeAttributes, getPseudoElementContent, logger, @@ -16,6 +17,8 @@ interface NodeDescriptor { export interface ElementInfo { id: string; + indexId: string; + nodeHashId: string; locator: string | void; attributes: { nodeType: NodeType; @@ -70,9 +73,12 @@ export function extractTextWithPositionDFS(initNode: Node = container): ElementI if (isInputElement(node)) { const attributes = getNodeAttributes(node); - const selector = setDataForNode(node, nodeIndex); + const nodeHashId = generateHash(attributes.placeholder, rect); + const selector = setDataForNode(node, nodeHashId); elementInfoArray.push({ - id: generateId(nodeIndex++), + id: nodeHashId, + indexId: generateId(nodeIndex++), + nodeHashId, locator: selector, attributes: { ...attributes, @@ -88,15 +94,19 @@ export function extractTextWithPositionDFS(initNode: Node = container): ElementI if (isButtonElement(node)) { const attributes = getNodeAttributes(node); const pseudo = getPseudoElementContent(node); - const selector = setDataForNode(node, nodeIndex); + const content = node.innerText || pseudo.before || pseudo.after || ''; + const nodeHashId = generateHash(content, rect); + const selector = setDataForNode(node, nodeHashId); elementInfoArray.push({ - id: generateId(nodeIndex++), + id: nodeHashId, + indexId: generateId(nodeIndex++), + nodeHashId, locator: selector, attributes: { ...attributes, nodeType: NodeType.BUTTON, }, - content: node.innerText || pseudo.before || pseudo.after || '', + content, rect, center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)], }); @@ -105,9 +115,12 @@ export function extractTextWithPositionDFS(initNode: Node = container): ElementI if (isImgElement(node)) { const attributes = getNodeAttributes(node); - const selector = setDataForNode(node, nodeIndex); + const nodeHashId = generateHash('', rect); + const selector = setDataForNode(node, nodeHashId); elementInfoArray.push({ - id: generateId(nodeIndex++), + id: nodeHashId, + indexId: generateId(nodeIndex++), + nodeHashId, locator: selector, attributes: { ...attributes, @@ -147,9 +160,12 @@ export function extractTextWithPositionDFS(initNode: Node = container): ElementI return; } const attributes = getNodeAttributes(node); - const selector = setDataForNode(node, nodeIndex); + const nodeHashId = generateHash(text, rect); + const selector = setDataForNode(node, nodeHashId); elementInfoArray.push({ - id: generateId(nodeIndex++), + id: nodeHashId, + indexId: generateId(nodeIndex++), + nodeHashId, attributes: { ...attributes, nodeType: NodeType.TEXT, diff --git a/packages/web-integration/src/extractor/util.ts b/packages/web-integration/src/extractor/util.ts index 5cc38c2f8..dbcb736ae 100644 --- a/packages/web-integration/src/extractor/util.ts +++ b/packages/web-integration/src/extractor/util.ts @@ -1,6 +1,7 @@ // import { TEXT_MAX_SIZE } from './constants'; +import SHA256 from 'js-sha256'; -export function logger(...msg: any[]): void { +export function logger(..._msg: any[]): void { // console.log(...msg); } @@ -10,11 +11,11 @@ const taskIdKey = '_midscene_retrieve_task_id'; // const nodeDataIdKey = 'data-midscene-task-'; // const nodeIndexKey = '_midscene_retrieve_node_index'; -function selectorForValue(val: number): string { +function selectorForValue(val: number | string): string { return `[${taskIdKey}='${val}']`; } -export function setDataForNode(node: HTMLElement | Node, nodeIndex: number): string { +export function setDataForNode(node: HTMLElement | Node, nodeHash: string): string { const taskId = taskIdKey; if (!(node instanceof HTMLElement)) { return ''; @@ -24,8 +25,8 @@ export function setDataForNode(node: HTMLElement | Node, nodeIndex: number): str return ''; } - const selector = selectorForValue(nodeIndex); - node.setAttribute(taskIdKey, nodeIndex.toString()); + const selector = selectorForValue(nodeHash); + node.setAttribute(taskIdKey, nodeHash.toString()); return selector; } @@ -158,3 +159,15 @@ export function getNodeAttributes(node: HTMLElement | Node): Record = await getElementInfosFromPage(page); const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => { return { - label: elementInfo.id.toString(), + label: elementInfo.indexId.toString(), x: elementInfo.rect.left, y: elementInfo.rect.top, width: elementInfo.rect.width, diff --git a/packages/web-integration/src/playwright/cache.ts b/packages/web-integration/src/playwright/cache.ts new file mode 100644 index 000000000..468383083 --- /dev/null +++ b/packages/web-integration/src/playwright/cache.ts @@ -0,0 +1,63 @@ +import path, { join } from 'path'; +import fs from 'fs'; +import { writeDumpFile, getDumpDirPath } from '@midscene/core/utils'; +import { AiTaskCache } from '@/common/task-cache'; +import { findNearestPackageJson } from '@/common/utils'; + +export function writeTestCache(taskFile: string, taskTitle: string, taskCacheJson: AiTaskCache) { + const packageJson = getPkgInfo(); + writeDumpFile({ + fileName: `${taskFile}(${taskTitle})`, + fileExt: 'json', + fileContent: JSON.stringify( + { + pkgName: packageJson.name, + pkgVersion: packageJson.version, + taskFile, + taskTitle, + ...taskCacheJson, + }, + null, + 2, + ), + type: 'cache', + }); +} + +export function readTestCache(taskFile: string, taskTitle: string) { + const cacheFile = join(getDumpDirPath('cache'), `${taskFile}(${taskTitle}).json`); + const pkgInfo = getPkgInfo(); + if (process.env.MIDSCENE_CACHE === 'true' && fs.existsSync(cacheFile)) { + try { + const data = fs.readFileSync(cacheFile, 'utf8'); + const jsonData = JSON.parse(data); + if (jsonData.pkgName !== pkgInfo.name || jsonData.pkgVersion !== pkgInfo.version) { + return undefined; + } + return jsonData as AiTaskCache; + } catch (err) { + return undefined; + } + } + return undefined; +} + +function getPkgInfo(): { name: string; version: string } { + const packageJsonDir = findNearestPackageJson(__dirname); + if (!packageJsonDir) { + console.error('Cannot find package.json directory: ', __dirname); + return { + name: '@midscene/web', + version: '0.0.0', + }; + } + + const packageJsonPath = path.join(packageJsonDir, 'package.json'); + const data = fs.readFileSync(packageJsonPath, 'utf8'); + const packageJson = JSON.parse(data); + + return { + name: packageJson.name, + version: packageJson.version, + }; +} diff --git a/packages/web-integration/src/playwright/index.ts b/packages/web-integration/src/playwright/index.ts index a4fdb6a5d..4fa73f48c 100644 --- a/packages/web-integration/src/playwright/index.ts +++ b/packages/web-integration/src/playwright/index.ts @@ -1,51 +1,61 @@ import { randomUUID } from 'crypto'; +import type { Page as PlaywrightPage } from 'playwright'; import { TestInfo, TestType } from '@playwright/test'; import { PageTaskExecutor } from '../common/tasks'; +import { readTestCache, writeTestCache } from './cache'; import { WebPage } from '@/common/page'; import { PageAgent } from '@/common/agent'; export type APITestType = Pick, 'step'>; const groupAndCaseForTest = (testInfo: TestInfo) => { - let groupName: string; - let caseName: string; + let taskFile: string; + let taskTitle: string; const titlePath = [...testInfo.titlePath]; if (titlePath.length > 1) { - caseName = titlePath.pop()!; - groupName = `${titlePath.join(' > ')}:${testInfo.line}`; + taskTitle = titlePath.pop()!; + taskFile = `${titlePath.join(' > ')}:${testInfo.line}`; } else if (titlePath.length === 1) { - caseName = titlePath[0]; - groupName = `${caseName}:${testInfo.line}`; + taskTitle = titlePath[0]; + taskFile = `${taskTitle}:${testInfo.line}`; } else { - caseName = 'unnamed'; - groupName = 'unnamed'; + taskTitle = 'unnamed'; + taskFile = 'unnamed'; } - return { groupName, caseName }; + return { taskFile, taskTitle }; }; const midSceneAgentKeyId = '_midSceneAgentId'; export const PlaywrightAiFixture = () => { const pageAgentMap: Record = {}; - const agentForPage = (page: WebPage, testId: string) => { + const agentForPage = (page: WebPage, opts: { testId: string; taskFile: string; taskTitle: string }) => { let idForPage = (page as any)[midSceneAgentKeyId]; if (!idForPage) { idForPage = randomUUID(); (page as any)[midSceneAgentKeyId] = idForPage; - pageAgentMap[idForPage] = new PageAgent(page, `${testId}-${idForPage}`); + const testCase = readTestCache(opts.taskFile, opts.taskTitle) || { aiTasks: [] }; + pageAgentMap[idForPage] = new PageAgent(page, { + testId: `${opts.testId}-${idForPage}`, + taskFile: opts.taskFile, + cache: testCase, + }); } return pageAgentMap[idForPage]; }; return { - ai: async ({ page }: any, use: any, testInfo: TestInfo) => { - const agent = agentForPage(page, testInfo.testId); + ai: async ({ page }: { page: PlaywrightPage }, use: any, testInfo: TestInfo) => { + const { taskFile, taskTitle } = groupAndCaseForTest(testInfo); + const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle }); await use(async (taskPrompt: string, opts?: { type?: 'action' | 'query' }) => { - const { groupName, caseName } = groupAndCaseForTest(testInfo); + await page.waitForLoadState('networkidle'); const actionType = opts?.type || 'action'; - const result = await agent.ai(taskPrompt, actionType, caseName, groupName); + const result = await agent.ai(taskPrompt, actionType); return result; }); + const taskCacheJson = agent.actionAgent.taskCache.generateTaskCache(); + writeTestCache(taskFile, taskTitle, taskCacheJson); if (agent.dumpFile) { testInfo.annotations.push({ type: 'MIDSCENE_AI_ACTION', @@ -56,11 +66,12 @@ export const PlaywrightAiFixture = () => { }); } }, - aiAction: async ({ page }: any, use: any, testInfo: TestInfo) => { - const agent = agentForPage(page, testInfo.testId); + aiAction: async ({ page }: { page: PlaywrightPage }, use: any, testInfo: TestInfo) => { + const { taskFile, taskTitle } = groupAndCaseForTest(testInfo); + const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle }); await use(async (taskPrompt: string) => { - const { groupName, caseName } = groupAndCaseForTest(testInfo); - await agent.aiAction(taskPrompt, caseName, groupName); + await page.waitForLoadState('networkidle'); + await agent.aiAction(taskPrompt); }); if (agent.dumpFile) { testInfo.annotations.push({ @@ -72,11 +83,12 @@ export const PlaywrightAiFixture = () => { }); } }, - aiQuery: async ({ page }: any, use: any, testInfo: TestInfo) => { - const agent = agentForPage(page, testInfo.testId); + aiQuery: async ({ page }: { page: PlaywrightPage }, use: any, testInfo: TestInfo) => { + const { taskFile, taskTitle } = groupAndCaseForTest(testInfo); + const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle }); await use(async function (demand: any) { - const { groupName, caseName } = groupAndCaseForTest(testInfo); - const result = await agent.aiQuery(demand, caseName, groupName); + await page.waitForLoadState('networkidle'); + const result = await agent.aiQuery(demand); return result; }); if (agent.dumpFile) { diff --git a/packages/web-integration/src/playwright/reporter/index.ts b/packages/web-integration/src/playwright/reporter/index.ts index eab6fa460..3df6582e4 100644 --- a/packages/web-integration/src/playwright/reporter/index.ts +++ b/packages/web-integration/src/playwright/reporter/index.ts @@ -18,7 +18,7 @@ function logger(...message: any[]) { } class MidSceneReporter implements Reporter { - onBegin(config: FullConfig, suite: Suite) { + async onBegin(config: FullConfig, suite: Suite) { const suites = suite.allTests(); logger(`Starting the run with ${suites.length} tests`); } diff --git a/packages/web-integration/src/playwright/reporter/select-cache-file.ts b/packages/web-integration/src/playwright/reporter/select-cache-file.ts new file mode 100644 index 000000000..94b4fa846 --- /dev/null +++ b/packages/web-integration/src/playwright/reporter/select-cache-file.ts @@ -0,0 +1,131 @@ +import fs from 'fs'; +import path from 'path'; +import inquirer from 'inquirer'; + +interface Task { + type: string; + prompt: string; + pageContext: { + url: string; + width: number; + height: number; + }; + response: any; +} + +interface TasksFile { + taskFile: string; + taskTitle: string; + aiTasks: Task[]; +} + +interface SelectedFilePrompt { + selectedFile: string; +} + +interface ActionPrompt { + action: 'select' | 'exclude'; +} + +interface SelectedTasksPrompt { + selectedTasks: number[]; +} + +// Get all JSON files in the current directory +const getJsonFiles = (dir: string): string[] => { + return fs.readdirSync(dir).filter((file) => path.extname(file) === '.json'); +}; + +// Read JSON file content +const readJsonFile = (filePath: string): TasksFile => { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); +}; + +// Write JSON file content +const writeJsonFile = (filePath: string, data: TasksFile): void => { + fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8'); +}; + +// Format tasks for display +const formatTasks = (tasks: Task[]): string => { + return tasks + .map((task, _index) => { + return `[ ] ${task.type}: ${task.prompt}`; + }) + .join('\n'); +}; + +// Main function +export const getTask = async (): Promise => { + const targetDir = path.join(process.cwd(), 'midscene_run/cache'); + const jsonFiles = getJsonFiles(targetDir); + + if (jsonFiles.length === 0) { + console.log('No JSON files found in the current directory.'); + return; + } + + // Let the user select the JSON file to process + const { selectedFile } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedFile', + message: 'Select the JSON file to process', + choices: jsonFiles, + }, + ] as any); + console.log('selectedFile', selectedFile); + + const filePath = path.join(targetDir, selectedFile); + const tasksFile: TasksFile = readJsonFile(filePath); + + // Provide options to select or exclude tasks + const { action } = await inquirer.prompt([ + { + type: 'list', + name: 'action', + message: 'Choose an action', + choices: [ + { name: 'Select tasks', value: 'select' }, + { name: 'Exclude tasks', value: 'exclude' }, + ], + }, + ] as any); + + // Extract task options + const taskChoices = tasksFile.aiTasks.map((task, index) => ({ + name: `${task.type}: ${task.prompt}`, + value: index, + checked: false, + })); + + // Provide command line interaction using Inquirer.js + const { selectedTasks } = await inquirer.prompt([ + { + type: 'checkbox', + message: action === 'select' ? 'Select tasks to run' : 'Select tasks to exclude', + name: 'selectedTasks', + choices: taskChoices, + }, + ] as any); + + if (action === 'select') { + // Retain tasks based on user selection + tasksFile.aiTasks = tasksFile.aiTasks.filter((_, index) => selectedTasks.includes(index)); + } else if (action === 'exclude') { + // Exclude tasks based on user selection + tasksFile.aiTasks = tasksFile.aiTasks.filter((_, index) => !selectedTasks.includes(index)); + } + + // Write the updated tasks back to the JSON file + writeJsonFile(filePath, tasksFile); + + console.log('Task file updated:', filePath); + console.log('Current task list:'); + console.log(formatTasks(tasksFile.aiTasks)); +}; + +// // Execute the main function +// main().catch((error) => { +// console.error('An error occurred:', error); +// }); diff --git a/packages/web-integration/tests/e2e/ai-xicha.spec.ts b/packages/web-integration/tests/e2e/ai-online-order.spec.ts similarity index 93% rename from packages/web-integration/tests/e2e/ai-xicha.spec.ts rename to packages/web-integration/tests/e2e/ai-online-order.spec.ts index a8f3883be..1dc38ebef 100644 --- a/packages/web-integration/tests/e2e/ai-xicha.spec.ts +++ b/packages/web-integration/tests/e2e/ai-online-order.spec.ts @@ -7,7 +7,7 @@ test.beforeEach(async ({ page }) => { await page.waitForLoadState('networkidle'); }); -test('ai order', async ({ ai, aiQuery }) => { +test('ai online order', async ({ ai, aiQuery }) => { await ai('点击左上角语言切换按钮(英文、中文),在弹出的下拉列表中点击中文'); await ai('在向下滚动一屏'); await ai('直接点击多肉葡萄的规格按钮'); @@ -18,9 +18,6 @@ test('ai order', async ({ ai, aiQuery }) => { await ai('点击选好了按钮'); await ai('点击右上角商品图标按钮'); - // // 随便滚动一下 - // await ai('滚动到最下面'); - const cardDetail = await aiQuery({ productName: '商品名称,在价格上面', productPrice: '商品价格, string', diff --git a/packages/web-integration/tests/e2e/generate-test-data.spec.ts b/packages/web-integration/tests/e2e/generate-test-data.spec.ts index 46fa17c02..a1216ba35 100644 --- a/packages/web-integration/tests/e2e/generate-test-data.spec.ts +++ b/packages/web-integration/tests/e2e/generate-test-data.spec.ts @@ -1,60 +1,60 @@ -// import { test } from '@playwright/test'; -// import { generateTestData, generateTestDataPath } from './tool'; - -// test('generate todo test data', async ({ page }) => { -// await page.goto('https://todomvc.com/examples/react/dist/'); -// // Add data -// await page.getByTestId('text-input').click(); -// await page.keyboard.type('Learn Python'); -// await page.keyboard.press('Enter'); -// await page.getByTestId('text-input').click(); -// await page.keyboard.type('Learn Rust'); -// await page.keyboard.press('Enter'); -// await page.getByTestId('text-input').click(); -// await page.keyboard.type('Learn AI'); -// await page.keyboard.press('Enter'); -// await page.getByText('Learn Rust').hover(); - -// const midsceneTestDataPath = generateTestDataPath('todo'); -// const buffer = await page.screenshot(); - -// const base64String = buffer.toString('base64'); -// await generateTestData(page, midsceneTestDataPath, base64String); -// }); - -// test('generate visualstudio test data', async ({ page }) => { -// await page.goto('https://code.visualstudio.com/'); -// await page.waitForLoadState('networkidle'); - -// const midsceneTestDataPath = generateTestDataPath('visualstudio'); -// const buffer = await page.screenshot(); - -// const base64String = buffer.toString('base64'); -// await generateTestData(page, midsceneTestDataPath, base64String); -// }); - -// test('generate githubstatus test data', async ({ page }) => { -// await page.setViewportSize({ width: 1920, height: 1080 }); -// await page.goto('https://www.githubstatus.com/'); -// await page.waitForLoadState('networkidle'); - -// const midsceneTestDataPath = generateTestDataPath('githubstatus'); -// const buffer = await page.screenshot(); - -// const base64String = buffer.toString('base64'); -// await generateTestData(page, midsceneTestDataPath, base64String); -// }); - -// test('generate xicha test data', async ({ page }) => { -// page.setViewportSize({ width: 400, height: 905 }); -// await page.goto('https://heyteavivocity.meuu.online/home'); -// await page.evaluate('window.localStorage.setItem("LOCALE", "zh-CN")'); -// await page.goto('https://heyteavivocity.meuu.online/home'); -// await page.waitForLoadState('networkidle'); - -// const midsceneTestDataPath = generateTestDataPath('xicha'); -// const buffer = await page.screenshot(); - -// const base64String = buffer.toString('base64'); -// await generateTestData(page, midsceneTestDataPath, base64String); -// }); +import { test } from '@playwright/test'; +import { generateTestData, generateTestDataPath } from './tool'; + +test('generate todo test data', async ({ page }) => { + await page.goto('https://todomvc.com/examples/react/dist/'); + // Add data + await page.getByTestId('text-input').click(); + await page.keyboard.type('Learn Python'); + await page.keyboard.press('Enter'); + await page.getByTestId('text-input').click(); + await page.keyboard.type('Learn Rust'); + await page.keyboard.press('Enter'); + await page.getByTestId('text-input').click(); + await page.keyboard.type('Learn AI'); + await page.keyboard.press('Enter'); + await page.getByText('Learn Rust').hover(); + + const midsceneTestDataPath = generateTestDataPath('todo'); + const buffer = await page.screenshot(); + + const base64String = buffer.toString('base64'); + await generateTestData(page, midsceneTestDataPath, base64String); +}); + +test('generate visualstudio test data', async ({ page }) => { + await page.goto('https://code.visualstudio.com/'); + await page.waitForLoadState('networkidle'); + + const midsceneTestDataPath = generateTestDataPath('visualstudio'); + const buffer = await page.screenshot(); + + const base64String = buffer.toString('base64'); + await generateTestData(page, midsceneTestDataPath, base64String); +}); + +test('generate githubstatus test data', async ({ page }) => { + await page.setViewportSize({ width: 1920, height: 1080 }); + await page.goto('https://www.githubstatus.com/'); + await page.waitForLoadState('networkidle'); + + const midsceneTestDataPath = generateTestDataPath('githubstatus'); + const buffer = await page.screenshot(); + + const base64String = buffer.toString('base64'); + await generateTestData(page, midsceneTestDataPath, base64String); +}); + +test('generate online order test data', async ({ page }) => { + page.setViewportSize({ width: 400, height: 905 }); + await page.goto('https://heyteavivocity.meuu.online/home'); + await page.evaluate('window.localStorage.setItem("LOCALE", "zh-CN")'); + await page.goto('https://heyteavivocity.meuu.online/home'); + await page.waitForLoadState('networkidle'); + + const midsceneTestDataPath = generateTestDataPath('online_order'); + const buffer = await page.screenshot(); + + const base64String = buffer.toString('base64'); + await generateTestData(page, midsceneTestDataPath, base64String); +}); diff --git a/packages/web-integration/tests/puppeteer/bing.spec.ts b/packages/web-integration/tests/puppeteer/bing.test.ts similarity index 100% rename from packages/web-integration/tests/puppeteer/bing.spec.ts rename to packages/web-integration/tests/puppeteer/bing.test.ts diff --git a/packages/web-integration/tests/task-cache.test.ts b/packages/web-integration/tests/task-cache.test.ts new file mode 100644 index 000000000..c9b1c03d5 --- /dev/null +++ b/packages/web-integration/tests/task-cache.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { AIElementParseResponse } from '@midscene/core'; +import { LocateTask, PlanTask, TaskCache } from '@/common/task-cache'; +import { WebElementInfo } from '@/web-element'; +import { WebUIContext } from '@/common/utils'; + +describe('TaskCache', () => { + let taskCache: TaskCache; + let formalPageContext: WebUIContext; + let pageContext: LocateTask['pageContext']; + + beforeEach(() => { + taskCache = new TaskCache(); + pageContext = { + url: 'https://example.com', + size: { width: 1024, height: 768 }, + }; + formalPageContext = { + ...pageContext, + screenshotBase64: '', + content: [{ id: 'element1' } as WebElementInfo], // 示例页面内容 + }; + }); + + it('should return false if no cache is available', async () => { + const result = taskCache.readCache(formalPageContext, 'plan', 'test prompt'); + expect(result).toBe(false); + }); + + it('should return false if the prompt does not match', async () => { + taskCache.cache = { + aiTasks: [{ type: 'plan', prompt: 'different prompt', pageContext, response: { plans: [] } }], + }; + const result = taskCache.readCache(formalPageContext, 'plan', 'test prompt'); + expect(result).toBe(false); + }); + + it('should return false if the element cannot be found in the new context', async () => { + taskCache.cache = { + aiTasks: [ + { + type: 'locate', + prompt: 'test prompt', + pageContext, + response: { elements: [{ id: 'element3' }] } as AIElementParseResponse, + }, + ], + }; + const result = taskCache.readCache(formalPageContext, 'locate', 'test prompt'); + expect(result).toBe(false); + }); + + it('should return cached response if the conditions match', async () => { + const cachedResponse = { plans: [{ type: 'Locate', thought: '', param: {} }] } as PlanTask['response']; + taskCache.cache = { + aiTasks: [{ type: 'plan', prompt: 'test prompt', pageContext, response: cachedResponse }], + }; + const result = taskCache.readCache(formalPageContext, 'plan', 'test prompt'); + expect(result).toEqual(cachedResponse); + }); + + it('should save cache correctly', () => { + const newCache: PlanTask = { + type: 'plan', + prompt: 'new prompt', + pageContext, + response: { plans: [{ type: 'Locate', thought: '', param: {} }] }, + }; + taskCache.saveCache(newCache); + expect(taskCache.newCache.aiTasks).toContain(newCache); + }); + + it('should check page context equality correctly', () => { + const isEqual = taskCache.pageContextEqual(pageContext, formalPageContext); + expect(isEqual).toBe(true); + + const differentContext = { ...formalPageContext, size: { width: 800, height: 600 } }; + const isNotEqual = taskCache.pageContextEqual(pageContext, differentContext); + expect(isNotEqual).toBe(false); + }); + + it('should generate task cache correctly', () => { + const generatedCache = taskCache.generateTaskCache(); + expect(generatedCache).toEqual(taskCache.newCache); + }); +}); diff --git a/packages/web-integration/tsconfig.json b/packages/web-integration/tsconfig.json index 6bb40d153..1839cf7c5 100644 --- a/packages/web-integration/tsconfig.json +++ b/packages/web-integration/tsconfig.json @@ -19,5 +19,5 @@ "strict": true }, "exclude": [ "node_modules"], - "include": ["src", "tests", "./playwright.config.ts", "midscene-reporter.ts", "./vitest.config"] + "include": ["src", "tests", "./playwright.config.ts", "./vitest.config"] } diff --git a/packages/web-integration/vitest.config.ts b/packages/web-integration/vitest.config.ts index 71508d36e..f728b3868 100644 --- a/packages/web-integration/vitest.config.ts +++ b/packages/web-integration/vitest.config.ts @@ -1,6 +1,10 @@ import { defineConfig } from 'vitest/config'; import path from 'path'; +const enableTest = process.env.AITEST; + +const aiModelTest = enableTest !== 'true' ? ['tests/puppeteer/bing.test.ts']: []; + export default defineConfig({ resolve: { alias: { @@ -8,6 +12,7 @@ export default defineConfig({ }, }, test: { - include: ['./tests/puppeteer/**/*.spec.ts'], + include: ['./tests/**/*.test.ts'], + exclude: [...aiModelTest] }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3b9da925..e189b4293 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -260,6 +260,9 @@ importers: '@midscene/visualizer-report': specifier: workspace:* version: link:../visualizer-report + inquirer: + specifier: 10.1.5 + version: 10.1.5 openai: specifier: 4.47.1 version: 4.47.1 @@ -282,6 +285,9 @@ importers: fs-extra: specifier: 11.2.0 version: 11.2.0 + js-sha256: + specifier: 0.11.0 + version: 0.11.0 playwright: specifier: 1.44.1 version: 1.44.1 @@ -1910,6 +1916,62 @@ packages: cpu: [x64] os: [win32] + '@inquirer/checkbox@2.4.4': + resolution: {integrity: sha512-2NWoY9NfFFfQZgNfisI4ttg5yfWB2NfxdQ6xC5prywPvyG1RWETKUNZlqzMnZv/HbNdE2CkhZPSK8hl6WBRiFA==} + engines: {node: '>=18'} + + '@inquirer/confirm@3.1.19': + resolution: {integrity: sha512-dcLbnxmhx3a72c4fM6CwhydG8rS8TZCXtCYU7kUraA+qU2Ue8gNCiYOxnlhb0H0wbTKL23lUo68fX0iMP8t2Dw==} + engines: {node: '>=18'} + + '@inquirer/core@9.0.7': + resolution: {integrity: sha512-wyqnTmlnd9p7cX6tfMlth+/Nx7vV2t/FvtO9VMSi2XjBkNy0MkPr19RSOyP3qrywdlJT+BQbEnXLPqq0wFMw3A==} + engines: {node: '>=18'} + + '@inquirer/editor@2.1.19': + resolution: {integrity: sha512-Tho5lqe3LNpPnZSC0B3KiK+pUMPt7sJAEf2bFr8891qhsOkymj/FFE0NKkdS5oFlPSdTalP0bgjvOhXVR73U5Q==} + engines: {node: '>=18'} + + '@inquirer/expand@2.1.19': + resolution: {integrity: sha512-emQTmMCmCbJx6fDS+VUIrujg8rFw2RVEdH8Qphi7zCgLnyn8Cig8SmSVd/mK5CDlk5BvSLu4EW8wbvyqIxp9QA==} + engines: {node: '>=18'} + + '@inquirer/figures@1.0.5': + resolution: {integrity: sha512-79hP/VWdZ2UVc9bFGJnoQ/lQMpL74mGgzSYX1xUqCVk7/v73vJCMw1VuyWN1jGkZ9B3z7THAbySqGbCNefcjfA==} + engines: {node: '>=18'} + + '@inquirer/input@2.2.6': + resolution: {integrity: sha512-32l4FxgY54O2YXVK6SHyC8gWZaemFBPHiMoKmJMqtwuicjHYF0meZKrTNPfHSOoxUzb6XVSICnXw0wKtsg7nKg==} + engines: {node: '>=18'} + + '@inquirer/number@1.0.7': + resolution: {integrity: sha512-SGABJSqLJeoYV078OwqZ74XwhSLc3Yn6xA3UZD093TXnIm5ggts3h2Pb3+EFWDFekKmUzuTeEkCo4cxaj+bnEg==} + engines: {node: '>=18'} + + '@inquirer/password@2.1.19': + resolution: {integrity: sha512-1BQqBlKFNfa+Cnio3cM1Qs78ho/eOek62ifiJBQZ1Q04K5lSGKwuoXxMZRHoafIN3ag9nicWCQZJxdWsvTNUkw==} + engines: {node: '>=18'} + + '@inquirer/prompts@5.3.5': + resolution: {integrity: sha512-YbaSt+PJF3g/bJzBQN6jAiFyLU3Sz01rYpWb5vDq5Y7yIEhVF3pN8I0zEXIzTpB+y5Df9w+ugeBSIAZAx01mcw==} + engines: {node: '>=18'} + + '@inquirer/rawlist@2.2.1': + resolution: {integrity: sha512-+Cy9tuSqdZiTo0rJ4BvGYjiANc9+70Xpk8aSNw7KQmzp2WnUBhXUNGhXeGxEHPuR9UwL4HCn61WEei9O+uSTVw==} + engines: {node: '>=18'} + + '@inquirer/search@1.0.4': + resolution: {integrity: sha512-h0dxuH+6t0BCHGoCqb66CM4pOQKmZuHZQARk8XYjSG/Tb8umZNVXyJUB1Rmx0s2voU4wd5A4QA0int0c9gkvAg==} + engines: {node: '>=18'} + + '@inquirer/select@2.4.4': + resolution: {integrity: sha512-TfC3M/sUesQnDUMSjCpp631e+Sl3LMzVV5Eyx0z3OoPd44ttF0QNUe/gE4XCX3ofg1LxUHOl/qYUukdfLUxXGw==} + engines: {node: '>=18'} + + '@inquirer/type@1.5.1': + resolution: {integrity: sha512-m3YgGQlKNS0BM+8AFiJkCsTqHEFCWn6s/Rqye3mYwvqY6LdfUv12eSwbsgNzrYyrLXiy7IrrjDLPysaSBwEfhw==} + engines: {node: '>=18'} + '@istanbuljs/load-nyc-config@1.1.0': resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} engines: {node: '>=8'} @@ -3681,6 +3743,9 @@ packages: '@types/ms@0.7.34': resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + '@types/mute-stream@0.0.4': + resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==} + '@types/node-fetch@2.6.11': resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} @@ -3696,6 +3761,9 @@ packages: '@types/node@20.5.1': resolution: {integrity: sha512-4tT2UrL5LBqDwoed9wZ6N3umC4Yhz3W3FloMmiiG4JwmUJWpie0c7lcnUNd4gtMKuDEO4wRVS8B6Xa0uMRsMKg==} + '@types/node@22.0.0': + resolution: {integrity: sha512-VT7KSYudcPOzP5Q0wfbowyNLaVR8QWUdw+088uFWwfvpY6uCWaXpqV6ieLAu9WBcnTa7H4Z5RLK8I5t2FuOcqw==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -3771,6 +3839,9 @@ packages: '@types/uuid@9.0.8': resolution: {integrity: sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==} + '@types/wrap-ansi@3.0.0': + resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} + '@types/yargs-parser@21.0.3': resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -4659,6 +4730,10 @@ packages: resolution: {integrity: sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==} engines: {node: '>= 10'} + cli-width@4.1.0: + resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==} + engines: {node: '>= 12'} + client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} @@ -6454,6 +6529,10 @@ packages: inline-style-parser@0.1.1: resolution: {integrity: sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==} + inquirer@10.1.5: + resolution: {integrity: sha512-ksPK04lcs0GcgkvfmZ5Sjxdm74Nf3Ksy37NTQDrdMZlB0IJgH6TB8cTCJpvGAvt6edsgHa07flRcvXwkr+tPqQ==} + engines: {node: '>=18'} + inquirer@7.3.3: resolution: {integrity: sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA==} engines: {node: '>=8.0.0'} @@ -6936,6 +7015,9 @@ packages: resolution: {integrity: sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==} hasBin: true + js-sha256@0.11.0: + resolution: {integrity: sha512-6xNlKayMZvds9h1Y1VWc0fQHQ82BxTXizWPEtEeGvmOUYpBRy4gbWroHLpzowe6xiQhHpelCQiE7HEdznyBL9Q==} + js-stringify@1.0.2: resolution: {integrity: sha512-rtS5ATOo2Q5k1G+DADISilDA6lv79zIiwFd6CcjuIxGKLFm5C+RLImRscVap9k55i+MOZwgliw+NejvkLuGD5g==} @@ -7682,6 +7764,10 @@ packages: mute-stream@0.0.8: resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} + mute-stream@1.0.0: + resolution: {integrity: sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==} + engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -9324,6 +9410,10 @@ packages: resolution: {integrity: sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==} engines: {node: '>=0.12.0'} + run-async@3.0.0: + resolution: {integrity: sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==} + engines: {node: '>=0.12.0'} + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -10249,6 +10339,9 @@ packages: undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.11.1: + resolution: {integrity: sha512-mIDEX2ek50x0OlRgxryxsenE5XaQD4on5U2inY7RApK3SOJpofyw7uW2AyfMKkhAxXIceo2DeWGVGwyvng1GNQ==} + unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} engines: {node: '>=4'} @@ -10689,6 +10782,10 @@ packages: resolution: {integrity: sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==} engines: {node: '>=12.20'} + yoctocolors-cjs@2.1.2: + resolution: {integrity: sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==} + engines: {node: '>=18'} + zod-validation-error@1.2.0: resolution: {integrity: sha512-laJkD/ugwEh8CpuH+xXv5L9Z+RLz3lH8alNxolfaHZJck611OJj97R4Rb+ZqA7WNly2kNtTo4QwjdjXw9scpiw==} engines: {node: ^14.17 || >=16.0.0} @@ -12593,6 +12690,103 @@ snapshots: '@img/sharp-win32-x64@0.33.3': optional: true + '@inquirer/checkbox@2.4.4': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/figures': 1.0.5 + '@inquirer/type': 1.5.1 + ansi-escapes: 4.3.2 + yoctocolors-cjs: 2.1.2 + + '@inquirer/confirm@3.1.19': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + + '@inquirer/core@9.0.7': + dependencies: + '@inquirer/figures': 1.0.5 + '@inquirer/type': 1.5.1 + '@types/mute-stream': 0.0.4 + '@types/node': 22.0.0 + '@types/wrap-ansi': 3.0.0 + ansi-escapes: 4.3.2 + cli-spinners: 2.9.2 + cli-width: 4.1.0 + mute-stream: 1.0.0 + signal-exit: 4.1.0 + strip-ansi: 6.0.1 + wrap-ansi: 6.2.0 + yoctocolors-cjs: 2.1.2 + + '@inquirer/editor@2.1.19': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + external-editor: 3.1.0 + + '@inquirer/expand@2.1.19': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + yoctocolors-cjs: 2.1.2 + + '@inquirer/figures@1.0.5': {} + + '@inquirer/input@2.2.6': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + + '@inquirer/number@1.0.7': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + + '@inquirer/password@2.1.19': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + ansi-escapes: 4.3.2 + + '@inquirer/prompts@5.3.5': + dependencies: + '@inquirer/checkbox': 2.4.4 + '@inquirer/confirm': 3.1.19 + '@inquirer/editor': 2.1.19 + '@inquirer/expand': 2.1.19 + '@inquirer/input': 2.2.6 + '@inquirer/number': 1.0.7 + '@inquirer/password': 2.1.19 + '@inquirer/rawlist': 2.2.1 + '@inquirer/search': 1.0.4 + '@inquirer/select': 2.4.4 + + '@inquirer/rawlist@2.2.1': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/type': 1.5.1 + yoctocolors-cjs: 2.1.2 + + '@inquirer/search@1.0.4': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/figures': 1.0.5 + '@inquirer/type': 1.5.1 + yoctocolors-cjs: 2.1.2 + + '@inquirer/select@2.4.4': + dependencies: + '@inquirer/core': 9.0.7 + '@inquirer/figures': 1.0.5 + '@inquirer/type': 1.5.1 + ansi-escapes: 4.3.2 + yoctocolors-cjs: 2.1.2 + + '@inquirer/type@1.5.1': + dependencies: + mute-stream: 1.0.0 + '@istanbuljs/load-nyc-config@1.1.0': dependencies: camelcase: 5.3.1 @@ -15907,6 +16101,10 @@ snapshots: '@types/ms@0.7.34': {} + '@types/mute-stream@0.0.4': + dependencies: + '@types/node': 16.11.68 + '@types/node-fetch@2.6.11': dependencies: '@types/node': 16.11.68 @@ -15922,6 +16120,10 @@ snapshots: '@types/node@20.5.1': {} + '@types/node@22.0.0': + dependencies: + undici-types: 6.11.1 + '@types/normalize-package-data@2.4.4': {} '@types/parse-json@4.0.2': {} @@ -15997,6 +16199,8 @@ snapshots: '@types/uuid@9.0.8': {} + '@types/wrap-ansi@3.0.0': {} + '@types/yargs-parser@21.0.3': {} '@types/yargs@17.0.32': @@ -17257,6 +17461,8 @@ snapshots: cli-width@3.0.0: {} + cli-width@4.1.0: {} + client-only@0.0.1: {} cliui@6.0.0: @@ -19589,6 +19795,16 @@ snapshots: inline-style-parser@0.1.1: {} + inquirer@10.1.5: + dependencies: + '@inquirer/prompts': 5.3.5 + '@inquirer/type': 1.5.1 + '@types/mute-stream': 0.0.4 + ansi-escapes: 4.3.2 + mute-stream: 1.0.0 + run-async: 3.0.0 + rxjs: 7.8.1 + inquirer@7.3.3: dependencies: ansi-escapes: 4.3.2 @@ -20261,6 +20477,8 @@ snapshots: jiti@1.21.6: optional: true + js-sha256@0.11.0: {} + js-stringify@1.0.2: {} js-tokens@4.0.0: {} @@ -21281,6 +21499,8 @@ snapshots: mute-stream@0.0.8: {} + mute-stream@1.0.0: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -23432,6 +23652,8 @@ snapshots: run-async@2.4.1: {} + run-async@3.0.0: {} + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -24477,6 +24699,8 @@ snapshots: undici-types@5.26.5: {} + undici-types@6.11.1: {} + unicode-canonical-property-names-ecmascript@2.0.0: {} unicode-match-property-ecmascript@2.0.0: @@ -25019,6 +25243,8 @@ snapshots: yocto-queue@1.1.1: {} + yoctocolors-cjs@2.1.2: {} + zod-validation-error@1.2.0(zod@3.23.8): dependencies: zod: 3.23.8