chore: optimize ai unit test and fix ui-tars keybord event

packages/midscene/tests/ai/evaluate/ai-data/assertion/online_order.json

@@ -1,10 +1,6 @@
 {
   "testDataPath": "test-data/online_order",
   "testCases": [
-    {
-      "prompt": "the 'select option' button background color is yellow",
-      "expected": true
-    },
     {
       "prompt": "there are three tabs in the page, named 'Menu', 'Reviews', 'Merchant'",
       "expected": true

packages/midscene/tests/ai/evaluate/inspect.test.ts

@@ -89,7 +89,6 @@ describe('ai inspect element', () => {
                   },
                 );
 
-                console.log('planning res', res);
                 prompt = res.actions[0].locate?.prompt as string;
                 console.log('prompt from planning', prompt);
                 expect(prompt).toBeTruthy();
@@ -98,8 +97,15 @@ describe('ai inspect element', () => {
                 if (!relocateAfterPlanning) {
                   const matchedId = res.actions[0].locate?.id;
                   if (matchedId) {
+                    const element = elementById(matchedId);
                     return {
-                      elements: [elementById(matchedId)],
+                      elements: [
+                        {
+                          id: element.id,
+                          reason: element.reason ?? '',
+                          text: element.content ?? '',
+                        },
+                      ],
                     };
                   }
 
@@ -117,7 +123,13 @@ describe('ai inspect element', () => {
               return {
                 ...parseResult,
                 elements: parseResult.elements.length
-                  ? [parseResult.elements[0]]
+                  ? [
+                      {
+                        ...parseResult.elements[0],
+                        reason: parseResult.elements[0].reason ?? '',
+                        text: parseResult.elements[0].text ?? '',
+                      },
+                    ]
                   : [],
               };
             },

packages/midscene/tests/ai/util.ts

@@ -1,6 +1,6 @@
 import type { PlanningAction } from '@/types';
 
-export const repeatTime = 2;
+export const repeatTime = 1;
 export function makePlanResultStable(plans: PlanningAction[]) {
   return plans.map((plan) => {
     // Removing thinking makes the results stable for snapshot testing

packages/shared/src/us-keyboard-layout.ts

@@ -712,15 +712,12 @@ const keyMap: Record<string, KeyInput> = {
 };
 
 export function transformHotkeyInput(keyInput: string): string[] {
+  // page down
+  if (keyMap[keyInput.toLowerCase()]) {
+    return [getKeyDefinition(keyMap[keyInput.toLowerCase()])];
+  }
+
   return keyInput.split(' ').map((key) => {
-    let newKey = key;
-    if (keyMap[key.toLowerCase()]) {
-      newKey = keyMap[key.toLowerCase()];
-    }
-    const keyDefinition = getKeyDefinition(newKey);
-    if (keyDefinition) {
-      return keyDefinition;
-    }
-    return newKey;
+    return getKeyDefinition(keyMap[key.toLowerCase()] || key);
   });
 }

packages/shared/tests/unit-test/__snapshots__/tree.test.ts.snap

@@ -0,0 +1,35 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`utils > should be able to describe tree 1`] = `
+"<container id="1" markerId="19"  left="0" top="0" width="100" height="100">
+  Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ...
+  <text id="2" markerId="999" ariaLabel="image description, it could be a long text, very loooooooooooooooooooooooooooooooooooooooooong" left="0" top="0" width="100" height="100">
+    world
+  </text>
+  <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ..." left="0" top="0" width="100" height="100">
+    world 2345
+    <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ..." left="0" top="0" width="100" height="100">
+    </img>
+      <>
+        <img id="3222" markerId="20" ariaLabel="image description" storyContent="Legend had it that the Whispering Woods held an ancient secret, one that connected the world of man and magic, of reality and dream. Each leaf, every ..." left="0" top="0" width="100" height="100">
+          world 2345
+        </img>
+      </>
+  </img>
+</container>"
+`;
+
+exports[`utils > should be able to describe tree, filterNonTextContent = true 1`] = `
+"<container id="1" markerId="19"  left="0" top="0" width="100" height="100">
+  Legend had it that t...
+  <text id="2" markerId="999" ariaLabel="image description, i..." left="0" top="0" width="100" height="100">
+    world
+  </text>
+  <img id="3" markerId="20" ariaLabel="image description" storyContent="Legend had it that t..." left="0" top="0" width="100" height="100">
+    world 2345
+        <img id="3222" markerId="20" ariaLabel="image description" storyContent="Legend had it that t..." left="0" top="0" width="100" height="100">
+          world 2345
+        </img>
+  </img>
+</container>"
+`;

packages/shared/tests/unit-test/keyboard.test.ts

@@ -59,4 +59,9 @@ describe('transformHotkeyInput', () => {
   it('should handle empty input', () => {
     expect(transformHotkeyInput('')).toEqual(['']);
   });
+
+  it('should handle page down', () => {
+    expect(transformHotkeyInput('page down')).toEqual(['PageDown']);
+    expect(transformHotkeyInput('page up')).toEqual(['PageUp']);
+  });
 });