refactor(extract): modify dependencies

Author: zhoushaw

.gitignore

@@ -96,6 +96,7 @@ test-results/
 playwright-report/
 blob-report/
 playwright/.cache/
+iife-script/
 
 # Midscene.js dump files
 __ai_responses__/

biome.json

@@ -16,6 +16,7 @@
       "**/doc_build",
       "*-dump.json",
       "test-results/**",
+      "iife-script/**",
       "script_get_all_texts.tmp.js",
       "**/playwright-report/**",
       "**/todo-report.spec.ts-snapshots/**",

packages/cli/tsconfig.json

@@ -7,7 +7,7 @@
     "forceConsistentCasingInFileNames": true,
     "isolatedModules": true,
     "jsx": "preserve",
-    "lib": ["ESNext"],
+    "lib": ["ESNext", "DOM"],
     "moduleResolution": "node",
     "paths": {
       "@/*": ["./src/*"]

packages/midscene/src/ai-model/prompt/util.ts

@@ -1,10 +1,10 @@
 import assert from 'node:assert';
 import { MATCH_BY_POSITION, getAIConfig } from '@/env';
 import { imageInfoOfBase64 } from '@/image';
-import { descriptionOfTree } from '@/tree';
 import type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';
 import { PromptTemplate } from '@langchain/core/prompts';
 import { NodeType } from '@midscene/shared/constants';
+import { descriptionOfTree, treeToList } from '@midscene/shared/extractor';
 import { generateHashId } from '@midscene/shared/utils';
 import type { ResponseFormatJSONSchema } from 'openai/resources';
 
@@ -261,21 +261,10 @@ export async function describeUserPage<
   const treeRoot = context.tree;
   // dfs tree, save the id and element info
   const idElementMap: Record<string, ElementType> = {};
-  const flatElements: ElementType[] = [];
-  function dfsTree(node: ElementTreeNode<ElementType>) {
-    if (node?.node) {
-      idElementMap[node.node.id] = node.node;
-
-      if (typeof node.node.indexId !== 'undefined') {
-        idElementMap[`${node.node.indexId}`] = node.node;
-      }
-      flatElements.push(node.node);
-    }
-    for (let i = 0; i < (node.children || []).length; i++) {
-      dfsTree(node.children[i]);
-    }
-  }
-  dfsTree(treeRoot);
+  const flatElements: ElementType[] = treeToList(treeRoot);
+  flatElements.forEach((element) => {
+    idElementMap[element.id] = element;
+  });
 
   const contentTree = await descriptionOfTree(
     treeRoot,

packages/shared/modern.config.ts

@@ -5,6 +5,8 @@ const commonConfig = {
     index: './src/index.ts',
     img: './src/img/index.ts',
     constants: './src/constants/index.ts',
+    extractor: './src/extractor/index.ts',
+    'extractor-debug': './src/extractor/debug.ts',
     fs: './src/fs/index.ts',
     utils: './src/utils.ts',
     'us-keyboard-layout': './src/us-keyboard-layout.ts',

packages/web-integration/modern.inspect.config.ts renamed to packages/shared/modern.inspect.config.ts

@@ -38,6 +38,16 @@
       "require": "./dist/lib/utils.js",
       "import": "./dist/es/utils.js"
     },
+    "./extractor": {
+      "types": "./src/extractor/index.ts",
+      "require": "./dist/lib/extractor.js",
+      "import": "./dist/es/extractor.js"
+    },
+    "./extractor-debug": {
+      "types": "./src/extractor/debug.ts",
+      "require": "./dist/lib/extractor-debug.js",
+      "import": "./dist/es/extractor-debug.js"
+    },
     "./keyboard-layout": {
       "types": "./src/us-keyboard-layout.ts",
       "require": "./dist/lib/us-keyboard-layout.js",
@@ -52,13 +62,17 @@
       "browser/img": ["./src/img/index.ts"],
       "fs": ["./src/fs/index.ts"],
       "utils": ["./src/utils.ts"],
+      "extractor": ["./src/extractor/index.ts"],
+      "extractor-debug": ["./src/extractor/debug.ts"],
       "keyboard-layout": ["./src/us-keyboard-layout.ts"]
     }
   },
   "files": ["dist", "src", "README.md"],
   "scripts": {
     "dev": "modern dev",
-    "build": "modern build",
+    "build": "npm run build:pkg && npm run build:script",
+    "build:pkg": "modern build -c ./modern.config.ts",
+    "build:script": "modern build -c ./modern.inspect.config.ts",
     "build:watch": "modern build -w",
     "reset": "rimraf ./**/node_modules",
     "lint": "modern lint",
@@ -120,6 +134,16 @@
         "require": "./dist/lib/utils.js",
         "import": "./dist/es/utils.js"
       },
+      "./extractor": {
+        "types": "./dist/lib/extractor.d.ts",
+        "require": "./dist/lib/extractor.js",
+        "import": "./dist/es/extractor.js"
+      },
+      "./extractor-debug": {
+        "types": "./dist/lib/extractor-debug.d.ts",
+        "require": "./dist/lib/extractor-debug.js",
+        "import": "./dist/es/extractor-debug.js"
+      },
       "./keyboard-layout": {
         "types": "./dist/lib/us-keyboard-layout.d.ts",
         "require": "./dist/lib/us-keyboard-layout.js",
@@ -134,6 +158,8 @@
         "browser/img": ["./dist/browser/img.d.ts"],
         "fs": ["./dist/lib/fs.d.ts"],
         "utils": ["./dist/lib/utils.d.ts"],
+        "extractor": ["./dist/lib/extractor.d.ts"],
+        "extractor-debug": ["./dist/lib/extractor-debug.d.ts"],
         "keyboard-layout": ["./dist/lib/us-keyboard-layout.d.ts"]
       }
     }

packages/shared/package.json

@@ -5,17 +5,17 @@ import { generateId, midsceneGenerateHash } from './util';
 // https://github.com/appium/appium/tree/master/packages/universal-xml-plugin
 // Definition of NodeDescriptor interface
 interface NodeDescriptor {
-  node: Node;
+  node: globalThis.Node;
   children: NodeDescriptor[];
 }
 
 // Retrieve attributes from a node
-function getNodeAttributes(node: Node): { [key: string]: string } {
+function getNodeAttributes(node: globalThis.Node): { [key: string]: string } {
   const attrs: { [key: string]: string } = {};
 
   // Check if node exists and its type is ELEMENT_NODE
   if (node && node.nodeType === 1) {
-    const element = node as Element;
+    const element = node as globalThis.Element;
 
     for (let i = 0; i < element.attributes.length; i++) {
       const attr = element.attributes[i];
@@ -115,7 +115,9 @@ function getXPathForElement(element: Node): string {
 }
 
 // Perform DFS traversal and collect element information
-export function extractTextWithPosition(initNode: Document): ElementInfo[] {
+export function extractTextWithPosition(
+  initNode: globalThis.Document,
+): ElementInfo[] {
   const elementInfoArray: ElementInfo[] = [];
   let nodeIndex = 1;
 

packages/web-integration/src/extractor/client-extractor.ts renamed to packages/shared/src/extractor/client-extractor.ts

@@ -2,4 +2,4 @@ export {
   NodeType,
   TEXT_MAX_SIZE,
   TEXT_SIZE_THRESHOLD,
-} from '@midscene/shared/constants';
+} from '../constants';

packages/web-integration/src/extractor/constants.ts renamed to packages/shared/src/extractor/constants.ts

@@ -1,4 +1,4 @@
-export function isFormElement(node: Node) {
+export function isFormElement(node: globalThis.Node) {
   return (
     node instanceof HTMLElement &&
     (node.tagName.toLowerCase() === 'input' ||
@@ -8,11 +8,15 @@ export function isFormElement(node: Node) {
   );
 }
 
-export function isButtonElement(node: Node): node is HTMLButtonElement {
+export function isButtonElement(
+  node: globalThis.Node,
+): node is globalThis.HTMLButtonElement {
   return node instanceof HTMLElement && node.tagName.toLowerCase() === 'button';
 }
 
-export function isImgElement(node: Node): node is HTMLImageElement {
+export function isImgElement(
+  node: globalThis.Node,
+): node is globalThis.HTMLImageElement {
   // check if the node is an image element
   if (!includeBaseElement(node) && node instanceof Element) {
     const computedStyle = window.getComputedStyle(node);
@@ -32,7 +36,7 @@ export function isImgElement(node: Node): node is HTMLImageElement {
   );
 }
 
-function isIconfont(node: Node): boolean {
+function isIconfont(node: globalThis.Node): boolean {
   if (node instanceof Element) {
     const computedStyle = window.getComputedStyle(node);
     const fontFamilyValue = computedStyle.fontFamily || '';
@@ -42,11 +46,15 @@ function isIconfont(node: Node): boolean {
   return false;
 }
 
-export function isTextElement(node: Node): node is HTMLTextAreaElement {
+export function isTextElement(
+  node: globalThis.Node,
+): node is globalThis.HTMLTextAreaElement {
   return node.nodeName.toLowerCase() === '#text' && !isIconfont(node);
 }
 
-export function isContainerElement(node: Node): node is HTMLElement {
+export function isContainerElement(
+  node: globalThis.Node,
+): node is globalThis.HTMLElement {
   if (!(node instanceof HTMLElement)) return false;
 
   // include other base elements
@@ -63,7 +71,7 @@ export function isContainerElement(node: Node): node is HTMLElement {
   return false;
 }
 
-function includeBaseElement(node: Node) {
+function includeBaseElement(node: globalThis.Node) {
   if (!(node instanceof HTMLElement)) return false;
 
   // include text

packages/web-integration/src/extractor/debug.ts renamed to packages/shared/src/extractor/debug.ts

@@ -1,4 +1,4 @@
-import type { NodeType } from '@midscene/shared/constants';
+import type { NodeType } from '../constants';
 
 export interface ElementInfo {
   id: string;
@@ -20,6 +20,8 @@ export interface ElementNode {
   children: ElementNode[];
 }
 
+export { descriptionOfTree, traverseTree, treeToList } from './tree';
+
 export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor';
 
 export { extractTextWithPosition as clientExtractTextWithPosition } from './client-extractor';