feat: show pointer position in chrome extension (#286)

---------

Co-authored-by: zhouxiao.shaw <[email protected]>

apps/site/docs/en/model-provider.md

@@ -86,6 +86,8 @@ There are some extra configs when using Azure OpenAI Service.
 
 ### Use ADT token provider
 
+This mode cannot be used in Chrome extension.
+
 ```bash
 # this is always true when using Azure OpenAI Service
 export MIDSCENE_USE_AZURE_OPENAI=1

apps/site/docs/en/quick-experience.mdx

@@ -49,5 +49,5 @@ It's mainly due to conflicts with other extensions injecting `<iframe />` or `<s
 To find the suspicious plugins:
 
 1. Open the Devtools of the page, find the `<script>` or `<iframe>` with a url like `chrome-extension://{ID-of-the-suspicious-plugin}/...`.
-2. Copy the ID from the url, open chrome://extensions/, find the plugin with the same ID, disable it.
+2. Copy the ID from the url, open `chrome://extensions/` , use cmd+f to find the plugin with the same ID, disable it.
 3. Refresh the page, try again.

apps/site/docs/zh/model-provider.md

@@ -73,7 +73,9 @@ import 'dotenv/config';
 
 ## 使用 Azure OpenAI 服务时的配置
 
-使用 ADT token provider
+### 使用 ADT token provider
+
+此种模式无法运行在浏览器插件中。
 
 ```bash
 # 使用 Azure OpenAI 服务时，配置为 1
@@ -85,7 +87,7 @@ export AZURE_OPENAI_API_VERSION="2024-05-01-preview"
 export AZURE_OPENAI_DEPLOYMENT="gpt-4o"
 ```
 
-使用 keyless 模式
+### 使用 keyless 模式
 
 ```bash
 export MIDSCENE_USE_AZURE_OPENAI=1

apps/site/docs/zh/quick-experience.mdx

@@ -47,5 +47,5 @@ OPENAI_API_KEY="sk-replace-by-your-own"
 找到可疑插件：
 
 1. 打开页面的调试器，找到被其他插件注入的 `<iframe />` 或 `<script />`，一般 URL 是 `chrome-extension://{这串就是ID}/...` 格式，复制其 ID。
-2. 打开 chrome://extensions/ ，找到相同 ID 的插件，禁用它。
+2. 打开 `chrome://extensions/` ，用 cmd+f 找到相同 ID 的插件，禁用它。
 3. 刷新页面，再次尝试。

packages/midscene/src/ai-model/prompt/llm-planning.ts

@@ -122,16 +122,16 @@ The JSON format is as follows:
 {{
   "actions": [
     {{
-      "thought": "Reasons for generating this task, and why this task is feasible on this page",
+      "thought": "Reasons for generating this task, and why this task is feasible on this page.", // Use the same language as the user's instruction.
       "type": "Tap",
       "param": null,
       "locate": {sample} | null,
     }},
     // ... more actions
   ],
   "taskWillBeAccomplished": boolean,
-  "furtherPlan": {{ "whatHaveDone": string, "whatToDoNext": string }} | null,
-  "error"?: string
+  "furtherPlan": {{ "whatHaveDone": string, "whatToDoNext": string }} | null, // Use the same language as the user's instruction.
+  "error"?: string // Use the same language as the user's instruction.
 }}
 Here is an example of how to decompose a task:
 

packages/midscene/src/insight/utils.ts

@@ -1,14 +1,15 @@
 import assert from 'node:assert';
-import { randomUUID } from 'node:crypto';
 /* eslint-disable @typescript-eslint/ban-types */
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
-import { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';
+import {
+  MIDSCENE_MODEL_NAME,
+  MIDSCENE_USE_VLM_UI_TARS,
+  getAIConfig,
+} from '@/env';
 import type {
-  BaseElement,
   DumpMeta,
   DumpSubscriber,
-  ElementById,
   InsightDump,
   PartialInsightDumpFromSDK,
 } from '@/types';
@@ -40,6 +41,9 @@ export function writeInsightDump(
     sdkVersion: getVersion(),
     logTime: Date.now(),
     model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',
+    model_description: getAIConfig(MIDSCENE_USE_VLM_UI_TARS)
+      ? 'vlm-ui-tars enabled'
+      : '',
   };
   const finalData: InsightDump = {
     logId: id,

packages/midscene/src/types.ts

@@ -140,6 +140,7 @@ export interface DumpMeta {
   sdkVersion: string;
   logTime: number;
   model_name: string;
+  model_description?: string;
 }
 
 export interface ReportDumpWithAttributes {

packages/visualizer/src/component/logo.tsx

@@ -1,13 +1,13 @@
 import './logo.less';
 
+export const LogoUrl =
+  'https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png';
+
 const Logo = ({ withGithubStar = false }: { withGithubStar?: boolean }) => {
   if (withGithubStar) {
     return (
       <div className="logo logo-with-star-wrapper">
-        <img
-          alt="Midscene_logo"
-          src="https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png"
-        />
+        <img alt="Midscene_logo" src={LogoUrl} />
         <a
           href="https://github.com/web-infra-dev/midscene"
           target="_blank"

packages/visualizer/src/component/playground-component.tsx

@@ -33,7 +33,7 @@ import type { WebUIContext } from '@midscene/web/utils';
 import type { MenuProps } from 'antd';
 import { Dropdown, Space } from 'antd';
 import { EnvConfig } from './env-config';
-import { type HistoryItem, useChromeTabInfo, useEnvConfig } from './store';
+import { type HistoryItem, useEnvConfig } from './store';
 
 import {
   ChromeExtensionProxyPage,
@@ -339,16 +339,15 @@ export function Playground({
         }
       }
     } catch (e: any) {
+      const errorMessage = e?.message || '';
       console.error(e);
-      if (typeof e === 'string') {
-        if (e.includes('of different extension')) {
-          result.error =
-            'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-experience.html#faq';
-        } else {
-          result.error = e;
-        }
-      } else if (!e.message?.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
-        result.error = e.message;
+      if (errorMessage.includes('of different extension')) {
+        result.error =
+          'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-experience.html#faq';
+      } else if (
+        !errorMessage?.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)
+      ) {
+        result.error = errorMessage;
       } else {
         result.error = 'Unknown error';
       }

packages/visualizer/src/component/replay-scripts.tsx

@@ -119,6 +119,7 @@ export interface ReplayScriptsInfo {
   height: number;
   sdkVersion?: string;
   modelName?: string;
+  modelDescription?: string;
 }
 
 export const allScriptsFromDump = (
@@ -129,6 +130,7 @@ export const allScriptsFromDump = (
   let height = 0;
   let sdkVersion = '';
   let modelName = '';
+  let modelDescription = '';
 
   dump.executions.forEach((execution) => {
     execution.tasks.forEach((task) => {
@@ -145,6 +147,10 @@ export const allScriptsFromDump = (
       if (insightTask.log?.dump?.model_name) {
         modelName = insightTask.log.dump.model_name;
       }
+
+      if (insightTask.log?.dump?.model_description) {
+        modelDescription = insightTask.log.dump.model_description;
+      }
     });
   });
 
@@ -176,6 +182,7 @@ export const allScriptsFromDump = (
     height,
     sdkVersion,
     modelName,
+    modelDescription,
   };
 };
 

packages/visualizer/src/component/sidebar.tsx

@@ -63,6 +63,7 @@ const SideItem = (props: {
 const Sidebar = (): JSX.Element => {
   const sdkVersion = useExecutionDump((store) => store.sdkVersion);
   const modelName = useExecutionDump((store) => store.modelName);
+  const modelDescription = useExecutionDump((store) => store.modelDescription);
   const groupedDump = useExecutionDump((store) => store.dump);
   const setActiveTask = useExecutionDump((store) => store.setActiveTask);
   const activeTask = useExecutionDump((store) => store.activeTask);
@@ -110,8 +111,9 @@ const Sidebar = (): JSX.Element => {
     };
   }, [currentSelectedIndex, allTasks, setActiveTask]);
 
+  const modelDescriptionText = modelDescription ? `, ${modelDescription}` : '';
   const envInfo = sdkVersion
-    ? `v${sdkVersion}, ${modelName || 'default model'}`
+    ? `v${sdkVersion}, ${modelName || 'default model'}${modelDescriptionText}`
     : '';
 
   const sideList = groupedDump ? (

packages/visualizer/src/component/store.tsx

@@ -91,6 +91,9 @@ export const useChromeTabInfo = create<{
   };
 
   Promise.resolve().then(async () => {
+    if (typeof window.chrome === 'undefined') {
+      return;
+    }
     const tab = await activeTab();
     const windowId = await currentWindowId();
     set({
@@ -206,6 +209,7 @@ export const useExecutionDump = create<{
   allExecutionAnimation: AnimationScript[] | null;
   sdkVersion: string | null;
   modelName: string | null;
+  modelDescription: string | null;
   insightWidth: number | null;
   insightHeight: number | null;
   activeExecution: ExecutionDump | null;
@@ -228,6 +232,7 @@ export const useExecutionDump = create<{
     allExecutionAnimation: null,
     sdkVersion: null,
     modelName: null,
+    modelDescription: null,
     insightWidth: null,
     insightHeight: null,
     activeTask: null,
@@ -297,6 +302,7 @@ export const useExecutionDump = create<{
           width,
           height,
           modelName,
+          modelDescription,
           sdkVersion,
         } = allScriptsInfo;
 
@@ -307,6 +313,7 @@ export const useExecutionDump = create<{
           insightWidth: width,
           insightHeight: height,
           modelName,
+          modelDescription,
           sdkVersion,
         });
       }

packages/visualizer/src/extension/scripts/stop-water-flow.ts

@@ -1,4 +1,3 @@
-const styleElements = document.querySelectorAll('[id="water-flow-animation"]');
-styleElements.forEach((element) => {
-  document.head.removeChild(element);
-});
+if (typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+  window.midsceneWaterFlowAnimation.disable();
+}