feat(web): export puppeteer integration (#11)

* feat: puppeteer integration * feat: export puppeteer agent * fix: bug of insight visualizer
web-infra-dev · Jul 28, 2024 · beeebba · beeebba
1 parent bb27afe
commit beeebba
Show file tree

Hide file tree

Showing 40 changed files with 303 additions and 399 deletions.
diff --git a/apps/site/docs/visualization/index.mdx b/apps/site/docs/visualization/index.mdx
@@ -3,4 +3,4 @@ pageType: custom
 ---
 import Visualizer from '@midscene/visualizer';
 
-<Visualizer showLogo={false} />
+<Visualizer hideLogo />
diff --git a/packages/midscene/src/utils.ts b/packages/midscene/src/utils.ts
@@ -87,8 +87,8 @@ export function getTmpDir() {
   return path;
 }
 
-export function getTmpFile(fileExt: string) {
-  const filename = `${randomUUID()}.${fileExt}`;
+export function getTmpFile(fileExtWithoutDot: string) {
+  const filename = `${randomUUID()}.${fileExtWithoutDot}`;
   return join(getTmpDir(), filename);
 }
 

diff --git a/packages/midscene/tests/fixtures/heytea.jpeg b/packages/midscene/tests/fixtures/heytea.jpeg
diff --git a/packages/midscene/tests/utils.ts b/packages/midscene/tests/utils.ts
@@ -23,35 +23,6 @@ export function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-// export async function launch(url: string, opt?: {
-//   viewport?: Viewport,
-// }) {
-//   const browser = await puppeteer.launch();
-
-//   const page = (await browser.pages())[0];
-//   const viewportConfig = {
-//     width: opt?.viewport?.pixelWidth || 1920,
-//     height: opt?.viewport?.pixelHeight || 1080,
-//     deviceScaleFactor: opt?.viewport?.dpr || 1,
-//   }
-//   await page.setViewport(viewportConfig);
-//   await Promise.all([
-//     page.waitForNavigation({
-//       timeout: 20 * 1000,
-//       waitUntil: 'networkidle0',
-//     }),
-//     (async () => {
-//       const response = await page.goto(url);
-//       if (response?.status) {
-//         assert(response.status() <= 399, `Page load failed: ${response.status()}`);
-//       }
-//     })(),
-//   ]);
-//   await sleep(2 * 1000);
-
-//   return browser;
-// }
-
 export function fakeInsight(content: string) {
   const screenshot = getFixture('baidu.png');
   const basicContext = {

diff --git a/packages/visualizer/docs/index.tsx b/packages/visualizer/docs/index.tsx
@@ -2,5 +2,6 @@ import React from 'react';
 import Tool from '@/index';
 
 export default () => {
+  // return <Tool hideLogo />;
   return <Tool />;
 };
diff --git a/packages/visualizer/src/component/common.less b/packages/visualizer/src/component/common.less
@@ -3,7 +3,7 @@
 
 @main-orange: #F9483E;
 
-@side-bg: #ECECEC;
+@side-bg: #f7f7f7;
 @title-bg: #DDDDDD;
 @border-color: #CCCCCC;
 @heavy-border-color: #888;

diff --git a/packages/visualizer/src/component/detail-panel.tsx b/packages/visualizer/src/component/detail-panel.tsx
@@ -18,15 +18,15 @@ const ScreenshotItem = (props: { time: string; img: string }) => {
   );
 };
 
+const VIEW_TYPE_BLACKBOARD = 'blackboard';
 const VIEW_TYPE_SCREENSHOT = 'screenshot';
 const VIEW_TYPE_JSON = 'json';
-const VIEW_TYPE_BLACKBOARD = 'blackboard';
 
 const DetailPanel = (): JSX.Element => {
   const dumpId = useInsightDump((store) => store._loadId);
   const blackboardViewAvailable = Boolean(dumpId);
   const activeTask = useExecutionDump((store) => store.activeTask);
-  const [preferredViewType, setViewType] = useState(dumpId ? VIEW_TYPE_BLACKBOARD : VIEW_TYPE_SCREENSHOT);
+  const [preferredViewType, setViewType] = useState(VIEW_TYPE_BLACKBOARD);
 
   const viewType =
     preferredViewType === VIEW_TYPE_BLACKBOARD && !dumpId ? VIEW_TYPE_SCREENSHOT : preferredViewType;

diff --git a/packages/visualizer/src/component/detail-side.less b/packages/visualizer/src/component/detail-side.less
@@ -119,10 +119,8 @@
     }
   }
 
-  .context {
-    pre {
-      text-wrap: balance;
-    }
+  pre {
+    text-wrap: balance;
   }
 
   .item-list-space-up {

diff --git a/packages/visualizer/src/component/detail-side.tsx b/packages/visualizer/src/component/detail-side.tsx
@@ -313,8 +313,14 @@ const DetailSide = (): JSX.Element => {
   ) : null;
 
   const dataCard = dump?.data ? (
-    <Card liteMode={true} onMouseEnter={noop} onMouseLeave={noop} content={<pre>{kv(dump.data)}</pre>}></Card>
+    <Card
+      liteMode={true}
+      onMouseEnter={noop}
+      onMouseLeave={noop}
+      content={<pre>{JSON.stringify(dump.data, undefined, 2)}</pre>}
+    ></Card>
   ) : null;
+  console.log('dump is', dump);
 
   const plans = (task as ExecutionTaskPlanning)?.output?.plans;
   let timelineData: TimelineItemProps[] = [];

diff --git a/packages/visualizer/src/component/sidebar.tsx b/packages/visualizer/src/component/sidebar.tsx
@@ -8,7 +8,7 @@ import {
   LogoutOutlined,
   MinusOutlined,
 } from '@ant-design/icons';
-import { ExecutionTask } from '@midscene/core';
+import { ExecutionTask, ExecutionTaskInsightQuery } from '@midscene/core';
 import { Button } from 'antd';
 import PanelTitle from './panel-title';
 import { timeCostStrElement } from './misc';
@@ -43,8 +43,17 @@ const SideItem = (props: {
     statusText = timeCostStrElement(task.timing.cost);
   }
 
-  const contentRow =
-    task.type === 'Planning' ? <div className="side-item-content">{task.param?.userPrompt} </div> : null;
+  let contentRow: JSX.Element | undefined;
+  if (task.type === 'Planning') {
+    contentRow = <div className="side-item-content">{task.param?.userPrompt}</div>;
+  } else if (task.type === 'Insight' && task.subType === 'Query') {
+    // debugger;
+    const demand = (task as ExecutionTaskInsightQuery).param?.dataDemand;
+    const contentToShow = typeof demand === 'string' ? demand : JSON.stringify(demand);
+    contentRow = <div className="side-item-content">{contentToShow}</div>;
+  } else {
+    // debugger;
+  }
   // add hover listener
   return (
     <div
@@ -72,7 +81,7 @@ const SideItem = (props: {
   );
 };
 
-const Sidebar = (): JSX.Element => {
+const Sidebar = (props: { hideLogo?: boolean }): JSX.Element => {
   const groupedDumps = useExecutionDump((store) => store.dump);
   const setActiveTask = useExecutionDump((store) => store.setActiveTask);
   const activeTask = useExecutionDump((store) => store.activeTask);
@@ -177,7 +186,7 @@ const Sidebar = (): JSX.Element => {
   return (
     <div className="side-bar">
       <div className="top-controls">
-        <div className="brand" onClick={reset}>
+        <div className="brand" onClick={reset} style={{ display: props?.hideLogo ? 'none' : 'block' }}>
           <img
             src={logo}
             alt="Logo"

diff --git a/packages/visualizer/src/component/store.tsx b/packages/visualizer/src/component/store.tsx
@@ -5,6 +5,7 @@ import {
   ExecutionTaskInsightLocate,
   ExecutionTask,
   GroupedActionDump,
+  ExecutionTaskInsightQuery,
 } from '../../../midscene/dist/types';
 
 export const useBlackboardPreference = create<{
@@ -33,7 +34,7 @@ export const useExecutionDump = create<{
   hoverPreviewConfig: { x: number; y: number } | null;
   setHoverPreviewConfig: (config: { x: number; y: number } | null) => void;
   reset: () => void;
-}>((set) => {
+}>((set, get) => {
   const initData = {
     dump: null,
     activeTask: null,
@@ -62,14 +63,15 @@ export const useExecutionDump = create<{
       // set the first one as selected
       for (const item of dump) {
         if (item.executions.length > 0 && item.executions[0].tasks.length > 0) {
-          set({ activeTask: item.executions[0].tasks[0] });
+          get().setActiveTask(item.executions[0].tasks[0]);
           break;
         }
       }
     },
     setActiveTask(task: ExecutionTask) {
       set({ activeTask: task });
-      if ((task as ExecutionTaskInsightLocate).log?.dump?.matchedElement) {
+      console.log('task set', task);
+      if (task.type === 'Insight') {
         syncToInsightDump((task as ExecutionTaskInsightLocate).log!.dump!);
       } else {
         resetInsightDump();

diff --git a/packages/visualizer/src/component/timeline.tsx b/packages/visualizer/src/component/timeline.tsx
@@ -67,7 +67,7 @@ const TimelineWidget = (props: {
   const sizeRatio = 2;
 
   const titleBg = 0xdddddd; // @title-bg
-  const sideBg = 0xececec;
+  const sideBg = 0xf7f7f7; // @side-bg
   const gridTextColor = 0;
   const shotBorderColor = 0x777777;
   const gridLineColor = 0xcccccc; // @border-color

diff --git a/packages/visualizer/src/index.less b/packages/visualizer/src/index.less
@@ -88,7 +88,7 @@ footer.mt-8{
   .main-canvas-container {
     flex-grow: 1;
     height: 100%;
-    background: #F5F5F5;
+    background: #ffffff;
     overflow-x: hidden;
     overflow-y: scroll;
     border-left: 1px solid @border-color;

diff --git a/packages/visualizer/src/index.tsx b/packages/visualizer/src/index.tsx
@@ -13,7 +13,7 @@ import DetailSide from '@/component/detail-side';
 import Sidebar from '@/component/sidebar';
 
 const { Dragger } = Upload;
-const Index = (): JSX.Element => {
+const Index = (props: { hideLogo?: boolean }): JSX.Element => {
   const executionDump = useExecutionDump((store) => store.dump);
   const setGroupedDump = useExecutionDump((store) => store.setGroupedDump);
   const reset = useExecutionDump((store) => store.reset);
@@ -148,8 +148,8 @@ const Index = (): JSX.Element => {
           }
         }}
       >
-        <Panel maxSize={95}>
-          <Sidebar />
+        <Panel maxSize={95} defaultSize={20}>
+          <Sidebar hideLogo={props?.hideLogo} />
         </Panel>
         <PanelResizeHandle
           onDragging={(isChanging) => {

diff --git a/packages/web-integration/modern.inspect.config.ts b/packages/web-integration/modern.inspect.config.ts
@@ -8,7 +8,7 @@ export default defineConfig({
     buildType: 'bundle',
     format: 'iife',
     input: {
-      htmlElement:'src/html-element/index.ts',
+      htmlElement:'src/extractor/index.ts',
     },
     outDir: 'dist/script',
     esbuildOptions: options => {

diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts
@@ -0,0 +1,80 @@
+import { ExecutionDump, GroupedActionDump } from '@midscene/core';
+import { groupedActionDumpFileExt, writeDumpFile } from '@midscene/core/utils';
+import { PageTaskExecutor } from '../common/tasks';
+import { WebPage } from '@/common/page';
+
+export class PageAgent {
+  page: WebPage;
+
+  dumps: GroupedActionDump[];
+
+  constructor(page: WebPage) {
+    this.page = page;
+    this.dumps = [];
+  }
+
+  appendDump(groupName: string, execution: ExecutionDump) {
+    let currentDump = this.dumps.find((dump) => dump.groupName === groupName);
+    if (!currentDump) {
+      currentDump = {
+        groupName,
+        executions: [],
+      };
+      this.dumps.push(currentDump);
+    }
+    currentDump.executions.push(execution);
+  }
+
+  writeOutActionDumps() {
+    writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(this.dumps));
+  }
+
+  async aiAction(taskPrompt: string, dumpCaseName = 'AI Action', dumpGroupName = 'MidScene / Web') {
+    const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName });
+    let error: Error | undefined;
+    try {
+      await actionAgent.action(taskPrompt);
+    } catch (e: any) {
+      error = e;
+    }
+    if (actionAgent.executionDump) {
+      this.appendDump(dumpGroupName, actionAgent.executionDump);
+      this.writeOutActionDumps();
+    }
+    if (error) {
+      // playwright cli won't print error cause, so we print it here
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+  }
+
+  async aiQuery(demand: any, dumpCaseName = 'AI Query', dumpGroupName = 'MidScene / Web') {
+    const actionAgent = new PageTaskExecutor(this.page, { taskName: dumpCaseName });
+    let error: Error | undefined;
+    let result: any;
+    try {
+      result = await actionAgent.query(demand);
+    } catch (e: any) {
+      error = e;
+    }
+    if (actionAgent.executionDump) {
+      this.appendDump(dumpGroupName, actionAgent.executionDump);
+      this.writeOutActionDumps();
+    }
+    if (error) {
+      // playwright cli won't print error cause, so we print it here
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+    return result;
+  }
+
+  async ai(taskPrompt: string, type = 'action', dumpCaseName = 'AI', dumpGroupName = 'MidScene / Web') {
+    if (type === 'action') {
+      return this.aiAction(taskPrompt, dumpCaseName, dumpGroupName);
+    } else if (type === 'query') {
+      return this.aiQuery(taskPrompt, dumpCaseName, dumpGroupName);
+    }
+    throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+  }
+}
diff --git a/...ges/web-integration/src/playwright/cdp.ts → packages/web-integration/src/common/cdp.ts b/...ges/web-integration/src/playwright/cdp.ts → packages/web-integration/src/common/cdp.ts
diff --git a/packages/web-integration/src/common/page.d.ts b/packages/web-integration/src/common/page.d.ts
@@ -0,0 +1,5 @@
+import type { Page as PlaywrightPage } from 'playwright';
+import type { Page as PuppeteerPage, KeyInput } from 'puppeteer';
+
+export type WebPage = PlaywrightPage | PuppeteerPage;
+export type WebKeyInput = KeyInput;