Merge branch 'main' into patch-1

Author: ycjcl868

multimodal/agent-tars/core/examples/browser-gui-agent-rt.ts

@@ -0,0 +1,71 @@
+/*
+ * Quick real-environment runner for BrowserGUIAgent
+ * - Launches a local Chrome (via Puppeteer)
+ * - Navigates to a page
+ * - Uses BrowserGUIAgent.screenshot() and saves a WEBP file
+ */
+import { AgentTARS } from '../src';
+
+async function main() {
+  const localAgent = new AgentTARS({
+    model: {
+      provider: 'volcengine',
+      id: 'ep-20250510145437-5sxhs',
+      apiKey: process.env.ARK_API_KEY,
+      displayName: 'doubao-1.5-thinking-vision-pro',
+    },
+    toolCallEngine: 'structured_outputs',
+  });
+  await localAgent.initialize();
+  const tools = localAgent.getTools();
+  console.log('\n📋 Available Tools:');
+  console.log('─'.repeat(80));
+  tools.forEach((tool, index) => {
+    const num = (index + 1).toString().padStart(2, ' ');
+    const name = tool.name.padEnd(30, ' ');
+    const desc = (tool.description || 'No description').substring(0, 45).replace(/\n/g, ' ');
+    console.log(`${num}. ${name} │ ${desc}`);
+  });
+  console.log('─'.repeat(80));
+  console.log(`Total: ${tools.length} tools\n`);
+
+  // Test tasks to run
+  const tasks = [
+    'Open https://seed-tars.com',
+    'Use gui to go to https://www.producthunt.com/, search the top products for "AI", from the results, identify the top-listed product (the top 3 result). Collect the following information from that product\'s card: 1. Product name 2. Short description 3. Number of upvotes summarize it and report to me.',
+    'Use gui action, go to https://sample-files.com/documents/pdf/, find the 65KB pdf file, preview it, scroll the file from top to bottom.',
+  ];
+
+  // Execute tasks iteratively
+  for (let i = 0; i < tasks.length; i++) {
+    const task = tasks[i];
+    console.log(`\n🚀 Executing Task ${i + 1}:`);
+    console.log(`📝 ${task}`);
+    console.log('─'.repeat(80));
+
+    try {
+      const response = await localAgent.run(task);
+      console.log(`✅ Task ${i + 1} Response:`, response);
+    } catch (error) {
+      console.error(`❌ Task ${i + 1} Failed:`, error);
+    }
+
+    console.log('─'.repeat(80));
+  }
+  console.log('🎉 All tasks completed. Exiting...');
+
+  // Clean up resources and exit
+  console.log('\n🧹 Cleaning up resources...');
+  try {
+    await localAgent.cleanup();
+    console.log('✅ Cleanup completed');
+  } catch (error) {
+    console.error('❌ Cleanup failed:', error);
+  }
+  process.exit(0);
+}
+
+main().catch((err) => {
+  console.error('Runner failed:', err);
+  process.exit(1);
+});

multimodal/agent-tars/core/package.json

@@ -17,7 +17,7 @@
     "prepublishOnly": "pnpm run build",
     "test": "vitest run",
     "test:watch": "vitest",
-    "agent:snapshot:genreate": "npx tsx snapshot/runner.ts generate all",
+    "agent:snapshot:generate": "npx tsx snapshot/runner.ts generate all",
     "agent:snapshot:test": "npx vitest snapshot/index.test.ts",
     "benchmark:crawl": "cd benchmark/crawl && pnpm start"
   },
@@ -29,6 +29,8 @@
   },
   "devDependencies": {
     "@gui-agent/operator-browser": "workspace:*",
+    "@gui-agent/action-parser": "workspace:*",
+    "@gui-agent/shared": "workspace:*",
     "@agent-infra/mcp-server-browser": "1.1.10",
     "@agent-infra/mcp-server-commands": "1.1.10",
     "@agent-infra/mcp-server-filesystem": "1.1.10",

multimodal/agent-tars/core/src/environments/local/browser/browser-gui-agent.ts

@@ -9,8 +9,12 @@ import { BrowserOperator } from '@gui-agent/operator-browser';
 import { ConsoleLogger, AgentEventStream, Tool, z } from '@tarko/mcp-agent';
 import { ImageCompressor, formatBytes } from '@tarko/shared-media-utils';
 import { ActionInputs, PredictionParsed } from '@agent-tars/interface';
+import { ActionParserHelper } from '@gui-agent/action-parser';
+import { Coordinates, NormalizeCoordinates } from '@gui-agent/shared/types';
+import { normalizeActionCoords } from '@gui-agent/shared/utils';
 import {
   convertToGUIResponse,
+  convertToAgentUIAction,
   createGUIErrorResponse,
   GUIExecuteResult,
 } from '@tarko/shared-utils';
@@ -37,6 +41,22 @@ export interface GUIAgentOptions {
   eventStream?: AgentEventStream.Processor;
 }
 
+const actionParserHelper = new ActionParserHelper();
+
+const defaultNormalizeCoords: NormalizeCoordinates = (rawCoords: Coordinates) => {
+  if (!rawCoords.raw) {
+    return { normalized: rawCoords };
+  }
+  const normalizedCoords = {
+    ...rawCoords,
+    normalized: {
+      x: rawCoords.raw.x / 1000,
+      y: rawCoords.raw.y / 1000,
+    },
+  };
+  return { normalized: normalizedCoords };
+};
+
 /**
  * Browser GUI Agent for visual browser automation
  */
@@ -90,7 +110,7 @@ scroll(point='<point>x1 y1</point>', direction='down or up or right or left') -
 wait()                                         - Wait 5 seconds and take a screenshot to check for changes
 
 ## Note
-- Folow user lanuage in in \`thought\` part.
+- Follow user language in in \`thought\` part.
 - Describe your thought in \`step\` part.
 - Describe your action in \`Step\` part.
 - Extract the data your see in \`pageData\` part.
@@ -109,30 +129,39 @@ wait()                                         - Wait 5 seconds and take a scree
       }),
       function: async ({ thought, step, action }) => {
         try {
-          const parsed = this.parseAction(action);
-          parsed.thought = thought;
+          const parsedAction = actionParserHelper.parseActionCallString(action);
+          if (!parsedAction) {
+            return createGUIErrorResponse(action, 'Invalid action format');
+          }
+          const normalizedCoordsAction = normalizeActionCoords(
+            parsedAction,
+            defaultNormalizeCoords,
+          );
 
           this.logger.debug({
             thought,
             step,
             action,
-            parsedAction: JSON.stringify(parsed, null, 2),
+            normalizedCoordsAction: JSON.stringify(normalizedCoordsAction, null, 2),
             screenDimensions: {
               width: this.screenWidth,
               height: this.screenHeight,
             },
           });
 
-          const operatorResult: GUIExecuteResult = await this.browserOperator.execute({
-            parsedPrediction: parsed,
-            screenWidth: this.screenWidth || 1920,
-            screenHeight: this.screenHeight || 1080,
+          const operatorResult = await this.browserOperator.doExecute({
+            actions: [normalizedCoordsAction],
           });
+          this.logger.debug('Browser action completed', operatorResult);
 
           await sleep(500);
 
-          const guiResponse = convertToGUIResponse(action, parsed, operatorResult);
-          return guiResponse;
+          return {
+            success: true,
+            action: action,
+            normalizedAction: convertToAgentUIAction(normalizedCoordsAction),
+            observation: undefined, // Reserved for future implementation
+          };
         } catch (error) {
           this.logger.error(
             `Browser action failed: ${error instanceof Error ? error.message : String(error)}`,
@@ -164,7 +193,7 @@ wait()                                         - Wait 5 seconds and take a scree
     // Record screenshot start time
     const startTime = performance.now();
 
-    const output = await this.browserOperator.screenshot();
+    const output = await this.browserOperator.doScreenshot();
 
     // Calculate screenshot time
     const endTime = performance.now();

multimodal/agent-tars/core/src/environments/local/browser/browser-manager.ts

@@ -80,7 +80,7 @@ export class BrowserManager {
       this.logger.info('🌐 Launching browser instance...');
       const browser = this.getBrowser();
       await browser.launch(options);
-      // FIXME: Create new page here to avoid the mcp server browser createing
+      // FIXME: Create new page here to avoid the mcp server browser creating
       // another browser instance, we need a better solution here.
       // const openingPage = await browser.createPage();
       // await openingPage.goto('about:blank', {

multimodal/agent-tars/core/src/webui-config.ts

@@ -146,4 +146,7 @@ export const AGENT_TARS_WEBUI_CONFIG: AgentWebUIImplementation = {
     defaultLayout: 'narrow-chat',
     enableLayoutSwitchButton: true,
   },
+  debug: {
+    enableEventStreamViewer: true,
+  },
 };

multimodal/agent-tars/core/tests/__snapshots__/browser_tools_hybrid.snap

@@ -15,7 +15,7 @@ scroll(point='<point>x1 y1</point>', direction='down or up or right or left') -
 wait()                                         - Wait 5 seconds and take a screenshot to check for changes
 
 ## Note
-- Folow user lanuage in in `thought` part.
+- Follow user language in in `thought` part.
 - Describe your thought in `step` part.
 - Describe your action in `Step` part.
 - Extract the data your see in `pageData` part.

multimodal/agent-tars/core/tests/__snapshots__/browser_tools_visual-grounding.snap

@@ -15,7 +15,7 @@ scroll(point='<point>x1 y1</point>', direction='down or up or right or left') -
 wait()                                         - Wait 5 seconds and take a screenshot to check for changes
 
 ## Note
-- Folow user lanuage in in `thought` part.
+- Follow user language in in `thought` part.
 - Describe your thought in `step` part.
 - Describe your action in `Step` part.
 - Extract the data your see in `pageData` part.

multimodal/gui-agent/agent-sdk/.gitignore

@@ -1 +1,2 @@
 snapshots
+snapshot

multimodal/gui-agent/agent-sdk/examples/configs/browser-ve-15vp.config.ts

@@ -6,14 +6,15 @@ import 'dotenv/config';
 import path from 'path';
 
 import { defineConfig } from '@tarko/agent-cli';
-import { browserOperator } from './operators';
+import { browserOperator, remoteBrowserOperator } from './operators';
 import { doubao_1_5_vp } from './models';
-import { systemPromptTemplate1 } from './promptTemps';
+import { systemPromptTemplate2 } from './promptTemps';
 
 export default defineConfig({
-  operator: browserOperator,
+  // operator: browserOperator,
+  operator: remoteBrowserOperator,
   model: doubao_1_5_vp,
-  systemPrompt: systemPromptTemplate1,
+  systemPrompt: systemPromptTemplate2,
   snapshot: {
     enable: true,
     storageDirectory: path.join(__dirname, '../snapshots/browser-ve-15vp'),

multimodal/gui-agent/agent-sdk/examples/configs/operators.ts

@@ -4,35 +4,27 @@
  */
 import { AdbOperator } from '@gui-agent/operator-adb';
 import { NutJSOperator } from '@gui-agent/operator-nutjs';
-import { Operator, ScreenContext } from '@gui-agent/shared/base';
 import {
-  SupportedActionType,
-  ScreenshotOutput,
-  ExecuteParams,
-  ExecuteOutput,
-} from 'gui-agent/shared/src/types';
+  LocalBrowserOperator,
+  RemoteBrowserOperator,
+  SearchEngine,
+} from '@gui-agent/operator-browser';
 
 const computerOperator = new NutJSOperator();
 const androidOperator = new AdbOperator();
+const browserOperator = new LocalBrowserOperator({
+  searchEngine: SearchEngine.GOOGLE,
+  showActionInfo: false,
+  showWaterFlow: false,
+  highlightClickableElements: false,
+});
 
-class MockedBrowserOperator extends Operator {
-  protected initialize(): Promise<void> {
-    throw new Error('Method not implemented.');
-  }
-  protected supportedActions(): Array<SupportedActionType> {
-    throw new Error('Method not implemented.');
-  }
-  protected screenContext(): ScreenContext {
-    throw new Error('Method not implemented.');
-  }
-  protected screenshot(): Promise<ScreenshotOutput> {
-    throw new Error('Method not implemented.');
-  }
-  protected execute(params: ExecuteParams): Promise<ExecuteOutput> {
-    throw new Error('Method not implemented.');
-  }
-}
+const remoteBrowserOperator = new RemoteBrowserOperator({
+  wsEndpoint: 'ws://localhost:9222/devtools/browser/<id>',
+  searchEngine: SearchEngine.GOOGLE,
+  showActionInfo: true,
+  showWaterFlow: true,
+  highlightClickableElements: true,
+});
 
-const browserOperator = new MockedBrowserOperator();
-
-export { computerOperator, androidOperator, browserOperator };
+export { computerOperator, androidOperator, browserOperator, remoteBrowserOperator };