feat(gui-agent): add configurable image detail calculator

Author: ZhaoHeh

multimodal/gui-agent/agent-sdk/src/GUIAgent.ts

@@ -13,7 +13,11 @@ import { GUIAgentToolCallEngine } from './ToolCallEngine';
 import { SYSTEM_PROMPT } from './prompts';
 import { Base64ImageParser } from '@agent-infra/media-utils';
 import { Operator, BaseGUIAgent } from '@gui-agent/shared/base';
-import { GUIAgentConfig, NormalizeCoordinates } from '@gui-agent/shared/types';
+import {
+  GUIAgentConfig,
+  NormalizeCoordinates,
+  ImageDetailCalculator,
+} from '@gui-agent/shared/types';
 import {
   assembleSystemPrompt,
   isSystemPromptTemplate,
@@ -23,6 +27,7 @@ import {
 } from '@gui-agent/shared/utils';
 import { GUI_ADAPTED_TOOL_NAME } from './constants';
 import { convertToAgentUIAction, createGUIErrorResponse } from './utils';
+import { defaultDetailCalculator } from './defaultImpls';
 
 const defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);
 
@@ -31,6 +36,7 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent {
 
   private operator: Operator | undefined;
   private normalizeCoordinates: NormalizeCoordinates;
+  private detailCalculator: ImageDetailCalculator;
   private loopIntervalInMs: number;
 
   constructor(config: GUIAgentConfig<T>) {
@@ -40,6 +46,7 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent {
       systemPrompt,
       customeActionParser,
       normalizeCoordinates,
+      detailCalculator,
       maxLoopCount,
       loopIntervalInMs = 500,
     } = config;
@@ -69,6 +76,8 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent {
     });
     this.operator = operator;
     this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;
+    // Default detail calculator implementation
+    this.detailCalculator = detailCalculator ?? defaultDetailCalculator;
     this.loopIntervalInMs = loopIntervalInMs;
     this.logger = this.logger.spawn('[GUIAgent]');
   }
@@ -163,11 +172,17 @@ export class GUIAgent<T extends Operator> extends BaseGUIAgent {
       return;
     }
 
+    const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() || {
+      width: -1,
+      height: -1,
+    };
+
     const content: ChatCompletionContentPart[] = [
       {
         type: 'image_url',
         image_url: {
           url: base64Uri,
+          detail: this.detailCalculator(imageWidth, imageHeight),
         },
       },
     ];

multimodal/gui-agent/agent-sdk/src/defaultImpls.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { ImageDetailCalculator } from '@gui-agent/shared/types';
+
+/**
+ * Default implementation for detail calculation based on pixel count
+ * detail:low mode: 1,048,576 px (1024×1024)
+ * detail:high mode: 4,014,080 px (2048×1960)
+ */
+export const defaultDetailCalculator: ImageDetailCalculator = (
+  width: number,
+  height: number,
+): 'low' | 'high' | 'auto' => {
+  const LOW_DETAIL_THRESHOLD = 1024 * 1024; // 1,048,576 px
+  const HIGH_DETAIL_THRESHOLD = 2048 * 1960; // 4,014,080 px
+
+  const pixelCount = width * height;
+  if (pixelCount <= LOW_DETAIL_THRESHOLD) {
+    return 'low';
+  } else if (pixelCount <= HIGH_DETAIL_THRESHOLD) {
+    return 'high';
+  } else {
+    // For images larger than high detail threshold, use high detail
+    return 'auto';
+  }
+};

multimodal/gui-agent/shared/src/types/agents.ts

@@ -64,6 +64,11 @@ export type ExecuteOutput = {
   url?: string; // url of the page
 } & Record<string, any>;
 
+/**
+ * Function type for calculating detail level based on image dimensions
+ */
+export type ImageDetailCalculator = (width: number, height: number) => 'low' | 'high' | 'auto';
+
 export interface ScreenshotOutput extends ExecuteOutput {
   /** screenshot base64, `keep screenshot size as physical pixels` */
   base64: string;
@@ -113,6 +118,8 @@ export interface GUIAgentConfig<TOperator> extends AgentOptions {
   customeActionParser?: CustomActionParser;
   /** The function to normalize raw coordinates */
   normalizeCoordinates?: NormalizeCoordinates;
+  /** The function to calculate detail level based on image dimensions */
+  detailCalculator?: ImageDetailCalculator;
   /** Maximum number of turns for Agent to execute, @default 1000 */
   maxLoopCount?: number;
   /** Time interval between two loop iterations (in milliseconds), @default 0 */