wip: model adaptor

packages/core/src/agent/agent.ts

@@ -1,4 +1,4 @@
-import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';
+import { getModelAdapter } from '@/ai-model/models';
 import yaml from 'js-yaml';
 import type { TUserPrompt } from '../ai-model/index';
 import { ScreenshotItem } from '../screenshot-item';
@@ -132,10 +132,6 @@ const normalizeScrollType = (
   return scrollType as ScrollParam['scrollType'];
 };
 
-const defaultReplanningCycleLimit = 20;
-const defaultVlmUiTarsReplanningCycleLimit = 40;
-const defaultAutoGlmReplanningCycleLimit = 100;
-
 export type AiActOptions = {
   cacheable?: boolean;
   fileChooserAccept?: string | string[];
@@ -244,11 +240,8 @@ export class Agent<
       return this.opts.replanningCycleLimit;
     }
 
-    return isUITars(modelConfigForPlanning.modelFamily)
-      ? defaultVlmUiTarsReplanningCycleLimit
-      : isAutoGLM(modelConfigForPlanning.modelFamily)
-        ? defaultAutoGlmReplanningCycleLimit
-        : defaultReplanningCycleLimit;
+    return getModelAdapter(modelConfigForPlanning.modelFamily).planning
+      .defaultReplanningCycleLimit;
   }
 
   constructor(interfaceInstance: InterfaceType, opts?: AgentOpt) {
@@ -927,11 +920,11 @@ export class Agent<
       const replanningCycleLimit = this.resolveReplanningCycleLimit(
         modelConfigForPlanning,
       );
-      // if vlm-ui-tars or auto-glm, plan cache is not used
-      const isVlmUiTars = isUITars(modelConfigForPlanning.modelFamily);
-      const isAutoGlm = isAutoGLM(modelConfigForPlanning.modelFamily);
+      const planCacheEnabled = getModelAdapter(
+        modelConfigForPlanning.modelFamily,
+      ).planning.cacheEnabled;
       const matchedCache =
-        isVlmUiTars || isAutoGlm || cacheable === false
+        !planCacheEnabled || cacheable === false
           ? undefined
           : this.taskCache?.matchPlanCache(taskPrompt);
       if (
@@ -1121,8 +1114,9 @@ export class Agent<
       // Don't pass deepLocate to verification locate — the description was generated
       // from a cropped view (deepLocate describe), but verification should use regular
       // locate on the full screenshot to confirm the description works universally.
-      // Passing deepLocate here would trigger AiLocateSection with an element-level
-      // description as a section prompt, which is semantically incorrect.
+      // Passing deepLocate here would add another first-pass locate and search-area
+      // crop around an already element-level description, which is not the intent of
+      // verification.
       verifyResult = await this.verifyLocator(
         resultPrompt,
         undefined,

packages/core/src/agent/task-builder.ts

@@ -1,5 +1,6 @@
 import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';
 import type { AbstractInterface } from '@/device';
+import { createLocateResultElementFromRect } from '@/locate-result-element';
 import type Service from '@/service';
 import { setTimingFieldOnce } from '@/task-timing';
 import type {
@@ -21,7 +22,6 @@ import type {
 import { ServiceError } from '@/types';
 import { sleep } from '@/utils';
 import type { IModelConfig } from '@midscene/shared/env';
-import { generateElementByRect } from '@midscene/shared/extractor';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
 import type { TaskCache } from './task-cache';
@@ -460,7 +460,7 @@ export class TaskBuilder {
         }
 
         const elementFromXpath = rectFromXpath
-          ? generateElementByRect(
+          ? createLocateResultElementFromRect(
               // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;
               transformLogicalRectToScreenshotRect(
                 rectFromXpath,

packages/core/src/agent/tasks.ts

@@ -1,11 +1,6 @@
-import {
-  AIResponseParseError,
-  ConversationHistory,
-  autoGLMPlanning,
-  plan,
-  uiTarsPlanning,
-} from '@/ai-model';
-import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';
+import { AIResponseParseError, ConversationHistory } from '@/ai-model';
+import { getModelAdapter } from '@/ai-model/models';
+import { genericXmlPlan } from '@/ai-model/workflows/planning';
 import {
   type TMultimodalPrompt,
   type TUserPrompt,
@@ -21,7 +16,6 @@ import type {
   ExecutionTaskInsightQueryApply,
   ExecutionTaskPlanningApply,
   ExecutionTaskProgressOptions,
-  InterfaceType,
   MidsceneYamlFlowItem,
   PlanningAIResponse,
   PlanningAction,
@@ -370,19 +364,18 @@ export class TaskExecutor {
               );
             }
 
-            const planImpl = isUITars(modelFamily)
-              ? uiTarsPlanning
-              : isAutoGLM(modelFamily)
-                ? autoGLMPlanning
-                : plan;
+            const adapter = getModelAdapter(modelFamily);
+            const planImpl =
+              adapter.planning.kind === 'custom'
+                ? adapter.planning.planFn
+                : genericXmlPlan;
 
             let planResult: Awaited<ReturnType<typeof planImpl>>;
             try {
               setTimingFieldOnce(timing, 'callAiStart');
               planResult = await planImpl(param.userInstruction, {
                 context: uiContext,
                 actionContext: param.aiActContext,
-                interfaceType: this.interface.interfaceType as InterfaceType,
                 actionSpace,
                 modelConfig: modelConfigForPlanning,
                 conversationHistory,

packages/core/src/agent/utils.ts

@@ -1,5 +1,6 @@
 import type { TMultimodalPrompt, TUserPrompt } from '@/common';
 import type { AbstractInterface } from '@/device';
+import { createLocateResultElementFromRect } from '@/locate-result-element';
 import { ScreenshotItem } from '@/screenshot-item';
 import type {
   ElementCacheFeature,
@@ -15,7 +16,6 @@ import {
   MIDSCENE_REPORT_TAG_NAME,
   globalConfigManager,
 } from '@midscene/shared/env';
-import { generateElementByRect } from '@midscene/shared/extractor';
 import { imageInfoOfBase64, resizeImgBase64 } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/logger';
 import { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';
@@ -201,7 +201,7 @@ export function matchElementFromPlan(
       height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,
     };
 
-    const element = generateElementByRect(
+    const element = createLocateResultElementFromRect(
       rect,
       typeof planLocateParam.prompt === 'string'
         ? planLocateParam.prompt

packages/core/src/ai-model/index.ts

@@ -10,16 +10,16 @@ export {
   type ConnectivityTestConfig,
   type ConnectivityTestResult,
 } from './connectivity';
-export { systemPromptToLocateElement } from './prompt/llm-locator';
+export { systemPromptToLocateElement } from './prompts/llm-locator';
 export {
   generatePlaywrightTest,
   generatePlaywrightTestStream,
-} from './prompt/playwright-generator';
+} from './workflows/generation/playwright';
 export {
   generateYamlTest,
   generateYamlTestStream,
-} from './prompt/yaml-generator';
-export type { YamlGenerationOptions } from './prompt/yaml-generator';
+} from './workflows/generation/yaml';
+export type { YamlGenerationOptions } from './workflows/generation/yaml';
 
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
 
@@ -28,19 +28,17 @@ export {
   AiExtractElementInfo,
   AiLocateSection,
   AiJudgeOrderSensitive,
-} from './inspect';
+} from './workflows/inspect';
 
-export { plan } from './llm-planning';
-export { autoGLMPlanning } from './auto-glm/planning';
-export { adaptBboxToRect } from '../common';
-export { uiTarsPlanning } from './ui-tars-planning';
+export { plan } from './workflows/planning/generic';
+export { adaptModelLocateResultToRect } from './workflows/inspect/locate-result-rect';
 export {
   ConversationHistory,
   type ConversationHistoryOptions,
 } from './conversation-history';
 export type { SubGoal, SubGoalStatus } from '@/types';
 
-export type { AIArgs } from '../common';
+export type { AIArgs } from './types';
 
 export {
   getMidsceneLocationSchema,