refactor(core): simplify model JSON parsing

packages/core/package.json

@@ -90,7 +90,7 @@
     "dotenv": "^16.4.5",
     "fetch-socks": "^1.3.0",
     "js-yaml": "4.1.0",
-    "jsonrepair": "3.12.0",
+    "jsonrepair": "3.14.0",
     "mdast-util-from-markdown": "^2.0.2",
     "mdast-util-to-markdown": "^2.1.2",
     "openai": "6.3.0",

packages/core/src/ai-model/model-adapter/resolve.ts

@@ -1,4 +1,4 @@
-import { normalJsonParser } from '../service-caller/json';
+import { parseModelResponseJson } from '../service-caller/json';
 import { resolveChatCompletion } from './chat-completion';
 import { resolveLocate } from './locate';
 import { resolveCustomPlanningDefinition, resolvePlanning } from './planning';
@@ -16,7 +16,7 @@ function resolveJsonParser(
   jsonParser: ModelAdapterDefinition['jsonParser'],
 ): JsonParser {
   if (!jsonParser || jsonParser === 'lenient-json') {
-    return normalJsonParser;
+    return parseModelResponseJson;
   }
 
   if (typeof jsonParser === 'function') {

packages/core/src/ai-model/models/doubao.ts

@@ -1,17 +1,11 @@
 import type { TModelFamily } from '@midscene/shared/env';
 import { assert } from '@midscene/shared/utils';
-import { jsonrepair } from 'jsonrepair';
 import type {
   ChatCompletionCallContext,
   ChatCompletionParamsResult,
-  JsonParserContext,
-  JsonParserSource,
   ModelAdapterDefinition,
 } from '../model-adapter/types';
-import {
-  extractJSONFromCodeBlock,
-  safeParseJson,
-} from '../service-caller/json';
+import { parseModelResponseJson } from '../service-caller/json';
 import {
   type LocateResultValue,
   createLocateResultValue,
@@ -29,85 +23,6 @@ const doubaoPointCoordinatesMeta = {
   normalizedBy: 1000,
 } as const;
 
-export function normalizeDoubaoJsonObject(
-  obj: any,
-  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},
-): any {
-  if (obj === null || obj === undefined) {
-    return obj;
-  }
-
-  if (Array.isArray(obj)) {
-    return obj.map((item) => normalizeDoubaoJsonObject(item, context));
-  }
-
-  if (typeof obj === 'object') {
-    const normalized: any = {};
-    for (const [key, value] of Object.entries(obj)) {
-      const trimmedKey = key.trim();
-      const preserveStringValue =
-        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;
-      const normalizedValue =
-        typeof value === 'string'
-          ? preserveStringValue
-            ? value
-            : value.trim()
-          : normalizeDoubaoJsonObject(value, context);
-      normalized[trimmedKey] = normalizedValue;
-    }
-    return normalized;
-  }
-
-  return typeof obj === 'string' ? obj.trim() : obj;
-}
-
-export function shouldRepairDoubaoLocateJson(source: JsonParserSource) {
-  return (
-    source === 'locate' ||
-    source === 'section-locator' ||
-    source === 'planning-action-param'
-  );
-}
-
-export function preprocessDoubaoLocateJson(input: string) {
-  if (input.includes('bbox')) {
-    while (/\d+\s+\d+/.test(input)) {
-      input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
-    }
-  }
-  return input;
-}
-
-const doubaoJsonParser: ModelAdapterDefinition['jsonParser'] = (
-  raw,
-  context = { source: 'generic-object' },
-) => {
-  const { source } = context;
-  try {
-    return safeParseJson(raw, context);
-  } catch (firstError) {
-    if (!shouldRepairDoubaoLocateJson(source)) {
-      throw firstError;
-    }
-
-    const jsonString = preprocessDoubaoLocateJson(
-      extractJSONFromCodeBlock(raw),
-    );
-    try {
-      return normalizeDoubaoJsonObject(
-        JSON.parse(jsonrepair(jsonString)),
-        context,
-      );
-    } catch (error) {
-      throw Error(
-        `failed to parse LLM response into JSON. Error - ${String(
-          error ?? firstError ?? 'unknown error',
-        )}. Response - \n ${raw}`,
-      );
-    }
-  }
-};
-
 export function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {
   const bbox = unwrapCoordinateListLikeInput(input as any);
   if (typeof bbox === 'string') {
@@ -210,7 +125,7 @@ const buildDoubaoChatCompletionParams = (
 };
 
 const doubaoVisionAdapter: ModelAdapterDefinition = {
-  jsonParser: doubaoJsonParser,
+  jsonParser: parseModelResponseJson,
   chatCompletion: {
     unsupportedUserConfig: ['reasoningBudget'],
     buildChatCompletionParams: buildDoubaoChatCompletionParams,

packages/core/src/ai-model/models/glm.ts

@@ -4,6 +4,7 @@ import type {
   ChatCompletionParamsResult,
   ModelAdapterDefinition,
 } from '../model-adapter/types';
+import { isLocateIntent } from './utils/intent';
 
 const buildGlmChatCompletionParams = (
   input: ChatCompletionCallContext,
@@ -16,6 +17,12 @@ const buildGlmChatCompletionParams = (
     commonOverrideConfig.temperature = userConfig.temperature;
   }
 
+  // Zhipu structured output JSON mode:
+  // https://docs.bigmodel.cn/cn/guide/capabilities/struct-output
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const modelSpecificConfig: Record<string, unknown> = {};
 
   if (reasoningEnabled !== 'default') {

packages/core/src/ai-model/models/gpt.ts

@@ -5,11 +5,12 @@ import type {
   ImageDetail,
   ModelAdapterDefinition,
 } from '../model-adapter/types';
+import { isLocateIntent } from './utils/intent';
 
 const originalImageDetailForDefaultIntent = (
   input: ChatCompletionCallContext,
 ): ImageDetail | undefined =>
-  input.intent === 'default' || input.requiresOriginalImageDetail
+  isLocateIntent(input.intent) || input.requiresOriginalImageDetail
     ? 'original'
     : undefined;
 
@@ -24,6 +25,12 @@ const buildGpt5ChatCompletionParams = (
     commonOverrideConfig.temperature = userConfig.temperature;
   }
 
+  // OpenAI Chat Completions JSON mode:
+  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const effectiveReasoningEffort =
     reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';
 

packages/core/src/ai-model/models/kimi.ts

@@ -9,6 +9,7 @@ import {
   createLocateResultValue,
   parseCoordinateList,
 } from '../shared/model-locate-result';
+import { isLocateIntent } from './utils/intent';
 
 const kimiNormalizedPointCoordinatesMeta = {
   shape: 'point',
@@ -46,6 +47,12 @@ const buildKimiChatCompletionParams = (
   // kimi disallow custom temperature
   commonOverrideConfig.temperature = undefined;
 
+  // Kimi Chat Completions response_format:
+  // https://platform.kimi.com/docs/api/chat
+  if (isLocateIntent(input.intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
+
   const modelSpecificConfig: Record<string, unknown> = {
     thinking: {
       type: effectiveReasoningEnabled ? 'enabled' : 'disabled',

packages/core/src/ai-model/models/mimo.ts

@@ -4,6 +4,7 @@ import type {
   ChatCompletionParamsResult,
   ModelAdapterDefinition,
 } from '../model-adapter/types';
+import { isLocateIntent } from './utils/intent';
 
 const buildMimoChatCompletionParams = (
   input: ChatCompletionCallContext,
@@ -14,9 +15,9 @@ const buildMimoChatCompletionParams = (
 
   // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api
   // Observed with thinking disabled: Mimo needs json_object to return JSON.
-  commonOverrideConfig.response_format = {
-    type: intent === 'default' ? 'json_object' : 'text',
-  };
+  if (isLocateIntent(intent)) {
+    commonOverrideConfig.response_format = { type: 'json_object' };
+  }
 
   if (userConfig.temperature !== undefined) {
     commonOverrideConfig.temperature = userConfig.temperature;

packages/core/src/ai-model/models/qwen.ts

@@ -87,6 +87,14 @@ const buildQwenChatCompletionParams = (
     commonOverrideConfig.temperature = userConfig.temperature;
   }
 
+  // Alibaba Cloud Model Studio JSON mode:
+  // https://help.aliyun.com/zh/model-studio/json-mode
+  // Observed in qwen3.6 grounding runs: enabling this can make the model
+  // return only ["bbox_2d"] without coordinates.
+  // if (isLocateIntent(input.intent)) {
+  //   commonOverrideConfig.response_format = { type: 'json_object' };
+  // }
+
   const modelSpecificConfig: Record<string, unknown> = {};
 
   if (reasoningEnabled !== 'default') {

packages/core/src/ai-model/models/ui-tars/adapter.ts

@@ -1,15 +1,7 @@
 import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env';
 import { assert } from '@midscene/shared/utils';
-import { jsonrepair } from 'jsonrepair';
-import type {
-  JsonParserContext,
-  JsonParserSource,
-  ModelAdapterDefinition,
-} from '../../model-adapter/types';
-import {
-  extractJSONFromCodeBlock,
-  safeParseJson,
-} from '../../service-caller/json';
+import type { ModelAdapterDefinition } from '../../model-adapter/types';
+import { parseModelResponseJson } from '../../service-caller/json';
 import {
   type LocateResultValue,
   createLocateResultValue,
@@ -29,82 +21,6 @@ const uiTarsPointCoordinatesMeta = {
   normalizedBy: 1000,
 } as const;
 
-function normalizeJsonObject(
-  obj: any,
-  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},
-): any {
-  if (obj === null || obj === undefined) {
-    return obj;
-  }
-
-  if (Array.isArray(obj)) {
-    return obj.map((item) => normalizeJsonObject(item, context));
-  }
-
-  if (typeof obj === 'object') {
-    const normalized: any = {};
-    for (const [key, value] of Object.entries(obj)) {
-      const trimmedKey = key.trim();
-      const preserveStringValue =
-        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;
-      const normalizedValue =
-        typeof value === 'string'
-          ? preserveStringValue
-            ? value
-            : value.trim()
-          : normalizeJsonObject(value, context);
-      normalized[trimmedKey] = normalizedValue;
-    }
-    return normalized;
-  }
-
-  return typeof obj === 'string' ? obj.trim() : obj;
-}
-
-function shouldRepairUiTarsLocateJson(source: JsonParserSource) {
-  return (
-    source === 'locate' ||
-    source === 'section-locator' ||
-    source === 'planning-action-param'
-  );
-}
-
-function preprocessUiTarsLocateJson(input: string) {
-  if (input.includes('bbox')) {
-    while (/\d+\s+\d+/.test(input)) {
-      input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
-    }
-  }
-  return input;
-}
-
-const uiTarsJsonParser: ModelAdapterDefinition['jsonParser'] = (
-  raw,
-  context = { source: 'generic-object' },
-) => {
-  const { source } = context;
-  try {
-    return safeParseJson(raw, context);
-  } catch (firstError) {
-    if (!shouldRepairUiTarsLocateJson(source)) {
-      throw firstError;
-    }
-
-    const jsonString = preprocessUiTarsLocateJson(
-      extractJSONFromCodeBlock(raw),
-    );
-    try {
-      return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);
-    } catch (error) {
-      throw Error(
-        `failed to parse LLM response into JSON. Error - ${String(
-          error ?? firstError ?? 'unknown error',
-        )}. Response - \n ${raw}`,
-      );
-    }
-  }
-};
-
 // UI-TARS has not received active updates for a long time, so this parser is
 // intentionally kept separate from Doubao even though the current logic is the
 // same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.
@@ -182,7 +98,7 @@ function createUiTarsAdapter(
   uiTarsModelVersion: UITarsModelVersion,
 ): ModelAdapterDefinition {
   return {
-    jsonParser: uiTarsJsonParser,
+    jsonParser: parseModelResponseJson,
     chatCompletion: {
       unsupportedUserConfig: [
         'reasoningEnabled',

packages/core/src/ai-model/models/utils/intent.ts

@@ -0,0 +1,5 @@
+import type { TIntent } from '@midscene/shared/env';
+
+export function isLocateIntent(intent?: TIntent): boolean {
+  return intent === 'default';
+}

packages/core/src/ai-model/prompt/extraction.ts

@@ -1,6 +1,6 @@
 import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';
 import { getPreferredLanguage } from '@midscene/shared/env';
-import { safeParseJson } from '../service-caller/json';
+import { parseModelResponseJson } from '../service-caller/json';
 import { extractXMLTag } from './util';
 
 export function buildTypeQueryDemandValue(
@@ -41,7 +41,10 @@ export function parseXMLExtractionResponse<T>(
 
   let data: T;
   try {
-    data = safeParseJson(dataJsonStr) as T;
+    data = parseModelResponseJson(dataJsonStr, {
+      source: 'generic-object',
+      requireObject: false,
+    }) as T;
   } catch (e) {
     throw new Error(`Failed to parse data-json: ${e}`);
   }
@@ -50,7 +53,10 @@ export function parseXMLExtractionResponse<T>(
   let errors: string[] | undefined;
   if (errorsStr) {
     try {
-      const parsedErrors = safeParseJson(errorsStr);
+      const parsedErrors = parseModelResponseJson(errorsStr, {
+        source: 'generic-object',
+        requireObject: false,
+      });
       if (Array.isArray(parsedErrors)) {
         errors = parsedErrors;
       }