From 5e352e11819eea2b86fae2c701e91c639ba3ef28 Mon Sep 17 00:00:00 2001 From: EAGzzyCSL Date: Wed, 17 Jun 2026 15:09:30 +0800 Subject: [PATCH 1/4] refactor(core): simplify model json parsing --- packages/core/package.json | 2 +- .../src/ai-model/model-adapter/resolve.ts | 4 +- packages/core/src/ai-model/models/doubao.ts | 89 +---- .../src/ai-model/models/ui-tars/adapter.ts | 90 +---- .../core/src/ai-model/prompt/extraction.ts | 12 +- .../core/src/ai-model/service-caller/index.ts | 3 +- .../core/src/ai-model/service-caller/json.ts | 127 ++++--- packages/core/tests/unit-test/json.test.ts | 341 ++++++++++++++++++ .../unit-test/model-adapter/doubao.test.ts | 48 +-- .../tests/unit-test/service-caller.test.ts | 201 ----------- packages/core/tests/unit-test/utils.test.ts | 104 ------ pnpm-lock.yaml | 13 +- 12 files changed, 448 insertions(+), 586 deletions(-) create mode 100644 packages/core/tests/unit-test/json.test.ts delete mode 100644 packages/core/tests/unit-test/service-caller.test.ts diff --git a/packages/core/package.json b/packages/core/package.json index d51b8668c6..cf38c51c21 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -90,7 +90,7 @@ "dotenv": "^16.4.5", "fetch-socks": "^1.3.0", "js-yaml": "4.1.0", - "jsonrepair": "3.12.0", + "jsonrepair": "3.14.0", "mdast-util-from-markdown": "^2.0.2", "mdast-util-to-markdown": "^2.1.2", "openai": "6.3.0", diff --git a/packages/core/src/ai-model/model-adapter/resolve.ts b/packages/core/src/ai-model/model-adapter/resolve.ts index 3b1329311c..6b8950e626 100644 --- a/packages/core/src/ai-model/model-adapter/resolve.ts +++ b/packages/core/src/ai-model/model-adapter/resolve.ts @@ -1,4 +1,4 @@ -import { normalJsonParser } from '../service-caller/json'; +import { parseModelResponseJson } from '../service-caller/json'; import { resolveChatCompletion } from './chat-completion'; import { resolveLocate } from './locate'; import { resolveCustomPlanningDefinition, resolvePlanning } from './planning'; @@ -16,7 +16,7 @@ function resolveJsonParser( jsonParser: ModelAdapterDefinition['jsonParser'], ): JsonParser { if (!jsonParser || jsonParser === 'lenient-json') { - return normalJsonParser; + return parseModelResponseJson; } if (typeof jsonParser === 'function') { diff --git a/packages/core/src/ai-model/models/doubao.ts b/packages/core/src/ai-model/models/doubao.ts index 0c97c782ba..c2c3f580ab 100644 --- a/packages/core/src/ai-model/models/doubao.ts +++ b/packages/core/src/ai-model/models/doubao.ts @@ -1,17 +1,11 @@ import type { TModelFamily } from '@midscene/shared/env'; import { assert } from '@midscene/shared/utils'; -import { jsonrepair } from 'jsonrepair'; import type { ChatCompletionCallContext, ChatCompletionParamsResult, - JsonParserContext, - JsonParserSource, ModelAdapterDefinition, } from '../model-adapter/types'; -import { - extractJSONFromCodeBlock, - safeParseJson, -} from '../service-caller/json'; +import { parseModelResponseJson } from '../service-caller/json'; import { type LocateResultValue, createLocateResultValue, @@ -29,85 +23,6 @@ const doubaoPointCoordinatesMeta = { normalizedBy: 1000, } as const; -export function normalizeDoubaoJsonObject( - obj: any, - context: Pick = {}, -): any { - if (obj === null || obj === undefined) { - return obj; - } - - if (Array.isArray(obj)) { - return obj.map((item) => normalizeDoubaoJsonObject(item, context)); - } - - if (typeof obj === 'object') { - const normalized: any = {}; - for (const [key, value] of Object.entries(obj)) { - const trimmedKey = key.trim(); - const preserveStringValue = - context.preserveStringValueKeys?.includes(trimmedKey) ?? false; - const normalizedValue = - typeof value === 'string' - ? preserveStringValue - ? value - : value.trim() - : normalizeDoubaoJsonObject(value, context); - normalized[trimmedKey] = normalizedValue; - } - return normalized; - } - - return typeof obj === 'string' ? obj.trim() : obj; -} - -export function shouldRepairDoubaoLocateJson(source: JsonParserSource) { - return ( - source === 'locate' || - source === 'section-locator' || - source === 'planning-action-param' - ); -} - -export function preprocessDoubaoLocateJson(input: string) { - if (input.includes('bbox')) { - while (/\d+\s+\d+/.test(input)) { - input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2'); - } - } - return input; -} - -const doubaoJsonParser: ModelAdapterDefinition['jsonParser'] = ( - raw, - context = { source: 'generic-object' }, -) => { - const { source } = context; - try { - return safeParseJson(raw, context); - } catch (firstError) { - if (!shouldRepairDoubaoLocateJson(source)) { - throw firstError; - } - - const jsonString = preprocessDoubaoLocateJson( - extractJSONFromCodeBlock(raw), - ); - try { - return normalizeDoubaoJsonObject( - JSON.parse(jsonrepair(jsonString)), - context, - ); - } catch (error) { - throw Error( - `failed to parse LLM response into JSON. Error - ${String( - error ?? firstError ?? 'unknown error', - )}. Response - \n ${raw}`, - ); - } - } -}; - export function parseDoubaoRawLocateValue(input: unknown): LocateResultValue { const bbox = unwrapCoordinateListLikeInput(input as any); if (typeof bbox === 'string') { @@ -210,7 +125,7 @@ const buildDoubaoChatCompletionParams = ( }; const doubaoVisionAdapter: ModelAdapterDefinition = { - jsonParser: doubaoJsonParser, + jsonParser: parseModelResponseJson, chatCompletion: { unsupportedUserConfig: ['reasoningBudget'], buildChatCompletionParams: buildDoubaoChatCompletionParams, diff --git a/packages/core/src/ai-model/models/ui-tars/adapter.ts b/packages/core/src/ai-model/models/ui-tars/adapter.ts index 07af11f065..51a1b2e1e8 100644 --- a/packages/core/src/ai-model/models/ui-tars/adapter.ts +++ b/packages/core/src/ai-model/models/ui-tars/adapter.ts @@ -1,15 +1,7 @@ import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env'; import { assert } from '@midscene/shared/utils'; -import { jsonrepair } from 'jsonrepair'; -import type { - JsonParserContext, - JsonParserSource, - ModelAdapterDefinition, -} from '../../model-adapter/types'; -import { - extractJSONFromCodeBlock, - safeParseJson, -} from '../../service-caller/json'; +import type { ModelAdapterDefinition } from '../../model-adapter/types'; +import { parseModelResponseJson } from '../../service-caller/json'; import { type LocateResultValue, createLocateResultValue, @@ -29,82 +21,6 @@ const uiTarsPointCoordinatesMeta = { normalizedBy: 1000, } as const; -function normalizeJsonObject( - obj: any, - context: Pick = {}, -): any { - if (obj === null || obj === undefined) { - return obj; - } - - if (Array.isArray(obj)) { - return obj.map((item) => normalizeJsonObject(item, context)); - } - - if (typeof obj === 'object') { - const normalized: any = {}; - for (const [key, value] of Object.entries(obj)) { - const trimmedKey = key.trim(); - const preserveStringValue = - context.preserveStringValueKeys?.includes(trimmedKey) ?? false; - const normalizedValue = - typeof value === 'string' - ? preserveStringValue - ? value - : value.trim() - : normalizeJsonObject(value, context); - normalized[trimmedKey] = normalizedValue; - } - return normalized; - } - - return typeof obj === 'string' ? obj.trim() : obj; -} - -function shouldRepairUiTarsLocateJson(source: JsonParserSource) { - return ( - source === 'locate' || - source === 'section-locator' || - source === 'planning-action-param' - ); -} - -function preprocessUiTarsLocateJson(input: string) { - if (input.includes('bbox')) { - while (/\d+\s+\d+/.test(input)) { - input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2'); - } - } - return input; -} - -const uiTarsJsonParser: ModelAdapterDefinition['jsonParser'] = ( - raw, - context = { source: 'generic-object' }, -) => { - const { source } = context; - try { - return safeParseJson(raw, context); - } catch (firstError) { - if (!shouldRepairUiTarsLocateJson(source)) { - throw firstError; - } - - const jsonString = preprocessUiTarsLocateJson( - extractJSONFromCodeBlock(raw), - ); - try { - return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context); - } catch (error) { - throw Error( - `failed to parse LLM response into JSON. Error - ${String( - error ?? firstError ?? 'unknown error', - )}. Response - \n ${raw}`, - ); - } - } -}; - // UI-TARS has not received active updates for a long time, so this parser is // intentionally kept separate from Doubao even though the current logic is the // same. This avoids coupling UI-TARS behavior to future Doubao adapter changes. @@ -182,7 +98,7 @@ function createUiTarsAdapter( uiTarsModelVersion: UITarsModelVersion, ): ModelAdapterDefinition { return { - jsonParser: uiTarsJsonParser, + jsonParser: parseModelResponseJson, chatCompletion: { unsupportedUserConfig: [ 'reasoningEnabled', diff --git a/packages/core/src/ai-model/prompt/extraction.ts b/packages/core/src/ai-model/prompt/extraction.ts index c7ea764a8f..c2368015bb 100644 --- a/packages/core/src/ai-model/prompt/extraction.ts +++ b/packages/core/src/ai-model/prompt/extraction.ts @@ -1,6 +1,6 @@ import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types'; import { getPreferredLanguage } from '@midscene/shared/env'; -import { safeParseJson } from '../service-caller/json'; +import { parseModelResponseJson } from '../service-caller/json'; import { extractXMLTag } from './util'; export function buildTypeQueryDemandValue( @@ -41,7 +41,10 @@ export function parseXMLExtractionResponse( let data: T; try { - data = safeParseJson(dataJsonStr) as T; + data = parseModelResponseJson(dataJsonStr, { + source: 'generic-object', + requireObject: false, + }) as T; } catch (e) { throw new Error(`Failed to parse data-json: ${e}`); } @@ -50,7 +53,10 @@ export function parseXMLExtractionResponse( let errors: string[] | undefined; if (errorsStr) { try { - const parsedErrors = safeParseJson(errorsStr); + const parsedErrors = parseModelResponseJson(errorsStr, { + source: 'generic-object', + requireObject: false, + }); if (Array.isArray(parsedErrors)) { errors = parsedErrors; } diff --git a/packages/core/src/ai-model/service-caller/index.ts b/packages/core/src/ai-model/service-caller/index.ts index 489d215423..bde8c66a5a 100644 --- a/packages/core/src/ai-model/service-caller/index.ts +++ b/packages/core/src/ai-model/service-caller/index.ts @@ -56,8 +56,7 @@ import { } from './request-timeout'; export { extractJSONFromCodeBlock, - normalJsonParser, - safeParseJson, + parseModelResponseJson, } from './json'; export type { JsonParser } from './json'; diff --git a/packages/core/src/ai-model/service-caller/json.ts b/packages/core/src/ai-model/service-caller/json.ts index 1488ffd2a2..ac4ec12d4e 100644 --- a/packages/core/src/ai-model/service-caller/json.ts +++ b/packages/core/src/ai-model/service-caller/json.ts @@ -1,5 +1,22 @@ import { jsonrepair } from 'jsonrepair'; +/** + * Extract the JSON portion from a model response. + * + * This mainly handles fenced JSON like ```json { ... } ``` by removing the + * fence. If that does not produce a JSON object, it falls back to a greedy regex + * that extracts from the first "{" to the last "}". If that still fails, the + * original response is returned. + * + * Expected model responses are JSON objects, possibly wrapped in markdown + * fences. Natural language mixed with JSON, or arrays like + * [{"type":"Tap"}, {"type":"Hover"}], are outside the supported contract and + * are not reliably recoverable. + * + * This legacy extractor is also used by extraction responses that can be any + * JSON value. In those cases, arrays/strings/numbers have no object braces and + * pass through unchanged. + */ export function extractJSONFromCodeBlock(response: string) { try { // First, try to match a JSON object directly in the response @@ -35,17 +52,18 @@ export type JsonParserSource = export interface JsonParserContext { source: JsonParserSource; preserveStringValueKeys?: string[]; + requireObject?: boolean; } export type JsonParser = (raw: string, context?: JsonParserContext) => unknown; /** - * Normalize a parsed JSON object by trimming whitespace from: + * Trim whitespace in parsed JSON by normalizing: * 1. All object keys (e.g., " prompt " -> "prompt") * 2. All string values (e.g., " Tap " -> "Tap") * This handles LLM output that may include leading/trailing spaces. */ -function normalizeJsonObject( +function trimParsedJsonStrings( obj: any, context: Pick = {}, ): any { @@ -56,7 +74,7 @@ function normalizeJsonObject( // Handle arrays - recursively normalize each element if (Array.isArray(obj)) { - return obj.map((item) => normalizeJsonObject(item, context)); + return obj.map((item) => trimParsedJsonStrings(item, context)); } // Handle objects @@ -74,7 +92,7 @@ function normalizeJsonObject( ? preserveStringValue ? value : value.trim() - : normalizeJsonObject(value, context); + : trimParsedJsonStrings(value, context); normalized[trimmedKey] = normalizedValue; } @@ -91,52 +109,71 @@ function normalizeJsonObject( return obj; } -const parseNormalJson = ( - input: string, - rawResponse: string, - context?: JsonParserContext, -) => { - if (input?.match(/\((\d+),(\d+)\)/)) { - return input - .match(/\((\d+),(\d+)\)/) - ?.slice(1) - .map(Number); - } +function repairKnownJsonIssues( + jsonBlock: string, + _rawResponse: string, +): string { + // TODO: Add project-specific repairs that jsonrepair cannot handle. + return jsonBlock; +} - let parsed: any; - let lastError: unknown; - try { - parsed = JSON.parse(input); - return normalizeJsonObject(parsed, context); - } catch (error) { - lastError = error; +function assertJsonObject( + parsed: unknown, +): asserts parsed is Record { + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error( + `expected parsed LLM response to be a JSON object, got ${JSON.stringify( + parsed, + )}`, + ); } +} + +function parseJsonWithRepair(jsonStr: string) { try { - parsed = JSON.parse(jsonrepair(input)); - return normalizeJsonObject(parsed, context); - } catch (error) { - lastError = error; + return JSON.parse(jsonStr); + } catch { + return JSON.parse(jsonrepair(jsonStr)); } +} - return { parsed: undefined, lastError, rawResponse }; -}; - -export function safeParseJson(raw: string, context?: JsonParserContext) { +export function parseModelResponseJson( + raw: string, + context?: JsonParserContext, +) { const cleanJsonString = extractJSONFromCodeBlock(raw); - const result = parseNormalJson(cleanJsonString, raw, context); - if ( - result && - typeof result === 'object' && - 'parsed' in result && - result.parsed === undefined - ) { - throw Error( - `failed to parse LLM response into JSON. Error - ${String( - result.lastError ?? 'unknown error', - )}. Response - \n ${raw}`, - ); + const requireObject = context?.requireObject ?? true; + + let parsedObj: unknown; + + try { + parsedObj = parseJsonWithRepair(cleanJsonString); + if (requireObject) { + assertJsonObject(parsedObj); + } + } catch (e1) { + const code = repairKnownJsonIssues(cleanJsonString, raw); + if (code === cleanJsonString) { + throw new Error( + `failed to parse LLM response into JSON. Error - ${String( + e1, + )}. Response - \n ${raw}`, + ); + } + + try { + parsedObj = parseJsonWithRepair(code); + if (requireObject) { + assertJsonObject(parsedObj); + } + } catch (e2) { + throw new Error( + `failed to parse LLM response into JSON. First error - ${String( + e1, + )}. Second error - ${String(e2)}. Response - \n ${raw}`, + ); + } } - return result; -} -export const normalJsonParser: JsonParser = safeParseJson; + return trimParsedJsonStrings(parsedObj, context); +} diff --git a/packages/core/tests/unit-test/json.test.ts b/packages/core/tests/unit-test/json.test.ts new file mode 100644 index 0000000000..f94b496d82 --- /dev/null +++ b/packages/core/tests/unit-test/json.test.ts @@ -0,0 +1,341 @@ +import { getModelAdapter } from '@/ai-model/models'; +import { + extractJSONFromCodeBlock, + parseModelResponseJson, +} from '@/ai-model/service-caller/json'; +import { describe, expect, it } from 'vitest'; + +describe('extractJSONFromCodeBlock', () => { + it('should extract JSON from a direct JSON object', () => { + const input = '{ "key": "value" }'; + const result = extractJSONFromCodeBlock(input); + expect(result).toBe('{ "key": "value" }'); + }); + + it('should extract JSON from a code block with json language specifier', () => { + const input = '```json\n{ "key": "value" }\n```'; + const result = extractJSONFromCodeBlock(input); + expect(result).toBe('{ "key": "value" }'); + + const input2 = ' ```JSON\n{ "key": "value" }\n```'; + const result2 = extractJSONFromCodeBlock(input2); + expect(result2).toBe('{ "key": "value" }'); + }); + + it('should extract JSON from a code block without language specifier', () => { + const input = '```\n{ "key": "value" }\n```'; + const result = extractJSONFromCodeBlock(input); + expect(result).toBe('{ "key": "value" }'); + }); + + it('should extract JSON-like structure from text', () => { + const input = 'Some text { "key": "value" } more text'; + const result = extractJSONFromCodeBlock(input); + expect(result).toBe('{ "key": "value" }'); + }); + + it('should keep legacy greedy extraction for multiple JSON-like objects', () => { + const input = 'first { "a": 1 } second { "b": 2 }'; + const result = extractJSONFromCodeBlock(input); + + expect(result).toBe('{ "a": 1 } second { "b": 2 }'); + }); + + it('should return the original response if no JSON structure is found', () => { + const input = 'This is just plain text'; + const result = extractJSONFromCodeBlock(input); + expect(result).toBe('This is just plain text'); + }); + + it('should handle multi-line JSON objects', () => { + const input = `{ + "key1": "value1", + "key2": { + "nestedKey": "nestedValue" + } + }`; + const result = extractJSONFromCodeBlock(input); + expect(result).toBe(input); + }); +}); + +describe('parseModelResponseJson', () => { + const parseJson = ( + input: string, + modelFamily?: Parameters[0], + ): any => + modelFamily === undefined + ? parseModelResponseJson(input) + : getModelAdapter(modelFamily).jsonParser(input, { + source: 'generic-object', + }); + + it('should parse valid JSON string', () => { + const input = '{"key": "value"}'; + const result = parseModelResponseJson(input); + expect(result).toEqual({ key: 'value' }); + }); + + it('should parse dirty JSON that jsonrepair can fix', () => { + const input = "{key: 'value'}"; + const result = parseModelResponseJson(input); + expect(result).toEqual({ key: 'value' }); + }); + + it('should throw error for unparseable content', () => { + const input = '{foo: true false}'; + expect(() => parseModelResponseJson(input)).toThrow( + /failed to parse LLM response into JSON/, + ); + }); + + it('should reject top-level non-object JSON values', () => { + expect(() => parseModelResponseJson('[1, 2]')).toThrow( + /expected parsed LLM response to be a JSON object/, + ); + }); + + it('should allow top-level non-object JSON values when object validation is disabled', () => { + expect( + parseModelResponseJson('[" todo 1 ", " todo 2 "]', { + source: 'generic-object', + requireObject: false, + }), + ).toEqual(['todo 1', 'todo 2']); + + expect( + parseModelResponseJson('" todo list "', { + source: 'generic-object', + requireObject: false, + }), + ).toBe('todo list'); + + expect( + parseModelResponseJson('42', { + source: 'generic-object', + requireObject: false, + }), + ).toBe(42); + }); + + it('should parse JSON from code block', () => { + const input = '```json\n{"key": "value"}\n```'; + const result = parseModelResponseJson(input); + expect(result).toEqual({ key: 'value' }); + }); + + it('should parse complex nested JSON', () => { + const input = `{ + "string": "value", + "number": 123, + "boolean": true, + "array": [1, 2, 3], + "object": { + "nested": "value" + } + }`; + const result = parseModelResponseJson(input); + expect(result).toEqual({ + string: 'value', + number: 123, + boolean: true, + array: [1, 2, 3], + object: { + nested: 'value', + }, + }); + }); + + it('should trim leading and trailing spaces from object keys', () => { + const input = + '{" type ": "Tap", "param": {" prompt ": "Login button"}}'; + const result = parseJson(input); + + expect(result).toEqual({ + type: 'Tap', + param: { + prompt: 'Login button', + }, + }); + expect(Object.keys(result)).toEqual(['type', 'param']); + expect(Object.keys(result.param)).toEqual(['prompt']); + }); + + it('should trim leading and trailing spaces from type field values', () => { + const input = '{"type": " Tap ", "param": {}}'; + const result = parseJson(input); + + expect(result.type).toBe('Tap'); + }); + + it('should trim leading and trailing spaces from prompt field values', () => { + const input = '{"param": {"prompt": " Click the button "}}'; + const result = parseJson(input); + + expect(result.param.prompt).toBe('Click the button'); + }); + + it('should handle the original error case with leading spaces', () => { + const input = + '{"type":" Tap","param":{"locate":{"bbox":[574,308,865,352]," prompt ":"The \'Login\' button"}}}'; + const result = parseJson(input); + + expect(result).toEqual({ + type: 'Tap', + param: { + locate: { + bbox: [574, 308, 865, 352], + prompt: "The 'Login' button", + }, + }, + }); + }); + + it('should handle nested objects and arrays', () => { + const input = JSON.stringify({ + ' type ': ' Tap ', + ' items ': [{ ' name ': ' item1 ' }, { ' name ': ' item2 ' }], + }); + const result = parseJson(input); + + expect(result).toEqual({ + type: 'Tap', + items: [{ name: 'item1' }, { name: 'item2' }], + }); + }); + + it('should trim string values inside arrays', () => { + const input = '{"items": [" first ", " second "]}'; + const result = parseJson(input); + + expect(result).toEqual({ + items: ['first', 'second'], + }); + }); + + it('should trim all string values including descriptions', () => { + const input = + '{"type": " Tap ", "description": " Some text with spaces "}'; + const result = parseJson(input); + + expect(result.type).toBe('Tap'); + expect(result.description).toBe('Some text with spaces'); + }); + + it('should preserve configured string value keys while trimming other fields', () => { + const input = + '{" value ": " test value ", "param": {" prompt ": " input field "}}'; + const result = parseModelResponseJson(input, { + source: 'generic-object', + preserveStringValueKeys: ['value'], + }); + + expect(result).toEqual({ + value: ' test value ', + param: { + prompt: 'input field', + }, + }); + }); + + it('should preserve configured string value keys from JSON code blocks', () => { + const input = `\`\`\`json +{ + "value": " test value ", + "locate": { + "prompt": " input field " + } +} +\`\`\``; + const result = parseModelResponseJson(input, { + source: 'generic-object', + preserveStringValueKeys: ['value'], + }); + + expect(result).toEqual({ + value: ' test value ', + locate: { + prompt: 'input field', + }, + }); + }); + + it('should preserve configured string value keys after jsonrepair', () => { + const input = + '{ value: " test value ", locate: {" prompt ": " input field ",}, }'; + const result = parseModelResponseJson(input, { + source: 'generic-object', + preserveStringValueKeys: ['value'], + }); + + expect(result).toEqual({ + value: ' test value ', + locate: { + prompt: 'input field', + }, + }); + }); + + it('should handle null values', () => { + const input = '{"type": "Tap", "value": null, "param": {}}'; + const result = parseJson(input); + + expect(result.type).toBe('Tap'); + expect(result.value).toBeNull(); + }); + + it('should work with malformed JSON that jsonrepair can fix', () => { + const input = '{type: " Tap ", param: {" prompt ": "Login"}}'; + const result = parseJson(input); + + expect(result.type).toBe('Tap'); + expect(result.param.prompt).toBe('Login'); + }); + + it('should handle deeply nested structures', () => { + const input = JSON.stringify({ + ' type ': ' Action ', + ' nested ': { + ' level1 ': { + ' level2 ': { + ' prompt ': ' deep value ', + }, + }, + }, + }); + const result = parseJson(input); + + expect(result.type).toBe('Action'); + expect(result.nested.level1.level2.prompt).toBe('deep value'); + }); + + it('should trim id field values', () => { + const input = '{"id": " element-123 ", "type": " Tap "}'; + const result = parseJson(input); + + expect(result.id).toBe('element-123'); + expect(result.type).toBe('Tap'); + }); + + it('should handle a single object wrapped in an array through legacy extraction', () => { + const input = '[{" type ": " Tap "}]'; + const result = parseJson(input); + + expect(result).toEqual({ type: 'Tap' }); + }); + + it('should preserve coordinate tuple text inside valid JSON strings', () => { + const input = '{"message": "target is near (100,200)"}'; + const result = parseJson(input); + + expect(result).toEqual({ message: 'target is near (100,200)' }); + }); + + it('should work with doubao-vision mode and trim spaces', () => { + const input = '{" type ": " Tap ", "param": {" prompt ": "Click"}}'; + const result = parseJson(input, 'doubao-vision'); + + expect(result.type).toBe('Tap'); + expect(result.param.prompt).toBe('Click'); + }); +}); diff --git a/packages/core/tests/unit-test/model-adapter/doubao.test.ts b/packages/core/tests/unit-test/model-adapter/doubao.test.ts index ed31f8a896..42201d5f95 100644 --- a/packages/core/tests/unit-test/model-adapter/doubao.test.ts +++ b/packages/core/tests/unit-test/model-adapter/doubao.test.ts @@ -1,10 +1,7 @@ import { ResolvedModelAdapter } from '@/ai-model/model-adapter/resolve'; import { doubaoAdapters, - normalizeDoubaoJsonObject, parseDoubaoRawLocateValue, - preprocessDoubaoLocateJson, - shouldRepairDoubaoLocateJson, } from '@/ai-model/models/doubao'; import { describe, expect, it } from 'vitest'; @@ -159,7 +156,7 @@ describe('doubao model adapter', () => { }); }); - it('repairs bbox coordinate strings for locate-like json parser sources', () => { + it('parses locate-like JSON through the shared parser', () => { const parser = doubaoVisionAdapter.jsonParser; const context = { source: 'locate' as const }; expect(parser('{"bbox": [123 456]}', context)).toEqual({ @@ -191,54 +188,13 @@ describe('doubao model adapter', () => { }); }); - it('normalizes Doubao repaired json objects recursively', () => { - expect( - normalizeDoubaoJsonObject( - { - ' value ': ' keep spaces ', - ' nested ': { - ' prompt ': ' submit ', - }, - ' list ': [{ ' item ': ' first ' }], - nil: null, - }, - { preserveStringValueKeys: ['value'] }, - ), - ).toEqual({ - value: ' keep spaces ', - nested: { - prompt: 'submit', - }, - list: [{ item: 'first' }], - nil: null, - }); - expect(normalizeDoubaoJsonObject(' text ')).toBe('text'); - expect(normalizeDoubaoJsonObject(undefined)).toBeUndefined(); - }); - - it('detects Doubao json parser sources that need locate repair', () => { - expect(shouldRepairDoubaoLocateJson('locate')).toBe(true); - expect(shouldRepairDoubaoLocateJson('section-locator')).toBe(true); - expect(shouldRepairDoubaoLocateJson('planning-action-param')).toBe(true); - expect(shouldRepairDoubaoLocateJson('generic-object')).toBe(false); - }); - - it('preprocesses Doubao locate json only when bbox text is present', () => { - expect(preprocessDoubaoLocateJson('{"bbox": [940 445 969 490]}')).toBe( - '{"bbox": [940,445,969,490]}', - ); - expect(preprocessDoubaoLocateJson('{"point": [940 445]}')).toBe( - '{"point": [940 445]}', - ); - }); - it('does not repair malformed json for generic parser sources', () => { const parser = doubaoVisionAdapter.jsonParser; expect(() => parser('```', { source: 'generic-object' })).toThrow(); }); - it('normalizes repaired doubao json while preserving configured string values', () => { + it('normalizes parsed doubao json while preserving configured string values', () => { const parser = doubaoVisionAdapter.jsonParser; expect( diff --git a/packages/core/tests/unit-test/service-caller.test.ts b/packages/core/tests/unit-test/service-caller.test.ts deleted file mode 100644 index 2d1a1c5d88..0000000000 --- a/packages/core/tests/unit-test/service-caller.test.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { getModelAdapter } from '@/ai-model/models'; -import { safeParseJson } from '@/ai-model/service-caller/json'; -import { describe, expect, it } from 'vitest'; - -describe('service-caller', () => { - const parseJson = ( - input: string, - modelFamily: Parameters[0], - ): any => - modelFamily === undefined - ? safeParseJson(input) - : getModelAdapter(modelFamily).jsonParser(input, { - source: 'generic-object', - }); - - describe('adapter json parser - JSON normalization', () => { - it('should trim leading and trailing spaces from object keys', () => { - const input = - '{" type ": "Tap", "param": {" prompt ": "Login button"}}'; - const result = parseJson(input, undefined); - - expect(result).toEqual({ - type: 'Tap', - param: { - prompt: 'Login button', - }, - }); - expect(Object.keys(result)).toEqual(['type', 'param']); - expect(Object.keys(result.param)).toEqual(['prompt']); - }); - - it('should trim leading and trailing spaces from type field values', () => { - const input = '{"type": " Tap ", "param": {}}'; - const result = parseJson(input, undefined); - - expect(result.type).toBe('Tap'); - }); - - it('should trim leading and trailing spaces from prompt field values', () => { - const input = '{"param": {"prompt": " Click the button "}}'; - const result = parseJson(input, undefined); - - expect(result.param.prompt).toBe('Click the button'); - }); - - it('should handle the original error case with leading spaces', () => { - const input = - '[{"type":" Tap","param":{"locate":{"bbox":[574,308,865,352]," prompt ":"The \'Login\' button"}}}]'; - const result = parseJson(input, undefined); - - expect(result).toEqual({ - type: 'Tap', - param: { - locate: { - bbox: [574, 308, 865, 352], - prompt: "The 'Login' button", - }, - }, - }); - }); - - it('should handle nested objects and arrays', () => { - const input = JSON.stringify({ - ' type ': ' Tap ', - ' items ': [{ ' name ': ' item1 ' }, { ' name ': ' item2 ' }], - }); - const result = parseJson(input, undefined); - - expect(result).toEqual({ - type: 'Tap', - items: [{ name: 'item1' }, { name: 'item2' }], - }); - }); - - it('should trim all string values including descriptions', () => { - const input = - '{"type": " Tap ", "description": " Some text with spaces "}'; - const result = parseJson(input, undefined); - - expect(result.type).toBe('Tap'); - expect(result.description).toBe('Some text with spaces'); - }); - - it('should preserve configured string value keys while trimming other fields', () => { - const input = - '{" value ": " test value ", "param": {" prompt ": " input field "}}'; - const result = safeParseJson(input, { - source: 'generic-object', - preserveStringValueKeys: ['value'], - }); - - expect(result).toEqual({ - value: ' test value ', - param: { - prompt: 'input field', - }, - }); - }); - - it('should preserve configured string value keys from JSON code blocks', () => { - const input = `\`\`\`json -{ - "value": " test value ", - "locate": { - "prompt": " input field " - } -} -\`\`\``; - const result = safeParseJson(input, { - source: 'generic-object', - preserveStringValueKeys: ['value'], - }); - - expect(result).toEqual({ - value: ' test value ', - locate: { - prompt: 'input field', - }, - }); - }); - - it('should preserve configured string value keys after jsonrepair', () => { - const input = - '{ value: " test value ", locate: {" prompt ": " input field ",}, }'; - const result = safeParseJson(input, { - source: 'generic-object', - preserveStringValueKeys: ['value'], - }); - - expect(result).toEqual({ - value: ' test value ', - locate: { - prompt: 'input field', - }, - }); - }); - - it('should handle null and undefined values', () => { - const input = '{"type": "Tap", "value": null, "param": {}}'; - const result = parseJson(input, undefined); - - expect(result.type).toBe('Tap'); - expect(result.value).toBeNull(); - }); - - it('should work with malformed JSON that jsonrepair can fix', () => { - const input = '{type: " Tap ", param: {" prompt ": "Login"}}'; - const result = parseJson(input, undefined); - - expect(result.type).toBe('Tap'); - expect(result.param.prompt).toBe('Login'); - }); - - it('should handle deeply nested structures', () => { - const input = JSON.stringify({ - ' type ': ' Action ', - ' nested ': { - ' level1 ': { - ' level2 ': { - ' prompt ': ' deep value ', - }, - }, - }, - }); - const result = parseJson(input, undefined); - - expect(result.type).toBe('Action'); - expect(result.nested.level1.level2.prompt).toBe('deep value'); - }); - - it('should trim id field values', () => { - const input = '{"id": " element-123 ", "type": " Tap "}'; - const result = parseJson(input, undefined); - - expect(result.id).toBe('element-123'); - expect(result.type).toBe('Tap'); - }); - - it('should handle arrays of actions with spaces', () => { - const input = '[{" type ": " Tap "}, {" type ": " Hover "}]'; - const result = parseJson(input, undefined); - - expect(result).toEqual([{ type: 'Tap' }, { type: 'Hover' }]); - }); - - it('should handle coordinate tuples without breaking them', () => { - const input = '(100,200)'; - const result = parseJson(input, undefined); - - expect(result).toEqual([100, 200]); - }); - - it('should work with doubao-vision mode and trim spaces', () => { - const input = '{" type ": " Tap ", "param": {" prompt ": "Click"}}'; - const result = parseJson(input, 'doubao-vision'); - - expect(result.type).toBe('Tap'); - expect(result.param.prompt).toBe('Click'); - }); - }); -}); diff --git a/packages/core/tests/unit-test/utils.test.ts b/packages/core/tests/unit-test/utils.test.ts index d0322b89fc..b5f63603ff 100644 --- a/packages/core/tests/unit-test/utils.test.ts +++ b/packages/core/tests/unit-test/utils.test.ts @@ -1,9 +1,5 @@ import { existsSync, readFileSync, statSync } from 'node:fs'; import * as fs from 'node:fs'; -import { - extractJSONFromCodeBlock, - safeParseJson, -} from '@/ai-model/service-caller/json'; import { dumpActionParam, findAllMidsceneLocatorField } from '@/common'; import { getMidsceneLocationSchema } from '@/index'; import { getMidsceneRunSubDir } from '@midscene/shared/common'; @@ -270,106 +266,6 @@ describe('utils', () => { }); }); -describe('extractJSONFromCodeBlock', () => { - it('should extract JSON from a direct JSON object', () => { - const input = '{ "key": "value" }'; - const result = extractJSONFromCodeBlock(input); - expect(result).toBe('{ "key": "value" }'); - }); - - it('should extract JSON from a code block with json language specifier', () => { - const input = '```json\n{ "key": "value" }\n```'; - const result = extractJSONFromCodeBlock(input); - expect(result).toBe('{ "key": "value" }'); - - const input2 = ' ```JSON\n{ "key": "value" }\n```'; - const result2 = extractJSONFromCodeBlock(input2); - expect(result2).toBe('{ "key": "value" }'); - }); - - it('should extract JSON from a code block without language specifier', () => { - const input = '```\n{ "key": "value" }\n```'; - const result = extractJSONFromCodeBlock(input); - expect(result).toBe('{ "key": "value" }'); - }); - - it('should extract JSON-like structure from text', () => { - const input = 'Some text { "key": "value" } more text'; - const result = extractJSONFromCodeBlock(input); - expect(result).toBe('{ "key": "value" }'); - }); - - it('should return the original response if no JSON structure is found', () => { - const input = 'This is just plain text'; - const result = extractJSONFromCodeBlock(input); - expect(result).toBe('This is just plain text'); - }); - - it('should handle multi-line JSON objects', () => { - const input = `{ - "key1": "value1", - "key2": { - "nestedKey": "nestedValue" - } - }`; - const result = extractJSONFromCodeBlock(input); - expect(result).toBe(input); - }); - - it('should handle JSON with point coordinates', () => { - const input = '(123,456)'; - const result = safeParseJson(input); - expect(result).toEqual([123, 456]); - }); - - it('should parse valid JSON string using JSON.parse', () => { - const input = '{"key": "value"}'; - const result = safeParseJson(input); - expect(result).toEqual({ key: 'value' }); - }); - - it('should parse dirty JSON using dirty-json parser', () => { - const input = "{key: 'value'}"; // Invalid JSON but valid dirty-json - const result = safeParseJson(input); - expect(result).toEqual({ key: 'value' }); - }); - - it('should throw error for unparseable content', () => { - const input = '{foo: true false}'; - expect(() => safeParseJson(input)).toThrow( - /failed to parse LLM response into JSON/, - ); - }); - - it('should parse JSON from code block', () => { - const input = '```json\n{"key": "value"}\n```'; - const result = safeParseJson(input); - expect(result).toEqual({ key: 'value' }); - }); - - it('should parse complex nested JSON', () => { - const input = `{ - "string": "value", - "number": 123, - "boolean": true, - "array": [1, 2, 3], - "object": { - "nested": "value" - } - }`; - const result = safeParseJson(input); - expect(result).toEqual({ - string: 'value', - number: 123, - boolean: true, - array: [1, 2, 3], - object: { - nested: 'value', - }, - }); - }); -}); - describe('buildDetailedLocateParam', () => { it('merges multimodal locate options into the prompt object', () => { const result = buildDetailedLocateParam('Click the icon', { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 026eff46c1..cb01517912 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1006,8 +1006,8 @@ importers: specifier: 4.1.0 version: 4.1.0 jsonrepair: - specifier: 3.12.0 - version: 3.12.0 + specifier: 3.14.0 + version: 3.14.0 mdast-util-from-markdown: specifier: ^2.0.2 version: 2.0.2 @@ -1996,19 +1996,16 @@ packages: '@computer-use/libnut-darwin@2.7.1': resolution: {integrity: sha512-7B/aPcIYS4a4S7D3IYIHSpZ4B4m8Z3CjlYq0efTr+/JYmEu+LlO67ZhPvisLKifhagxf7goqEfnphg1F4jq5jw==} engines: {node: '>=10.15.3'} - cpu: [x64, arm64] os: [darwin, linux, win32] '@computer-use/libnut-linux@2.7.1': resolution: {integrity: sha512-QJD5URTFJ/2+JwBwRyajRF2BB+3eXpd4+t5btGeRVeiRQLKQ4lgorbMHySo6IrfAbSfnU1OVOrxAUygGxj0cFg==} engines: {node: '>=10.15.3'} - cpu: [x64, arm64] os: [darwin, linux, win32] '@computer-use/libnut-win32@2.7.1': resolution: {integrity: sha512-nDvH5kP1zoO2cBtFYWV0om9xtTu523cc1LIk8r/wizqPrIAm0wCizTU+odF3Fi42zcKJWT6J+Pguy8fKrZyIuA==} engines: {node: '>=10.15.3'} - cpu: [x64, arm64] os: [darwin, linux, win32] '@computer-use/libnut@4.2.0': @@ -7616,8 +7613,8 @@ packages: resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} engines: {node: '>=0.10.0'} - jsonrepair@3.12.0: - resolution: {integrity: sha512-SWfjz8SuQ0wZjwsxtSJ3Zy8vvLg6aO/kxcp9TWNPGwJKgTZVfhNEQBMk/vPOpYCDFWRxD6QWuI6IHR1t615f0w==} + jsonrepair@3.14.0: + resolution: {integrity: sha512-tWPGKMZf/8UPim+fcW2EfcQ/d/7aKUrP6IECz9G3Tu6Q5dX0orSleqJ9z6sSw7qrQkjF8/Edo4DvsWBZ8H+HNg==} hasBin: true jszip@3.10.1: @@ -18801,7 +18798,7 @@ snapshots: jsonpointer@5.0.1: optional: true - jsonrepair@3.12.0: {} + jsonrepair@3.14.0: {} jszip@3.10.1: dependencies: From e88e8f8728aa2b5957ac87cf443b22c209b81780 Mon Sep 17 00:00:00 2001 From: EAGzzyCSL Date: Wed, 17 Jun 2026 21:50:34 +0800 Subject: [PATCH 2/4] feat(core): request json response format for locate adapters --- packages/core/src/ai-model/models/glm.ts | 7 +++++++ packages/core/src/ai-model/models/gpt.ts | 9 ++++++++- packages/core/src/ai-model/models/kimi.ts | 7 +++++++ packages/core/src/ai-model/models/mimo.ts | 7 ++++--- packages/core/src/ai-model/models/qwen.ts | 7 +++++++ packages/core/src/ai-model/models/utils/intent.ts | 5 +++++ .../core/tests/unit-test/model-adapter/glm.test.ts | 13 +++++++++++++ .../core/tests/unit-test/model-adapter/gpt.test.ts | 9 +++++++++ .../core/tests/unit-test/model-adapter/kimi.test.ts | 9 +++++++++ .../core/tests/unit-test/model-adapter/mimo.test.ts | 13 ++----------- .../core/tests/unit-test/model-adapter/qwen.test.ts | 12 ++++++++++++ 11 files changed, 83 insertions(+), 15 deletions(-) create mode 100644 packages/core/src/ai-model/models/utils/intent.ts diff --git a/packages/core/src/ai-model/models/glm.ts b/packages/core/src/ai-model/models/glm.ts index 8152a90d32..30d5bf975e 100644 --- a/packages/core/src/ai-model/models/glm.ts +++ b/packages/core/src/ai-model/models/glm.ts @@ -4,6 +4,7 @@ import type { ChatCompletionParamsResult, ModelAdapterDefinition, } from '../model-adapter/types'; +import { isLocateIntent } from './utils/intent'; const buildGlmChatCompletionParams = ( input: ChatCompletionCallContext, @@ -16,6 +17,12 @@ const buildGlmChatCompletionParams = ( commonOverrideConfig.temperature = userConfig.temperature; } + // Zhipu structured output JSON mode: + // https://docs.bigmodel.cn/cn/guide/capabilities/struct-output + if (isLocateIntent(input.intent)) { + commonOverrideConfig.response_format = { type: 'json_object' }; + } + const modelSpecificConfig: Record = {}; if (reasoningEnabled !== 'default') { diff --git a/packages/core/src/ai-model/models/gpt.ts b/packages/core/src/ai-model/models/gpt.ts index a8bab86945..c47f66ab36 100644 --- a/packages/core/src/ai-model/models/gpt.ts +++ b/packages/core/src/ai-model/models/gpt.ts @@ -5,11 +5,12 @@ import type { ImageDetail, ModelAdapterDefinition, } from '../model-adapter/types'; +import { isLocateIntent } from './utils/intent'; const originalImageDetailForDefaultIntent = ( input: ChatCompletionCallContext, ): ImageDetail | undefined => - input.intent === 'default' || input.requiresOriginalImageDetail + isLocateIntent(input.intent) || input.requiresOriginalImageDetail ? 'original' : undefined; @@ -24,6 +25,12 @@ const buildGpt5ChatCompletionParams = ( commonOverrideConfig.temperature = userConfig.temperature; } + // OpenAI Chat Completions JSON mode: + // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode + if (isLocateIntent(input.intent)) { + commonOverrideConfig.response_format = { type: 'json_object' }; + } + const effectiveReasoningEffort = reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none'; diff --git a/packages/core/src/ai-model/models/kimi.ts b/packages/core/src/ai-model/models/kimi.ts index dbd171176c..07bece82d6 100644 --- a/packages/core/src/ai-model/models/kimi.ts +++ b/packages/core/src/ai-model/models/kimi.ts @@ -9,6 +9,7 @@ import { createLocateResultValue, parseCoordinateList, } from '../shared/model-locate-result'; +import { isLocateIntent } from './utils/intent'; const kimiNormalizedPointCoordinatesMeta = { shape: 'point', @@ -46,6 +47,12 @@ const buildKimiChatCompletionParams = ( // kimi disallow custom temperature commonOverrideConfig.temperature = undefined; + // Kimi Chat Completions response_format: + // https://platform.kimi.com/docs/api/chat + if (isLocateIntent(input.intent)) { + commonOverrideConfig.response_format = { type: 'json_object' }; + } + const modelSpecificConfig: Record = { thinking: { type: effectiveReasoningEnabled ? 'enabled' : 'disabled', diff --git a/packages/core/src/ai-model/models/mimo.ts b/packages/core/src/ai-model/models/mimo.ts index 62d20171b6..5bfe5ad906 100644 --- a/packages/core/src/ai-model/models/mimo.ts +++ b/packages/core/src/ai-model/models/mimo.ts @@ -4,6 +4,7 @@ import type { ChatCompletionParamsResult, ModelAdapterDefinition, } from '../model-adapter/types'; +import { isLocateIntent } from './utils/intent'; const buildMimoChatCompletionParams = ( input: ChatCompletionCallContext, @@ -14,9 +15,9 @@ const buildMimoChatCompletionParams = ( // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api // Observed with thinking disabled: Mimo needs json_object to return JSON. - commonOverrideConfig.response_format = { - type: intent === 'default' ? 'json_object' : 'text', - }; + if (isLocateIntent(intent)) { + commonOverrideConfig.response_format = { type: 'json_object' }; + } if (userConfig.temperature !== undefined) { commonOverrideConfig.temperature = userConfig.temperature; diff --git a/packages/core/src/ai-model/models/qwen.ts b/packages/core/src/ai-model/models/qwen.ts index c1927792e6..cfb61225c5 100644 --- a/packages/core/src/ai-model/models/qwen.ts +++ b/packages/core/src/ai-model/models/qwen.ts @@ -11,6 +11,7 @@ import { isBboxLocateResultValue, unwrapCoordinateListLikeInput, } from '../shared/model-locate-result'; +import { isLocateIntent } from './utils/intent'; const defaultBboxSize = 20; const qwen25BboxCoordinatesMeta = { @@ -87,6 +88,12 @@ const buildQwenChatCompletionParams = ( commonOverrideConfig.temperature = userConfig.temperature; } + // Alibaba Cloud Model Studio JSON mode: + // https://help.aliyun.com/zh/model-studio/json-mode + if (isLocateIntent(input.intent)) { + commonOverrideConfig.response_format = { type: 'json_object' }; + } + const modelSpecificConfig: Record = {}; if (reasoningEnabled !== 'default') { diff --git a/packages/core/src/ai-model/models/utils/intent.ts b/packages/core/src/ai-model/models/utils/intent.ts new file mode 100644 index 0000000000..6d28fcbb94 --- /dev/null +++ b/packages/core/src/ai-model/models/utils/intent.ts @@ -0,0 +1,5 @@ +import type { TIntent } from '@midscene/shared/env'; + +export function isLocateIntent(intent?: TIntent): boolean { + return intent === 'default'; +} diff --git a/packages/core/tests/unit-test/model-adapter/glm.test.ts b/packages/core/tests/unit-test/model-adapter/glm.test.ts index 5d3808be44..1721d1eb39 100644 --- a/packages/core/tests/unit-test/model-adapter/glm.test.ts +++ b/packages/core/tests/unit-test/model-adapter/glm.test.ts @@ -87,4 +87,17 @@ describe('glm model adapter', () => { thinking: { type: 'disabled' }, }); }); + + it('uses json_object response format for glm-v locate intent', () => { + const result = glmAdapter.chatCompletion.buildChatCompletionParams({ + intent: 'default', + userConfig: {}, + }); + + expect(result.config).toEqual({ + temperature: 0, + response_format: { type: 'json_object' }, + thinking: { type: 'disabled' }, + }); + }); }); diff --git a/packages/core/tests/unit-test/model-adapter/gpt.test.ts b/packages/core/tests/unit-test/model-adapter/gpt.test.ts index 5bbd7eeb53..318c9b8776 100644 --- a/packages/core/tests/unit-test/model-adapter/gpt.test.ts +++ b/packages/core/tests/unit-test/model-adapter/gpt.test.ts @@ -133,4 +133,13 @@ describe('gpt model adapter', () => { reasoning_effort: 'none', }); }); + + it('uses json_object response format for gpt-5 locate intent', () => { + const result = gpt5Adapter.chatCompletion.buildChatCompletionParams({ + intent: 'default', + userConfig: {}, + }); + + expect(result.config.response_format).toEqual({ type: 'json_object' }); + }); }); diff --git a/packages/core/tests/unit-test/model-adapter/kimi.test.ts b/packages/core/tests/unit-test/model-adapter/kimi.test.ts index b6b4f69deb..b38911ac83 100644 --- a/packages/core/tests/unit-test/model-adapter/kimi.test.ts +++ b/packages/core/tests/unit-test/model-adapter/kimi.test.ts @@ -149,4 +149,13 @@ describe('kimi model adapter', () => { thinking: { type: 'disabled' }, }); }); + + it('uses json_object response format for kimi locate intent', () => { + const result = kimiAdapter.chatCompletion.buildChatCompletionParams({ + intent: 'default', + userConfig: {}, + }); + + expect(result.config.response_format).toEqual({ type: 'json_object' }); + }); }); diff --git a/packages/core/tests/unit-test/model-adapter/mimo.test.ts b/packages/core/tests/unit-test/model-adapter/mimo.test.ts index 32303a9860..7e582d0fc2 100644 --- a/packages/core/tests/unit-test/model-adapter/mimo.test.ts +++ b/packages/core/tests/unit-test/model-adapter/mimo.test.ts @@ -15,7 +15,6 @@ describe('mimo model adapter', () => { expect(result.config).toEqual({ temperature: 0, - response_format: { type: 'text' }, thinking: { type: 'disabled' }, }); }); @@ -48,7 +47,6 @@ describe('mimo model adapter', () => { expect(result.config).toEqual({ temperature: 0.7, seed: 123, - response_format: { type: 'text' }, thinking: { type: 'disabled' }, }); }); @@ -69,12 +67,10 @@ describe('mimo model adapter', () => { expect(disabledResult.config).toEqual({ temperature: 0, - response_format: { type: 'text' }, thinking: { type: 'disabled' }, }); expect(enabledResult.config).toEqual({ temperature: 0, - response_format: { type: 'text' }, thinking: { type: 'enabled' }, }); }); @@ -93,7 +89,6 @@ describe('mimo model adapter', () => { ]); expect(result.config).toEqual({ temperature: 0, - response_format: { type: 'text' }, thinking: { type: 'disabled' }, }); }); @@ -107,12 +102,11 @@ describe('mimo model adapter', () => { expect(result.config).toEqual({ temperature: 0.7, - response_format: { type: 'text' }, thinking: { type: 'disabled' }, }); }); - it('uses text response format for planning intent', () => { + it('does not set response format for planning intent', () => { const result = mimoAdapter.chatCompletion.buildChatCompletionParams({ intent: 'planning', userConfig: {}, @@ -120,7 +114,6 @@ describe('mimo model adapter', () => { expect(result.config).toEqual({ temperature: 0, - response_format: { type: 'text' }, thinking: { type: 'disabled' }, }); }); @@ -138,8 +131,6 @@ describe('mimo model adapter', () => { expect(defaultResult.config.response_format).toEqual({ type: 'json_object', }); - expect(insightResult.config.response_format).toEqual({ - type: 'text', - }); + expect(insightResult.config.response_format).toBeUndefined(); }); }); diff --git a/packages/core/tests/unit-test/model-adapter/qwen.test.ts b/packages/core/tests/unit-test/model-adapter/qwen.test.ts index 53d4557947..7d137cb058 100644 --- a/packages/core/tests/unit-test/model-adapter/qwen.test.ts +++ b/packages/core/tests/unit-test/model-adapter/qwen.test.ts @@ -298,6 +298,18 @@ describe('qwen model adapter', () => { }); }); + it('uses json_object response format for qwen3 locate intent', () => { + const qwen3VlResult = + qwen3VlAdapter.chatCompletion.buildChatCompletionParams({ + intent: 'default', + userConfig: {}, + }); + + expect(qwen3VlResult.config.response_format).toEqual({ + type: 'json_object', + }); + }); + it('normalizes actual-pixel bbox coordinates for qwen2.5-vl', () => { const locateAdapter = qwen25Adapter.locate; expect(locateAdapter.kind).toBe('standard'); From 7cc31682b774bfafd240c93c24dbad5286372719 Mon Sep 17 00:00:00 2001 From: EAGzzyCSL Date: Thu, 18 Jun 2026 17:18:48 +0800 Subject: [PATCH 3/4] fix(core): disable qwen json response format --- packages/core/src/ai-model/models/qwen.ts | 9 +++++---- .../core/tests/unit-test/model-adapter/qwen.test.ts | 12 ------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/packages/core/src/ai-model/models/qwen.ts b/packages/core/src/ai-model/models/qwen.ts index cfb61225c5..de4ffb0cd2 100644 --- a/packages/core/src/ai-model/models/qwen.ts +++ b/packages/core/src/ai-model/models/qwen.ts @@ -11,7 +11,6 @@ import { isBboxLocateResultValue, unwrapCoordinateListLikeInput, } from '../shared/model-locate-result'; -import { isLocateIntent } from './utils/intent'; const defaultBboxSize = 20; const qwen25BboxCoordinatesMeta = { @@ -90,9 +89,11 @@ const buildQwenChatCompletionParams = ( // Alibaba Cloud Model Studio JSON mode: // https://help.aliyun.com/zh/model-studio/json-mode - if (isLocateIntent(input.intent)) { - commonOverrideConfig.response_format = { type: 'json_object' }; - } + // Observed in qwen3.6 grounding runs: enabling this can make the model + // return only ["bbox_2d"] without coordinates. + // if (isLocateIntent(input.intent)) { + // commonOverrideConfig.response_format = { type: 'json_object' }; + // } const modelSpecificConfig: Record = {}; diff --git a/packages/core/tests/unit-test/model-adapter/qwen.test.ts b/packages/core/tests/unit-test/model-adapter/qwen.test.ts index 7d137cb058..53d4557947 100644 --- a/packages/core/tests/unit-test/model-adapter/qwen.test.ts +++ b/packages/core/tests/unit-test/model-adapter/qwen.test.ts @@ -298,18 +298,6 @@ describe('qwen model adapter', () => { }); }); - it('uses json_object response format for qwen3 locate intent', () => { - const qwen3VlResult = - qwen3VlAdapter.chatCompletion.buildChatCompletionParams({ - intent: 'default', - userConfig: {}, - }); - - expect(qwen3VlResult.config.response_format).toEqual({ - type: 'json_object', - }); - }); - it('normalizes actual-pixel bbox coordinates for qwen2.5-vl', () => { const locateAdapter = qwen25Adapter.locate; expect(locateAdapter.kind).toBe('standard'); From 4be8f049c53ab1fe986a40e5d96e241b28465849 Mon Sep 17 00:00:00 2001 From: EAGzzyCSL Date: Tue, 23 Jun 2026 10:53:27 +0800 Subject: [PATCH 4/4] fix(core): fix ut --- .../ui-tars/adapter-json-repair.test.ts | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/packages/core/tests/unit-test/model-adapter/ui-tars/adapter-json-repair.test.ts b/packages/core/tests/unit-test/model-adapter/ui-tars/adapter-json-repair.test.ts index 7c62bbcf4d..72a3e185c9 100644 --- a/packages/core/tests/unit-test/model-adapter/ui-tars/adapter-json-repair.test.ts +++ b/packages/core/tests/unit-test/model-adapter/ui-tars/adapter-json-repair.test.ts @@ -1,26 +1,14 @@ import { ResolvedModelAdapter } from '@/ai-model/model-adapter/resolve'; import { uiTarsAdapters } from '@/ai-model/models/ui-tars/adapter'; -import { describe, expect, it, vi } from 'vitest'; - -vi.mock('@/ai-model/service-caller/json', async (importOriginal) => { - const actual = - await importOriginal(); - return { - ...actual, - extractJSONFromCodeBlock: vi.fn((raw: string) => raw), - safeParseJson: vi.fn(() => { - throw new Error('first safe parse failed'); - }), - }; -}); +import { describe, expect, it } from 'vitest'; const uiTarsAdapter = new ResolvedModelAdapter( uiTarsAdapters['vlm-ui-tars'], 'vlm-ui-tars', ); -describe('ui-tars json repair fallback', () => { - it('repairs bbox whitespace after the first parser fails', () => { +describe('ui-tars json parser', () => { + it('repairs bbox whitespace for locate results', () => { expect( uiTarsAdapter.jsonParser('{"bbox": [123 456 789 100]}', { source: 'locate', @@ -30,9 +18,9 @@ describe('ui-tars json repair fallback', () => { }); }); - it('rethrows the first parse error for generic parser sources', () => { - expect(() => - uiTarsAdapter.jsonParser('{"bbox": [123 456 789 100]}'), - ).toThrow('first safe parse failed'); + it('uses the unified parser for generic object responses', () => { + expect(uiTarsAdapter.jsonParser('{"bbox": [123 456 789 100]}')).toEqual({ + bbox: [123, 456, 789, 100], + }); }); });