diff --git a/apps/site/docs/en/model-config.mdx b/apps/site/docs/en/model-config.mdx index 33f43c8054..c0b636d2ba 100644 --- a/apps/site/docs/en/model-config.mdx +++ b/apps/site/docs/en/model-config.mdx @@ -28,6 +28,7 @@ If you configure a dedicated Insight or Planning model, model-related `MIDSCENE_ | `MIDSCENE_MODEL_REASONING_EFFORT` | Controls model-native thinking effort, supported by some models. Common values: `low`, `medium`, `high`. See [Model-native thinking](#model-native-reasoning) | | `MIDSCENE_MODEL_REASONING_BUDGET` | Thinking token budget (number), supported by some models. See [Model-native thinking](#model-native-reasoning) | | `MIDSCENE_MODEL_RESPONSE_FORMAT` | Structured response strategy: `auto` (default) lets Midscene automatically use `response_format` in appropriate scenarios to specify a structured output format (usually JSON), making the model response as suitable for structured parsing as possible; `none` does not set `response_format`, for models that do not support structured output. | +| `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` | Encoding for inline images sent to the model: `webp` (default) or `jpeg`. Use `jpeg` as a compatibility fallback when an OpenAI-compatible provider rejects WebP. This only changes the request payload; persisted screenshots and reports remain WebP. | | `MIDSCENE_MODEL_HTTP_PROXY` | HTTP/HTTPS proxy, e.g., `http://127.0.0.1:8080` or `https://proxy.example.com:8080`. Takes precedence over `MIDSCENE_MODEL_SOCKS_PROXY` | | `MIDSCENE_MODEL_SOCKS_PROXY` | SOCKS proxy, e.g., `socks5://127.0.0.1:1080` | | `MIDSCENE_MODEL_INIT_CONFIG_JSON` | JSON blob that overrides the OpenAI SDK initialization config. Use `defaultHeaders` for custom auth headers; `extra_headers` and `extraHeaders` are accepted as aliases | @@ -54,6 +55,7 @@ Set the following if the Insight intent needs a different model: | `MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON` | Optional; same effect as `MIDSCENE_MODEL_INIT_CONFIG_JSON` | | `MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON` | Optional; same effect as `MIDSCENE_MODEL_EXTRA_BODY_JSON` | | `MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT` | Optional; controls the structured response strategy in appropriate Insight scenarios | +| `MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT` | Optional; same effect as `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` for the dedicated Insight model | ### Configure a dedicated Planning model @@ -74,6 +76,7 @@ Set the following if the Planning intent needs a different model: | `MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON` | Optional; same effect as `MIDSCENE_MODEL_INIT_CONFIG_JSON` | | `MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON` | Optional; same effect as `MIDSCENE_MODEL_EXTRA_BODY_JSON` | | `MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT` | Optional; controls the structured response strategy in appropriate Planning scenarios | +| `MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT` | Optional; same effect as `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` for the dedicated Planning model | ### Model-native thinking {#model-native-reasoning} diff --git a/apps/site/docs/en/reference/index.mdx b/apps/site/docs/en/reference/index.mdx index b5ab19802d..3cf4066c88 100644 --- a/apps/site/docs/en/reference/index.mdx +++ b/apps/site/docs/en/reference/index.mdx @@ -1348,7 +1348,7 @@ interface RecordToReportOptions { screenshotBase64?: string; screenshots?: { /** - * PNG/JPEG data URI, or raw PNG base64 body. + * PNG/JPEG/WebP data URI, or raw PNG/JPEG/WebP base64 body. */ base64: string; description?: string; @@ -1366,7 +1366,7 @@ function recordToReport( - `title?: string` — Optional title for the report entry. Default: `'untitled'`. - `options?: RecordToReportOptions` — Optional configuration: - `content?: string` — Description of the screenshot. - - `screenshots?: Array<{ base64: string; description?: string }>` — One or more screenshots to record under the same report entry. When this option is set, Midscene does not capture another screenshot automatically. `base64` accepts a PNG/JPEG data URI such as `data:image/png;base64,...` or a raw base64 body, which Midscene treats as PNG. + - `screenshots?: Array<{ base64: string; description?: string }>` — One or more screenshots to record under the same report entry. When this option is set, Midscene does not capture another screenshot automatically. `base64` accepts a PNG/JPEG/WebP data URI such as `data:image/webp;base64,...` or a raw PNG/JPEG/WebP base64 body. Midscene infers known raw image signatures and treats an unrecognized raw body as PNG for compatibility. - Compatibility: diff --git a/apps/site/docs/zh/model-config.mdx b/apps/site/docs/zh/model-config.mdx index 7fb3c20d8d..fa3f00c017 100644 --- a/apps/site/docs/zh/model-config.mdx +++ b/apps/site/docs/zh/model-config.mdx @@ -27,6 +27,7 @@ | `MIDSCENE_MODEL_REASONING_EFFORT` | 控制模型的原生思考力度,部分模型支持。常用值:`low`、`medium`、`high`。详见[模型原生思考](#model-native-reasoning) | | `MIDSCENE_MODEL_REASONING_BUDGET` | 思考 Token 预算(数字),部分模型支持。详见[模型原生思考](#model-native-reasoning) | | `MIDSCENE_MODEL_RESPONSE_FORMAT` | 结构化响应策略:`auto`(默认)表示 Midscene 会在合适的场景自动使用 `response_format` 参数指定模型输出的结构化格式(一般是 JSON),来尽可能保证模型返回值能够被结构化解析;`none` 表示不指定 `response_format` 参数,适用于模型不支持结构化输出的情况。 | +| `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` | 发送给模型的内联图片编码:`webp`(默认)或 `jpeg`。当 OpenAI 兼容服务不接受 WebP 时,可设为 `jpeg` 作为兼容回退。该配置只改变请求体,落盘截图和报告仍保持 WebP。 | | `MIDSCENE_MODEL_HTTP_PROXY` | HTTP/HTTPS 代理配置,如 `http://127.0.0.1:8080` 或 `https://proxy.example.com:8080`,优先级高于 `MIDSCENE_MODEL_SOCKS_PROXY` | | `MIDSCENE_MODEL_SOCKS_PROXY` | SOCKS 代理配置,如 `socks5://127.0.0.1:1080` | | `MIDSCENE_MODEL_INIT_CONFIG_JSON` | 覆盖 OpenAI SDK 初始化配置的 JSON。自定义鉴权 header 请使用 `defaultHeaders`;`extra_headers` 和 `extraHeaders` 也会作为别名兼容 | @@ -53,6 +54,7 @@ | `MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON` | 可选,效果等同于 `MIDSCENE_MODEL_INIT_CONFIG_JSON` | | `MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON` | 可选,效果等同于 `MIDSCENE_MODEL_EXTRA_BODY_JSON` | | `MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT` | 可选,在合适的 Insight 场景中控制结构化响应策略 | +| `MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT` | 可选,为独立 Insight 模型配置图片输入格式,效果等同于 `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` | ### 为 Planning 意图单独配置模型 @@ -73,6 +75,7 @@ | `MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON` | 可选,效果等同于 `MIDSCENE_MODEL_INIT_CONFIG_JSON` | | `MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON` | 可选,效果等同于 `MIDSCENE_MODEL_EXTRA_BODY_JSON` | | `MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT` | 可选,在合适的 Planning 场景中控制结构化响应策略 | +| `MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT` | 可选,为独立 Planning 模型配置图片输入格式,效果等同于 `MIDSCENE_MODEL_IMAGE_INPUT_FORMAT` | ### 模型原生思考 {#model-native-reasoning} diff --git a/apps/site/docs/zh/reference/index.mdx b/apps/site/docs/zh/reference/index.mdx index 54e00ee7ae..fd4125f245 100644 --- a/apps/site/docs/zh/reference/index.mdx +++ b/apps/site/docs/zh/reference/index.mdx @@ -1331,7 +1331,7 @@ interface RecordToReportOptions { screenshotBase64?: string; screenshots?: { /** - * PNG/JPEG data URI,或裸 PNG base64 body。 + * PNG/JPEG/WebP data URI,或裸 PNG/JPEG/WebP base64 body。 */ base64: string; description?: string; @@ -1349,7 +1349,7 @@ function recordToReport( - `title?: string` - 可选,截图的标题,如果未提供,则标题为 'untitled'。 - `options?: RecordToReportOptions` - 可选,一个配置对象,包含: - `content?: string` - 截图的描述。 - - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后,Midscene 不会再自动截图。`base64` 推荐使用 PNG/JPEG data URI,例如 `data:image/png;base64,...`;也可以传裸 base64 body,此时会按 PNG 处理。 + - `screenshots?: Array<{ base64: string; description?: string }>` - 在同一个报告条目下记录一张或多张传入的截图。设置该选项后,Midscene 不会再自动截图。`base64` 支持 PNG/JPEG/WebP data URI,例如 `data:image/webp;base64,...`,也支持裸 PNG/JPEG/WebP base64 body。Midscene 会根据已知图片签名推断裸数据格式;无法识别时为保持兼容仍按 PNG 处理。 - 兼容性: diff --git a/packages/core/src/agent/screenshot-preparation.ts b/packages/core/src/agent/screenshot-preparation.ts index fce36888f6..4478018a6f 100644 --- a/packages/core/src/agent/screenshot-preparation.ts +++ b/packages/core/src/agent/screenshot-preparation.ts @@ -1,19 +1,20 @@ import { - type JpegBase64DataUrl, - convertBase64ImageToJpeg, + type WebpBase64DataUrl, + convertBase64ImageToWebp, imageInfoOfBase64, - resizeBase64ImageToJpeg, + resizeBase64ImageToWebp, } from '@midscene/shared/img'; import type { Size } from '../types'; -const SCREENSHOT_JPEG_QUALITY = 90; +const SCREENSHOT_WEBP_QUALITY = 90; +const SCREENSHOT_WEBP_EFFORT = 1; export interface PrepareRawScreenshotOptions { shrinkFactor?: number; } export interface PreparedScreenshot { - base64: JpegBase64DataUrl; + base64: WebpBase64DataUrl; originalSize: Size; shotSize: Size; } @@ -58,10 +59,11 @@ export async function prepareRawScreenshot( : { ...originalSize }; assertValidSize(shotSize, 'prepared screenshot dimensions'); - const base64 = await resizeBase64ImageToJpeg(screenshotBase64, { + const base64 = await resizeBase64ImageToWebp(screenshotBase64, { sourceSize: originalSize, targetSize: shotSize, - jpegQuality: SCREENSHOT_JPEG_QUALITY, + webpQuality: SCREENSHOT_WEBP_QUALITY, + webpEffort: SCREENSHOT_WEBP_EFFORT, }); return { @@ -72,16 +74,19 @@ export async function prepareRawScreenshot( } /** - * Prepare a screenshot for persistence when only JPEG output is required. + * Prepare a screenshot for persistence when only WebP output is required. * Unscaled frames avoid dimension reads; scaled frames use the full pipeline. */ export async function prepareScreenshotForPersistence( screenshotBase64: string, options?: PrepareRawScreenshotOptions, -): Promise { +): Promise { const shrinkFactor = options?.shrinkFactor ?? 1; if (shrinkFactor === 1) { - return convertBase64ImageToJpeg(screenshotBase64, SCREENSHOT_JPEG_QUALITY); + return convertBase64ImageToWebp(screenshotBase64, { + webpQuality: SCREENSHOT_WEBP_QUALITY, + webpEffort: SCREENSHOT_WEBP_EFFORT, + }); } return (await prepareRawScreenshot(screenshotBase64, options)).base64; } diff --git a/packages/core/src/agent/ui-observer.ts b/packages/core/src/agent/ui-observer.ts index 55e339c2d7..bebeb48c6d 100644 --- a/packages/core/src/agent/ui-observer.ts +++ b/packages/core/src/agent/ui-observer.ts @@ -186,7 +186,7 @@ interface BufferedFrame extends DeviceFrameRef { function isImageDataUrl(value: unknown): value is string { return ( typeof value === 'string' && - /^data:image\/(?:png|jpe?g);base64,/i.test(value) + /^data:image\/(?:png|jpe?g|webp);base64,/i.test(value) ); } diff --git a/packages/core/src/agent/utils.ts b/packages/core/src/agent/utils.ts index 4481713951..105e17e523 100644 --- a/packages/core/src/agent/utils.ts +++ b/packages/core/src/agent/utils.ts @@ -22,10 +22,7 @@ import { globalConfigManager, } from '@midscene/shared/env'; import { generateElementByRect } from '@midscene/shared/extractor'; -import { - imageInfoOfBase64, - normalizeScreenshotBase64, -} from '@midscene/shared/img'; +import { normalizeScreenshotBase64 } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import { _keyDefinitions } from '@midscene/shared/us-keyboard-layout'; import { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils'; @@ -165,9 +162,10 @@ export async function createScreenshotBoundUIContext( ): Promise { const normalizedScreenshotBase64 = normalizeScreenshotBase64(screenshotBase64); - const actualScreenshotSize = await imageInfoOfBase64( + const preparedScreenshot = await prepareRawScreenshot( normalizedScreenshotBase64, ); + const actualScreenshotSize = preparedScreenshot.originalSize; if ( opt.screenshotSize && (opt.screenshotSize.width !== actualScreenshotSize.width || @@ -183,8 +181,8 @@ export async function createScreenshotBoundUIContext( } return { - screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()), - shotSize: actualScreenshotSize, + screenshot: ScreenshotItem.create(preparedScreenshot.base64, Date.now()), + shotSize: preparedScreenshot.shotSize, shrunkShotToLogicalRatio: 1, _isFrozen: true, }; diff --git a/packages/core/src/ai-model/model-adapter/image-preprocess.ts b/packages/core/src/ai-model/model-adapter/image-preprocess.ts index 8321a54b14..9ac5bce05c 100644 --- a/packages/core/src/ai-model/model-adapter/image-preprocess.ts +++ b/packages/core/src/ai-model/model-adapter/image-preprocess.ts @@ -37,10 +37,15 @@ export async function prepareModelImage(options: { let modelWidth = width; let modelHeight = height; - if (policy.padBlockSize !== undefined) { + const padBlockSize = policy.padBlockSize; + const requiresPadding = + padBlockSize !== undefined && + (width % padBlockSize !== 0 || height % padBlockSize !== 0); + if (requiresPadding) { const paddedResult = await paddingToMatchBlockByBase64( imageBase64, - policy.padBlockSize, + padBlockSize, + 'webp', ); preparedImageBase64 = paddedResult.imageBase64; modelWidth = paddedResult.width; diff --git a/packages/core/src/ai-model/service-caller/index.ts b/packages/core/src/ai-model/service-caller/index.ts index 162909f568..ef74576971 100644 --- a/packages/core/src/ai-model/service-caller/index.ts +++ b/packages/core/src/ai-model/service-caller/index.ts @@ -52,6 +52,7 @@ import { isModelCallRecordingEnabled, recordModelCallEvent, } from './model-call-recorder'; +import { prepareModelMessagesImageInput } from './model-image-input'; import { type OpenAIErrorResponseContext, formatOpenAIAPIErrorDetails, @@ -401,6 +402,10 @@ export async function callAI( }); } : undefined; + const modelMessages = await prepareModelMessagesImageInput( + messages, + modelConfig.imageInputFormat, + ); const chatCompletionInput = { intent: modelConfig.intent, userConfig: { @@ -443,7 +448,7 @@ export async function callAI( try { const codexResult = await callAIWithCodexAppServer( - messages, + modelMessages, modelConfig, { stream: options?.stream, @@ -597,10 +602,10 @@ export async function callAI( // resolution for localization-sensitive tasks. const messagesWithImageDetail: ChatCompletionMessageParam[] = (() => { if (!imageDetail) { - return messages; + return modelMessages; } - return messages.map((msg) => { + return modelMessages.map((msg) => { if (!Array.isArray(msg.content)) { return msg; } diff --git a/packages/core/src/ai-model/service-caller/model-image-input.ts b/packages/core/src/ai-model/service-caller/model-image-input.ts new file mode 100644 index 0000000000..a7a7511276 --- /dev/null +++ b/packages/core/src/ai-model/service-caller/model-image-input.ts @@ -0,0 +1,99 @@ +import type { TModelImageInputFormat } from '@midscene/shared/env'; +import { + convertBase64ImageToJpeg, + convertBase64ImageToWebp, +} from '@midscene/shared/img'; +import type { ChatCompletionMessageParam } from 'openai/resources/index'; + +const supportedBase64ImageUrlPattern = + /^data:image\/(?:png|jpe?g|webp);base64,/i; +type MessageContentPart = Exclude< + NonNullable, + string +>[number]; + +async function convertModelImageUrl( + url: string, + imageInputFormat: TModelImageInputFormat, + conversionCache: Map>, +): Promise { + if (!supportedBase64ImageUrlPattern.test(url)) { + return url; + } + + const cacheKey = `${imageInputFormat}\0${url}`; + const cachedConversion = conversionCache.get(cacheKey); + if (cachedConversion) { + return cachedConversion; + } + + const conversion = + imageInputFormat === 'jpeg' + ? convertBase64ImageToJpeg(url) + : convertBase64ImageToWebp(url); + conversionCache.set(cacheKey, conversion); + return conversion; +} + +/** + * Applies the selected encoding to inline screenshot inputs at the final model + * request boundary. Remote URLs are intentionally left unchanged. + */ +export async function prepareModelMessagesImageInput( + messages: ChatCompletionMessageParam[], + imageInputFormat: TModelImageInputFormat = 'webp', +): Promise { + let messagesChanged = false; + const preparedMessages: ChatCompletionMessageParam[] = []; + const conversionCache = new Map>(); + + // Process sequentially to bound memory when one request contains many + // full-resolution observation frames. + for (const message of messages) { + if (!Array.isArray(message.content)) { + preparedMessages.push(message); + continue; + } + + let contentChanged = false; + const preparedContent: MessageContentPart[] = []; + for (const part of message.content) { + if (part.type !== 'image_url' || !part.image_url?.url) { + preparedContent.push(part); + continue; + } + + const preparedUrl = await convertModelImageUrl( + part.image_url.url, + imageInputFormat, + conversionCache, + ); + if (preparedUrl === part.image_url.url) { + preparedContent.push(part); + continue; + } + + contentChanged = true; + preparedContent.push({ + ...part, + image_url: { + ...part.image_url, + url: preparedUrl, + }, + }); + } + + if (!contentChanged) { + preparedMessages.push(message); + continue; + } + + messagesChanged = true; + preparedMessages.push({ + ...message, + content: preparedContent, + } as ChatCompletionMessageParam); + } + + return messagesChanged ? preparedMessages : messages; +} diff --git a/packages/core/src/ai-model/workflows/grounding/search-area.ts b/packages/core/src/ai-model/workflows/grounding/search-area.ts index 6574b29601..8684fa1c46 100644 --- a/packages/core/src/ai-model/workflows/grounding/search-area.ts +++ b/packages/core/src/ai-model/workflows/grounding/search-area.ts @@ -78,9 +78,14 @@ export async function buildSearchAreaConfig(options: { const croppedResult = await cropByRect( context.screenshot.base64, sectionRect, + 'webp', ); - const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio); + const scaledResult = await scaleImage( + croppedResult.imageBase64, + scaleRatio, + 'webp', + ); return { sourceRect: sectionRect, image: { diff --git a/packages/core/src/ai-model/workflows/recorder-generation/markdown.ts b/packages/core/src/ai-model/workflows/recorder-generation/markdown.ts index 2d902b1e6e..d744286d3e 100644 --- a/packages/core/src/ai-model/workflows/recorder-generation/markdown.ts +++ b/packages/core/src/ai-model/workflows/recorder-generation/markdown.ts @@ -2,7 +2,7 @@ import type { IModelConfig } from '@midscene/shared/env'; import { imageInfoOfBase64, parseBase64, - resizeBase64ImageToJpeg, + resizeBase64ImageToWebp, } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import { @@ -54,7 +54,7 @@ async function compressScreenshotAssetForMarkdownReplay( } const scale = MARKDOWN_REPLAY_SCREENSHOT_MAX_EDGE / longestEdge; - const dataUrl = await resizeBase64ImageToJpeg(asset.dataUrl, { + const dataUrl = await resizeBase64ImageToWebp(asset.dataUrl, { sourceSize: { width, height }, targetSize: { width: Math.max(1, Math.round(width * scale)), diff --git a/packages/core/src/service/index.ts b/packages/core/src/service/index.ts index e7f7e696ce..8160315036 100644 --- a/packages/core/src/service/index.ts +++ b/packages/core/src/service/index.ts @@ -33,7 +33,7 @@ import { compositeElementInfoImg, compositePointMarkerImg, cropByRect, - resizeBase64ImageToJpeg, + resizeBase64ImageToWebp, } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/logger'; import { assert } from '@midscene/shared/utils'; @@ -476,6 +476,7 @@ export default class Service { inputImgBase64: screenshotBase64, size: shotSize, point: targetPoint!, + outputFormat: 'webp', }) : await compositeElementInfoImg({ inputImgBase64: screenshotBase64, @@ -487,6 +488,7 @@ export default class Service { ], borderThickness: getDescribeMarkerBorderThickness(targetRect), centerPoint: true, + outputFormat: 'webp', }); const shouldDeepDescribe = opt?.deepDescribe; @@ -496,7 +498,11 @@ export default class Service { const contextImages = await Promise.all( contextAreas.map(async (area) => { debug('describe: cropping deep context area', area); - const croppedResult = await cropByRect(screenshotBase64, area.rect); + const croppedResult = await cropByRect( + screenshotBase64, + area.rect, + 'webp', + ); const cropSize = { width: croppedResult.width, height: croppedResult.height, @@ -510,6 +516,7 @@ export default class Service { x: targetPoint!.x - area.rect.left, y: targetPoint!.y - area.rect.top, }, + outputFormat: 'webp', }) : await compositeElementInfoImg({ inputImgBase64: croppedResult.imageBase64, @@ -521,12 +528,13 @@ export default class Service { ], borderThickness: getDescribeMarkerBorderThickness(targetInCrop), centerPoint: true, + outputFormat: 'webp', }); const resizeSize = getDescribeDeepLocateResizeSize(croppedResult); return { kind: area.kind, imageBase64: resizeSize - ? await resizeBase64ImageToJpeg(markedCropPayload, { + ? await resizeBase64ImageToWebp(markedCropPayload, { sourceSize: cropSize, targetSize: resizeSize, }) diff --git a/packages/core/tests/unit-test/agent-custom-model.test.ts b/packages/core/tests/unit-test/agent-custom-model.test.ts index eb7cd478a7..f593b182a7 100644 --- a/packages/core/tests/unit-test/agent-custom-model.test.ts +++ b/packages/core/tests/unit-test/agent-custom-model.test.ts @@ -64,6 +64,7 @@ describe('Agent with custom OpenAI client', () => { "createOpenAIClient": undefined, "extraBody": undefined, "httpProxy": undefined, + "imageInputFormat": "webp", "intent": "default", "modelDescription": "qwen2.5-vl mode", "modelFamily": "qwen2.5-vl", @@ -93,6 +94,7 @@ describe('Agent with custom OpenAI client', () => { "createOpenAIClient": undefined, "extraBody": undefined, "httpProxy": undefined, + "imageInputFormat": "webp", "intent": "planning", "modelDescription": "qwen2.5-vl mode", "modelFamily": "qwen2.5-vl", @@ -122,6 +124,7 @@ describe('Agent with custom OpenAI client', () => { "createOpenAIClient": undefined, "extraBody": undefined, "httpProxy": undefined, + "imageInputFormat": "webp", "intent": "insight", "modelDescription": "qwen2.5-vl mode", "modelFamily": "qwen2.5-vl", @@ -163,6 +166,7 @@ describe('Agent with custom OpenAI client', () => { "createOpenAIClient": undefined, "extraBody": undefined, "httpProxy": undefined, + "imageInputFormat": "webp", "intent": "default", "modelDescription": "qwen2.5-vl mode", "modelFamily": "qwen2.5-vl", @@ -192,6 +196,7 @@ describe('Agent with custom OpenAI client', () => { "createOpenAIClient": undefined, "extraBody": undefined, "httpProxy": undefined, + "imageInputFormat": "webp", "intent": "planning", "modelDescription": "", "modelFamily": undefined, @@ -221,6 +226,7 @@ describe('Agent with custom OpenAI client', () => { "createOpenAIClient": undefined, "extraBody": undefined, "httpProxy": undefined, + "imageInputFormat": "webp", "intent": "insight", "modelDescription": "", "modelFamily": undefined, diff --git a/packages/core/tests/unit-test/agent-describe-element.test.ts b/packages/core/tests/unit-test/agent-describe-element.test.ts index 967a597955..ddebf64bd5 100644 --- a/packages/core/tests/unit-test/agent-describe-element.test.ts +++ b/packages/core/tests/unit-test/agent-describe-element.test.ts @@ -424,7 +424,7 @@ describe('element describer utils', () => { const describeContext = describe.mock.calls[0][2]?.context; expect(describeContext?.screenshot.base64).toMatch( - /^data:image\/png;base64,/, + /^data:image\/webp;base64,/, ); expect(describeContext?.shotSize).toEqual(fixtureScreenshotSize); diff --git a/packages/core/tests/unit-test/agent-ui-observer.test.ts b/packages/core/tests/unit-test/agent-ui-observer.test.ts index 5fc8c5c039..2c2a95b048 100644 --- a/packages/core/tests/unit-test/agent-ui-observer.test.ts +++ b/packages/core/tests/unit-test/agent-ui-observer.test.ts @@ -92,10 +92,10 @@ describe('Agent.startObserving', () => { expect(sequence.length).toBeGreaterThanOrEqual(2); expect(sequence[0].hasBase64()).toBe(false); expect(sequence[0].toSerializable()).not.toHaveProperty('path'); - expect(sequence[0].format).toBe('jpeg'); - expect(Buffer.from(sequence[0].rawBase64, 'base64').subarray(0, 3)).toEqual( - Buffer.from([0xff, 0xd8, 0xff]), - ); + expect(sequence[0].format).toBe('webp'); + const frameBuffer = Buffer.from(sequence[0].rawBase64, 'base64'); + expect(frameBuffer.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(frameBuffer.subarray(8, 12).toString('ascii')).toBe('WEBP'); }); it('rejects a second active observer but permits another after stop', async () => { diff --git a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts index 1c9de642c6..59ab46c40a 100644 --- a/packages/core/tests/unit-test/common-context-parser-orientation.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-orientation.test.ts @@ -6,9 +6,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('@midscene/shared/img', async (importOriginal) => ({ ...(await importOriginal()), imageInfoOfBase64: vi.fn(), - resizeBase64ImageToJpeg: vi + resizeBase64ImageToWebp: vi .fn() - .mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'), + .mockResolvedValue('data:image/webp;base64,UklGRgAAAABXRUJQ'), })); import { imageInfoOfBase64 } from '@midscene/shared/img'; diff --git a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts index 846258b2e3..56255f042d 100644 --- a/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts +++ b/packages/core/tests/unit-test/common-context-parser-shrink-factor.test.ts @@ -5,20 +5,20 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; vi.mock('@midscene/shared/img', async (importOriginal) => ({ ...(await importOriginal()), imageInfoOfBase64: vi.fn(), - resizeBase64ImageToJpeg: vi + resizeBase64ImageToWebp: vi .fn() - .mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'), + .mockResolvedValue('data:image/webp;base64,UklGRgAAAABXRUJQ'), })); import { imageInfoOfBase64, - resizeBase64ImageToJpeg, + resizeBase64ImageToWebp, } from '@midscene/shared/img'; const mockScreenshotBase64 = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA'; const mockedImageInfo = vi.mocked(imageInfoOfBase64); -const mockedResizeToJpeg = vi.mocked(resizeBase64ImageToJpeg); +const mockedResizeToWebp = vi.mocked(resizeBase64ImageToWebp); function createMockInterface( logicalWidth: number, @@ -39,19 +39,24 @@ describe('commonContextParser screenshotShrinkFactor', () => { vi.clearAllMocks(); }); - it('converts PNG screenshots to JPEG quality 90 when not shrinking', async () => { + it('converts PNG screenshots to WebP quality 90 when not shrinking', async () => { const mockInterface = createMockInterface(800, 400); mockedImageInfo.mockResolvedValue({ width: 2400, height: 1200 }); - mockedResizeToJpeg.mockResolvedValue('data:image/jpeg;base64,/9j/4AAQ'); + mockedResizeToWebp.mockResolvedValue( + 'data:image/webp;base64,UklGRgAAAABXRUJQ', + ); const result = await commonContextParser(mockInterface, {}); - expect(mockedResizeToJpeg).toHaveBeenCalledWith(mockScreenshotBase64, { + expect(mockedResizeToWebp).toHaveBeenCalledWith(mockScreenshotBase64, { sourceSize: { width: 2400, height: 1200 }, targetSize: { width: 2400, height: 1200 }, - jpegQuality: 90, + webpQuality: 90, + webpEffort: 1, }); - expect(result.screenshot.base64).toBe('data:image/jpeg;base64,/9j/4AAQ'); + expect(result.screenshot.base64).toBe( + 'data:image/webp;base64,UklGRgAAAABXRUJQ', + ); }); it('does not shrink when screenshotShrinkFactor is not provided', async () => { @@ -60,10 +65,11 @@ describe('commonContextParser screenshotShrinkFactor', () => { const result = await commonContextParser(mockInterface, {}); - expect(mockedResizeToJpeg).toHaveBeenCalledWith(mockScreenshotBase64, { + expect(mockedResizeToWebp).toHaveBeenCalledWith(mockScreenshotBase64, { sourceSize: { width: 2400, height: 1200 }, targetSize: { width: 2400, height: 1200 }, - jpegQuality: 90, + webpQuality: 90, + webpEffort: 1, }); expect(result.shotSize).toEqual({ width: 2400, height: 1200 }); }); @@ -76,10 +82,11 @@ describe('commonContextParser screenshotShrinkFactor', () => { screenshotShrinkFactor: 2, }); - expect(mockedResizeToJpeg).toHaveBeenCalledWith(mockScreenshotBase64, { + expect(mockedResizeToWebp).toHaveBeenCalledWith(mockScreenshotBase64, { sourceSize: { width: 2400, height: 1200 }, targetSize: { width: 1200, height: 600 }, - jpegQuality: 90, + webpQuality: 90, + webpEffort: 1, }); expect(result.shotSize).toEqual({ width: 1200, height: 600 }); }); @@ -93,10 +100,11 @@ describe('commonContextParser screenshotShrinkFactor', () => { screenshotShrinkFactor: 2, }); - expect(mockedResizeToJpeg).toHaveBeenCalledWith(mockScreenshotBase64, { + expect(mockedResizeToWebp).toHaveBeenCalledWith(mockScreenshotBase64, { sourceSize: { width: 1216, height: 2688 }, targetSize: { width: 608, height: 1344 }, - jpegQuality: 90, + webpQuality: 90, + webpEffort: 1, }); expect(result.shotSize).toEqual({ width: 608, height: 1344 }); // dpr=1, shrunkShotToLogicalRatio = 1/2 = 0.5 @@ -111,10 +119,11 @@ describe('commonContextParser screenshotShrinkFactor', () => { const result = await commonContextParser(mockInterface, {}); - expect(mockedResizeToJpeg).toHaveBeenCalledWith(mockScreenshotBase64, { + expect(mockedResizeToWebp).toHaveBeenCalledWith(mockScreenshotBase64, { sourceSize: { width: 1216, height: 2688 }, targetSize: { width: 1216, height: 2688 }, - jpegQuality: 90, + webpQuality: 90, + webpEffort: 1, }); expect(result.shotSize).toEqual({ width: 1216, height: 2688 }); expect(result.shrunkShotToLogicalRatio).toBeCloseTo(1, 5); diff --git a/packages/core/tests/unit-test/gpt-image-detail.test.ts b/packages/core/tests/unit-test/gpt-image-detail.test.ts index 8820872d88..4d123893c6 100644 --- a/packages/core/tests/unit-test/gpt-image-detail.test.ts +++ b/packages/core/tests/unit-test/gpt-image-detail.test.ts @@ -43,6 +43,8 @@ const imageMessage = [ ], }, ]; +const webpDataUrl = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; describe('GPT image detail handling', () => { beforeEach(() => { @@ -120,6 +122,32 @@ describe('GPT image detail handling', () => { ); }); + it('applies the configured JPEG image fallback before setting image detail', async () => { + await callAI( + [ + { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { url: webpDataUrl, detail: 'high' }, + }, + ], + }, + ], + getModelRuntime({ + ...baseModelConfig, + imageInputFormat: 'jpeg', + }), + ); + + const sentMessages = mockCreate.mock.calls[0][0].messages; + expect(sentMessages[0].content[0].image_url).toEqual({ + url: expect.stringMatching(/^data:image\/jpeg;base64,/), + detail: 'original', + }); + }); + it('overrides image detail to original when required by the caller and adapter', async () => { await callAI( imageMessage, diff --git a/packages/core/tests/unit-test/image-preprocess.test.ts b/packages/core/tests/unit-test/image-preprocess.test.ts index 88c9211863..d1b8049444 100644 --- a/packages/core/tests/unit-test/image-preprocess.test.ts +++ b/packages/core/tests/unit-test/image-preprocess.test.ts @@ -60,6 +60,7 @@ describe('prepareModelImage', () => { expect(paddingToMatchBlockByBase64).toHaveBeenCalledWith( 'original-image', 28, + 'webp', ); expect(image).toEqual({ imageBase64: 'padded-image', @@ -73,6 +74,24 @@ describe('prepareModelImage', () => { }, }); }); + + it('does not decode or encode an image that already matches the block size', async () => { + const image = await prepareModelImage({ + imageBase64: 'already-aligned-webp', + width: 112, + height: 84, + policy: { + padBlockSize: 28, + }, + }); + + expect(paddingToMatchBlockByBase64).not.toHaveBeenCalled(); + expect(image).toEqual({ + imageBase64: 'already-aligned-webp', + preparedSize: { width: 112, height: 84 }, + contentSize: { width: 112, height: 84 }, + }); + }); }); describe('buildSearchAreaConfig', () => { @@ -117,6 +136,7 @@ describe('buildSearchAreaConfig', () => { expect(cropByRect).toHaveBeenCalledWith( 'full-screenshot', expect.any(Object), + 'webp', ); expect(cropCalls).toEqual([ { @@ -126,7 +146,7 @@ describe('buildSearchAreaConfig', () => { height: 400, }, ]); - expect(scaleImage).toHaveBeenCalledWith('cropped-image', 2); + expect(scaleImage).toHaveBeenCalledWith('cropped-image', 2, 'webp'); expect(searchArea).toEqual({ sourceRect: { left: 300, diff --git a/packages/core/tests/unit-test/model-image-input.test.ts b/packages/core/tests/unit-test/model-image-input.test.ts new file mode 100644 index 0000000000..e6b79b6043 --- /dev/null +++ b/packages/core/tests/unit-test/model-image-input.test.ts @@ -0,0 +1,91 @@ +import { + convertBase64ImageToJpeg, + convertBase64ImageToWebp, +} from '@midscene/shared/img'; +import type { ChatCompletionMessageParam } from 'openai/resources/index'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { prepareModelMessagesImageInput } from '../../src/ai-model/service-caller/model-image-input'; + +vi.mock('@midscene/shared/img', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + convertBase64ImageToJpeg: vi.fn(actual.convertBase64ImageToJpeg), + convertBase64ImageToWebp: vi.fn(actual.convertBase64ImageToWebp), + }; +}); + +const pngDataUrl = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAIAAABxZ0isAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEUlEQVR4nGMQqbiDFTEMpAQAorNDgTX/VEoAAAAASUVORK5CYII='; +const webpDataUrl = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; + +function imageMessage(url: string): ChatCompletionMessageParam[] { + return [ + { + role: 'user', + content: [ + { type: 'text', text: 'inspect this screenshot' }, + { type: 'image_url', image_url: { url, detail: 'high' } }, + ], + }, + ]; +} + +function preparedImageUrl(messages: ChatCompletionMessageParam[]): string { + const content = messages[0].content; + if (!Array.isArray(content)) { + throw new Error('expected multimodal content'); + } + const imagePart = content.find((part) => part.type === 'image_url'); + if (!imagePart || imagePart.type !== 'image_url') { + throw new Error('expected image content'); + } + return imagePart.image_url.url; +} + +describe('prepareModelMessagesImageInput', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('normalizes inline screenshots to WebP by default', async () => { + const messages = imageMessage(pngDataUrl); + const prepared = await prepareModelMessagesImageInput(messages); + + expect(preparedImageUrl(prepared)).toMatch(/^data:image\/webp;base64,/); + expect(prepared).not.toBe(messages); + expect(preparedImageUrl(messages)).toBe(pngDataUrl); + }); + + it('provides a JPEG fallback without changing the original messages', async () => { + const messages = imageMessage(webpDataUrl); + const prepared = await prepareModelMessagesImageInput(messages, 'jpeg'); + + expect(preparedImageUrl(prepared)).toMatch(/^data:image\/jpeg;base64,/); + expect(preparedImageUrl(messages)).toBe(webpDataUrl); + }); + + it('leaves remote image URLs and message identity unchanged', async () => { + const messages = imageMessage('https://example.com/screenshot.png'); + + await expect( + prepareModelMessagesImageInput(messages, 'jpeg'), + ).resolves.toBe(messages); + }); + + it('converts duplicate inline images only once per model request', async () => { + const messages = [ + ...imageMessage(webpDataUrl), + ...imageMessage(webpDataUrl), + ]; + + const prepared = await prepareModelMessagesImageInput(messages, 'jpeg'); + + expect(convertBase64ImageToJpeg).toHaveBeenCalledTimes(1); + expect(convertBase64ImageToWebp).not.toHaveBeenCalled(); + expect(preparedImageUrl([prepared[0]])).toBe( + preparedImageUrl([prepared[1]]), + ); + }); +}); diff --git a/packages/core/tests/unit-test/screenshot-persistence-preparation.test.ts b/packages/core/tests/unit-test/screenshot-persistence-preparation.test.ts index b4196e97ba..e5ad223753 100644 --- a/packages/core/tests/unit-test/screenshot-persistence-preparation.test.ts +++ b/packages/core/tests/unit-test/screenshot-persistence-preparation.test.ts @@ -2,9 +2,9 @@ import { prepareScreenshotForPersistence } from '@/agent/screenshot-preparation' import { beforeEach, describe, expect, it, vi } from 'vitest'; const imageMocks = vi.hoisted(() => ({ - convertBase64ImageToJpeg: vi.fn(), + convertBase64ImageToWebp: vi.fn(), imageInfoOfBase64: vi.fn(), - resizeBase64ImageToJpeg: vi.fn(), + resizeBase64ImageToWebp: vi.fn(), })); vi.mock('@midscene/shared/img', () => imageMocks); @@ -12,45 +12,46 @@ vi.mock('@midscene/shared/img', () => imageMocks); describe('prepareScreenshotForPersistence', () => { beforeEach(() => { vi.clearAllMocks(); - imageMocks.convertBase64ImageToJpeg.mockResolvedValue( - 'data:image/jpeg;base64,prepared', + imageMocks.convertBase64ImageToWebp.mockResolvedValue( + 'data:image/webp;base64,prepared', ); }); it('normalizes an unscaled frame without reading image dimensions', async () => { await expect( - prepareScreenshotForPersistence('data:image/jpeg;base64,frame'), - ).resolves.toBe('data:image/jpeg;base64,prepared'); + prepareScreenshotForPersistence('data:image/webp;base64,frame'), + ).resolves.toBe('data:image/webp;base64,prepared'); - expect(imageMocks.convertBase64ImageToJpeg).toHaveBeenCalledWith( - 'data:image/jpeg;base64,frame', - 90, + expect(imageMocks.convertBase64ImageToWebp).toHaveBeenCalledWith( + 'data:image/webp;base64,frame', + { webpQuality: 90, webpEffort: 1 }, ); expect(imageMocks.imageInfoOfBase64).not.toHaveBeenCalled(); - expect(imageMocks.resizeBase64ImageToJpeg).not.toHaveBeenCalled(); + expect(imageMocks.resizeBase64ImageToWebp).not.toHaveBeenCalled(); }); it('uses the full preparation pipeline when the frame must be shrunk', async () => { imageMocks.imageInfoOfBase64.mockResolvedValue({ width: 8, height: 6 }); - imageMocks.resizeBase64ImageToJpeg.mockResolvedValue( - 'data:image/jpeg;base64,resized', + imageMocks.resizeBase64ImageToWebp.mockResolvedValue( + 'data:image/webp;base64,resized', ); await expect( prepareScreenshotForPersistence('data:image/png;base64,frame', { shrinkFactor: 2, }), - ).resolves.toBe('data:image/jpeg;base64,resized'); + ).resolves.toBe('data:image/webp;base64,resized'); expect(imageMocks.imageInfoOfBase64).toHaveBeenCalledOnce(); - expect(imageMocks.resizeBase64ImageToJpeg).toHaveBeenCalledWith( + expect(imageMocks.resizeBase64ImageToWebp).toHaveBeenCalledWith( 'data:image/png;base64,frame', { sourceSize: { width: 8, height: 6 }, targetSize: { width: 4, height: 3 }, - jpegQuality: 90, + webpQuality: 90, + webpEffort: 1, }, ); - expect(imageMocks.convertBase64ImageToJpeg).not.toHaveBeenCalled(); + expect(imageMocks.convertBase64ImageToWebp).not.toHaveBeenCalled(); }); }); diff --git a/packages/core/tests/unit-test/screenshot-preparation.test.ts b/packages/core/tests/unit-test/screenshot-preparation.test.ts index ae92d12577..58cf7e8ae5 100644 --- a/packages/core/tests/unit-test/screenshot-preparation.test.ts +++ b/packages/core/tests/unit-test/screenshot-preparation.test.ts @@ -6,14 +6,16 @@ const pngDataUrl = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAIAAABxZ0isAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEUlEQVR4nGMQqbiDFTEMpAQAorNDgTX/VEoAAAAASUVORK5CYII='; const jpegDataUrl = 'data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAADAAQDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAACP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AJ0AWYyP/9k='; +const webpDataUrl = + 'data:image/webp;base64,UklGRjQAAABXRUJQVlA4ICgAAACQAQCdASoCAAMAAMASJQBOl0AAjNAA/v4icv1difCfoP7mxzi2QwAA'; describe('prepareRawScreenshot', () => { - it('normalizes a PNG to JPEG without changing its dimensions', async () => { + it('normalizes a PNG to WebP without changing its dimensions', async () => { const prepared = await prepareRawScreenshot(pngDataUrl); expect(prepared.originalSize).toEqual({ width: 8, height: 6 }); expect(prepared.shotSize).toEqual({ width: 8, height: 6 }); - expect(prepared.base64).toMatch(/^data:image\/jpeg;base64,/); + expect(prepared.base64).toMatch(/^data:image\/webp;base64,/); await expect(imageInfoOfBase64(prepared.base64)).resolves.toEqual( prepared.shotSize, ); @@ -26,16 +28,16 @@ describe('prepareRawScreenshot', () => { expect(prepared.originalSize).toEqual({ width: 8, height: 6 }); expect(prepared.shotSize).toEqual({ width: 4, height: 3 }); - expect(prepared.base64).toMatch(/^data:image\/jpeg;base64,/); + expect(prepared.base64).toMatch(/^data:image\/webp;base64,/); await expect(imageInfoOfBase64(prepared.base64)).resolves.toEqual( prepared.shotSize, ); }); - it('keeps JPEG dimensions while preserving the JPEG output contract', async () => { + it('converts JPEG to the final WebP output contract', async () => { const prepared = await prepareRawScreenshot(jpegDataUrl); - expect(prepared.base64).toBe(jpegDataUrl); + expect(prepared.base64).toMatch(/^data:image\/webp;base64,/); expect(prepared.originalSize).toEqual({ width: 4, height: 3 }); expect(prepared.shotSize).toEqual(prepared.originalSize); await expect(imageInfoOfBase64(prepared.base64)).resolves.toEqual( @@ -43,6 +45,14 @@ describe('prepareRawScreenshot', () => { ); }); + it('reuses an unchanged WebP byte-for-byte', async () => { + const prepared = await prepareRawScreenshot(webpDataUrl); + + expect(prepared.base64).toBe(webpDataUrl); + expect(prepared.originalSize).toEqual({ width: 2, height: 3 }); + expect(prepared.shotSize).toEqual(prepared.originalSize); + }); + it.each([0, 0.5, Number.NaN, Number.POSITIVE_INFINITY])( 'rejects invalid shrink factor %s', async (shrinkFactor) => { diff --git a/packages/core/tests/unit-test/service-describe.test.ts b/packages/core/tests/unit-test/service-describe.test.ts index 176a1faafe..2b8cb1481c 100644 --- a/packages/core/tests/unit-test/service-describe.test.ts +++ b/packages/core/tests/unit-test/service-describe.test.ts @@ -13,12 +13,12 @@ const { mockCompositeElementInfoImg, mockCompositePointMarkerImg, mockCropByRect, - mockResizeBase64ImageToJpeg, + mockResizeBase64ImageToWebp, } = vi.hoisted(() => ({ mockCompositeElementInfoImg: vi.fn(), mockCompositePointMarkerImg: vi.fn(), mockCropByRect: vi.fn(), - mockResizeBase64ImageToJpeg: vi.fn(), + mockResizeBase64ImageToWebp: vi.fn(), })); vi.mock('@/ai-model/service-caller', async (importOriginal) => { @@ -37,7 +37,7 @@ vi.mock('@midscene/shared/img', async (importOriginal) => { compositeElementInfoImg: mockCompositeElementInfoImg, compositePointMarkerImg: mockCompositePointMarkerImg, cropByRect: mockCropByRect, - resizeBase64ImageToJpeg: mockResizeBase64ImageToJpeg, + resizeBase64ImageToWebp: mockResizeBase64ImageToWebp, }; }); @@ -67,9 +67,9 @@ describe('service.describe', () => { height: rect.height, imageBase64: 'data:image/png;base64,cropped', })); - mockResizeBase64ImageToJpeg.mockReset(); - mockResizeBase64ImageToJpeg.mockResolvedValue( - 'data:image/jpeg;base64,resized', + mockResizeBase64ImageToWebp.mockReset(); + mockResizeBase64ImageToWebp.mockResolvedValue( + 'data:image/webp;base64,resized', ); }); @@ -265,12 +265,16 @@ describe('service.describe', () => { { deepDescribe: true }, ); - expect(mockCropByRect).toHaveBeenCalledWith(expect.any(String), { - left: 1313, - top: 325, - width: 431, - height: 371, - }); + expect(mockCropByRect).toHaveBeenCalledWith( + expect.any(String), + { + left: 1313, + top: 325, + width: 431, + height: 371, + }, + 'webp', + ); expect(mockCropByRect.mock.calls.map(([image]) => image)).not.toContain( 'data:image/png;base64,boxed', ); @@ -318,12 +322,16 @@ describe('service.describe', () => { description: 'point target with row context', }); - expect(mockCropByRect).toHaveBeenCalledWith(expect.any(String), { - left: 1300, - top: 300, - width: 400, - height: 400, - }); + expect(mockCropByRect).toHaveBeenCalledWith( + expect.any(String), + { + left: 1300, + top: 300, + width: 400, + height: 400, + }, + 'webp', + ); expect(mockCropByRect.mock.calls.map(([image]) => image)).not.toContain( 'data:image/png;base64,point', ); diff --git a/packages/core/tests/unit-test/ui-observer.test.ts b/packages/core/tests/unit-test/ui-observer.test.ts index b55d833b8b..b12112b984 100644 --- a/packages/core/tests/unit-test/ui-observer.test.ts +++ b/packages/core/tests/unit-test/ui-observer.test.ts @@ -18,6 +18,8 @@ const largeTestPngDataUrl = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAGCAIAAABxZ0isAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAAEUlEQVR4nGMQqbiDFTEMpAQAorNDgTX/VEoAAAAASUVORK5CYII='; const testJpegDataUrl = 'data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAf/xAAdEAACAgMAAwAAAAAAAAAAAAABAgAEAwUGEiEx/8QAFQEBAQAAAAAAAAAAAAAAAAAABgj/xAAcEQABAwUAAAAAAAAAAAAAAAAAAQIDBTI0crH/2gAMAwEAAhEDEQA/AL5zGpo5Ob1LvTrs7VMRLNiUknwHv5ERIdqObNs7qiplqH//2Q='; +const testWebpDataUrl = + 'data:image/webp;base64,UklGRkYAAABXRUJQVlA4IDoAAACQAQCdASoCAAIAAMASJaAAAzoO6YAA/vdwyEKF/jvhD9bXt18+f//efP/+58//7nz/t//3SfqfygAA'; const shrunkTestJpegDataUrl = 'data:image/jpeg;base64,/9j/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAADAAQDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFAEBAAAAAAAAAAAAAAAAAAAACP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AJ0AWYyP/9k='; @@ -33,7 +35,7 @@ function options(extra: Record = {}) { const fakeRepresentative = (): UIContext => ({ - screenshot: ScreenshotItem.create(testJpegDataUrl, 9999), + screenshot: ScreenshotItem.create(testWebpDataUrl, 9999), shotSize: { width: 2, height: 2 }, shrunkShotToLogicalRatio: 1, }) as UIContext; @@ -83,8 +85,10 @@ const makeDeps = (fake: ReturnType | null) => { }; }; -function expectJpegFrame(path: string): void { - expect([...readFileSync(path).subarray(0, 3)]).toEqual([0xff, 0xd8, 0xff]); +function expectWebpFrame(path: string): void { + const buffer = readFileSync(path); + expect(buffer.subarray(0, 4).toString('ascii')).toBe('RIFF'); + expect(buffer.subarray(8, 12).toString('ascii')).toBe('WEBP'); } function fixedRecord(): UIObservationRecord { @@ -147,10 +151,10 @@ describe('UIObserver', () => { expect(fake.stop).toHaveBeenCalledOnce(); expect(record.frames.length).toBeGreaterThanOrEqual(3); expect( - record.frames.every((frame) => frame.mimeType === 'image/jpeg'), + record.frames.every((frame) => frame.mimeType === 'image/webp'), ).toBe(true); - expectJpegFrame(record.frames[0].path); - expectJpegFrame(record.frames.at(-1)!.path); + expectWebpFrame(record.frames[0].path); + expectWebpFrame(record.frames.at(-1)!.path); expect(record.frames[0]).not.toHaveProperty('base64'); }); @@ -198,10 +202,10 @@ describe('UIObserver', () => { 30, ); expect( - record.frames.every((frame) => frame.mimeType === 'image/jpeg'), + record.frames.every((frame) => frame.mimeType === 'image/webp'), ).toBe(true); for (const frame of record.frames) { - expectJpegFrame(frame.path); + expectWebpFrame(frame.path); const base64 = readFileSync(frame.path).toString('base64'); await expect( imageInfoOfBase64(`data:${frame.mimeType};base64,${base64}`), @@ -416,8 +420,9 @@ describe('UIObserver', () => { expect(record.frames).toHaveLength(56); }); - it('persists fallback screenshots immediately as JPEG files', async () => { + it('persists fallback screenshots immediately as WebP files', async () => { const { deps, screenshot } = makeDeps(null); + screenshot.mockResolvedValue(testJpegDataUrl); const observer = new UIObserverImpl(deps, options({ intervalMs: 200 })); await observer.start(); await sleep(250); @@ -426,9 +431,9 @@ describe('UIObserver', () => { expect(screenshot).toHaveBeenCalled(); expect( - record.frames.every((frame) => frame.mimeType === 'image/jpeg'), + record.frames.every((frame) => frame.mimeType === 'image/webp'), ).toBe(true); - expectJpegFrame(record.frames[0].path); + expectWebpFrame(record.frames[0].path); expect((observer as any).frames[0].ref).not.toContain('base64'); }); diff --git a/packages/core/tests/unit-test/workflows/recorder-generation/markdown.test.ts b/packages/core/tests/unit-test/workflows/recorder-generation/markdown.test.ts index 0a6d687f6f..39962fe097 100644 --- a/packages/core/tests/unit-test/workflows/recorder-generation/markdown.test.ts +++ b/packages/core/tests/unit-test/workflows/recorder-generation/markdown.test.ts @@ -67,7 +67,7 @@ describe('markdown-generator', () => { vi.clearAllMocks(); }); - it('compresses oversized screenshots to JPEG before model generation', async () => { + it('compresses oversized screenshots to WebP before model generation', async () => { const largePng = await sharp({ create: { width: 1024, @@ -116,7 +116,7 @@ describe('markdown-generator', () => { if (imagePart?.type !== 'image_url') { throw new Error('Expected a compressed screenshot in the prompt'); } - expect(imagePart.image_url.url).toMatch(/^data:image\/jpeg;base64,/); + expect(imagePart.image_url.url).toMatch(/^data:image\/webp;base64,/); await expect(imageInfoOfBase64(imagePart.image_url.url)).resolves.toEqual({ width: 768, height: 384, diff --git a/packages/shared/src/env/constants.ts b/packages/shared/src/env/constants.ts index c1d3fb1455..fb8417842f 100644 --- a/packages/shared/src/env/constants.ts +++ b/packages/shared/src/env/constants.ts @@ -4,6 +4,7 @@ import { MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON, MIDSCENE_INSIGHT_MODEL_FAMILY, MIDSCENE_INSIGHT_MODEL_HTTP_PROXY, + MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_INSIGHT_MODEL_INIT_CONFIG_JSON, MIDSCENE_INSIGHT_MODEL_NAME, MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET, @@ -20,6 +21,7 @@ import { MIDSCENE_MODEL_EXTRA_BODY_JSON, MIDSCENE_MODEL_FAMILY, MIDSCENE_MODEL_HTTP_PROXY, + MIDSCENE_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_MODEL_INIT_CONFIG_JSON, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_REASONING_BUDGET, @@ -39,6 +41,7 @@ import { MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON, MIDSCENE_PLANNING_MODEL_FAMILY, MIDSCENE_PLANNING_MODEL_HTTP_PROXY, + MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_PLANNING_MODEL_INIT_CONFIG_JSON, MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_MODEL_REASONING_BUDGET, @@ -89,6 +92,7 @@ interface IModelConfigKeys { reasoningEnabled: string; reasoningBudget: string; responseFormat: string; + imageInputFormat: string; } export const INSIGHT_MODEL_CONFIG_KEYS: IModelConfigKeys = { @@ -126,6 +130,7 @@ export const INSIGHT_MODEL_CONFIG_KEYS: IModelConfigKeys = { reasoningEnabled: MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED, reasoningBudget: MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET, responseFormat: MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT, + imageInputFormat: MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT, } as const; export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = { @@ -163,6 +168,7 @@ export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = { reasoningEnabled: MIDSCENE_PLANNING_MODEL_REASONING_ENABLED, reasoningBudget: MIDSCENE_PLANNING_MODEL_REASONING_BUDGET, responseFormat: MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT, + imageInputFormat: MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT, } as const; // modelConfig return default @@ -201,6 +207,7 @@ export const DEFAULT_MODEL_CONFIG_KEYS: IModelConfigKeys = { reasoningEnabled: MIDSCENE_MODEL_REASONING_ENABLED, reasoningBudget: MIDSCENE_MODEL_REASONING_BUDGET, responseFormat: MIDSCENE_MODEL_RESPONSE_FORMAT, + imageInputFormat: MIDSCENE_MODEL_IMAGE_INPUT_FORMAT, } as const; // read from process.env @@ -239,4 +246,5 @@ export const DEFAULT_MODEL_CONFIG_KEYS_LEGACY: IModelConfigKeys = { reasoningEnabled: MIDSCENE_MODEL_REASONING_ENABLED, reasoningBudget: MIDSCENE_MODEL_REASONING_BUDGET, responseFormat: MIDSCENE_MODEL_RESPONSE_FORMAT, + imageInputFormat: MIDSCENE_MODEL_IMAGE_INPUT_FORMAT, } as const; diff --git a/packages/shared/src/env/parse-model-config.ts b/packages/shared/src/env/parse-model-config.ts index 99936284d0..6825807972 100644 --- a/packages/shared/src/env/parse-model-config.ts +++ b/packages/shared/src/env/parse-model-config.ts @@ -296,6 +296,14 @@ export const parseOpenaiSdkConfig = ({ `${keys.responseFormat} must be one of: none, auto. Got: ${val}`, ); })(), + imageInputFormat: (() => { + const val = provider[keys.imageInputFormat]?.trim()?.toLowerCase(); + if (!val || val === 'webp') return 'webp'; + if (val === 'jpeg') return 'jpeg'; + throw new Error( + `${keys.imageInputFormat} must be one of: webp, jpeg. Got: ${val}`, + ); + })(), }; }; diff --git a/packages/shared/src/env/types.ts b/packages/shared/src/env/types.ts index 4037b52b9d..d2d3f928fa 100644 --- a/packages/shared/src/env/types.ts +++ b/packages/shared/src/env/types.ts @@ -36,9 +36,12 @@ export const MIDSCENE_MODEL_REASONING_ENABLED = export const MIDSCENE_MODEL_REASONING_BUDGET = 'MIDSCENE_MODEL_REASONING_BUDGET'; export const MIDSCENE_MODEL_RESPONSE_FORMAT = 'MIDSCENE_MODEL_RESPONSE_FORMAT'; +export const MIDSCENE_MODEL_IMAGE_INPUT_FORMAT = + 'MIDSCENE_MODEL_IMAGE_INPUT_FORMAT'; export type TModelReasoningEnabled = boolean | 'default'; export type TModelResponseFormat = 'none' | 'auto'; +export type TModelImageInputFormat = 'webp' | 'jpeg'; /** * @deprecated Use MIDSCENE_MODEL_API_KEY instead. This is kept for backward compatibility. @@ -124,6 +127,8 @@ export const MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET = 'MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET'; export const MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT = 'MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT'; +export const MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT = + 'MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT'; // PLANNING export const MIDSCENE_PLANNING_MODEL_NAME = 'MIDSCENE_PLANNING_MODEL_NAME'; @@ -156,6 +161,8 @@ export const MIDSCENE_PLANNING_MODEL_REASONING_BUDGET = 'MIDSCENE_PLANNING_MODEL_REASONING_BUDGET'; export const MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT = 'MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT'; +export const MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT = + 'MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT'; export const MIDSCENE_MODEL_FAMILY = 'MIDSCENE_MODEL_FAMILY'; /** @@ -236,6 +243,7 @@ export const MODEL_ENV_KEYS = [ MIDSCENE_MODEL_REASONING_ENABLED, MIDSCENE_MODEL_REASONING_BUDGET, MIDSCENE_MODEL_RESPONSE_FORMAT, + MIDSCENE_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_QWEN3_VL, @@ -265,6 +273,7 @@ export const MODEL_ENV_KEYS = [ MIDSCENE_INSIGHT_MODEL_REASONING_ENABLED, MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET, MIDSCENE_INSIGHT_MODEL_RESPONSE_FORMAT, + MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT, // PLANNING MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_MODEL_SOCKS_PROXY, @@ -282,6 +291,7 @@ export const MODEL_ENV_KEYS = [ MIDSCENE_PLANNING_MODEL_REASONING_ENABLED, MIDSCENE_PLANNING_MODEL_REASONING_BUDGET, MIDSCENE_PLANNING_MODEL_RESPONSE_FORMAT, + MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_MODEL_FAMILY, ] as const; @@ -358,6 +368,7 @@ export interface IModelConfigForInsight { [MIDSCENE_INSIGHT_MODEL_TEMPERATURE]?: string; // model family [MIDSCENE_INSIGHT_MODEL_FAMILY]?: TModelFamily; + [MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT]?: TModelImageInputFormat; } export interface IModelConfigForPlanning { @@ -377,6 +388,7 @@ export interface IModelConfigForPlanning { [MIDSCENE_PLANNING_MODEL_TEMPERATURE]?: string; // model family [MIDSCENE_PLANNING_MODEL_FAMILY]?: TModelFamily; + [MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT]?: TModelImageInputFormat; } /** @@ -410,6 +422,8 @@ export interface IModelConfigForDefault { [MIDSCENE_MODEL_REASONING_BUDGET]?: string; // Response format strategy (none/auto) [MIDSCENE_MODEL_RESPONSE_FORMAT]?: TModelResponseFormat; + // Image format sent to the model (webp/jpeg) + [MIDSCENE_MODEL_IMAGE_INPUT_FORMAT]?: TModelImageInputFormat; } export interface IModelConfigForDefaultLegacy { @@ -540,6 +554,11 @@ export interface IModelConfig { * provider-supported structured response format for eligible intents. */ responseFormat?: TModelResponseFormat; + /** + * Encoding used for Base64 image inputs at the model request boundary. + * Defaults to WebP. Set to JPEG for providers that reject WebP inputs. + */ + imageInputFormat?: TModelImageInputFormat; /** * Model family - unified model configuration * Maps directly to model families like 'qwen2.5-vl', 'qwen3-vl', 'doubao-vision', 'doubao-seed', etc. diff --git a/packages/shared/tests/unit-test/env/decide-sdk.test.ts b/packages/shared/tests/unit-test/env/decide-sdk.test.ts index 652e860cd1..4fa3df1d24 100644 --- a/packages/shared/tests/unit-test/env/decide-sdk.test.ts +++ b/packages/shared/tests/unit-test/env/decide-sdk.test.ts @@ -3,6 +3,7 @@ import { MIDSCENE_MODEL_API_KEY, MIDSCENE_MODEL_BASE_URL, MIDSCENE_MODEL_EXTRA_BODY_JSON, + MIDSCENE_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_MODEL_INIT_CONFIG_JSON, MIDSCENE_MODEL_RESPONSE_FORMAT, } from '../../../src/env'; @@ -85,6 +86,29 @@ describe('decideOpenaiSdkConfig', () => { ).toThrow('MIDSCENE_MODEL_RESPONSE_FORMAT must be one of: none, auto'); }); + it('defaults model image input to WebP and accepts the JPEG fallback', () => { + const webpResult = parseOpenaiSdkConfig({ + keys: DEFAULT_MODEL_CONFIG_KEYS, + provider: {}, + }); + const jpegResult = parseOpenaiSdkConfig({ + keys: DEFAULT_MODEL_CONFIG_KEYS, + provider: { [MIDSCENE_MODEL_IMAGE_INPUT_FORMAT]: 'jpeg' }, + }); + + expect(webpResult.imageInputFormat).toBe('webp'); + expect(jpegResult.imageInputFormat).toBe('jpeg'); + }); + + it('rejects unsupported model image input formats', () => { + expect(() => + parseOpenaiSdkConfig({ + keys: DEFAULT_MODEL_CONFIG_KEYS, + provider: { [MIDSCENE_MODEL_IMAGE_INPUT_FORMAT]: 'png' }, + }), + ).toThrow('MIDSCENE_MODEL_IMAGE_INPUT_FORMAT must be one of: webp, jpeg'); + }); + it('throws on invalid extraBody JSON', () => { expect(() => parseOpenaiSdkConfig({ diff --git a/packages/shared/tests/unit-test/env/modle-config-manager.test.ts b/packages/shared/tests/unit-test/env/modle-config-manager.test.ts index 64599db6b1..1bf0a086eb 100644 --- a/packages/shared/tests/unit-test/env/modle-config-manager.test.ts +++ b/packages/shared/tests/unit-test/env/modle-config-manager.test.ts @@ -5,6 +5,7 @@ import { MIDSCENE_INSIGHT_MODEL_API_KEY, MIDSCENE_INSIGHT_MODEL_BASE_URL, MIDSCENE_INSIGHT_MODEL_EXTRA_BODY_JSON, + MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_INSIGHT_MODEL_NAME, MIDSCENE_INSIGHT_MODEL_REASONING_BUDGET, MIDSCENE_INSIGHT_MODEL_REASONING_EFFORT, @@ -14,6 +15,7 @@ import { MIDSCENE_MODEL_BASE_URL, MIDSCENE_MODEL_EXTRA_BODY_JSON, MIDSCENE_MODEL_FAMILY, + MIDSCENE_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_MODEL_INIT_CONFIG_JSON, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_REASONING_BUDGET, @@ -23,6 +25,7 @@ import { MIDSCENE_PLANNING_MODEL_API_KEY, MIDSCENE_PLANNING_MODEL_BASE_URL, MIDSCENE_PLANNING_MODEL_EXTRA_BODY_JSON, + MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT, MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_MODEL_REASONING_BUDGET, MIDSCENE_PLANNING_MODEL_REASONING_EFFORT, @@ -90,6 +93,19 @@ describe('ModelConfigManager', () => { expect(planningConfig.modelName).toBe('gpt-4'); }); + it('resolves image input formats independently for each model slot', () => { + const manager = new ModelConfigManager({ + ...baseMap, + [MIDSCENE_MODEL_IMAGE_INPUT_FORMAT]: 'jpeg', + [MIDSCENE_INSIGHT_MODEL_IMAGE_INPUT_FORMAT]: 'webp', + [MIDSCENE_PLANNING_MODEL_IMAGE_INPUT_FORMAT]: 'jpeg', + }); + + expect(manager.getModelConfig('default').imageInputFormat).toBe('jpeg'); + expect(manager.getModelConfig('insight').imageInputFormat).toBe('webp'); + expect(manager.getModelConfig('planning').imageInputFormat).toBe('jpeg'); + }); + it('prefer MIDSCENE_MODEL', () => { rs.stubEnv(MIDSCENE_MODEL_NAME, 'env-model'); rs.stubEnv(MIDSCENE_MODEL_API_KEY, 'env-key');