refactor(core): model adaptor

Author: EAGzzyCSL

apps/chrome-extension/src/extension/recorder/generators/playwrightGenerator.ts

@@ -6,7 +6,10 @@ import type {
   StreamingCodeGenerationOptions,
 } from '@midscene/core';
 // Note: Streaming support will be added once the core package exports are updated
-import { generatePlaywrightTestStream as coreGeneratePlaywrightTestStream } from '@midscene/core/ai-model';
+import {
+  generatePlaywrightTestStream as coreGeneratePlaywrightTestStream,
+  getModelRuntime,
+} from '@midscene/core/ai-model';
 import type { IModelConfig } from '@midscene/shared/env';
 import { recordLogger } from '../logger';
 import { handleTestGenerationError } from './shared/testGenerationUtils';
@@ -98,7 +101,7 @@ export const generatePlaywrightTestStream = async (
     const result = await coreGeneratePlaywrightTestStream(
       events,
       enhancedOptions,
-      modelConfig,
+      getModelRuntime(modelConfig),
     );
 
     recordLogger.success('Streaming Playwright test generated successfully', {

apps/chrome-extension/src/extension/recorder/generators/yamlGenerator.ts

@@ -5,6 +5,7 @@ import type {
 import {
   generateYamlTest as generateYamlTestCore,
   generateYamlTestStream as generateYamlTestStreamCore,
+  getModelRuntime,
 } from '@midscene/core/ai-model';
 import type { ChromeRecordedEvent } from '@midscene/recorder';
 import type { IModelConfig } from '@midscene/shared/env';
@@ -39,7 +40,7 @@ export const generateYamlTest = async (
     const yamlContent = await generateYamlTestCore(
       events,
       enhancedOptions,
-      modelConfig,
+      getModelRuntime(modelConfig),
     );
 
     recordLogger.success('AI-powered YAML test generated successfully', {
@@ -125,7 +126,7 @@ export const generateYamlTestStream = async (
     const result = await generateYamlTestStreamCore(
       events,
       enhancedOptions,
-      modelConfig,
+      getModelRuntime(modelConfig),
     );
 
     recordLogger.success(

apps/chrome-extension/src/extension/recorder/utils.ts

@@ -2,6 +2,7 @@ import {
   type AIArgs,
   callAIWithObjectResponse,
   callAIWithStringResponse,
+  getModelRuntime,
 } from '@midscene/core/ai-model';
 import type { ChromeRecordedEvent } from '@midscene/recorder';
 import type { IModelConfig } from '@midscene/shared/env';
@@ -623,7 +624,10 @@ const generateAIMindmap = async (
       },
     ];
 
-    const response = await callAIWithStringResponse(prompt, modelConfig);
+    const response = await callAIWithStringResponse(
+      prompt,
+      getModelRuntime(modelConfig),
+    );
 
     if (response?.content && typeof response.content === 'string') {
       return response.content as string;

apps/chrome-extension/src/utils/eventOptimizer.ts

@@ -1,6 +1,7 @@
 import Service from '@midscene/core';
 import type { Rect, UIContext } from '@midscene/core';
 import { ScreenshotItem } from '@midscene/core';
+import { getModelRuntime } from '@midscene/core/ai-model';
 import type { RecordedEvent } from '@midscene/recorder';
 import { globalModelConfigManager } from '@midscene/shared/env';
 import { compositeElementInfoImg } from '@midscene/shared/img';
@@ -121,7 +122,7 @@ export const generateAIDescription = async (
     for (let attempt = 1; attempt <= maxRetries; attempt++) {
       try {
         const modelConfig = globalModelConfigManager.getModelConfig('default');
-        return await service.describe(rect, modelConfig);
+        return await service.describe(rect, getModelRuntime(modelConfig));
       } catch (err) {
         lastError = err;
         if (attempt < maxRetries) {

apps/studio/src/main/playground/connectivity-test.ts

@@ -1,6 +1,7 @@
 import {
   type ChatCompletionMessageParam,
   callAIWithStringResponse,
+  getModelRuntime,
 } from '@midscene/core/ai-model';
 import type {
   ConnectivityTestRequest,
@@ -40,15 +41,16 @@ export async function runConnectivityTest(
   );
 
   try {
+    const modelRuntime = getModelRuntime({
+      ...resolvedWithConfig.modelConfig,
+      timeout: Math.min(
+        resolvedWithConfig.modelConfig.timeout ?? CONNECTIVITY_TIMEOUT_MS,
+        CONNECTIVITY_TIMEOUT_MS,
+      ),
+    });
     const { content } = await callAIWithStringResponse(
       CONNECTIVITY_PROMPT,
-      {
-        ...resolvedWithConfig.modelConfig,
-        timeout: Math.min(
-          resolvedWithConfig.modelConfig.timeout ?? CONNECTIVITY_TIMEOUT_MS,
-          CONNECTIVITY_TIMEOUT_MS,
-        ),
-      },
+      modelRuntime,
       {
         abortSignal: controller.signal,
       },

apps/studio/tests/connectivity-test.test.ts

@@ -2,9 +2,16 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
 
 vi.mock('@midscene/core/ai-model', () => ({
   callAIWithStringResponse: vi.fn(),
+  getModelRuntime: vi.fn((config) => ({
+    config,
+    adapter: {},
+  })),
 }));
 
-import { callAIWithStringResponse } from '@midscene/core/ai-model';
+import {
+  callAIWithStringResponse,
+  getModelRuntime,
+} from '@midscene/core/ai-model';
 import { runConnectivityTest } from '../src/main/playground/connectivity-test';
 
 describe('runConnectivityTest', () => {
@@ -27,6 +34,16 @@ describe('runConnectivityTest', () => {
       ok: true,
       sample: 'CONNECTIVITY_OK',
     });
+    expect(getModelRuntime).toHaveBeenCalledWith(
+      expect.objectContaining({
+        openaiApiKey: 'sk-test',
+        openaiBaseURL: 'https://api.example.com/v1',
+        modelName: 'gpt-4o',
+        intent: 'default',
+        slot: 'default',
+        timeout: 30_000,
+      }),
+    );
     expect(callAIWithStringResponse).toHaveBeenCalledWith(
       [
         {
@@ -39,12 +56,14 @@ describe('runConnectivityTest', () => {
         },
       ],
       expect.objectContaining({
-        openaiApiKey: 'sk-test',
-        openaiBaseURL: 'https://api.example.com/v1',
-        modelName: 'gpt-4o',
-        intent: 'default',
-        slot: 'default',
-        timeout: 30_000,
+        config: expect.objectContaining({
+          openaiApiKey: 'sk-test',
+          openaiBaseURL: 'https://api.example.com/v1',
+          modelName: 'gpt-4o',
+          intent: 'default',
+          slot: 'default',
+          timeout: 30_000,
+        }),
       }),
       expect.objectContaining({
         abortSignal: expect.any(AbortSignal),