Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/ai-unit-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,9 @@ jobs:
run: pnpm exec nx run-many --target=build --projects="@midscene/report,@midscene/core,@midscene/web,@midscene/cli"

- name: Run tests with coverage
# Worker caps live in each package's own test config so the flags stay
# runner-agnostic: rstest rejects bare --minWorkers/--maxWorkers, so they
# must not be passed here.
# Worker caps live in the core/web-integration/cli rstest.config.ts files:
# rstest rejects bare --minWorkers/--maxWorkers, so they must not be
# passed here.
run: MIDSCENE_COVERAGE_DIR=coverage-ai pnpm exec nx run-many --target=test:ai --projects=@midscene/core,@midscene/web,@midscene/cli --verbose -- --coverage
id: test-ai
continue-on-error: true
Expand Down
26 changes: 18 additions & 8 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,18 @@
"main": "./dist/lib/index.js",
"types": "./dist/types/index.d.ts",
"module": "./dist/es/index.mjs",
"imports": {
"#proxy-deps": {
"node": {
"import": "./dist/es/ai-model/service-caller/proxy-deps.node.mjs",
"require": "./dist/lib/ai-model/service-caller/proxy-deps.node.js"
},
"default": {
"import": "./dist/es/ai-model/service-caller/proxy-deps.stub.mjs",
"require": "./dist/lib/ai-model/service-caller/proxy-deps.stub.js"
}
}
},
"files": ["dist", "README.md"],
"exports": {
".": {
Expand Down Expand Up @@ -90,11 +102,10 @@
"build": "rslib build",
"build:watch": "USE_DEV_REPORT=1 rslib build --watch --no-clean",
"sync-report-template": "node ../../scripts/sync-core-report-template.mjs",
"test": "vitest --run",
"test:u": "vitest --run -u",
"test:ai": "AITEST=true vitest --run",
"computer": "TEST_COMPUTER=true AITEST=true vitest --run tests/ai/evaluate/computer.test.ts",
"test:parse-action": "vitest --run tests/unit-test/parse-action.test.ts"
"test": "rstest",
"test:u": "rstest -u",
"test:ai": "AITEST=true rstest",
"test:parse-action": "rstest tests/unit-test/parse-action.test.ts"
},
"nx": {
"targets": {
Expand Down Expand Up @@ -131,15 +142,14 @@
},
"devDependencies": {
"@rslib/core": "^0.18.3",
"@rstest/core": "0.11.5",
"@types/js-yaml": "4.0.9",
"@types/node": "^18.0.0",
"@types/node-fetch": "2.6.11",
"@types/semver": "7.7.0",
"@vitest/runner": "3.0.5",
"langsmith": "^0.3.74",
"sharp": "^0.34.3",
"typescript": "^5.8.3",
"vitest": "3.0.5"
"typescript": "^5.8.3"
},
"engines": {
"node": ">=18.19.0"
Expand Down
1 change: 1 addition & 0 deletions packages/core/rslib.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ export default defineConfig({
},
},
output: {
externals: ['#proxy-deps', 'undici', 'fetch-socks'],
sourceMap: true,
},
plugins: [createTypeCheckPlugin(), writeExistingReportTemplate()],
Expand Down
47 changes: 47 additions & 0 deletions packages/core/rstest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import path from 'node:path';
import { defineConfig } from '@rstest/core';
import dotenv from 'dotenv';
import { createCoverageConfig } from '../../scripts/rstest-coverage';
import { defineVersion, photonExternal } from '../../scripts/rstest-shared';
import { version } from './package.json';

/**
* Read environment variables from file.
* https://github.com/motdotla/dotenv
*/
dotenv.config({
path: path.join(__dirname, '../../.env'),
override: true,
});

const enableAiTest = Boolean(process.env.AITEST);
const basicTest = ['tests/unit-test/**/*.test.ts'];

export default defineConfig({
coverage: createCoverageConfig(__dirname),
include: enableAiTest ? ['tests/ai/**/*.test.ts'] : basicTest,
retry: process.env.CI ? 1 : 0,
...(enableAiTest && process.env.CI ? { pool: { maxWorkers: 4 } } : {}),
resolve: {
alias: {
'@': path.resolve(__dirname, 'src'),
// Tests must not require a prior package build: route the conditional
// '#proxy-deps' subpath import to the Node source implementation.
'#proxy-deps': path.resolve(
__dirname,
'src/ai-model/service-caller/proxy-deps.node.ts',
),
},
Comment thread
fi3ework marked this conversation as resolved.
},
source: {
define: {
...defineVersion(version),
__DEV_REPORT_PATH__: JSON.stringify(
path.resolve(__dirname, '../../apps/report/dist/index.html'),
),
},
},
output: {
externals: photonExternal,
},
});
10 changes: 4 additions & 6 deletions packages/core/src/ai-model/service-caller/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,9 +208,8 @@ export async function createChatClient({
'HTTP proxy is configured but not supported in browser environment',
);
} else {
// Dynamic import with variable to avoid bundler static analysis
const moduleName = 'undici';
const { ProxyAgent } = await import(moduleName);
const { loadUndici } = await import('#proxy-deps');
const { ProxyAgent } = await loadUndici();
proxyAgent = new ProxyAgent({
uri: httpProxy,
// Note: authentication is handled via the URI (e.g., http://user:pass@proxy.com:8080)
Expand All @@ -224,9 +223,8 @@ export async function createChatClient({
);
} else {
try {
// Dynamic import with variable to avoid bundler static analysis
const moduleName = 'fetch-socks';
const { socksDispatcher } = await import(moduleName);
const { loadFetchSocks } = await import('#proxy-deps');
const { socksDispatcher } = await loadFetchSocks();
// Parse SOCKS proxy URL (e.g., socks5://127.0.0.1:1080)
const proxyUrl = new URL(socksProxy);

Expand Down
4 changes: 4 additions & 0 deletions packages/core/src/ai-model/service-caller/proxy-deps.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
declare module '#proxy-deps' {
export function loadUndici(): Promise<typeof import('undici')>;
export function loadFetchSocks(): Promise<typeof import('fetch-socks')>;
}
7 changes: 7 additions & 0 deletions packages/core/src/ai-model/service-caller/proxy-deps.node.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export function loadUndici(): Promise<typeof import('undici')> {
return import('undici');
}

export function loadFetchSocks(): Promise<typeof import('fetch-socks')> {
return import('fetch-socks');
}
11 changes: 11 additions & 0 deletions packages/core/src/ai-model/service-caller/proxy-deps.stub.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
function unavailable(): never {
throw new Error('proxy dependencies are unavailable in browser builds');
}

export function loadUndici(): Promise<typeof import('undici')> {
return unavailable();
}

export function loadFetchSocks(): Promise<typeof import('fetch-socks')> {
return unavailable();
}
4 changes: 2 additions & 2 deletions packages/core/tests/ai/connectivity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@ import { getModelRuntime } from '@/ai-model/models';
import { callAI, callAIWithObjectResponse } from '@/ai-model/service-caller';
import { globalModelConfigManager } from '@midscene/shared/env';
import { localImg2Base64 } from '@midscene/shared/img';
import { beforeAll, describe, expect, it, rs } from '@rstest/core';
import dotenv from 'dotenv';
import { beforeAll, describe, expect, it, vi } from 'vitest';
import { getFixture } from '../utils';

dotenv.config({
debug: true,
override: true,
});

vi.setConfig({
rs.setConfig({
testTimeout: 20 * 1000,
});
[
Expand Down
14 changes: 7 additions & 7 deletions packages/core/tests/ai/extract/extract.test.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
import { AiExtractElementInfo, getModelRuntime } from '@/ai-model';
import { globalModelConfigManager } from '@midscene/shared/env';
import { beforeAll, describe, expect, it, vi } from 'vitest';
import { describe, expect, it, rs } from '@rstest/core';
import { getContextFromFixture } from '../../evaluation';

vi.setConfig({
rs.setConfig({
testTimeout: 240 * 1000,
hookTimeout: 30 * 1000,
});

const defaultModelConfig = globalModelConfigManager.getModelConfig('default');
const defaultModelRuntime = getModelRuntime(defaultModelConfig);
const defaultModelRuntime = () =>
getModelRuntime(globalModelConfigManager.getModelConfig('default'));

describe('extract', () => {
it('todo', async () => {
Expand All @@ -18,7 +18,7 @@ describe('extract', () => {
const { parseResult } = await AiExtractElementInfo({
dataQuery: 'Array<string>, task list, task name as string',
context,
modelRuntime: defaultModelRuntime,
modelRuntime: defaultModelRuntime(),
});
expect(parseResult).toBeDefined();
expect((parseResult.data as string[]).length).toBeGreaterThanOrEqual(3);
Expand All @@ -31,7 +31,7 @@ describe('extract', () => {
const { parseResult } = await AiExtractElementInfo({
dataQuery: '{name: string, price: string}[], 饮品名称和价格',
context,
modelRuntime: defaultModelRuntime,
modelRuntime: defaultModelRuntime(),
});

// Remove the thought field since it's generated dynamically by AI
Expand All @@ -50,7 +50,7 @@ describe('extract', () => {
dataQuery:
'{checked: boolean; text: string;}[], Task list with checkbox ahead of the task name (checkbox is a round box), task name as string and `checked` is true if the task is completed. Exclude the fist row if there is no round checkbox ahead of the task name.',
context,
modelRuntime: defaultModelRuntime,
modelRuntime: defaultModelRuntime(),
});

// Remove the thought field since it's generated dynamically by AI
Expand Down
12 changes: 6 additions & 6 deletions packages/core/tests/ai/llm-inspect.test.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
import { AiLocateElement, AiLocateSection } from '@/ai-model';
import { getModelRuntime } from '@/ai-model/models';
import { globalModelConfigManager } from '@midscene/shared/env';
import { expect, test, vi } from 'vitest';
import { expect, rs, test } from '@rstest/core';
import { getContextFromFixture } from '../evaluation';

vi.setConfig({
rs.setConfig({
testTimeout: 120 * 1000,
});

const defaultModelConfig = globalModelConfigManager.getModelConfig('default');
const defaultModelRuntime = getModelRuntime(defaultModelConfig);
const defaultModelRuntime = () =>
getModelRuntime(globalModelConfigManager.getModelConfig('default'));

test(
'basic inspect',
Expand All @@ -22,7 +22,7 @@ test(
const { parseResult } = await AiLocateElement({
context,
targetElementDescription: 'input 输入框',
modelRuntime: defaultModelRuntime,
modelRuntime: defaultModelRuntime(),
});
expect(parseResult.element).toBeDefined();
},
Expand All @@ -33,7 +33,7 @@ test('locate section', { timeout: 120 * 1000 }, async () => {
const { searchAreaConfig } = await AiLocateSection({
context,
sectionDescription: '搜索框',
modelRuntime: defaultModelRuntime,
modelRuntime: defaultModelRuntime(),
});
expect(searchAreaConfig?.sourceRect).toBeDefined();
});
29 changes: 18 additions & 11 deletions packages/core/tests/ai/llm-planning/basic.test.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,30 @@
import { ConversationHistory, standardPlan } from '@/ai-model';
import { getModelRuntime } from '@/ai-model/models';
import { globalModelConfigManager } from '@midscene/shared/env';
import { describe, expect, it, vi } from 'vitest';
import { describe, expect, it, rs } from '@rstest/core';
import { mockActionSpace } from '../../common';
import { getContextFromFixture } from '../../evaluation';

vi.setConfig({
rs.setConfig({
testTimeout: 180 * 1000,
hookTimeout: 30 * 1000,
});

const modelConfig = globalModelConfigManager.getModelConfig('default');
const modelRuntime = getModelRuntime(modelConfig);
const modelConfig = () => globalModelConfigManager.getModelConfig('default');
const modelRuntime = () => getModelRuntime(modelConfig());
const hasModelFamily = (() => {
try {
return Boolean(modelConfig().modelFamily);
} catch {
return false;
}
})();

// These assertions check a deterministic next-action shape. In real
// model-family runs, planning may choose a valid intermediate Tap before Input
// or include a whole-page locate for page-level scroll, so keep this suite out
// of AI CI until that prompt contract is tightened.
describe.skipIf(modelConfig.modelFamily)('automation - llm planning', () => {
describe.skipIf(hasModelFamily)('automation - llm planning', () => {
it('basic run', async () => {
const { context } = await getContextFromFixture('todo');

Expand All @@ -26,7 +33,7 @@ describe.skipIf(modelConfig.modelFamily)('automation - llm planning', () => {
{
context,
actionSpace: mockActionSpace,
modelRuntime,
modelRuntime: modelRuntime(),
conversationHistory: new ConversationHistory(),
includeLocateInPlanning: true,
effort: 'balance',
Expand All @@ -46,7 +53,7 @@ describe.skipIf(modelConfig.modelFamily)('automation - llm planning', () => {
{
context,
actionSpace: mockActionSpace,
modelRuntime,
modelRuntime: modelRuntime(),
conversationHistory: new ConversationHistory(),
includeLocateInPlanning: true,
effort: 'balance',
Expand Down Expand Up @@ -96,7 +103,7 @@ describe('planning', () => {
const { actions } = await standardPlan(instruction, {
context,
actionSpace: mockActionSpace,
modelRuntime,
modelRuntime: modelRuntime(),
conversationHistory: new ConversationHistory(),
includeLocateInPlanning: true,
effort: 'balance',
Expand All @@ -118,7 +125,7 @@ describe('planning', () => {
{
context,
actionSpace: mockActionSpace,
modelRuntime,
modelRuntime: modelRuntime(),
conversationHistory: new ConversationHistory(),
includeLocateInPlanning: true,
effort: 'balance',
Expand All @@ -137,7 +144,7 @@ describe('planning', () => {
{
context,
actionSpace: mockActionSpace,
modelRuntime,
modelRuntime: modelRuntime(),
conversationHistory: new ConversationHistory(),
includeLocateInPlanning: true,
effort: 'balance',
Expand All @@ -156,7 +163,7 @@ describe('planning', () => {
{
context,
actionSpace: mockActionSpace,
modelRuntime,
modelRuntime: modelRuntime(),
conversationHistory: new ConversationHistory(),
includeLocateInPlanning: true,
effort: 'balance',
Expand Down
Loading
Loading