chore(ava): add llm validate for extract method in test case, support invoke tbox api with ci (#841)

* chore(ava): fix most test cases bug

* chore(ava): add llm validate for extract method in test case, support invoke tbox api with ci

---------

Co-authored-by: yanzhi.yan <yanzhi.yan@antgroup.com>

Author: Yzing

__tests__/unit/extract/index.test.ts

@@ -1,28 +1,12 @@
 import _ from 'lodash';
 import { extractData } from '../../../src/extract';
-
-const EXTRACT_MOCK_RESULT = [
-  {
-    shape: 'plain',
-    data: [
-      { id: 1001, name: '张三', age: 25 },
-      { id: 1002, name: '李四', age: 30 },
-      { id: 1003, name: '王五', age: 28 },
-      { id: 1004, name: '赵六', age: 35 },
-    ],
-    meta: [
-      { id: 'id', name: '用户ID', dataType: 'number' },
-      { id: 'name', name: '姓名', dataType: 'string' },
-      { id: 'age', name: '年龄', dataType: 'number' },
-    ],
-  },
-];
+import { extract } from './mock-extract';
 
 jest.mock('../../../src/extract/extract', () => ({
   __esModule: true,
-  extract: async () => {
-    return EXTRACT_MOCK_RESULT;
-  },
+  extract: jest.fn().mockImplementation(async (value: string) => {
+    return await extract(value);
+  }),
 }));
 
 describe('extractData', () => {
@@ -38,10 +22,21 @@ describe('extractData', () => {
       3. 用户ID: 1003, 姓名: 王五, 年龄: 28
       4. 用户ID: 1004, 姓名: 赵六, 年龄: 35
     `;
-    const expectResult = EXTRACT_MOCK_RESULT;
 
     const result = await extractData(input);
-    expect(result).toEqual(expectResult);
+    const shard = result?.[0];
+    expect(shard.shape).toBe('plain');
+    expect(shard.data).toEqual([
+      { 用户ID: 1001, 姓名: '张三', 年龄: 25 },
+      { 用户ID: 1002, 姓名: '李四', 年龄: 30 },
+      { 用户ID: 1003, 姓名: '王五', 年龄: 28 },
+      { 用户ID: 1004, 姓名: '赵六', 年龄: 35 },
+    ]);
+    expect(shard.metas).toEqual([
+      { id: '用户ID', name: '用户ID', dataType: 'number' },
+      { id: '姓名', name: '姓名', dataType: 'string' },
+      { id: '年龄', name: '年龄', dataType: 'number' },
+    ]);
   });
 
   it('should return plain dataType if input is plain data', async () => {

__tests__/unit/extract/mock-extract.ts

@@ -0,0 +1,21 @@
+import { requestTboxLLM } from '@ava/utils/llm';
+import { getExtractPrompt } from '@ava/extract/extract/prompt';
+
+export const extract = async (value: string) => {
+  const llmAuth = process.env.LLM_AUTH;
+  const llmAppId = process.env.LLM_APP_ID;
+  // @ts-ignore for debug at ci
+  console.debug(process.env);
+  if (process.env?.NODE_ENV === 'test' && llmAuth && llmAppId) {
+    const query = getExtractPrompt(value);
+    const result = await requestTboxLLM({
+      config: {
+        authorization: llmAuth,
+        appId: llmAppId,
+      },
+      prompt: query,
+    });
+    return JSON.parse(result);
+  }
+  return {};
+};

jest.config.js

@@ -1,6 +1,7 @@
 module.exports = {
-  testRegex: '(/__tests__/.*.(test|spec)).(js?|jsx?|tsx?|ts?)$',
+  setupFiles: ['<rootDir>/jest.setup.js'],
   collectCoverage: false,
+  testRegex: '(/__tests__/.*.(test|spec)).(js?|jsx?|tsx?|ts?)$',
   coveragePathIgnorePatterns: ['(tests/.*.mock).(jsx?|tsx?)$'],
   transform: {
     '^.+\\.m?[tj]sx?$': [

jest.setup.js

@@ -0,0 +1,5 @@
+
+require('dotenv').config({
+  path: ['.env.local']
+});
+jest.setTimeout(30000);

package.json

@@ -38,20 +38,20 @@
     ]
   },
   "dependencies": {
-    "@antv/color-schema": "^0.2.3",
-    "@antv/smart-color": "^0.2.1",
     "@antv/algorithm": "^0.1.26",
+    "@antv/color-schema": "^0.2.3",
     "@antv/gpt-vis": "^0.5.9",
+    "@antv/smart-color": "^0.2.1",
+    "axios": "^1.12.2",
     "bayesian-changepoint": "^1.0.1",
     "csstype": "^3.1.2",
     "heap-js": "^2.1.6",
     "lodash": "^4.17.21",
+    "moment": "^2.29.4",
+    "openai": "^6.2.0",
     "regression": "^2.0.1",
     "tapable": "^2.2.1",
-    "tslib": "^2.3.1",
-    "moment": "^2.29.4",
-    "axios": "^1.12.2",
-    "openai": "^6.2.0"
+    "tslib": "^2.3.1"
   },
   "devDependencies": {
     "@antv/data-samples": "^1.0.1",
@@ -66,14 +66,15 @@
     "@types/numeral": "^2.0.2",
     "@typescript-eslint/eslint-plugin": "^4.29.2",
     "@typescript-eslint/parser": "^4.29.2",
+    "dotenv": "^17.2.3",
     "eslint": "^7.32.0",
     "eslint-config-airbnb": "^18.2.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-import-resolver-typescript": "^2.4.0",
     "eslint-plugin-import": "^2.24.1",
     "eslint-plugin-react": "^7.24.0",
-    "husky": "^7.0.1",
     "father": "^4.1.0",
+    "husky": "^7.0.1",
     "jest": "^30.0.0",
     "lint-staged": "^11.0.1",
     "npm-run-all": "^4.1.5",

src/extract/extract/prompt.ts

@@ -50,8 +50,8 @@ export const getExtractPrompt = (input: string) => {
       shape: DATA_SHAPE;
       data: FieldDataType<DATA_SHAPE>;
       metas: Array<{
+        id: string; // 字段在数据中的 id
         name: string; // 字段的名称
-        key: string; // 字段在数据中的 key
         dataType: 'number' | 'string' | 'date' | 'geo'; // 字段的类型
       }>; // 字段元信息
       purpose?: {