fix(ava): make extract work (#835)

* fix(ava): make extract work

* fix(ava): fix unit test

---------

Co-authored-by: xueren.dt <xueren.dt@antgroup.com>

Author: leondt1

packages/ava/src/advisor/advise-chart-pipeline/plugins/advisePlugin.ts

@@ -28,7 +28,7 @@ export class AdvisePlugin implements AdvisorPlugin<AdviseChartParams> {
 
   execute = async (input: AdviseChartPluginInput) => {
     const { dataStore } = input;
-    const { dataShards } = dataStore.data;
+    const { dataShards } = dataStore.extract;
     // create all valid chart configs using field data
     const shard = dataShards[0];
     if (shard.shape === DATA_SHAPE.PLAIN) {

packages/ava/src/advisor/advise-chart-pipeline/plugins/extractPlugin.ts

@@ -13,10 +13,9 @@ export class ExtractPlugin implements AdvisorPlugin<AdviseChartParams> {
 
   async execute(ctx: AdviseChartPluginInput) {
     try {
-      const { purpose } = ctx.context;
-      const { data } = ctx.context;
+      const { data, llm, purpose } = ctx.context;
       const input = `${purpose}\n${JSON.stringify(data)}`;
-      const dataShards = await extractData(input);
+      const dataShards = await extractData(input, { llmConfig: llm });
       ctx.dataStore.extract.dataShards = dataShards;
     } catch (e) {
       ctx.dataStore.extract.dataShards = [];

packages/ava/src/advisor/chartAdvise/index.ts

@@ -676,7 +676,7 @@ export const optimizeChartConfig = (params: {
       case CHART_NAME.wordCloud: {
         const hasNegatives = metas.some((field) => {
           if (field.dataType === COLUMN_TYPE.number) {
-            return (field.statisticsFeature as NumberColumnFeature).minimum < 0;
+            return (field.statisticsFeature as NumberColumnFeature)?.minimum < 0;
           }
           return false;
         });

packages/ava/src/constants/pipeline.ts

@@ -1,6 +1,5 @@
 export enum AdviseChartPluginEnum {
   ExtractPlugin = 'extractPlugin',
-  DataPlugin = 'dataPlugin',
   AdvisePlugin = 'advisePlugin',
   GeneratePlugin = 'generatePlugin',
 }

packages/ava/src/extract/extract/index.ts

@@ -1,12 +1,12 @@
-import { requestTboxLLM } from '@ava/utils/llm';
+import { requestLLM } from '../../utils';
 
 import { getExtractPrompt } from './prompt';
 
-import type { TboxLLM } from '@ava/types';
+import type { AdvisorConfig } from '../../types';
 
-export const extract = async (input: string, config: TboxLLM) => {
+export const extract = async (input: string, config: AdvisorConfig['llm']) => {
   try {
-    const res = await requestTboxLLM({
+    const res = await requestLLM({
       config,
       prompt: getExtractPrompt(input),
     });

packages/ava/src/extract/extract/prompt.ts

@@ -2,7 +2,7 @@ export const getExtractPrompt = (input: string) => {
   return `
     # 你是一个数据解析专家，你的任务是将用户输入的文本解析为结构化数据，并理解用户的分析意图。
     ## 通过 typescript 类型来定义输出的类型，如下:
-    type PlainDataType = Array<Record<string, string | number>> | Array<Array<string | number>>; // 常规的二维明细数据
+    type PlainLikeDataType = Array<Record<string, string | number>> | Array<Array<string | number>>; // 常规的二维明细数据
     type DATA_SHAPE = 'plain' | 'graph' | 'tree' | 'flow'; // 数据形状
     type HierarchyDataType = Array<{
       id: string;
@@ -44,7 +44,7 @@ export const getExtractPrompt = (input: string) => {
       ? FlowDataType
       : T extends 'graph'
       ? RelationDataType
-      : PlainDataType; // 数据类型统一定义
+      : PlainLikeDataType; // 数据类型统一定义
 
     type DataShards = Array<{
       shape: DATA_SHAPE;
@@ -61,10 +61,10 @@ export const getExtractPrompt = (input: string) => {
         purposeDesc?: string; // 分析意图的简要说明
       };
     }>;
-    ## 需要你根据用户的输入文本，解析出其中的数据，并根据用户的意图，将其转化为上述 ts 类型中的 DataShards，并用标准 JSON 输出
+    ## 需要你根据用户的输入文本，解析出其中的数据，并根据用户的意图，将其转化为上述 ts 类型中的 DataShards，并用标准 JSON 字符串输出，不要使用\`\`\`JSON等任何代码块包裹
     ## 建议的执行步骤是
     ### 一、分离数据和文本
-    ### 二、解析数据，判断数据形状，并将其转化为 PlainDataType、TreeDataType、GraphDataType、FlowDataType 中的一种
+    ### 二、解析数据，判断数据形状，并将其转化为 PlainLikeDataType、TreeDataType、GraphDataType、FlowDataType 中的一种
     ### 三、理解用户意图，分析字段的名词和key，并将其转化为 DataShards 中的 purpose 信息
     ### 四、如果用户没有意图，请根据你对这份数据的理解，拆解出一到两个分析意图，并构建 purpose 信息
     ### 五、将数据和文本转化为 DataShards 数组结构，注意 DataShards 需要是一个数组，并用纯文本的 JSON 字符串输出

packages/ava/src/extract/features/plain.ts

@@ -70,42 +70,23 @@ export function isOrdinal(info: ColumnFeature): boolean {
   const { rawData, recommendation } = info;
   if (recommendation !== COLUMN_TYPE.string) return false;
   if (isConst(info)) return false;
-  const list = rawData.filter((item) => !isNil(item) && isBasicType(item));
+  const list = rawData.filter((item) => !isNil(item) && isBasicType(item)).map((item) => `${item}`);
   if (list.length === 0) return false;
-  let start: null | string = null;
-  let end: null | string = null;
+  // Compute common prefix and suffix lengths safely to avoid infinite loops
+  const minLen = Math.min(...list.map((s) => s.length));
   let startIndex = -1;
-  let endIndex = -1;
-
-  let through = true;
-  while (through) {
-    let through = true;
-    for (let i = 0; i < list.length; i += 1) {
-      const item = list[i];
-      const char = item[startIndex + 1];
-      if (start === null || i === 0) start = char;
-      if (char !== start) {
-        through = false;
-        break;
-      }
-    }
-    if (!through) break;
-    startIndex += 1;
+  for (let idx = 0; idx < minLen; idx += 1) {
+    const c0 = list[0][idx];
+    const allSame = list.every((s) => s[idx] === c0);
+    if (!allSame) break;
+    startIndex = idx;
   }
-  through = true;
-  while (through) {
-    let through = true;
-    for (let i = 0; i < list.length; i += 1) {
-      const item = list[i];
-      const char = item[item.length - 1 - (endIndex + 1)];
-      if (end === null || i === 0) end = char;
-      if (char !== end) {
-        through = false;
-        break;
-      }
-    }
-    if (!through) break;
-    endIndex += 1;
+  let endIndex = -1;
+  for (let idx = 0; idx < minLen; idx += 1) {
+    const c0 = list[0][list[0].length - 1 - idx];
+    const allSame = list.every((s) => s[s.length - 1 - idx] === c0);
+    if (!allSame) break;
+    endIndex = idx;
   }
   const patterns = [/\d+/, /(零|一|二|三|四|五|六|七|八|九|十)+/, /(一|二|三|四|五|六|日)/, /^[a-z]$/, /^[A-Z]$/];
   if (startIndex === -1 && endIndex === -1) return false;

packages/ava/src/extract/index.ts

@@ -9,23 +9,22 @@ import { extract } from '@ava/extract/extract';
 import type {
   DataShard,
   Meta,
-  TboxLLM,
-  OpenAiLLM,
   PlainLikeDataType,
   HierarchyLikeDataType,
   RelationLikeDataType,
-} from '@ava/types';
+  AdvisorConfig,
+} from '../types';
 
 export const extractData: (
   input: string | Record<string, any> | Record<string, any>[],
   config?: {
-    llmConfig?: TboxLLM | OpenAiLLM;
+    llmConfig?: AdvisorConfig['llm'];
   }
 ) => Promise<DataShard[]> = async (input, config) => {
   const shards: DataShard[] = [];
   if (typeof input === 'string') {
-    const res = await extract(input, config?.llmConfig as TboxLLM);
-    return res;
+    const res = await extract(input, config?.llmConfig);
+    return [res];
   }
   const inferRes = matchDataShape(input);
   if (inferRes.shape === DATA_SHAPE.PLAIN) {

packages/ava/src/utils/llm.ts

@@ -7,7 +7,7 @@ import { logError, sleep } from './common';
 import type { OpenAiLLM, TboxLLM } from '@ava/types';
 
 const DEFAULT_MAX_RETRY_COUNT = 3;
-const DEFAULT_TIMEOUT = 10000;
+const DEFAULT_TIMEOUT = 60000;
 const DEFAULT_DELAY = 500;
 
 /**

playground/src/examples/advise/index.tsx

@@ -7,8 +7,8 @@ import { render } from '../../utils';
 // 创建 advisor 实例并为该实例绑定渲染器
 const advisor = new Advisor({
   llm: {
-    appId: '202510APxPmo00551539',
-    authorization: 'TBox-c4ae8a71224e42baaafb1c01d15395a7',
+    appId: '202511APkFwG00560135',
+    authorization: 'TBox-174d46eaa4374e96b3fd99b6fec527d7',
   },
 });