Skip to content

Commit d039d1e

Browse files
dcc123456dcc123456
authored
fix: Added dataset generation logging functionality infiniflow#9869 (infiniflow#10180)
### What problem does this PR solve? fix: Added dataset generation logging functionality infiniflow#9869 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
1 parent d050ef5 commit d039d1e

File tree

15 files changed

+491
-119
lines changed

15 files changed

+491
-119
lines changed

‎web/src/components/data-pipeline-select/index.tsx‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,10 @@ interface IProps {
2020
isMult?: boolean;
2121
}
2222

23-
const data = [
24-
{ id: '1', name: 'data-pipeline-1' },
25-
{ id: '2', name: 'data-pipeline-2' },
26-
{ id: '3', name: 'data-pipeline-3' },
27-
{ id: '4', name: 'data-pipeline-4' },
28-
];
2923
export function DataFlowSelect(props: IProps) {
3024
const { toDataPipeline, formFieldName, isMult = true } = props;
3125
const { t } = useTranslate('knowledgeConfiguration');
3226
const form = useFormContext();
33-
console.log('data-pipline form', form);
3427
const toDataPipLine = () => {
3528
toDataPipeline?.();
3629
};

‎web/src/components/parse-configuration/graph-rag-form-fields.tsx‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { DocumentParserType } from '@/constants/knowledge';
22
import { useTranslate } from '@/hooks/common-hooks';
33
import { cn } from '@/lib/utils';
4+
import {
5+
GenerateLogButton,
6+
GenerateType,
7+
} from '@/pages/dataset/dataset/generate-button/generate';
48
import { upperFirst } from 'lodash';
59
import { useCallback, useMemo } from 'react';
610
import { useFormContext, useWatch } from 'react-hook-form';
@@ -47,6 +51,7 @@ export const showGraphRagItems = (parserId: DocumentParserType | undefined) => {
4751
type GraphRagItemsProps = {
4852
marginBottom?: boolean;
4953
className?: string;
54+
showGenerateItem?: boolean;
5055
};
5156

5257
export function UseGraphRagFormField() {
@@ -88,6 +93,7 @@ export function UseGraphRagFormField() {
8893
// The three types "table", "resume" and "one" do not display this configuration.
8994
const GraphRagItems = ({
9095
marginBottom = false,
96+
showGenerateItem = false,
9197
className = 'p-10',
9298
}: GraphRagItemsProps) => {
9399
const { t } = useTranslate('knowledgeConfiguration');
@@ -210,6 +216,18 @@ const GraphRagItems = ({
210216
</FormItem>
211217
)}
212218
/>
219+
{showGenerateItem && (
220+
<div className="w-full flex items-center">
221+
<div className="text-sm whitespace-nowrap w-1/4">
222+
{t('extractKnowledgeGraph')}
223+
</div>
224+
<GenerateLogButton
225+
className="w-3/4 text-text-secondary"
226+
status={1}
227+
type={GenerateType.KnowledgeGraph}
228+
/>
229+
</div>
230+
)}
213231
</>
214232
)}
215233
</FormContainer>

‎web/src/components/parse-configuration/raptor-form-fields.tsx‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { FormLayout } from '@/constants/form';
22
import { DocumentParserType } from '@/constants/knowledge';
33
import { useTranslate } from '@/hooks/common-hooks';
4+
import {
5+
GenerateLogButton,
6+
GenerateType,
7+
} from '@/pages/dataset/dataset/generate-button/generate';
48
import random from 'lodash/random';
59
import { Shuffle } from 'lucide-react';
610
import { useCallback } from 'react';
@@ -52,7 +56,11 @@ const Prompt = 'parser_config.raptor.prompt';
5256

5357
// The three types "table", "resume" and "one" do not display this configuration.
5458

55-
const RaptorFormFields = () => {
59+
const RaptorFormFields = ({
60+
showGenerateItem = false,
61+
}: {
62+
showGenerateItem?: boolean;
63+
}) => {
5664
const form = useFormContext();
5765
const { t } = useTranslate('knowledgeConfiguration');
5866
const useRaptor = useWatch({ name: UseRaptorField });
@@ -211,6 +219,18 @@ const RaptorFormFields = () => {
211219
</FormItem>
212220
)}
213221
/>
222+
{showGenerateItem && (
223+
<div className="w-full flex items-center">
224+
<div className="text-sm whitespace-nowrap w-1/4">
225+
{t('extractRaptor')}
226+
</div>
227+
<GenerateLogButton
228+
className="w-3/4 text-text-secondary"
229+
status={1}
230+
type={GenerateType.Raptor}
231+
/>
232+
</div>
233+
)}
214234
</div>
215235
)}
216236
</>

‎web/src/components/ui/modal/modal.tsx‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,21 +75,21 @@ const Modal: ModalType = ({
7575

7676
const handleCancel = useCallback(() => {
7777
onOpenChange?.(false);
78-
onCancel?.();
79-
}, [onOpenChange, onCancel]);
78+
// onCancel?.();
79+
}, [onOpenChange]);
8080

8181
const handleOk = useCallback(() => {
8282
onOpenChange?.(true);
83-
onOk?.();
84-
}, [onOpenChange, onOk]);
83+
// onOk?.();
84+
}, [onOpenChange]);
8585
const handleChange = (open: boolean) => {
8686
onOpenChange?.(open);
8787
console.log('open', open, onOpenChange);
8888
if (open) {
89-
handleOk();
89+
onOk?.();
9090
}
9191
if (!open) {
92-
handleCancel();
92+
onCancel?.();
9393
}
9494
};
9595
const footEl = useMemo(() => {

‎web/src/locales/en.ts‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,15 @@ export default {
102102
noMoreData: `That's all. Nothing more.`,
103103
},
104104
knowledgeDetails: {
105+
notGenerated: 'Not generated',
106+
generatedOn: 'Generated on',
107+
subbarFiles: 'Files',
105108
generateKnowledgeGraph:
106109
'This will extract entities and relationships from all your documents in this dataset. The process may take a while to complete.',
107110
generateRaptor:
108111
'This will extract entities and relationships from all your documents in this dataset. The process may take a while to complete.',
109112
generate: 'Generate',
110113
raptor: 'Raptor',
111-
knowledgeGraph: 'Knowledge Graph',
112114
processingType: 'Processing Type',
113115
dataPipeline: 'Data Pipeline',
114116
operations: 'Operations',
@@ -138,12 +140,12 @@ export default {
138140
testing: 'Retrieval testing',
139141
files: 'files',
140142
configuration: 'Configuration',
141-
knowledgeGraph: 'Knowledge graph',
143+
knowledgeGraph: 'Knowledge Graph',
142144
name: 'Name',
143145
namePlaceholder: 'Please input name!',
144146
doc: 'Docs',
145147
datasetDescription:
146-
'😉 Please wait for your files to finish parsing before starting an AI-powered chat.',
148+
'Please wait for your files to finish parsing before starting an AI-powered chat.',
147149
addFile: 'Add file',
148150
searchFiles: 'Search your files',
149151
localFiles: 'Local files',
@@ -261,6 +263,22 @@ export default {
261263
reRankModelWaring: 'Re-rank model is very time consuming.',
262264
},
263265
knowledgeConfiguration: {
266+
deleteGenerateModalContent: `
267+
<p>Deleting the generated <strong class='text-text-primary'>{{type}}</strong> results
268+
will remove all derived entities and relationships from this dataset.
269+
Your original files will remain intact.<p>
270+
<br/>
271+
Do you want to continue?
272+
`,
273+
extractRaptor: 'Extract Raptor',
274+
extractKnowledgeGraph: 'Extract Knowledge Graph',
275+
filterPlaceholder: 'please input filter',
276+
fileFilterTip: '',
277+
fileFilter: 'File Filter',
278+
setDefaultTip: '',
279+
setDefault: 'Set as Default',
280+
eidtLinkDataPipeline: 'Edit Data Pipeline',
281+
linkPipelineSetTip: 'Manage data pipeline linkage with this dataset',
264282
default: 'Default',
265283
dataPipeline: 'Data Pipeline',
266284
linkDataPipeline: 'Link Data Pipeline',
@@ -1646,6 +1664,13 @@ This delimiter is used to split the input text into several text pieces echo of
16461664
<p>To keep them, please click Rerun to re-run the current stage.</p> `,
16471665
changeStepModalConfirmText: 'Switch Anyway',
16481666
changeStepModalCancelText: 'Cancel',
1667+
unlinkPipelineModalTitle: 'Unlink data pipeline',
1668+
unlinkPipelineModalContent: `
1669+
<p>Once unlinked, this Dataset will no longer be connected to the current Data Pipeline.</p>
1670+
<p>Files that are already being parsed will continue until completion</p>
1671+
<p>Files that are not yet parsed will no longer be processed</p> <br/>
1672+
<p>Are you sure you want to proceed?</p> `,
1673+
unlinkPipelineModalConfirmText: 'Unlink',
16491674
},
16501675
dataflow: {
16511676
parser: 'Parser',

‎web/src/locales/zh.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,11 @@ export default {
9494
noMoreData: '没有更多数据了',
9595
},
9696
knowledgeDetails: {
97+
notGenerated: '未生成',
98+
generatedOn: '生成于',
99+
subbarFiles: '文件列表',
97100
generate: '生成',
98101
raptor: 'Raptor',
99-
knowledgeGraph: '知识图谱',
100102
processingType: '处理类型',
101103
dataPipeline: '数据管道',
102104
operations: '操作',
@@ -130,7 +132,7 @@ export default {
130132
name: '名称',
131133
namePlaceholder: '请输入名称',
132134
doc: '文档',
133-
datasetDescription: '😉 解析成功后才能问答哦。',
135+
datasetDescription: '解析成功后才能问答哦。',
134136
addFile: '新增文件',
135137
searchFiles: '搜索文件',
136138
localFiles: '本地文件',
@@ -246,6 +248,22 @@ export default {
246248
theDocumentBeingParsedCannotBeDeleted: '正在解析的文档不能被删除',
247249
},
248250
knowledgeConfiguration: {
251+
deleteGenerateModalContent: `
252+
<p>删除生成的 <strong class='text-text-primary'>{{type}}</strong> 结果
253+
将从此数据集中移除所有派生实体和关系。
254+
您的原始文件将保持不变。<p>
255+
<br/>
256+
是否要继续？
257+
`,
258+
extractRaptor: '从文档中提取Raptor',
259+
extractKnowledgeGraph: '从文档中提取知识图谱',
260+
filterPlaceholder: '请输入',
261+
fileFilterTip: '',
262+
fileFilter: '正则匹配表达式',
263+
setDefaultTip: '',
264+
setDefault: '设置默认',
265+
eidtLinkDataPipeline: '编辑数据流',
266+
linkPipelineSetTip: '管理与此数据集的数据管道链接',
249267
default: '默认',
250268
dataPipeline: '数据流',
251269
linkDataPipeline: '关联数据流',
@@ -1556,6 +1574,13 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
15561574
<p>要保留这些更改，请点击“重新运行”以重新运行当前阶段。</p> `,
15571575
changeStepModalConfirmText: '继续切换',
15581576
changeStepModalCancelText: '取消',
1577+
unlinkPipelineModalTitle: '解绑数据流',
1578+
unlinkPipelineModalContent: `
1579+
<p>一旦取消链接，该数据集将不再连接到当前数据管道。</p>
1580+
<p>正在解析的文件将继续解析，直到完成。</p>
1581+
<p>尚未解析的文件将不再被处理。</p> <br/>
1582+
<p>你确定要继续吗?</p> `,
1583+
unlinkPipelineModalConfirmText: '解绑',
15591584
},
15601585
dataflow: {
15611586
parser: '解析器',

‎web/src/pages/dataset/dataset-overview/index.tsx‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import SvgIcon from '@/components/svg-icon';
22
import { useIsDarkTheme } from '@/components/theme-provider';
33
import { parseColorToRGBA } from '@/utils/common-util';
44
import { CircleQuestionMark } from 'lucide-react';
5-
import { FC, useMemo, useState } from 'react';
5+
import { FC, useEffect, useMemo, useState } from 'react';
66
import { useTranslation } from 'react-i18next';
77
import { LogTabs } from './dataset-common';
88
import { DatasetFilter } from './dataset-filter';
@@ -74,25 +74,35 @@ const FileLogsPage: FC = () => {
7474
const [active, setActive] = useState<(typeof LogTabs)[keyof typeof LogTabs]>(
7575
LogTabs.FILE_LOGS,
7676
);
77-
const topMockData = {
77+
const [topAllData, setTopAllData] = useState({
7878
totalFiles: {
79-
value: 2827,
80-
precent: 12.5,
79+
value: 0,
80+
precent: 0,
8181
},
8282
downloads: {
83-
value: 28,
84-
success: 8,
85-
failed: 2,
83+
value: 0,
84+
success: 0,
85+
failed: 0,
8686
},
8787
processing: {
88-
value: 156,
89-
success: 8,
90-
failed: 2,
88+
value: 0,
89+
success: 0,
90+
failed: 0,
9191
},
92-
};
92+
});
9393

9494
const { data: topData } = useFetchOverviewTital();
9595
console.log('topData --> ', topData);
96+
useEffect(() => {
97+
setTopAllData({
98+
...topAllData,
99+
processing: {
100+
value: topData?.processing || 0,
101+
success: topData?.finished || 0,
102+
failed: topData?.failed || 0,
103+
},
104+
});
105+
}, [topData, topAllData]);
96106

97107
const mockData = useMemo(() => {
98108
if (active === LogTabs.FILE_LOGS) {
@@ -161,7 +171,7 @@ const FileLogsPage: FC = () => {
161171
<div className="grid grid-cols-3 md:grid-cols-3 gap-4 mb-6">
162172
<StatCard
163173
title="Total Files"
164-
value={topMockData.totalFiles.value}
174+
value={topAllData.totalFiles.value}
165175
icon={
166176
isDark ? (
167177
<SvgIcon name="data-flow/total-files-icon" width={40} />
@@ -172,15 +182,15 @@ const FileLogsPage: FC = () => {
172182
>
173183
<div>
174184
<span className="text-accent-primary">
175-
{topMockData.totalFiles.precent > 0 ? '+' : ''}
176-
{topMockData.totalFiles.precent}%{' '}
185+
{topAllData.totalFiles.precent > 0 ? '+' : ''}
186+
{topAllData.totalFiles.precent}%{' '}
177187
</span>
178188
from last week
179189
</div>
180190
</StatCard>
181191
<StatCard
182192
title="Downloading"
183-
value={topMockData.downloads.value}
193+
value={topAllData.downloads.value}
184194
icon={
185195
isDark ? (
186196
<SvgIcon name="data-flow/data-icon" width={40} />
@@ -190,13 +200,13 @@ const FileLogsPage: FC = () => {
190200
}
191201
>
192202
<CardFooterProcess
193-
success={topMockData.downloads.success}
194-
failed={topMockData.downloads.failed}
203+
success={topAllData.downloads.success}
204+
failed={topAllData.downloads.failed}
195205
/>
196206
</StatCard>
197207
<StatCard
198208
title="Processing"
199-
value={topMockData.processing.value}
209+
value={topAllData.processing.value}
200210
icon={
201211
isDark ? (
202212
<SvgIcon name="data-flow/processing-icon" width={40} />
@@ -206,8 +216,8 @@ const FileLogsPage: FC = () => {
206216
}
207217
>
208218
<CardFooterProcess
209-
success={topMockData.processing.success}
210-
failed={topMockData.processing.failed}
219+
success={topAllData.processing.success}
220+
failed={topAllData.processing.failed}
211221
/>
212222
</StatCard>
213223
</div>

0 commit comments

Comments
 (0)