Skip to content

Commit d982e8d

Browse files
Kota-MaedaKota-Maeda
committed
feat: add batch sync for Notion and Website documents in knowledge base
Add "Sync" button to the batch action bar when Notion/Website documents are selected, and a "Sync All" button in the document list header for knowledge bases backed by Notion or Website sources. Backend adds a new `/datasets/<id>/website-sync` API endpoint mirroring the existing `/datasets/<id>/notion/sync` endpoint. Frontend wires up both sync mutations, filters syncable documents by data source type, polls for completion, and shows loading state with toast notifications.
1 parent fa1ac75 commit d982e8d

36 files changed

Lines changed: 485 additions & 13 deletions

File tree

‎api/controllers/console/datasets/data_source.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,30 @@ def get(self, dataset_id: UUID) -> tuple[dict[str, str], int]:
411411
return {"result": "success"}, 200
412412

413413

414+
@console_ns.route("/datasets/<uuid:dataset_id>/website-sync")
415+
class DataSourceWebsiteDatasetSyncApi(Resource):
416+
@setup_required
417+
@login_required
418+
@account_initialization_required
419+
@console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])
420+
@rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_CREATE_AND_MANAGEMENT)
421+
def get(self, dataset_id: UUID) -> tuple[dict[str, str], int]:
422+
dataset_id_str = str(dataset_id)
423+
dataset = DatasetService.get_dataset(dataset_id_str)
424+
if dataset is None:
425+
raise NotFound("Dataset not found.")
426+
427+
documents = DocumentService.get_document_by_dataset_id(dataset_id_str)
428+
for document in documents:
429+
if document.data_source_type != "website_crawl" or document.archived:
430+
continue
431+
try:
432+
DocumentService.sync_website_document(dataset_id_str, document)
433+
except ValueError:
434+
pass
435+
return {"result": "success"}, 200
436+
437+
414438
@console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/notion/sync")
415439
class DataSourceNotionDocumentSyncApi(Resource):
416440
@setup_required

‎web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-actions.spec.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ const mockDisable = vi.fn()
1515
const mockDelete = vi.fn()
1616
const mockRetryIndex = vi.fn()
1717
const mockDownloadZip = vi.fn()
18+
const mockBatchSyncNotion = vi.fn()
19+
const mockBatchSyncWebsite = vi.fn()
1820
let mockIsDownloadingZip = false
1921

2022
vi.mock('@/service/knowledge/use-document', () => ({
@@ -25,6 +27,8 @@ vi.mock('@/service/knowledge/use-document', () => ({
2527
useDocumentDelete: () => ({ mutateAsync: mockDelete }),
2628
useDocumentBatchRetryIndex: () => ({ mutateAsync: mockRetryIndex }),
2729
useDocumentDownloadZip: () => ({ mutateAsync: mockDownloadZip, isPending: mockIsDownloadingZip }),
30+
useBatchSyncNotion: () => ({ mutateAsync: mockBatchSyncNotion }),
31+
useBatchSyncWebsite: () => ({ mutateAsync: mockBatchSyncWebsite }),
2832
}))
2933

3034
vi.mock('@langgenius/dify-ui/toast', () => ({
@@ -44,6 +48,7 @@ describe('useDocumentActions', () => {
4448
datasetId: 'ds-1',
4549
selectedIds: ['doc-1', 'doc-2'],
4650
downloadableSelectedIds: ['doc-1'],
51+
syncableSelectedDocs: [],
4752
onUpdate: vi.fn(),
4853
onClearSelection: vi.fn(),
4954
}

‎web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-actions.spec.tsx‎

Lines changed: 155 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ import type { ReactNode } from 'react'
22
import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
33
import { act, renderHook, waitFor } from '@testing-library/react'
44
import { beforeEach, describe, expect, it, vi } from 'vitest'
5-
import { DocumentActionType } from '@/models/datasets'
5+
import { DataSourceType, DocumentActionType } from '@/models/datasets'
6+
import type { SimpleDocumentDetail } from '@/models/datasets'
67
import * as useDocument from '@/service/knowledge/use-document'
78
import { useDocumentActions } from '../use-document-actions'
89

@@ -15,6 +16,26 @@ const mockUseDocumentDisable = vi.mocked(useDocument.useDocumentDisable)
1516
const mockUseDocumentDelete = vi.mocked(useDocument.useDocumentDelete)
1617
const mockUseDocumentBatchRetryIndex = vi.mocked(useDocument.useDocumentBatchRetryIndex)
1718
const mockUseDocumentDownloadZip = vi.mocked(useDocument.useDocumentDownloadZip)
19+
const mockUseBatchSyncNotion = vi.mocked(useDocument.useBatchSyncNotion)
20+
const mockUseBatchSyncWebsite = vi.mocked(useDocument.useBatchSyncWebsite)
21+
22+
type LocalDoc = SimpleDocumentDetail & { percent?: number }
23+
const createMockDoc = (overrides: Partial<LocalDoc> = {}): LocalDoc => ({
24+
id: 'doc1',
25+
name: 'Test.txt',
26+
data_source_type: DataSourceType.FILE,
27+
data_source_info: {},
28+
word_count: 100,
29+
hit_count: 0,
30+
created_at: 0,
31+
position: 1,
32+
doc_form: 'text_model',
33+
enabled: true,
34+
archived: false,
35+
display_status: 'available',
36+
created_from: 'api',
37+
...overrides,
38+
} as LocalDoc)
1839

1940
const createTestQueryClient = () => new QueryClient({
2041
defaultOptions: {
@@ -67,6 +88,8 @@ describe('useDocumentActions', () => {
6788
...createMockMutation(),
6889
isPending: false,
6990
} as unknown as ReturnType<typeof useDocument.useDocumentDownloadZip>)
91+
mockUseBatchSyncNotion.mockReturnValue(createMockMutation() as unknown as ReturnType<typeof useDocument.useBatchSyncNotion>)
92+
mockUseBatchSyncWebsite.mockReturnValue(createMockMutation() as unknown as ReturnType<typeof useDocument.useBatchSyncWebsite>)
7093
})
7194

7295
describe('handleAction', () => {
@@ -80,6 +103,7 @@ describe('useDocumentActions', () => {
80103
datasetId: 'ds1',
81104
selectedIds: ['doc1'],
82105
downloadableSelectedIds: [],
106+
syncableSelectedDocs: [],
83107
onUpdate,
84108
onClearSelection,
85109
}),
@@ -106,6 +130,7 @@ describe('useDocumentActions', () => {
106130
datasetId: 'ds1',
107131
selectedIds: ['doc1'],
108132
downloadableSelectedIds: [],
133+
syncableSelectedDocs: [],
109134
onUpdate,
110135
onClearSelection,
111136
}),
@@ -131,6 +156,7 @@ describe('useDocumentActions', () => {
131156
datasetId: 'ds1',
132157
selectedIds: ['doc1'],
133158
downloadableSelectedIds: [],
159+
syncableSelectedDocs: [],
134160
onUpdate,
135161
onClearSelection,
136162
}),
@@ -158,6 +184,7 @@ describe('useDocumentActions', () => {
158184
datasetId: 'ds1',
159185
selectedIds: ['doc1', 'doc2'],
160186
downloadableSelectedIds: [],
187+
syncableSelectedDocs: [],
161188
onUpdate,
162189
onClearSelection,
163190
}),
@@ -184,6 +211,7 @@ describe('useDocumentActions', () => {
184211
datasetId: 'ds1',
185212
selectedIds: ['doc1'],
186213
downloadableSelectedIds: [],
214+
syncableSelectedDocs: [],
187215
onUpdate,
188216
onClearSelection,
189217
}),
@@ -213,6 +241,7 @@ describe('useDocumentActions', () => {
213241
datasetId: 'ds1',
214242
selectedIds: ['doc1'],
215243
downloadableSelectedIds: ['doc1'],
244+
syncableSelectedDocs: [],
216245
onUpdate: vi.fn(),
217246
onClearSelection: vi.fn(),
218247
}),
@@ -240,6 +269,7 @@ describe('useDocumentActions', () => {
240269
datasetId: 'ds1',
241270
selectedIds: ['doc1', 'doc2'],
242271
downloadableSelectedIds: ['doc1'],
272+
syncableSelectedDocs: [],
243273
onUpdate: vi.fn(),
244274
onClearSelection: vi.fn(),
245275
}),
@@ -269,6 +299,7 @@ describe('useDocumentActions', () => {
269299
datasetId: 'ds1',
270300
selectedIds: [],
271301
downloadableSelectedIds: [],
302+
syncableSelectedDocs: [],
272303
onUpdate: vi.fn(),
273304
onClearSelection: vi.fn(),
274305
}),
@@ -290,6 +321,7 @@ describe('useDocumentActions', () => {
290321
datasetId: 'ds1',
291322
selectedIds: ['doc1'],
292323
downloadableSelectedIds: [],
324+
syncableSelectedDocs: [],
293325
onUpdate,
294326
onClearSelection,
295327
}),
@@ -314,6 +346,7 @@ describe('useDocumentActions', () => {
314346
datasetId: 'ds1',
315347
selectedIds: ['doc1'],
316348
downloadableSelectedIds: [],
349+
syncableSelectedDocs: [],
317350
onUpdate,
318351
onClearSelection,
319352
}),
@@ -342,6 +375,7 @@ describe('useDocumentActions', () => {
342375
datasetId: 'ds1',
343376
selectedIds: ['doc1'],
344377
downloadableSelectedIds: ['doc1'],
378+
syncableSelectedDocs: [],
345379
onUpdate: vi.fn(),
346380
onClearSelection: vi.fn(),
347381
}),
@@ -369,6 +403,7 @@ describe('useDocumentActions', () => {
369403
datasetId: 'ds1',
370404
selectedIds: ['doc1'],
371405
downloadableSelectedIds: ['doc1'],
406+
syncableSelectedDocs: [],
372407
onUpdate: vi.fn(),
373408
onClearSelection: vi.fn(),
374409
}),
@@ -384,6 +419,123 @@ describe('useDocumentActions', () => {
384419
})
385420
})
386421

422+
describe('handleBatchSync', () => {
423+
it('should call notion sync when Notion docs are selected', async () => {
424+
const notionMutateAsync = vi.fn().mockResolvedValue({ result: 'success' })
425+
mockUseBatchSyncNotion.mockReturnValue({
426+
mutateAsync: notionMutateAsync,
427+
} as unknown as ReturnType<typeof useDocument.useBatchSyncNotion>)
428+
429+
const onUpdate = vi.fn()
430+
431+
const { result } = renderHook(
432+
() => useDocumentActions({
433+
datasetId: 'ds1',
434+
selectedIds: ['doc1'],
435+
downloadableSelectedIds: [],
436+
syncableSelectedDocs: [createMockDoc({ id: 'doc1', data_source_type: DataSourceType.NOTION })],
437+
onUpdate,
438+
onClearSelection: vi.fn(),
439+
}),
440+
{ wrapper: createWrapper() },
441+
)
442+
443+
await act(async () => {
444+
await result.current.handleBatchSync()
445+
})
446+
447+
expect(notionMutateAsync).toHaveBeenCalledWith({ datasetId: 'ds1' })
448+
await waitFor(() => expect(onUpdate).toHaveBeenCalled())
449+
})
450+
451+
it('should call website sync when Website docs are selected', async () => {
452+
const websiteMutateAsync = vi.fn().mockResolvedValue({ result: 'success' })
453+
mockUseBatchSyncWebsite.mockReturnValue({
454+
mutateAsync: websiteMutateAsync,
455+
} as unknown as ReturnType<typeof useDocument.useBatchSyncWebsite>)
456+
457+
const onUpdate = vi.fn()
458+
459+
const { result } = renderHook(
460+
() => useDocumentActions({
461+
datasetId: 'ds1',
462+
selectedIds: ['doc2'],
463+
downloadableSelectedIds: [],
464+
syncableSelectedDocs: [createMockDoc({ id: 'doc2', data_source_type: DataSourceType.WEB })],
465+
onUpdate,
466+
onClearSelection: vi.fn(),
467+
}),
468+
{ wrapper: createWrapper() },
469+
)
470+
471+
await act(async () => {
472+
await result.current.handleBatchSync()
473+
})
474+
475+
expect(websiteMutateAsync).toHaveBeenCalledWith({ datasetId: 'ds1' })
476+
await waitFor(() => expect(onUpdate).toHaveBeenCalled())
477+
})
478+
479+
it('should call both sync APIs when mixed Notion and Website docs are selected', async () => {
480+
const notionMutateAsync = vi.fn().mockResolvedValue({ result: 'success' })
481+
const websiteMutateAsync = vi.fn().mockResolvedValue({ result: 'success' })
482+
mockUseBatchSyncNotion.mockReturnValue({
483+
mutateAsync: notionMutateAsync,
484+
} as unknown as ReturnType<typeof useDocument.useBatchSyncNotion>)
485+
mockUseBatchSyncWebsite.mockReturnValue({
486+
mutateAsync: websiteMutateAsync,
487+
} as unknown as ReturnType<typeof useDocument.useBatchSyncWebsite>)
488+
489+
const { result } = renderHook(
490+
() => useDocumentActions({
491+
datasetId: 'ds1',
492+
selectedIds: ['doc1', 'doc2'],
493+
downloadableSelectedIds: [],
494+
syncableSelectedDocs: [
495+
createMockDoc({ id: 'doc1', data_source_type: DataSourceType.NOTION }),
496+
createMockDoc({ id: 'doc2', data_source_type: DataSourceType.WEB }),
497+
],
498+
onUpdate: vi.fn(),
499+
onClearSelection: vi.fn(),
500+
}),
501+
{ wrapper: createWrapper() },
502+
)
503+
504+
await act(async () => {
505+
await result.current.handleBatchSync()
506+
})
507+
508+
expect(notionMutateAsync).toHaveBeenCalledWith({ datasetId: 'ds1' })
509+
expect(websiteMutateAsync).toHaveBeenCalledWith({ datasetId: 'ds1' })
510+
})
511+
512+
it('should call onUpdate even when sync succeeds', async () => {
513+
mockUseBatchSyncNotion.mockReturnValue({
514+
mutateAsync: vi.fn().mockResolvedValue({ result: 'success' }),
515+
} as unknown as ReturnType<typeof useDocument.useBatchSyncNotion>)
516+
517+
const onUpdate = vi.fn()
518+
519+
const { result } = renderHook(
520+
() => useDocumentActions({
521+
datasetId: 'ds1',
522+
selectedIds: ['doc1'],
523+
downloadableSelectedIds: [],
524+
syncableSelectedDocs: [createMockDoc({ id: 'doc1', data_source_type: DataSourceType.NOTION })],
525+
onUpdate,
526+
onClearSelection: vi.fn(),
527+
}),
528+
{ wrapper: createWrapper() },
529+
)
530+
531+
await act(async () => {
532+
await result.current.handleBatchSync()
533+
})
534+
535+
await waitFor(() => expect(onUpdate).toHaveBeenCalled())
536+
})
537+
})
538+
387539
describe('all action types', () => {
388540
it('should handle summary action', async () => {
389541
mockMutateAsync.mockResolvedValue({ result: 'success' })
@@ -394,6 +546,7 @@ describe('useDocumentActions', () => {
394546
datasetId: 'ds1',
395547
selectedIds: ['doc1'],
396548
downloadableSelectedIds: [],
549+
syncableSelectedDocs: [],
397550
onUpdate,
398551
onClearSelection: vi.fn(),
399552
}),
@@ -419,6 +572,7 @@ describe('useDocumentActions', () => {
419572
datasetId: 'ds1',
420573
selectedIds: ['doc1'],
421574
downloadableSelectedIds: [],
575+
syncableSelectedDocs: [],
422576
onUpdate,
423577
onClearSelection: vi.fn(),
424578
}),

0 commit comments

Comments
 (0)