Skip to content

Commit e849eaa

Browse files
feat(datasource/dingtalk): 保留来源目录结构(开关控制,默认关) (#4422)
1 parent 0aa1a51 commit e849eaa

5 files changed

Lines changed: 787 additions & 14 deletions

File tree

‎internal/datasource/connector/dingtalk/connector.go‎

Lines changed: 138 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,12 @@ type syncScope struct {
647647
Reference resourceReference
648648
StartNodeID string
649649
Document *node
650+
// FolderPath is the folder hierarchy between the workspace root and the
651+
// scope's own node, root-first, with the scope's own folder name last. It
652+
// anchors every document the scope yields, so selecting a subfolder and
653+
// selecting the whole workspace file the same document under the same
654+
// folder instead of under two.
655+
FolderPath []string
650656
}
651657

652658
func (s syncScope) contains(candidate syncScope) bool {
@@ -688,6 +694,9 @@ func (c *Connector) sync(
688694

689695
api := c.api(cfg)
690696
settings := parseDocumentSettings(dataSourceConfig)
697+
if settings.PreserveFolderStructure {
698+
logger.Infof(ctx, "[DingTalk] source folder structure is encoded into document file names")
699+
}
691700
workspaces, err := api.listWorkspaces(ctx)
692701
if err != nil {
693702
return nil, nil, err
@@ -696,6 +705,13 @@ func (c *Connector) sync(
696705
if err != nil {
697706
return nil, nil, err
698707
}
708+
// The workspace name is the first folder segment of every encoded path. It
709+
// is read from the listing the sync already made, so the leading level
710+
// costs no extra request.
711+
workspaceFolderNames := make(map[string]string, len(workspaces))
712+
for _, item := range workspaces {
713+
workspaceFolderNames[item.ID] = workspaceFolderName(item)
714+
}
699715

700716
next := &cursorState{
701717
Version: cursorVersion,
@@ -757,7 +773,8 @@ func (c *Connector) sync(
757773
synced := 0
758774
failed := 0
759775

760-
for _, document := range documents {
776+
for _, scanned := range documents {
777+
document := scanned.node
761778
if document.ID == "" {
762779
continue
763780
}
@@ -770,7 +787,13 @@ func (c *Connector) sync(
770787
continue
771788
}
772789

773-
item, err := readDocument(ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document)
790+
folders := documentFolders(
791+
settings, workspaceFolderNames[scope.Reference.WorkspaceID],
792+
scope.FolderPath, scanned.folders,
793+
)
794+
item, err := readDocument(
795+
ctx, api, scope.ResourceID, scope.Reference.WorkspaceID, document, folders,
796+
)
774797
if err != nil {
775798
if isContextError(err) {
776799
return nil, nil, err
@@ -894,6 +917,7 @@ func resolveSyncScopes(
894917
}, nil
895918
}
896919

920+
var ancestorFolders []string
897921
parentNodeID := rootNodeID
898922
for _, ancestorID := range ref.Ancestors {
899923
children, err := listChildren(parentNodeID)
@@ -908,6 +932,11 @@ func resolveSyncScopes(
908932
if ancestor.WorkspaceID != "" && ancestor.WorkspaceID != ref.WorkspaceID {
909933
return syncScope{}, fmt.Errorf("DingTalk ancestor %q belongs to a different workspace", ancestorID)
910934
}
935+
// The name is read off a listing this resolution has to make
936+
// anyway, so anchoring the path at the workspace root costs no
937+
// extra request. A selection saved without ancestors yields a
938+
// shallower path rather than an error.
939+
ancestorFolders = append(ancestorFolders, ancestor.title())
911940
parentNodeID = ancestor.ID
912941
}
913942
children, err := listChildren(parentNodeID)
@@ -926,11 +955,13 @@ func resolveSyncScopes(
926955
case selectedNode.isFolder():
927956
return syncScope{
928957
ResourceID: canonicalID, Reference: ref, StartNodeID: selectedNode.ID,
958+
FolderPath: appendFolder(ancestorFolders, selectedNode.title()),
929959
}, nil
930960
case selectedNode.isIngestible(settings):
931961
document := selectedNode
932962
return syncScope{
933963
ResourceID: canonicalID, Reference: ref, Document: &document,
964+
FolderPath: ancestorFolders,
934965
}, nil
935966
default:
936967
return syncScope{}, fmt.Errorf("DingTalk node %q is not a supported online document or folder",
@@ -990,6 +1021,26 @@ func childByID(children []node, nodeID string) (node, bool) {
9901021
return node{}, false
9911022
}
9921023

1024+
// scannedDocument pairs an ingestible node with the folder hierarchy the walk
1025+
// found it under, relative to the scan's root node. The hierarchy is carried
1026+
// alongside the node rather than recomputed later: the breadth-first walk is
1027+
// the only place that knows which parent listed the node.
1028+
type scannedDocument struct {
1029+
node
1030+
// folders holds the folder titles between the scan root and the document,
1031+
// root-first. A document sitting directly in the scan root has none.
1032+
folders []string
1033+
}
1034+
1035+
// appendFolder returns parent plus one folder title. The result is always a
1036+
// fresh slice: appending in place would let one sibling's name overwrite
1037+
// another's inside a shared backing array.
1038+
func appendFolder(parent []string, name string) []string {
1039+
folders := make([]string, 0, len(parent)+1)
1040+
folders = append(folders, parent...)
1041+
return append(folders, name)
1042+
}
1043+
9931044
// scanScope lists every ingestible document in one scope, together with the
9941045
// nodes it saw but cannot ingest. The skipped nodes are returned rather than
9951046
// discarded so the caller can report them: a full sync must never look clean
@@ -999,26 +1050,34 @@ func scanScope(
9991050
api dingTalkAPI,
10001051
scope syncScope,
10011052
settings documentSettings,
1002-
) ([]node, []node, error) {
1053+
) ([]scannedDocument, []node, error) {
10031054
if scope.Document != nil {
1004-
return []node{*scope.Document}, nil, nil
1055+
// A single-document scope has no tree to walk; the scope's own
1056+
// FolderPath already carries the folders above it.
1057+
return []scannedDocument{{node: *scope.Document}}, nil, nil
10051058
}
10061059
return scanWorkspace(ctx, api, scope.Reference.WorkspaceID, scope.StartNodeID, settings)
10071060
}
10081061

10091062
// scanWorkspace walks a workspace subtree breadth-first. Folders are traversal
1010-
// only; every file is either an ingestible document or a skip.
1063+
// only; every file is either an ingestible document or a skip. Each document is
1064+
// reported with the folder titles the walk descended through to reach it, so
1065+
// the caller can mirror the source hierarchy in the knowledge base.
10111066
func scanWorkspace(
10121067
ctx context.Context,
10131068
api dingTalkAPI,
10141069
workspaceID string,
10151070
rootNodeID string,
10161071
settings documentSettings,
1017-
) ([]node, []node, error) {
1072+
) ([]scannedDocument, []node, error) {
10181073
queue := []string{rootNodeID}
10191074
visitedParents := make(map[string]struct{})
10201075
seenNodes := make(map[string]struct{})
1021-
var documents []node
1076+
// foldersByParent maps a node ID to the folder titles between the scan root
1077+
// and that node, root-first. Only nodes the walk descends through get an
1078+
// entry; the scan root itself has none.
1079+
foldersByParent := map[string][]string{rootNodeID: nil}
1080+
var documents []scannedDocument
10221081
var skipped []node
10231082

10241083
for len(queue) > 0 {
@@ -1052,14 +1111,18 @@ func scanWorkspace(
10521111
}
10531112
switch {
10541113
case child.isIngestible(settings):
1055-
documents = append(documents, child)
1114+
documents = append(documents, scannedDocument{
1115+
node: child,
1116+
folders: foldersByParent[parentID],
1117+
})
10561118
case child.isFolder():
10571119
// Containers hold no content of their own, so they are never
10581120
// reported as skipped.
10591121
default:
10601122
skipped = append(skipped, child)
10611123
}
10621124
if child.isFolder() || child.HasChildren {
1125+
foldersByParent[child.ID] = appendFolder(foldersByParent[parentID], child.title())
10631126
queue = append(queue, child.ID)
10641127
}
10651128
}
@@ -1141,13 +1204,16 @@ func warnUnknownBlockTypes(ctx context.Context, document node, rendered renderRe
11411204

11421205
// readDocument turns one supported wiki node into a FetchedItem. Native adoc
11431206
// documents are rendered from their blocks, and uploaded files are transferred
1144-
// byte-for-byte so WeKnora's own parsers can read them.
1207+
// byte-for-byte so WeKnora's own parsers can read them. folders is the source
1208+
// folder hierarchy to encode in the item's FileName, or nil to keep the
1209+
// historical flat name.
11451210
func readDocument(
11461211
ctx context.Context,
11471212
api dingTalkAPI,
11481213
sourceResourceID string,
11491214
workspaceID string,
11501215
document node,
1216+
folders []string,
11511217
) (types.FetchedItem, error) {
11521218
switch {
11531219
case document.isOnlineDocument():
@@ -1160,14 +1226,71 @@ func readDocument(
11601226
// both the request context and the node identity are known. Uploaded
11611227
// files take the default branch and have no rendering step to lose.
11621228
warnUnknownBlockTypes(ctx, document, rendered)
1163-
return fetchedDocument(sourceResourceID, workspaceID, document, rendered), nil
1229+
return fetchedDocument(sourceResourceID, workspaceID, document, rendered, folders), nil
11641230
default:
11651231
data, err := api.downloadDocument(ctx, document.ID)
11661232
if err != nil {
11671233
return types.FetchedItem{}, err
11681234
}
1169-
return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data), nil
1235+
return fetchedBinaryDocument(sourceResourceID, workspaceID, document, data, folders), nil
1236+
}
1237+
}
1238+
1239+
// documentFolders assembles the folder segments a document's FileName encodes:
1240+
// the workspace, the folders between the workspace root and the scope's own
1241+
// node, then the folders the scan descended through.
1242+
//
1243+
// It returns nil when the switch is off, which is what keeps the emitted
1244+
// FileName byte-for-byte identical to previous releases for every data source
1245+
// that never turned it on.
1246+
func documentFolders(
1247+
settings documentSettings,
1248+
workspaceName string,
1249+
scopeFolders []string,
1250+
scanFolders []string,
1251+
) []string {
1252+
if !settings.PreserveFolderStructure {
1253+
return nil
11701254
}
1255+
folders := make([]string, 0, len(scopeFolders)+len(scanFolders)+1)
1256+
if workspaceName != "" {
1257+
folders = append(folders, workspaceName)
1258+
}
1259+
folders = append(folders, scopeFolders...)
1260+
return append(folders, scanFolders...)
1261+
}
1262+
1263+
// workspaceFolderName names a workspace the way the resource picker does: the
1264+
// workspace name, falling back to its ID, which is what an operator sees when a
1265+
// tenant exposes no name.
1266+
func workspaceFolderName(item workspace) string {
1267+
if name := strings.TrimSpace(item.Name); name != "" {
1268+
return name
1269+
}
1270+
return item.ID
1271+
}
1272+
1273+
// buildFolderFileName prepends the folder hierarchy to a document's file name.
1274+
//
1275+
// folder_path downstream is derived from FileName alone
1276+
// (SplitKnowledgeRelativePath → NormalizeKnowledgeFolderPath), so the hierarchy
1277+
// has to be encoded here as a relative path.
1278+
//
1279+
// Every level is sanitised on its own, as the Yuque connector's own folder
1280+
// encoding does, so a "/" inside a folder name cannot silently create an extra
1281+
// nesting level. Folder names the downstream normaliser rejects outright — ".",
1282+
// "..", an empty name — drop their level instead of failing the sync. An empty
1283+
// folder list, which is what the default settings produce, returns the file
1284+
// name unchanged.
1285+
func buildFolderFileName(folders []string, fileName string) string {
1286+
if len(folders) == 0 {
1287+
return fileName
1288+
}
1289+
segments := make([]string, 0, len(folders))
1290+
for _, folder := range folders {
1291+
segments = append(segments, datasource.SanitizeFileName(folder))
1292+
}
1293+
return strings.Join(segments, "/") + "/" + fileName
11711294
}
11721295

11731296
// documentMetadata describes a wiki node the same way for rendered and
@@ -1195,6 +1318,7 @@ func fetchedDocument(
11951318
workspaceID string,
11961319
document node,
11971320
rendered renderResult,
1321+
folders []string,
11981322
) types.FetchedItem {
11991323
metadata := documentMetadata(workspaceID, document)
12001324
if len(rendered.UnknownTypes) > 0 {
@@ -1205,7 +1329,7 @@ func fetchedDocument(
12051329
Title: document.title(),
12061330
Content: []byte(rendered.Markdown),
12071331
ContentType: "text/markdown",
1208-
FileName: renderedDocumentFileName(document.title()),
1332+
FileName: buildFolderFileName(folders, renderedDocumentFileName(document.title())),
12091333
URL: documentURL(document),
12101334
UpdatedAt: document.modifiedAt(),
12111335
Metadata: metadata,
@@ -1234,6 +1358,7 @@ func fetchedBinaryDocument(
12341358
workspaceID string,
12351359
document node,
12361360
data []byte,
1361+
folders []string,
12371362
) types.FetchedItem {
12381363
contentType, ok := document.binaryContentType()
12391364
if !ok {
@@ -1244,7 +1369,7 @@ func fetchedBinaryDocument(
12441369
Title: document.title(),
12451370
Content: data,
12461371
ContentType: contentType,
1247-
FileName: binaryDocumentFileName(document),
1372+
FileName: buildFolderFileName(folders, binaryDocumentFileName(document)),
12481373
URL: documentURL(document),
12491374
UpdatedAt: document.modifiedAt(),
12501375
Metadata: documentMetadata(workspaceID, document),

0 commit comments

Comments
 (0)