Skip to content

Commit ba586c0

Browse files
ZhengYa-0110ZhengYa-0110
committed
feat: supports batch update when syncing resource
1 parent ecaf35d commit ba586c0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1006
-58
lines changed

‎server/controller/recorder/common/metadata.go

+4
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ func (m *Metadata) Copy() *Metadata {
5151
}
5252
}
5353

54+
func (m *Metadata) GetDB() *mysql.DB {
55+
return m.DB
56+
}
57+
5458
func (m *Metadata) GetORGID() int {
5559
return m.ORGID
5660
}

‎server/controller/recorder/db/operator.go

+31-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ package db
1919
import (
2020
"time"
2121

22+
"gorm.io/gorm/clause"
23+
2224
"github.com/deepflowio/deepflow/server/controller/common"
2325
"github.com/deepflowio/deepflow/server/controller/db/mysql"
2426
rcommon "github.com/deepflowio/deepflow/server/controller/recorder/common"
@@ -34,10 +36,14 @@ type Operator[MPT constraint.MySQLModelPtr[MT], MT constraint.MySQLModel] interf
3436
AddBatch(dbItems []*MT) ([]*MT, bool)
3537
// 更新数据
3638
Update(lcuuid string, updateInfo map[string]interface{}) (*MT, bool)
39+
// 批量更新数据
40+
UpdateBatch([]*MT, map[string]map[string]interface{}) ([]*MT, bool)
3741
// 批量删除数据
3842
DeleteBatch(lcuuids []string) ([]*MT, bool)
3943

4044
GetSoftDelete() bool
45+
SetFieldsToUpdate([]string)
46+
SetMetadata(*rcommon.Metadata)
4147
}
4248

4349
type OperatorBase[MPT constraint.MySQLModelPtr[MT], MT constraint.MySQLModel] struct {
@@ -47,6 +53,7 @@ type OperatorBase[MPT constraint.MySQLModelPtr[MT], MT constraint.MySQLModel] st
4753
softDelete bool
4854
allocateID bool
4955
fieldsNeededAfterCreate []string // fields needed to be used after create
56+
fieldsToUpdate []string // fields needed to be updated
5057
}
5158

5259
func newOperatorBase[MPT constraint.MySQLModelPtr[MT], MT constraint.MySQLModel](resourceTypeName string, softDelete, allocateID bool) OperatorBase[MPT, MT] {
@@ -57,9 +64,14 @@ func newOperatorBase[MPT constraint.MySQLModelPtr[MT], MT constraint.MySQLModel]
5764
}
5865
}
5966

60-
func (o *OperatorBase[MPT, MT]) SetMetadata(md *rcommon.Metadata) Operator[MPT, MT] {
67+
func (o *OperatorBase[MPT, MT]) SetFieldsToUpdate(fs []string) {
68+
o.fieldsToUpdate = fs
69+
return
70+
}
71+
72+
func (o *OperatorBase[MPT, MT]) SetMetadata(md *rcommon.Metadata) {
6173
o.metadata = md
62-
return o
74+
return
6375
}
6476

6577
func (o *OperatorBase[MPT, MT]) GetSoftDelete() bool {
@@ -117,6 +129,23 @@ func (o *OperatorBase[MPT, MT]) Update(lcuuid string, updateInfo map[string]inte
117129
return dbItem, true
118130
}
119131

132+
func (o *OperatorBase[MPT, MT]) UpdateBatch(dbItems []*MT, lcuuidToUpdateInfo map[string]map[string]interface{}) ([]*MT, bool) {
133+
startTime := time.Now()
134+
err := o.metadata.DB.Clauses(clause.OnConflict{
135+
Columns: []clause.Column{{Name: "lcuuid"}},
136+
DoUpdates: clause.AssignmentColumns(o.fieldsToUpdate),
137+
}).Create(&dbItems).Error
138+
log.Infof("update batch cost: %v", time.Since(startTime))
139+
if err != nil {
140+
log.Errorf("%s batch failed: %v", rcommon.LogUpdate(o.resourceTypeName), err.Error(), o.metadata.LogPrefixes)
141+
return nil, false
142+
}
143+
for lcuuid, updateInfo := range lcuuidToUpdateInfo {
144+
log.Infof("%s (lcuuid: %s, detail: %+v) success", rcommon.LogUpdate(o.resourceTypeName), lcuuid, updateInfo, o.metadata.LogPrefixes)
145+
}
146+
return dbItems, true
147+
}
148+
120149
func (o *OperatorBase[MPT, MT]) DeleteBatch(lcuuids []string) ([]*MT, bool) {
121150
var deletedItems []*MT
122151
err := o.metadata.DB.Where("lcuuid IN ?", lcuuids).Find(&deletedItems).Error

‎server/controller/recorder/pubsub/message/constraint/constraint.go

+9-3
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,16 @@ type UpdatePtr[T Update] interface {
4545

4646
SetFields(interface{})
4747
GetFields() interface{} // return *FieldsUpdate
48-
SetDiffBase(interface{})
49-
GetDiffBase() interface{} // return *constraint.DiffBase
50-
SetCloudItem(interface{})
48+
SetFieldsMap(map[string]interface{})
49+
GetFieldsMap() map[string]interface{}
50+
SetDiffBase(interface{}) // TODO remove this
51+
GetDiffBase() interface{} // return *constraint.DiffBase
52+
SetCloudItem(interface{}) // TODO remove this
5153
GetCloudItem() interface{} // return *constraint.CloudModel
54+
// SetNewMySQL(interface{}) // TODO: remove this
55+
// GetNewMySQL() interface{}
56+
// SetOldMySQL(interface{})
57+
// GetOldMySQL() interface{}
5258
}
5359

5460
// Update是所有资源更新消息的泛型约束

‎server/controller/recorder/pubsub/message/update.go

+22-10
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,27 @@ func (k *Key) GetLcuuid() string {
4747
}
4848

4949
type Fields[T any] struct {
50-
data *T
50+
structData *T
51+
mapData map[string]interface{}
5152
}
5253

5354
func (f *Fields[T]) SetFields(data interface{}) {
54-
f.data = data.(*T)
55+
f.structData = data.(*T)
5556
}
5657

5758
func (f *Fields[T]) GetFields() interface{} {
58-
return f.data
59+
return f.structData
5960
}
6061

61-
type fieldDetail[T any] struct {
62+
func (f *Fields[T]) SetFieldsMap(data map[string]interface{}) {
63+
f.mapData = data
64+
}
65+
66+
func (f *Fields[T]) GetFieldsMap() map[string]interface{} {
67+
return f.mapData
68+
}
69+
70+
type fieldDetail[T any] struct { // TODO add name
6271
different bool
6372
new T
6473
old T
@@ -102,20 +111,20 @@ type MySQLData[MT constraint.MySQLModel] struct {
102111
old *MT
103112
}
104113

105-
func (m *MySQLData[MT]) GetNewMySQL() *MT {
114+
func (m *MySQLData[MT]) GetNewMySQL() interface{} {
106115
return m.new
107116
}
108117

109-
func (m *MySQLData[MT]) SetNewMySQL(new *MT) {
110-
m.new = new
118+
func (m *MySQLData[MT]) SetNewMySQL(new interface{}) {
119+
m.new = new.(*MT)
111120
}
112121

113-
func (m *MySQLData[MT]) GetOldMySQL() *MT {
122+
func (m *MySQLData[MT]) GetOldMySQL() interface{} {
114123
return m.old
115124
}
116125

117-
func (m *MySQLData[MT]) SetOldMySQL(old *MT) {
118-
m.old = old
126+
func (m *MySQLData[MT]) SetOldMySQL(old interface{}) {
127+
m.old = old.(*MT)
119128
}
120129

121130
type DiffBase[DT constraint.DiffBase] struct {
@@ -338,6 +347,7 @@ type VInterfaceFieldsUpdate struct {
338347
Name fieldDetail[string]
339348
TapMac fieldDetail[string]
340349
Type fieldDetail[int]
350+
DeviceID fieldDetail[int]
341351
NetnsID fieldDetail[uint32]
342352
VTapID fieldDetail[uint32]
343353
NetworkID fieldDetail[int]
@@ -743,6 +753,8 @@ type ProcessFieldsUpdate struct {
743753
Key
744754
Name fieldDetail[string]
745755
ContainerID fieldDetail[string]
756+
DeviceType fieldDetail[int]
757+
DeviceID fieldDetail[int]
746758
OSAPPTags fieldDetail[string]
747759
VMID fieldDetail[int]
748760
VPCID fieldDetail[int]

‎server/controller/recorder/updater/az.go

+18-1
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,13 @@ func NewAZ(wholeCache *cache.Cache, cloudData []cloudmodel.AZ) *AZ {
5959
](
6060
ctrlrcommon.RESOURCE_TYPE_AZ_EN,
6161
wholeCache,
62-
db.NewAZ().SetMetadata(wholeCache.GetMetadata()),
62+
db.NewAZ(),
6363
wholeCache.DiffBaseDataSet.AZs,
6464
cloudData,
6565
),
6666
}
6767
updater.dataGenerator = updater
68+
updater.initDBOperator()
6869
return updater
6970
}
7071

@@ -84,6 +85,10 @@ func (z *AZ) generateDBItemToAdd(cloudItem *cloudmodel.AZ) (*mysqlmodel.AZ, bool
8485
return dbItem, true
8586
}
8687

88+
func (z *AZ) getUpdateableFields() []string {
89+
return []string{"name", "label", "region"}
90+
}
91+
8792
func (z *AZ) generateUpdateInfo(diffBase *diffbase.AZ, cloudItem *cloudmodel.AZ) (*message.AZFieldsUpdate, map[string]interface{}, bool) {
8893
structInfo := new(message.AZFieldsUpdate)
8994
mapInfo := make(map[string]interface{})
@@ -102,3 +107,15 @@ func (z *AZ) generateUpdateInfo(diffBase *diffbase.AZ, cloudItem *cloudmodel.AZ)
102107

103108
return structInfo, mapInfo, len(mapInfo) > 0
104109
}
110+
111+
func (z *AZ) setUpdatedFields(dbItem *mysqlmodel.AZ, updateInfo *message.AZFieldsUpdate) {
112+
if updateInfo.Name.IsDifferent() {
113+
dbItem.Name = updateInfo.Name.GetNew()
114+
}
115+
if updateInfo.Label.IsDifferent() {
116+
dbItem.Label = updateInfo.Label.GetNew()
117+
}
118+
if updateInfo.RegionLcuuid.IsDifferent() {
119+
dbItem.Region = updateInfo.RegionLcuuid.GetNew()
120+
}
121+
}

‎server/controller/recorder/updater/cen.go

+15-1
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,13 @@ func NewCEN(wholeCache *cache.Cache, cloudData []cloudmodel.CEN) *CEN {
6060
](
6161
ctrlrcommon.RESOURCE_TYPE_CEN_EN,
6262
wholeCache,
63-
db.NewCEN().SetMetadata(wholeCache.GetMetadata()),
63+
db.NewCEN(),
6464
wholeCache.DiffBaseDataSet.CENs,
6565
cloudData,
6666
),
6767
}
6868
updater.dataGenerator = updater
69+
updater.initDBOperator()
6970
return updater
7071
}
7172

@@ -97,6 +98,10 @@ func (c *CEN) generateDBItemToAdd(cloudItem *cloudmodel.CEN) (*mysqlmodel.CEN, b
9798
return dbItem, true
9899
}
99100

101+
func (c *CEN) getUpdateableFields() []string {
102+
return []string{"name", "epc_ids"}
103+
}
104+
100105
func (c *CEN) generateUpdateInfo(diffBase *diffbase.CEN, cloudItem *cloudmodel.CEN) (*message.CENFieldsUpdate, map[string]interface{}, bool) {
101106
structInfo := new(message.CENFieldsUpdate)
102107
mapInfo := make(map[string]interface{})
@@ -124,3 +129,12 @@ func (c *CEN) generateUpdateInfo(diffBase *diffbase.CEN, cloudItem *cloudmodel.C
124129

125130
return structInfo, mapInfo, len(mapInfo) > 0
126131
}
132+
133+
func (c *CEN) setUpdatedFields(dbItem *mysqlmodel.CEN, updateInfo *message.CENFieldsUpdate) {
134+
if updateInfo.Name.IsDifferent() {
135+
dbItem.Name = updateInfo.Name.GetNew()
136+
}
137+
if updateInfo.VPCIDs.IsDifferent() {
138+
dbItem.VPCIDs = rcommon.IntSliceToString(updateInfo.VPCIDs.GetNew())
139+
}
140+
}

‎server/controller/recorder/updater/dhcp_port.go

+21-1
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,13 @@ func NewDHCPPort(wholeCache *cache.Cache, cloudData []cloudmodel.DHCPPort) *DHCP
5959
](
6060
ctrlrcommon.RESOURCE_TYPE_DHCP_PORT_EN,
6161
wholeCache,
62-
db.NewDHCPPort().SetMetadata(wholeCache.GetMetadata()),
62+
db.NewDHCPPort(),
6363
wholeCache.DiffBaseDataSet.DHCPPorts,
6464
cloudData,
6565
),
6666
}
6767
updater.dataGenerator = updater
68+
updater.initDBOperator()
6869
return updater
6970
}
7071

@@ -93,6 +94,10 @@ func (p *DHCPPort) generateDBItemToAdd(cloudItem *cloudmodel.DHCPPort) (*mysqlmo
9394
return dbItem, true
9495
}
9596

97+
func (p *DHCPPort) getUpdateableFields() []string {
98+
return []string{"epc_id", "name", "region", "az"}
99+
}
100+
96101
func (p *DHCPPort) generateUpdateInfo(diffBase *diffbase.DHCPPort, cloudItem *cloudmodel.DHCPPort) (*message.DHCPPortFieldsUpdate, map[string]interface{}, bool) {
97102
structInfo := new(message.DHCPPortFieldsUpdate)
98103
mapInfo := make(map[string]interface{})
@@ -124,3 +129,18 @@ func (p *DHCPPort) generateUpdateInfo(diffBase *diffbase.DHCPPort, cloudItem *cl
124129

125130
return structInfo, mapInfo, len(mapInfo) > 0
126131
}
132+
133+
func (p *DHCPPort) setUpdatedFields(dbItem *mysqlmodel.DHCPPort, updateInfo *message.DHCPPortFieldsUpdate) {
134+
if updateInfo.VPCID.IsDifferent() {
135+
dbItem.VPCID = updateInfo.VPCID.GetNew()
136+
}
137+
if updateInfo.Name.IsDifferent() {
138+
dbItem.Name = updateInfo.Name.GetNew()
139+
}
140+
if updateInfo.RegionLcuuid.IsDifferent() {
141+
dbItem.Region = updateInfo.RegionLcuuid.GetNew()
142+
}
143+
if updateInfo.AZLcuuid.IsDifferent() {
144+
dbItem.AZ = updateInfo.AZLcuuid.GetNew()
145+
}
146+
}

‎server/controller/recorder/updater/floating_ip.go

+15-1
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,13 @@ func NewFloatingIP(wholeCache *cache.Cache, cloudData []cloudmodel.FloatingIP) *
6060
](
6161
ctrlrcommon.RESOURCE_TYPE_FLOATING_IP_EN,
6262
wholeCache,
63-
db.NewFloatingIP().SetMetadata(wholeCache.GetMetadata()),
63+
db.NewFloatingIP(),
6464
wholeCache.DiffBaseDataSet.FloatingIPs,
6565
cloudData,
6666
),
6767
}
6868
updater.dataGenerator = updater
69+
updater.initDBOperator()
6970
return updater
7071
}
7172

@@ -118,6 +119,10 @@ func (f *FloatingIP) generateDBItemToAdd(cloudItem *cloudmodel.FloatingIP) (*mys
118119
return dbItem, true
119120
}
120121

122+
func (f *FloatingIP) getUpdateableFields() []string {
123+
return []string{"epc_id", "region"}
124+
}
125+
121126
func (f *FloatingIP) generateUpdateInfo(diffBase *diffbase.FloatingIP, cloudItem *cloudmodel.FloatingIP) (*message.FloatingIPFieldsUpdate, map[string]interface{}, bool) {
122127
structInfo := new(message.FloatingIPFieldsUpdate)
123128
mapInfo := make(map[string]interface{})
@@ -140,3 +145,12 @@ func (f *FloatingIP) generateUpdateInfo(diffBase *diffbase.FloatingIP, cloudItem
140145
}
141146
return structInfo, mapInfo, len(mapInfo) > 0
142147
}
148+
149+
func (f *FloatingIP) setUpdatedFields(dbItem *mysqlmodel.FloatingIP, updateInfo *message.FloatingIPFieldsUpdate) {
150+
if updateInfo.VPCID.IsDifferent() {
151+
dbItem.VPCID = updateInfo.VPCID.GetNew()
152+
}
153+
if updateInfo.RegionLcuuid.IsDifferent() {
154+
dbItem.Region = updateInfo.RegionLcuuid.GetNew()
155+
}
156+
}

0 commit comments

Comments
 (0)