Skip to content

Commit 1e215b1

Browse files
huanghaoyuanhhyhuanghaoyuanhhy
authored
filter: unify filter implementation between backup and restore (#917)
Add ParseDBFilter, ParseCollFilter, InferFilterRuleType, InferMapperRuleType to internal/filter package. Simplify restore.Plan to use filter.Filter instead of separate DB/Coll maps, making it consistent with backup module. Signed-off-by: huanghaoyuanhhy <haoyuan.huang@zilliz.com>
1 parent ce600ba commit 1e215b1

8 files changed

Lines changed: 209 additions & 511 deletions

File tree

‎cmd/restore/restore.go‎

Lines changed: 29 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"encoding/json"
66
"errors"
77
"fmt"
8-
"regexp"
98
"strings"
109
"time"
1110

@@ -16,6 +15,7 @@ import (
1615
"github.com/zilliztech/milvus-backup/core/paramtable"
1716
"github.com/zilliztech/milvus-backup/core/restore"
1817
"github.com/zilliztech/milvus-backup/internal/client/milvus"
18+
"github.com/zilliztech/milvus-backup/internal/filter"
1919
"github.com/zilliztech/milvus-backup/internal/log"
2020
"github.com/zilliztech/milvus-backup/internal/meta"
2121
"github.com/zilliztech/milvus-backup/internal/namespace"
@@ -132,49 +132,11 @@ func (o *options) toOption() *restore.Option {
132132
}
133133
}
134134

135-
func (o *options) toTaskFilter() (map[string]struct{}, map[string]restore.CollFilter, error) {
136-
if o.filter == "" {
137-
return nil, nil, nil
138-
}
139-
140-
filterStrs := strings.Split(o.filter, ",")
141-
dbFilter := make(map[string]struct{})
142-
collFilter := make(map[string]restore.CollFilter)
143-
144-
for _, filterStr := range filterStrs {
145-
ruleType, err := inferFilterRuleType(filterStr)
146-
if err != nil {
147-
return nil, nil, err
148-
}
149-
150-
switch ruleType {
151-
case 1:
152-
db := filterStr[:len(filterStr)-2]
153-
dbFilter[db] = struct{}{}
154-
collFilter[db] = restore.CollFilter{AllowAll: true}
155-
case 2, 3:
156-
ns, err := namespace.Parse(filterStr)
157-
if err != nil {
158-
return nil, nil, fmt.Errorf("invalid collection name %s", filterStr)
159-
}
160-
161-
dbFilter[ns.DBName()] = struct{}{}
162-
if _, ok := collFilter[ns.DBName()]; !ok {
163-
collFilter[ns.DBName()] = restore.CollFilter{CollName: make(map[string]struct{})}
164-
}
165-
collFilter[ns.DBName()].CollName[ns.CollName()] = struct{}{}
166-
case 4:
167-
db := filterStr[:len(filterStr)-1]
168-
dbFilter[db] = struct{}{}
169-
default:
170-
return nil, nil, fmt.Errorf("invalid filter rule: %s", filterStr)
171-
}
172-
}
173-
174-
return dbFilter, collFilter, nil
135+
func (o *options) toTaskFilter() (filter.Filter, error) {
136+
return filter.Parse(o.filter)
175137
}
176138

177-
func (o *options) toBackupFilter() (map[string]struct{}, map[string]restore.CollFilter, error) {
139+
func (o *options) toBackupFilter() (filter.Filter, error) {
178140
if o.collectionNames != "" {
179141
return o.collectionNamesToBackupFilter()
180142
}
@@ -187,7 +149,7 @@ func (o *options) toBackupFilter() (map[string]struct{}, map[string]restore.Coll
187149
return o.dbCollectionsToBackupFilter()
188150
}
189151

190-
return nil, nil, nil
152+
return filter.Filter{}, nil
191153
}
192154

193155
func (o *options) toCollMapper() (restore.CollMapper, error) {
@@ -203,7 +165,7 @@ func (o *options) toCollMapper() (restore.CollMapper, error) {
203165
}
204166

205167
func (o *options) toPlan() (*restore.Plan, error) {
206-
dbFilter, collFilter, err := o.toBackupFilter()
168+
backupFilter, err := o.toBackupFilter()
207169
if err != nil {
208170
return nil, err
209171
}
@@ -213,76 +175,71 @@ func (o *options) toPlan() (*restore.Plan, error) {
213175
return nil, err
214176
}
215177

216-
dbTaskFilter, collTaskFilter, err := o.toTaskFilter()
178+
taskFilter, err := o.toTaskFilter()
217179
if err != nil {
218180
return nil, err
219181
}
220182

221183
return &restore.Plan{
222-
DBBackupFilter: dbFilter,
223-
CollBackupFilter: collFilter,
184+
BackupFilter: backupFilter,
224185

225186
// not support db mapping now
226187
CollMapper: collMapper,
227188

228-
DBTaskFilter: dbTaskFilter,
229-
CollTaskFilter: collTaskFilter,
189+
TaskFilter: taskFilter,
230190
}, nil
231191
}
232192

233-
func (o *options) collectionNamesToBackupFilter() (map[string]struct{}, map[string]restore.CollFilter, error) {
234-
dbFilter := make(map[string]struct{})
235-
collFilter := make(map[string]restore.CollFilter)
193+
func (o *options) collectionNamesToBackupFilter() (filter.Filter, error) {
194+
collFilter := make(map[string]filter.CollFilter)
236195

237196
nsStrs := strings.Split(o.collectionNames, ",")
238197
for _, nsStr := range nsStrs {
239198
ns, err := namespace.Parse(nsStr)
240199
if err != nil {
241-
return nil, nil, fmt.Errorf("invalid collection name %s", nsStr)
200+
return filter.Filter{}, fmt.Errorf("invalid collection name %s", nsStr)
242201
}
243202

244-
dbFilter[ns.DBName()] = struct{}{}
245-
collFilter[ns.DBName()] = restore.CollFilter{CollName: map[string]struct{}{ns.CollName(): {}}}
203+
if _, ok := collFilter[ns.DBName()]; !ok {
204+
collFilter[ns.DBName()] = filter.CollFilter{CollName: make(map[string]struct{})}
205+
}
206+
collFilter[ns.DBName()].CollName[ns.CollName()] = struct{}{}
246207
}
247208

248-
return dbFilter, collFilter, nil
209+
return filter.Filter{DBCollFilter: collFilter}, nil
249210
}
250211

251-
func (o *options) databasesToBackupFilter() (map[string]struct{}, map[string]restore.CollFilter, error) {
252-
dbFilter := make(map[string]struct{})
253-
collFilter := make(map[string]restore.CollFilter)
212+
func (o *options) databasesToBackupFilter() (filter.Filter, error) {
213+
collFilter := make(map[string]filter.CollFilter)
254214

255215
splits := strings.Split(o.databases, ",")
256216
for _, db := range splits {
257-
dbFilter[db] = struct{}{}
258-
collFilter[db] = restore.CollFilter{AllowAll: true}
217+
collFilter[db] = filter.CollFilter{AllowAll: true}
259218
}
260219

261-
return dbFilter, collFilter, nil
220+
return filter.Filter{DBCollFilter: collFilter}, nil
262221
}
263222

264-
func (o *options) dbCollectionsToBackupFilter() (map[string]struct{}, map[string]restore.CollFilter, error) {
223+
func (o *options) dbCollectionsToBackupFilter() (filter.Filter, error) {
265224
dbColls := make(map[string][]string)
266225
if err := json.Unmarshal([]byte(o.databaseCollections), &dbColls); err != nil {
267-
return nil, nil, fmt.Errorf("unmarshal dbCollections: %w", err)
226+
return filter.Filter{}, fmt.Errorf("unmarshal dbCollections: %w", err)
268227
}
269228

270-
dbFilter := make(map[string]struct{})
271-
collFilter := make(map[string]restore.CollFilter)
229+
collFilter := make(map[string]filter.CollFilter)
272230
for dbName, colls := range dbColls {
273-
dbFilter[dbName] = struct{}{}
274231
if len(colls) == 0 {
275-
collFilter[dbName] = restore.CollFilter{AllowAll: true}
232+
collFilter[dbName] = filter.CollFilter{AllowAll: true}
276233
} else {
277234
collName := make(map[string]struct{}, len(colls))
278235
for _, coll := range colls {
279236
collName[coll] = struct{}{}
280237
}
281-
collFilter[dbName] = restore.CollFilter{CollName: collName}
238+
collFilter[dbName] = filter.CollFilter{CollName: collName}
282239
}
283240
}
284241

285-
return dbFilter, collFilter, nil
242+
return filter.Filter{DBCollFilter: collFilter}, nil
286243
}
287244

288245
func (o *options) renameCollectionNamesToMapper() (*restore.TableMapper, error) {
@@ -378,69 +335,14 @@ func (o *options) run(cmd *cobra.Command, params *paramtable.BackupParams) error
378335
return nil
379336
}
380337

381-
// mapping and filter format:
382-
// mapping: key: oldName, value: newName
383-
//
384-
// rule 1. key: db1.*
385-
// rule 2. key: db1.coll1
386-
// rule 3. key: coll1, means use default db
387-
// rule 4. key: db1.
388-
389-
var (
390-
_rule1Regex = regexp.MustCompile(`^(\w+)\.\*$`)
391-
_rule2Regex = regexp.MustCompile(`^(\w+)\.(\w+)$`)
392-
_rule3Regex = regexp.MustCompile(`^(\w+)$`)
393-
_rule4Regex = regexp.MustCompile(`^(\w+)\.$`)
394-
)
395-
396-
func inferFilterRuleType(rule string) (int, error) {
397-
if _rule1Regex.MatchString(rule) {
398-
return 1, nil
399-
}
400-
401-
if _rule2Regex.MatchString(rule) {
402-
return 2, nil
403-
}
404-
405-
if _rule3Regex.MatchString(rule) {
406-
return 3, nil
407-
}
408-
409-
if _rule4Regex.MatchString(rule) {
410-
return 4, nil
411-
}
412-
413-
return 0, fmt.Errorf("restore: invalid filter rule: %s", rule)
414-
}
415-
416-
func inferMapperRuleType(k, v string) (int, error) {
417-
if _rule1Regex.MatchString(k) && _rule1Regex.MatchString(v) {
418-
return 1, nil
419-
}
420-
421-
if _rule2Regex.MatchString(k) && _rule2Regex.MatchString(v) {
422-
return 2, nil
423-
}
424-
425-
if _rule3Regex.MatchString(k) && _rule3Regex.MatchString(v) {
426-
return 3, nil
427-
}
428-
429-
if _rule4Regex.MatchString(k) && _rule4Regex.MatchString(v) {
430-
return 4, nil
431-
}
432-
433-
return 0, fmt.Errorf("restore: invalid mapper rule: %s -> %s", k, v)
434-
}
435-
436-
// newRenameGenerator creates a new mapRenamer with the given rename map.
338+
// newTableMapperFromCollRename creates a new TableMapper with the given rename map.
437339
func newTableMapperFromCollRename(collRename map[string]string) (*restore.TableMapper, error) {
438340
// add default db in collection_renames if not set
439341
nsMapping := make(map[string][]namespace.NS)
440342
dbWildcard := make(map[string]string)
441343

442344
for k, v := range collRename {
443-
rule, err := inferMapperRuleType(k, v)
345+
rule, err := filter.InferMapperRuleType(k, v)
444346
if err != nil {
445347
return nil, err
446348
}

‎cmd/restore/restore_test.go‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55

66
"github.com/stretchr/testify/assert"
77

8-
"github.com/zilliztech/milvus-backup/core/restore"
8+
"github.com/zilliztech/milvus-backup/internal/filter"
99
)
1010

1111
func TestOptions_validate(t *testing.T) {
@@ -44,22 +44,21 @@ func TestOptions_validate(t *testing.T) {
4444
func TestOptions_toTaskFilter(t *testing.T) {
4545
t.Run("NoFilter", func(t *testing.T) {
4646
var o options
47-
dbFilter, collFilter, err := o.toTaskFilter()
47+
f, err := o.toTaskFilter()
4848
assert.NoError(t, err)
49-
assert.Empty(t, dbFilter)
50-
assert.Empty(t, collFilter)
49+
assert.Empty(t, f.DBCollFilter)
5150
})
5251

5352
t.Run("Normal", func(t *testing.T) {
5453
var o options
5554
o.filter = "db1.*,db2.coll1,coll3,db3."
56-
dbFilter, collFilter, err := o.toTaskFilter()
55+
f, err := o.toTaskFilter()
5756
assert.NoError(t, err)
58-
assert.Equal(t, map[string]struct{}{"db1": {}, "db2": {}, "db3": {}, "default": {}}, dbFilter)
59-
assert.Equal(t, map[string]restore.CollFilter{
57+
assert.Equal(t, map[string]filter.CollFilter{
6058
"db1": {AllowAll: true},
6159
"db2": {CollName: map[string]struct{}{"coll1": {}}},
6260
"default": {CollName: map[string]struct{}{"coll3": {}}},
63-
}, collFilter)
61+
"db3": {}, // db3. rule produces empty CollFilter (database only, no collections)
62+
}, f.DBCollFilter)
6463
})
6564
}

0 commit comments

Comments
 (0)