From 0eb95fcb75d5a36400f8ecbd6c8d0ff253ceba17 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Fri, 3 Feb 2023 08:34:59 +0530 Subject: [PATCH 01/12] [metadata manager m8s] fixing expires date coming as zero value if no value is provided --- metadata/pkg/model/model.go | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/metadata/pkg/model/model.go b/metadata/pkg/model/model.go index eb3c6065e..fe37997d6 100644 --- a/metadata/pkg/model/model.go +++ b/metadata/pkg/model/model.go @@ -21,27 +21,24 @@ import ( ) type MetaBackend struct { - Id bson.ObjectId `json:"id" bson:"_id"` - BackendName string `json:"backendName" bson:"backendName"` - Type string `json:"type" bson:"type"` - Region string `json:"region" bson:"region"` - Buckets []*MetaBucket `json:"buckets" bson:"buckets"` - NumberOfBuckets int32 `json:"numberOfBuckets" bson:"numberOfBuckets"` - NumberOfFilteredBuckets int32 `json:"numberOFilteredBuckets" bson:"numberOFilteredBuckets"` + Id bson.ObjectId `json:"id,omitempty" bson:"_id,omitempty"` + BackendName string `json:"backendName,omitempty" bson:"backendName,omitempty"` + Type string `json:"type,omitempty" bson:"type,omitempty"` + Region string `json:"region" bson:"region"` + Buckets []*MetaBucket `json:"buckets,omitempty" bson:"buckets,omitempty"` + NumberOfBuckets int32 `json:"numberOfBuckets,omitempty" bson:"numberOfBuckets,omitempty"` } type MetaBucket struct { - CreationDate *time.Time `type:"timestamp" json:"creationDate,omitempty" bson:"creationDate,omitempty"` - Name string `type:"string" json:"name" bson:"name"` - Type string `json:"type,omitempty" bson:"type,omitempty"` - Region string `json:"region,omitempty" bson:"region,omitempty"` - Access string `json:"access,omitempty" bson:"access,omitempty"` - NumberOfObjects int `json:"numberOfObjects" bson:"numberOfObjects,omitempty"` - NumberOfFilteredObjects int `json:"numberOfFilteredObjects,omitempty" bson:"numberOfFilteredObjects,omitempty"` - Objects []*MetaObject `json:"objects" bson:"objects"` - TotalSize int64 `json:"totalSize" bson:"totalSize"` - FilteredBucketSize int64 `json:"filteredBucketSize,omitempty" bson:"filteredBucketSize"` - BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` + CreationDate *time.Time `type:"timestamp" json:"creationDate,omitempty" bson:"creationDate,omitempty"` + Name string `type:"string" json:"name" bson:"name"` + Type string `json:"type,omitempty" bson:"type,omitempty"` + Region string `json:"region,omitempty" bson:"region,omitempty"` + Access string `json:"access,omitempty" bson:"access,omitempty"` + NumberOfObjects int `json:"numberOfObjects,omitempty" bson:"numberOfObjects,omitempty"` + Objects []*MetaObject `json:"objects,omitempty" bson:"objects"` + TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize,omitempty"` + BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` } type MetaObject struct { From 84f9567c80802cfd14a904a54cc52543f0f02309 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Fri, 3 Feb 2023 10:30:20 +0530 Subject: [PATCH 02/12] [metadata manager m8s] empty buckets are missing in o/p for without any filter case fix --- metadata/pkg/model/model.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata/pkg/model/model.go b/metadata/pkg/model/model.go index fe37997d6..1c6f016ac 100644 --- a/metadata/pkg/model/model.go +++ b/metadata/pkg/model/model.go @@ -25,8 +25,8 @@ type MetaBackend struct { BackendName string `json:"backendName,omitempty" bson:"backendName,omitempty"` Type string `json:"type,omitempty" bson:"type,omitempty"` Region string `json:"region" bson:"region"` - Buckets []*MetaBucket `json:"buckets,omitempty" bson:"buckets,omitempty"` - NumberOfBuckets int32 `json:"numberOfBuckets,omitempty" bson:"numberOfBuckets,omitempty"` + Buckets []*MetaBucket `json:"buckets,omitempty" bson:"buckets"` + NumberOfBuckets int32 `json:"numberOfBuckets" bson:"numberOfBuckets"` } type MetaBucket struct { @@ -37,7 +37,7 @@ type MetaBucket struct { Access string `json:"access,omitempty" bson:"access,omitempty"` NumberOfObjects int `json:"numberOfObjects,omitempty" bson:"numberOfObjects,omitempty"` Objects []*MetaObject `json:"objects,omitempty" bson:"objects"` - TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize,omitempty"` + TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize"` BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` } From 86dc920599ed9252d31aa76da8453298bc288bc1 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Fri, 3 Feb 2023 15:04:28 +0530 Subject: [PATCH 03/12] [metadata manager m8s] adding document Id to o/p --- metadata/pkg/model/model.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata/pkg/model/model.go b/metadata/pkg/model/model.go index 1c6f016ac..057c1c9c4 100644 --- a/metadata/pkg/model/model.go +++ b/metadata/pkg/model/model.go @@ -25,7 +25,7 @@ type MetaBackend struct { BackendName string `json:"backendName,omitempty" bson:"backendName,omitempty"` Type string `json:"type,omitempty" bson:"type,omitempty"` Region string `json:"region" bson:"region"` - Buckets []*MetaBucket `json:"buckets,omitempty" bson:"buckets"` + Buckets []*MetaBucket `json:"buckets" bson:"buckets"` NumberOfBuckets int32 `json:"numberOfBuckets" bson:"numberOfBuckets"` } @@ -36,7 +36,7 @@ type MetaBucket struct { Region string `json:"region,omitempty" bson:"region,omitempty"` Access string `json:"access,omitempty" bson:"access,omitempty"` NumberOfObjects int `json:"numberOfObjects,omitempty" bson:"numberOfObjects,omitempty"` - Objects []*MetaObject `json:"objects,omitempty" bson:"objects"` + Objects []*MetaObject `json:"objects" bson:"objects"` TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize"` BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` } From 21bfb7e640d382c840ac1df17eef490f7dd5315f Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Fri, 3 Feb 2023 16:11:25 +0530 Subject: [PATCH 04/12] [metadata manager m8s] [query translator] adding filtered aggregation fields --- metadata/pkg/model/model.go | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/metadata/pkg/model/model.go b/metadata/pkg/model/model.go index 057c1c9c4..752fc40dd 100644 --- a/metadata/pkg/model/model.go +++ b/metadata/pkg/model/model.go @@ -21,24 +21,27 @@ import ( ) type MetaBackend struct { - Id bson.ObjectId `json:"id,omitempty" bson:"_id,omitempty"` - BackendName string `json:"backendName,omitempty" bson:"backendName,omitempty"` - Type string `json:"type,omitempty" bson:"type,omitempty"` - Region string `json:"region" bson:"region"` - Buckets []*MetaBucket `json:"buckets" bson:"buckets"` - NumberOfBuckets int32 `json:"numberOfBuckets" bson:"numberOfBuckets"` + Id bson.ObjectId `json:"id,omitempty" bson:"_id,omitempty"` + BackendName string `json:"backendName,omitempty" bson:"backendName,omitempty"` + Type string `json:"type,omitempty" bson:"type,omitempty"` + Region string `json:"region" bson:"region"` + Buckets []*MetaBucket `json:"buckets" bson:"buckets"` + NumberOfBuckets int32 `json:"numberOfBuckets" bson:"numberOfBuckets"` + NumberOfFilteredBuckets int32 `json:"numberOFilteredBuckets" bson:"numberOFilteredBuckets"` } type MetaBucket struct { - CreationDate *time.Time `type:"timestamp" json:"creationDate,omitempty" bson:"creationDate,omitempty"` - Name string `type:"string" json:"name" bson:"name"` - Type string `json:"type,omitempty" bson:"type,omitempty"` - Region string `json:"region,omitempty" bson:"region,omitempty"` - Access string `json:"access,omitempty" bson:"access,omitempty"` - NumberOfObjects int `json:"numberOfObjects,omitempty" bson:"numberOfObjects,omitempty"` - Objects []*MetaObject `json:"objects" bson:"objects"` - TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize"` - BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` + CreationDate *time.Time `type:"timestamp" json:"creationDate,omitempty" bson:"creationDate,omitempty"` + Name string `type:"string" json:"name" bson:"name"` + Type string `json:"type,omitempty" bson:"type,omitempty"` + Region string `json:"region,omitempty" bson:"region,omitempty"` + Access string `json:"access,omitempty" bson:"access,omitempty"` + NumberOfObjects int `json:"numberOfObjects,omitempty" bson:"numberOfObjects,omitempty"` + NumberOfFilteredObjects int `json:"numberOfFilteredObjects,omitempty" bson:"numberOfFilteredObjects,omitempty"` + Objects []*MetaObject `json:"objects" bson:"objects"` + TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize"` + FilteredBucketSize int64 `json:"filteredBucketSize,omitempty" bson:"filteredBucketSize"` + BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` } type MetaObject struct { From 64560357d4cf06c26964c1f5aa8ad96ac9006b9c Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Fri, 3 Feb 2023 19:32:00 +0530 Subject: [PATCH 05/12] [metadata manager m8s] [query translator] fixing buckets with no objects after filtering also getting displayed when querying object level --- metadata/pkg/model/model.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metadata/pkg/model/model.go b/metadata/pkg/model/model.go index 752fc40dd..eb3c6065e 100644 --- a/metadata/pkg/model/model.go +++ b/metadata/pkg/model/model.go @@ -21,9 +21,9 @@ import ( ) type MetaBackend struct { - Id bson.ObjectId `json:"id,omitempty" bson:"_id,omitempty"` - BackendName string `json:"backendName,omitempty" bson:"backendName,omitempty"` - Type string `json:"type,omitempty" bson:"type,omitempty"` + Id bson.ObjectId `json:"id" bson:"_id"` + BackendName string `json:"backendName" bson:"backendName"` + Type string `json:"type" bson:"type"` Region string `json:"region" bson:"region"` Buckets []*MetaBucket `json:"buckets" bson:"buckets"` NumberOfBuckets int32 `json:"numberOfBuckets" bson:"numberOfBuckets"` @@ -36,10 +36,10 @@ type MetaBucket struct { Type string `json:"type,omitempty" bson:"type,omitempty"` Region string `json:"region,omitempty" bson:"region,omitempty"` Access string `json:"access,omitempty" bson:"access,omitempty"` - NumberOfObjects int `json:"numberOfObjects,omitempty" bson:"numberOfObjects,omitempty"` + NumberOfObjects int `json:"numberOfObjects" bson:"numberOfObjects,omitempty"` NumberOfFilteredObjects int `json:"numberOfFilteredObjects,omitempty" bson:"numberOfFilteredObjects,omitempty"` Objects []*MetaObject `json:"objects" bson:"objects"` - TotalSize int64 `json:"totalSize,omitempty" bson:"totalSize"` + TotalSize int64 `json:"totalSize" bson:"totalSize"` FilteredBucketSize int64 `json:"filteredBucketSize,omitempty" bson:"filteredBucketSize"` BucketTags map[string]string `json:"tags,omitempty" bson:"tags,omitempty"` } From ded94184de0ecae842a8b07df679f2ffee3f3b64 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Tue, 31 Jan 2023 14:36:35 +0530 Subject: [PATCH 06/12] [metadata manager m8s] adding UT --- metadata/pkg/query-manager/paginator_test.go | 92 +++++++++++ metadata/pkg/query-manager/validator_test.go | 165 +++++++++++++++++++ metadata/pkg/utils/utils_test.go | 72 ++++++++ 3 files changed, 329 insertions(+) create mode 100644 metadata/pkg/query-manager/paginator_test.go create mode 100644 metadata/pkg/query-manager/validator_test.go create mode 100644 metadata/pkg/utils/utils_test.go diff --git a/metadata/pkg/query-manager/paginator_test.go b/metadata/pkg/query-manager/paginator_test.go new file mode 100644 index 000000000..33cec9079 --- /dev/null +++ b/metadata/pkg/query-manager/paginator_test.go @@ -0,0 +1,92 @@ +/* + * // Copyright 2023 The SODA Authors. + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + * + */ + +package query_manager + +import ( + "github.com/opensds/multi-cloud/metadata/pkg/model" + "testing" +) + +func TestPaginate(t *testing.T) { + tests := []struct { + unPaginatedResult []*model.MetaBackend + limit int32 + offset int32 + expectedResult []*model.MetaBackend + }{ + { + unPaginatedResult: []*model.MetaBackend{ + {Id: "1"}, + {Id: "2"}, + {Id: "3"}, + {Id: "4"}, + {Id: "5"}, + }, + limit: 2, + offset: 0, + expectedResult: []*model.MetaBackend{ + {Id: "1"}, + {Id: "2"}, + }, + }, + { + unPaginatedResult: []*model.MetaBackend{ + {Id: "1"}, + {Id: "2"}, + {Id: "3"}, + {Id: "4"}, + {Id: "5"}, + }, + limit: 2, + offset: 2, + expectedResult: []*model.MetaBackend{ + {Id: "3"}, + {Id: "4"}, + }, + }, + { + unPaginatedResult: []*model.MetaBackend{ + {Id: "1"}, + {Id: "2"}, + {Id: "3"}, + {Id: "4"}, + {Id: "5"}, + }, + limit: 2, + offset: 5, + expectedResult: []*model.MetaBackend{}, + }, + { + unPaginatedResult: []*model.MetaBackend{}, + limit: 2, + offset: 0, + expectedResult: []*model.MetaBackend{}, + }, + } + for i, test := range tests { + result := Paginate(test.unPaginatedResult, test.limit, test.offset) + if len(result) != len(test.expectedResult) { + t.Errorf("Test case %d: expected length of result to be %d but got %d", i, len(test.expectedResult), len(result)) + } + for j, r := range result { + if r.Id != test.expectedResult[j].Id { + t.Errorf("Test case %d: expected result at index %d to be %s but got %s", i, j, test.expectedResult[j].Id, r.Id) + } + } + } +} diff --git a/metadata/pkg/query-manager/validator_test.go b/metadata/pkg/query-manager/validator_test.go new file mode 100644 index 000000000..f12e19467 --- /dev/null +++ b/metadata/pkg/query-manager/validator_test.go @@ -0,0 +1,165 @@ +/* + * // Copyright 2023 The SODA Authors. + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + * + */ + +package query_manager + +import ( + "github.com/stretchr/testify/assert" + "testing" + + "github.com/opensds/multi-cloud/metadata/pkg/constants" + pb "github.com/opensds/multi-cloud/metadata/proto" +) + +func TestValidateInput(t *testing.T) { + tests := []struct { + input *pb.ListMetadataRequest + want bool + err string + }{ + { + input: &pb.ListMetadataRequest{ + SizeOfBucketInBytes: 0, + BucketSizeOperator: "", + SizeOfObjectInBytes: 0, + ObjectSizeOperator: "", + SortOrder: constants.ASC, + Region: "us-west-2", + Type: constants.AWS_S3, + }, + want: true, + err: "", + }, + { + input: &pb.ListMetadataRequest{ + SizeOfBucketInBytes: 0, + BucketSizeOperator: "", + SizeOfObjectInBytes: 0, + ObjectSizeOperator: "", + SortOrder: constants.ASC, + Region: "invalid-region", + Type: constants.AWS_S3, + }, + want: false, + err: "Not a valid AWS_S3 region", + }, + { + input: &pb.ListMetadataRequest{ + SizeOfBucketInBytes: 0, + BucketSizeOperator: "", + SizeOfObjectInBytes: 0, + ObjectSizeOperator: "", + SortOrder: constants.ASC, + Region: "us-west-2", + Type: "invalid-type", + }, + want: false, + err: "Not a valid cloud type", + }, + { + input: &pb.ListMetadataRequest{ + SizeOfBucketInBytes: 0, + BucketSizeOperator: "", + SizeOfObjectInBytes: 0, + ObjectSizeOperator: "", + SortOrder: "invalid-sort", + Region: "us-west-2", + Type: constants.AWS_S3, + }, + want: false, + err: "Invalid sort order", + }, + } + + for _, tc := range tests { + result, _ := ValidateInput(tc.input) + assert.Equal(t, tc.want, result) + } +} + +func TestIsSortParamValid(t *testing.T) { + tests := []struct { + sortOrder string + expected bool + }{ + {"", true}, + {"ASC", true}, + {"DESC", true}, + {"abc", false}, + } + + for _, test := range tests { + result, _ := isSortParamValid(test.sortOrder) + if result != test.expected { + t.Errorf("isSortParamValid(%q) = %v, want %v", test.sortOrder, result, test.expected) + } + } +} + +func TestIsSizeParamsValid(t *testing.T) { + tests := []struct { + sizeInBytes int64 + operator string + expected bool + }{ + {0, "", true}, + {0, "gt", true}, + {0, "lte", true}, + {0, "eq", true}, + {0, "gte", true}, + {0, "lt", true}, + {1, "gt", true}, + {1, "lte", true}, + {1, "eq", true}, + {1, "gte", true}, + {1, "lt", true}, + {-1, "gt", false}, + {-1, "lte", false}, + {-1, "eq", false}, + {-1, "gte", false}, + {-1, "lt", false}, + {1, "abc", false}, + } + + for _, test := range tests { + result, _ := isSizeParamsValid(test.sizeInBytes, test.operator) + if result != test.expected { + t.Errorf("isSizeParamsValid(%d, %q) = %v, want %v", test.sizeInBytes, test.operator, result, test.expected) + } + } +} + +func TestIsValidRegion(t *testing.T) { + tests := []struct { + cloudType string + region string + expected bool + }{ + {"AWS-S3", "us-west-1", true}, + {"AWS-S3", "us-west-2", true}, + {"AWS-S3", "abc", false}, + {"", "", true}, + {"abc", "us-west-1", false}, + } + + for _, test := range tests { + result, _ := isValidRegion(test.region, test.cloudType) + if result != test.expected { + t.Errorf("isValidRegion(%q, %q) = %v, want %v", test.region, test.cloudType, result, test.expected) + } + } +} diff --git a/metadata/pkg/utils/utils_test.go b/metadata/pkg/utils/utils_test.go new file mode 100644 index 000000000..cb4c40869 --- /dev/null +++ b/metadata/pkg/utils/utils_test.go @@ -0,0 +1,72 @@ +/* + * // Copyright 2023 The SODA Authors. + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + * + */ + +package utils + +import ( + "github.com/opensds/multi-cloud/metadata/pkg/model" + "testing" + "time" +) + +func TestGetBackends(t *testing.T) { + now := time.Now() + unPaginatedResult := []*model.MetaBackend{{ + BackendName: "backend1", + Region: "region1", + Type: "type1", + NumberOfBuckets: 1, + Buckets: []*model.MetaBucket{{ + Name: "bucket1", + Type: "type1", + Region: "region1", + TotalSize: 100, + NumberOfObjects: 1, + CreationDate: &now, + BucketTags: map[string]string{"key1": "value1"}, + Objects: []*model.MetaObject{{ + ObjectName: "object1", + LastModifiedDate: &now, + ServerSideEncryption: "true", + ExpiresDate: &now, + GrantControl: "grantControl1", + RedirectLocation: "redirect1", + ReplicationStatus: "replicated", + ObjectTags: map[string]string{"key1": "value1"}, + Metadata: map[string]string{"key1": "value1"}, + }}, + }}, + }} + + protoBackends := GetBackends(unPaginatedResult) + + if len(protoBackends) != 1 { + t.Errorf("Expected 1 proto backend, but got %d", len(protoBackends)) + } + + if protoBackends[0].BackendName != "backend1" { + t.Errorf("Expected backend name to be backend1, but got %s", protoBackends[0].BackendName) + } + + if len(protoBackends[0].Buckets) != 1 { + t.Errorf("Expected 1 proto bucket, but got %d", len(protoBackends[0].Buckets)) + } + + if len(protoBackends[0].Buckets[0].Objects) != 1 { + t.Errorf("Expected 1 proto object, but got %d", len(protoBackends[0].Buckets[0].Objects)) + } +} From 323d32302b50882f3ea48e5028f05c45abd46ef3 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Tue, 7 Feb 2023 14:25:59 +0530 Subject: [PATCH 07/12] [metadata manager m8s] remove directives in service.go --- metadata/pkg/service/service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/metadata/pkg/service/service.go b/metadata/pkg/service/service.go index c385fdbe1..6df9e1711 100755 --- a/metadata/pkg/service/service.go +++ b/metadata/pkg/service/service.go @@ -83,7 +83,6 @@ func (myc *S3Cred) Retrieve() (credentials.Value, error) { } func Sync(ctx context.Context, backend *backend.BackendDetail, in *pb.SyncMetadataRequest) error { - log.Debugln("the backend we got now....:%+v", backend) sd, err := driver.CreateStorageDriver(backend.Type, backend) if err != nil { log.Errorln("failed to create driver. err:", err) @@ -99,7 +98,6 @@ func Sync(ctx context.Context, backend *backend.BackendDetail, in *pb.SyncMetada } func (f *metadataService) SyncMetadata(ctx context.Context, in *pb.SyncMetadataRequest, out *pb.BaseResponse) error { - log.Infoln("received sncMetadata request in metadata service:%+v", in) if in.Id != "" { backend, err := utils.GetBackend(ctx, f.backendClient, in.Id) if err != nil { From ddb53cea31be73e03b7f0a0f3161759fc6ecad55 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Tue, 7 Feb 2023 17:22:18 +0530 Subject: [PATCH 08/12] [metadata manager m8s] correction in testcases --- metadata/pkg/query-manager/validator_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata/pkg/query-manager/validator_test.go b/metadata/pkg/query-manager/validator_test.go index f12e19467..7c16ed048 100644 --- a/metadata/pkg/query-manager/validator_test.go +++ b/metadata/pkg/query-manager/validator_test.go @@ -149,9 +149,9 @@ func TestIsValidRegion(t *testing.T) { region string expected bool }{ - {"AWS-S3", "us-west-1", true}, - {"AWS-S3", "us-west-2", true}, - {"AWS-S3", "abc", false}, + {"aws-s3", "us-west-1", true}, + {"aws-s3", "us-west-2", true}, + {"aws-s3", "abc", false}, {"", "", true}, {"abc", "us-west-1", false}, } From 496c6c6c62045f06833ed182a96198d2579babec Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Wed, 8 Feb 2023 12:28:02 +0530 Subject: [PATCH 09/12] [metadata manager m8s] adding UT for service layer for listmetadata --- metadata/pkg/query-manager/validator.go | 14 ++--- metadata/pkg/service/service_test.go | 79 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 7 deletions(-) create mode 100644 metadata/pkg/service/service_test.go diff --git a/metadata/pkg/query-manager/validator.go b/metadata/pkg/query-manager/validator.go index b95098c9b..1a6da4ae9 100644 --- a/metadata/pkg/query-manager/validator.go +++ b/metadata/pkg/query-manager/validator.go @@ -21,11 +21,11 @@ import ( ) type ValidationError struct { - errMsg string + ErrMsg string } func (e *ValidationError) Error() string { - return e.errMsg + return e.ErrMsg } func ValidateInput(in *pb.ListMetadataRequest) (okie bool, err error) { @@ -62,7 +62,7 @@ func isSortParamValid(sortOrder string) (bool, error) { if sortOrder == "" || sortOrder == constants.ASC || sortOrder == constants.DESC { return true, nil } - return false, &ValidationError{errMsg: "Invalid sort order"} + return false, &ValidationError{ErrMsg: "Invalid sort order"} } func isSizeParamsValid(sizeInBytes int64, operator string) (bool, error) { @@ -83,11 +83,11 @@ func isSizeParamsValid(sizeInBytes int64, operator string) (bool, error) { case constants.GREATER_THAN_OPERATOR: default: - return false, &ValidationError{errMsg: "Operator for size should be lte, gt, eq, gte or lt"} + return false, &ValidationError{ErrMsg: "Operator for size should be lte, gt, eq, gte or lt"} } if sizeInBytes < constants.ZERO { - return false, &ValidationError{errMsg: "Size should be always positive."} + return false, &ValidationError{ErrMsg: "Size should be always positive."} } return true, nil @@ -105,7 +105,7 @@ func isValidRegion(region string, cloudType string) (bool, error) { return true, nil default: - return false, &ValidationError{errMsg: "Not a valid cloud type"} + return false, &ValidationError{ErrMsg: "Not a valid cloud type"} } if region == "" { @@ -117,5 +117,5 @@ func isValidRegion(region string, cloudType string) (bool, error) { return true, nil } } - return false, &ValidationError{errMsg: "Not a valid " + cloudType + " region"} + return false, &ValidationError{ErrMsg: "Not a valid " + cloudType + " region"} } diff --git a/metadata/pkg/service/service_test.go b/metadata/pkg/service/service_test.go new file mode 100644 index 000000000..8be62a250 --- /dev/null +++ b/metadata/pkg/service/service_test.go @@ -0,0 +1,79 @@ +/* + * // Copyright 2023 The SODA Authors. + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + * + */ + +package service + +import ( + "context" + "github.com/opensds/multi-cloud/metadata/pkg/db" + "github.com/opensds/multi-cloud/metadata/pkg/model" + query_manager "github.com/opensds/multi-cloud/metadata/pkg/query-manager" + pb "github.com/opensds/multi-cloud/metadata/proto" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + "testing" +) + +var listService = NewMetaService() + +type listMetaDataTestcase struct { + title string + input pb.ListMetadataRequest + expectedOutput pb.ListMetadataResponse + expectedErr error +} + +type mockDbAdapter struct{} + +func (mockDbAdapter) CreateMetadata(ctx context.Context, metaBackend model.MetaBackend) error { + return nil +} + +func (mockDbAdapter) ListMetadata(ctx context.Context, query []bson.D) ([]*model.MetaBackend, error) { + return nil, nil +} + +func Test_ListMetadata(t *testing.T) { + db.DbAdapter = &mockDbAdapter{} + + testcases := []listMetaDataTestcase{ + //* Empty list metadata request + { + title: "Empty request", + input: pb.ListMetadataRequest{}, + expectedOutput: pb.ListMetadataResponse{}, + expectedErr: nil, + }, + //* Invalid list metadata request + { + title: "Invalid request", + input: pb.ListMetadataRequest{BucketSizeOperator: "invalid"}, + expectedOutput: pb.ListMetadataResponse{}, + expectedErr: &query_manager.ValidationError{ErrMsg: "Operator for size should be lte, gt, eq, gte or lt"}, + }, + } + + for _, test := range testcases { + output := pb.ListMetadataResponse{} + err := listService.ListMetadata(context.TODO(), &test.input, &output) + if err != nil { + assert.Equalf(t, err, test.expectedErr, test.title) + } else { + assert.Equalf(t, output, test.expectedOutput, test.title) + } + } +} From 0dbd46141969fda081c837cdf28f9d541cc991c2 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Wed, 8 Feb 2023 14:07:58 +0530 Subject: [PATCH 10/12] [metadata manager m8s] adding UT for service layer for listmetadata in api hander --- api/pkg/metadata/service_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 api/pkg/metadata/service_test.go diff --git a/api/pkg/metadata/service_test.go b/api/pkg/metadata/service_test.go new file mode 100644 index 000000000..8d0c8c4e3 --- /dev/null +++ b/api/pkg/metadata/service_test.go @@ -0,0 +1,26 @@ +/* + * // Copyright 2023 The SODA Authors. + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + * + */ + +package metadata + +import "testing" + +func TestAPIService_ListMetadata(t *testing.T) { + var service = NewAPIService(nil) + + service.ListMetadata() +} \ No newline at end of file From bd0742ae94dc3b7c124a6c26d911061402146cd4 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Thu, 29 Dec 2022 14:16:31 +0530 Subject: [PATCH 11/12] adding unit test for testing ListMetaData Rest Api for Api m8s --- api/pkg/metadata/service.go | 4 +- api/pkg/metadata/service_test.go | 64 +++++++++++++++++++++++++++++--- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/api/pkg/metadata/service.go b/api/pkg/metadata/service.go index 0e2c7211a..a4ed5a3e8 100755 --- a/api/pkg/metadata/service.go +++ b/api/pkg/metadata/service.go @@ -61,7 +61,7 @@ func (s *APIService) ListMetadata(request *restful.Request, response *restful.Re listMetadataRequest, err := GetListMetaDataRequest(request) if err != nil { - log.Errorf("Failed to construct list metadata request err: \n", err) + log.Errorf("Failed to construct list metadata request err: %v\n", err) response.WriteEntity("Invalid type for sizeOfObject or sizeOfBucket request params. It should be integer type.") return } @@ -69,7 +69,7 @@ func (s *APIService) ListMetadata(request *restful.Request, response *restful.Re //* calling the ListMetaData method from metadata manager m8s res, err := s.metaClient.ListMetadata(ctx, &listMetadataRequest) if err != nil { - log.Errorf("Failed to get metadata details err: \n", err) + log.Errorf("Failed to get metadata details err: %v\n", err) response.WriteEntity(err) return } diff --git a/api/pkg/metadata/service_test.go b/api/pkg/metadata/service_test.go index 8d0c8c4e3..1b5a544c0 100644 --- a/api/pkg/metadata/service_test.go +++ b/api/pkg/metadata/service_test.go @@ -17,10 +17,64 @@ package metadata -import "testing" +import ( + "github.com/emicklei/go-restful" + mt "github.com/opensds/multi-cloud/metadata/proto" + "github.com/stretchr/testify/assert" + "net/http" + "net/url" + "strconv" + "testing" +) -func TestAPIService_ListMetadata(t *testing.T) { - var service = NewAPIService(nil) +func GetRequest(uri string, param url.Values, t *testing.T) *restful.Request { + httpReq, err := http.NewRequest("GET", uri+param.Encode(), nil) + if err != nil { + t.Fail() + } + req := restful.NewRequest(httpReq) + return req +} - service.ListMetadata() -} \ No newline at end of file +func Test_ListMetaData_Api_Param_Fetching(t *testing.T) { + + validParams := make(url.Values) + validParams["limit"] = []string{"100"} + validParams["offset"] = []string{"1"} + + invalidParams := make(url.Values) + invalidParams["limit"] = []string{"invalid"} + invalidParams["offset"] = []string{"1"} + + tests := []struct { + title string + input interface{} + expectedOutput interface{} + expectedErr error + }{ + { + title: "UT to test whether the request params are read correctly for valid values", + input: validParams, + expectedOutput: mt.ListMetadataRequest{Limit: 100, Offset: 1}, + expectedErr: nil, + }, + { + title: "UT to test whether the request params is giving error for invalid values", + input: invalidParams, + expectedOutput: nil, + expectedErr: &strconv.NumError{}, + }, + } + + for _, test := range tests { + uri := "http://localhost:41651/v1/metadata/metadata/?" + values := test.input.(url.Values) + req := GetRequest(uri, values, t) + res, err := GetListMetaDataRequest(req) + if err != nil { + assert.NotNil(t, test.expectedErr, test.title) + } else { + assert.Equal(t, test.expectedOutput, res, test.title) + } + } +} From 51b9276a12406bde32c7f5954955ef9eb7d9b657 Mon Sep 17 00:00:00 2001 From: Joseph Jacob Date: Wed, 8 Feb 2023 16:23:23 +0530 Subject: [PATCH 12/12] [metadata manager m8s] translator unit test cases --- metadata/pkg/query-manager/translator_test.go | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 metadata/pkg/query-manager/translator_test.go diff --git a/metadata/pkg/query-manager/translator_test.go b/metadata/pkg/query-manager/translator_test.go new file mode 100644 index 000000000..7313330bc --- /dev/null +++ b/metadata/pkg/query-manager/translator_test.go @@ -0,0 +1,175 @@ +/* + * // Copyright 2023 The SODA Authors. + * // + * // Licensed under the Apache License, Version 2.0 (the "License"); + * // you may not use this file except in compliance with the License. + * // You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, software + * // distributed under the License is distributed on an "AS IS" BASIS, + * // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * // See the License for the specific language governing permissions and + * // limitations under the License. + * + */ + +package query_manager + +import ( + pb "github.com/opensds/multi-cloud/metadata/proto" + "github.com/stretchr/testify/assert" + "go.mongodb.org/mongo-driver/bson" + "testing" +) + +func Test_Translator(t *testing.T) { + sortAscBasedOnBackName := bson.D{{"$sort", bson.D{{"backendName", 1}}}} + + backendNameMatch := bson.D{ + bson.E{Key: "$match", Value: bson.D{ + bson.E{Key: "backendName", Value: "backendName"}}}} + + bucketNameMatch := bson.D{ + bson.E{Key: "$match", Value: bson.D{ + bson.E{Key: "buckets", Value: bson.D{ + bson.E{Key: "$elemMatch", Value: bson.D{ + bson.E{Key: "name", Value: bson.D{ + bson.E{Key: "$eq", Value: "bucketName"}}}}}}}}}} + + bucketNameFilter := bson.D{ + bson.E{Key: "$project", Value: bson.D{ + bson.E{Key: "_id", Value: 1}, + bson.E{Key: "backendName", Value: 1}, + bson.E{Key: "region", Value: 1}, + bson.E{Key: "type", Value: 1}, + bson.E{Key: "buckets", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$buckets"}, + bson.E{Key: "as", Value: "bucket"}, + bson.E{Key: "cond", Value: bson.D{ + bson.E{Key: "$and", Value: bson.A{bson.D{ + bson.E{Key: "$eq", Value: bson.A{"$$bucket.name", "bucketName"}}}}}}}}}}}, + bson.E{Key: "numberOfBuckets", Value: 1}, + bson.E{Key: "numberOFilteredBuckets", Value: bson.D{ + bson.E{Key: "$size", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$buckets"}, + bson.E{Key: "as", Value: "bucket"}, + bson.E{Key: "cond", + Value: bson.D{ + bson.E{Key: "$and", + Value: bson.A{bson.D{ + bson.E{Key: "$eq", Value: bson.A{"$$bucket.name", "bucketName"}}}}}}}}}}}}}}}} + + objectNameMatch := bson.D{ + bson.E{Key: "$match", Value: bson.D{ + bson.E{Key: "buckets", Value: bson.D{ + bson.E{Key: "$elemMatch", Value: bson.D{ + bson.E{Key: "objects.name", Value: bson.D{ + bson.E{Key: "$eq", Value: "objectName"}}}}}}}}}} + + objectNameFilter := bson.D{ + bson.E{Key: "$project", Value: bson.D{ + bson.E{Key: "_id", Value: 1}, + bson.E{Key: "backendName", Value: 1}, + bson.E{Key: "region", Value: 1}, + bson.E{Key: "type", Value: 1}, + bson.E{Key: "numberOfBuckets", Value: 1}, + bson.E{Key: "buckets", + Value: bson.D{ + bson.E{Key: "$map", Value: bson.D{ + bson.E{Key: "input", Value: "$buckets"}, + bson.E{Key: "as", Value: "bucket"}, + bson.E{Key: "in", Value: bson.D{ + bson.E{Key: "creationDate", Value: "$$bucket.creationDate"}, + bson.E{Key: "name", Value: "$$bucket.name"}, + bson.E{Key: "region", Value: "$$bucket.region"}, + bson.E{Key: "type", Value: "$$bucket.type"}, + bson.E{Key: "access", Value: "$$bucket.access"}, + bson.E{Key: "numberOfObjects", Value: "$$bucket.numberOfObjects"}, + bson.E{Key: "numberOfFilteredObjects", Value: bson.D{ + bson.E{Key: "$size", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$$bucket.objects"}, + bson.E{Key: "as", Value: "object"}, + bson.E{Key: "cond", Value: bson.D{ + bson.E{Key: "$and", Value: bson.A{bson.D{ + bson.E{Key: "$eq", Value: bson.A{"$$object.name", "objectName"}}}}}}}}}}}}}, + bson.E{Key: "totalSize", Value: "$$bucket.totalSize"}, + bson.E{Key: "filteredBucketSize", Value: bson.D{ + bson.E{Key: "$sum", Value: bson.D{ + bson.E{Key: "$map", Value: bson.D{ + bson.E{Key: "input", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$$bucket.objects"}, + bson.E{Key: "as", Value: "object"}, + bson.E{Key: "cond", Value: bson.D{ + bson.E{Key: "$and", Value: bson.A{bson.D{ + bson.E{Key: "$eq", Value: bson.A{"$$object.name", "objectName"}}}}}}}}}}}, + bson.E{Key: "as", Value: "object"}, + bson.E{Key: "in", Value: "$$object.size"}}}}}}}, + bson.E{Key: "tags", Value: "$$bucket.tags"}, + bson.E{Key: "objects", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$$bucket.objects"}, + bson.E{Key: "as", Value: "object"}, + bson.E{Key: "cond", Value: bson.D{ + bson.E{Key: "$and", Value: bson.A{bson.D{ + bson.E{Key: "$eq", Value: bson.A{"$$object.name", "objectName"}}}}}}}}}}}}}}}}}}}} + removeEmptyBucketsFilter := bson.D{ + bson.E{Key: "$project", Value: bson.D{ + bson.E{Key: "_id", Value: 1}, + bson.E{Key: "backendName", Value: 1}, + bson.E{Key: "region", Value: 1}, + bson.E{Key: "type", Value: 1}, + bson.E{Key: "buckets", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$buckets"}, + bson.E{Key: "as", Value: "bucket"}, + bson.E{Key: "cond", Value: bson.D{ + bson.E{Key: "$and", Value: bson.A{bson.D{ + bson.E{Key: "$ifNull", Value: bson.A{"$$bucket.numberOfFilteredObjects", false}}}}}}}}}}}, + bson.E{Key: "numberOfBuckets", Value: 1}, + bson.E{Key: "numberOFilteredBuckets", Value: bson.D{ + bson.E{Key: "$size", Value: bson.D{ + bson.E{Key: "$filter", Value: bson.D{ + bson.E{Key: "input", Value: "$buckets"}, + bson.E{Key: "as", Value: "bucket"}, + bson.E{Key: "cond", Value: bson.D{ + bson.E{Key: "$and", Value: bson.A{bson.D{ + bson.E{Key: "$ifNull", Value: bson.A{"$$bucket.numberOfFilteredObjects", false}}}}}}}}}}}}}}}} + + tests := []struct { + title string + input pb.ListMetadataRequest + expectedOutput interface{} + }{ + { + title: "Empty Request", + input: pb.ListMetadataRequest{}, + expectedOutput: []bson.D{sortAscBasedOnBackName}, + }, + { + title: "Query Backend Name", + input: pb.ListMetadataRequest{BackendName: "backendName"}, + expectedOutput: []bson.D{backendNameMatch, sortAscBasedOnBackName}, + }, + { + title: "Query Bucket Name", + input: pb.ListMetadataRequest{BucketName: "bucketName"}, + expectedOutput: []bson.D{bucketNameMatch, bucketNameFilter, sortAscBasedOnBackName}, + }, + { + title: "Query Object Name", + input: pb.ListMetadataRequest{ObjectName: "objectName"}, + expectedOutput: []bson.D{objectNameMatch, objectNameFilter, removeEmptyBucketsFilter, sortAscBasedOnBackName}, + }, + } + + for _, test := range tests { + output := Translate(&test.input) + assert.Equal(t, test.expectedOutput, output, test.title) + } +}