Skip to content

Commit e15c3c9

Browse files
tikazyqtikazyq
committed
feat: added performance monitoring for database
1 parent 64d664b commit e15c3c9

4 files changed

Lines changed: 188 additions & 2 deletions

File tree

‎core/grpc/server/model_base_service_v2_server.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ var (
1919
*new(models2.TestModelV2),
2020
*new(models2.DataCollectionV2),
2121
*new(models2.DatabaseV2),
22+
*new(models2.DatabaseMetricV2),
2223
*new(models2.DependencyV2),
2324
*new(models2.DependencyLogV2),
2425
*new(models2.DependencySettingV2),

‎core/models/common/index_service_v2.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,4 +202,33 @@ func CreateIndexesV2() {
202202
},
203203
},
204204
})
205+
206+
// databases
207+
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseV2{})).MustCreateIndexes([]mongo2.IndexModel{
208+
{
209+
Keys: bson.D{
210+
{"data_source_id", 1},
211+
},
212+
},
213+
})
214+
215+
// database metrics
216+
mongo.GetMongoCol(service.GetCollectionNameByInstance(models2.DatabaseMetricV2{})).MustCreateIndexes([]mongo2.IndexModel{
217+
{
218+
Keys: bson.D{
219+
{"created_ts", -1},
220+
},
221+
Options: (&options.IndexOptions{}).SetExpireAfterSeconds(60 * 60 * 24 * 30),
222+
},
223+
{
224+
Keys: bson.D{
225+
{"database_id", 1},
226+
},
227+
},
228+
{
229+
Keys: bson.D{
230+
{"type", 1},
231+
},
232+
},
233+
})
205234
}

‎core/models/models/v2/database_metric_v2.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package models
2+
3+
import "go.mongodb.org/mongo-driver/bson/primitive"
4+
5+
type DatabaseMetricV2 struct {
6+
any `collection:"database_metrics"`
7+
BaseModelV2[DatabaseMetricV2] `bson:",inline"`
8+
DatabaseId primitive.ObjectID `json:"database_id" bson:"database_id"`
9+
TotalMemory uint64 `json:"total_memory" bson:"total_memory"`
10+
AvailableMemory uint64 `json:"available_memory" bson:"available_memory"`
11+
UsedMemory uint64 `json:"used_memory" bson:"used_memory"`
12+
UsedMemoryPercent float32 `json:"used_memory_percent" bson:"used_memory_percent"`
13+
TotalDisk uint64 `json:"total_disk" bson:"total_disk"`
14+
AvailableDisk uint64 `json:"available_disk" bson:"available_disk"`
15+
UsedDisk uint64 `json:"used_disk" bson:"used_disk"`
16+
UsedDiskPercent float32 `json:"used_disk_percent" bson:"used_disk_percent"`
17+
Connections int `json:"connections" bson:"connections"`
18+
QueryPerSecond float64 `json:"query_per_second" bson:"query_per_second"`
19+
TotalQuery uint64 `json:"total_query,omitempty" bson:"total_query,omitempty"`
20+
CacheHitRatio float64 `json:"cache_hit_ratio" bson:"cache_hit_ratio"`
21+
ReplicationLag float64 `json:"replication_lag" bson:"replication_lag"`
22+
LockWaitTime float64 `json:"lock_wait_time" bson:"lock_wait_time"`
23+
}

‎core/models/service/base_service_v2.go‎

Lines changed: 135 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
11
package service
22

33
import (
4+
"context"
45
"fmt"
6+
"go.mongodb.org/mongo-driver/mongo/options"
7+
"reflect"
8+
"sync"
9+
510
"github.com/crawlab-team/crawlab/core/interfaces"
611
"github.com/crawlab-team/crawlab/db/mongo"
712
"go.mongodb.org/mongo-driver/bson"
813
"go.mongodb.org/mongo-driver/bson/primitive"
9-
"reflect"
10-
"sync"
1114
)
1215

1316
var (
@@ -30,6 +33,15 @@ func (svc *ModelServiceV2[T]) GetById(id primitive.ObjectID) (model *T, err erro
3033
return &result, nil
3134
}
3235

36+
func (svc *ModelServiceV2[T]) GetByIdContext(ctx context.Context, id primitive.ObjectID) (model *T, err error) {
37+
var result T
38+
err = svc.col.GetCollection().FindOne(ctx, bson.M{"_id": id}).Decode(&result)
39+
if err != nil {
40+
return nil, err
41+
}
42+
return &result, nil
43+
}
44+
3345
func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (model *T, err error) {
3446
var result T
3547
err = svc.col.Find(query, options).One(&result)
@@ -39,6 +51,25 @@ func (svc *ModelServiceV2[T]) GetOne(query bson.M, options *mongo.FindOptions) (
3951
return &result, nil
4052
}
4153

54+
func (svc *ModelServiceV2[T]) GetOneContext(ctx context.Context, query bson.M, opts *mongo.FindOptions) (model *T, err error) {
55+
var result T
56+
_opts := &options.FindOneOptions{}
57+
if opts != nil {
58+
if opts.Skip != 0 {
59+
skipInt64 := int64(opts.Skip)
60+
_opts.Skip = &skipInt64
61+
}
62+
if opts.Sort != nil {
63+
_opts.Sort = opts.Sort
64+
}
65+
}
66+
err = svc.col.GetCollection().FindOne(ctx, query, _opts).Decode(&result)
67+
if err != nil {
68+
return nil, err
69+
}
70+
return &result, nil
71+
}
72+
4273
func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions) (models []T, err error) {
4374
var result []T
4475
err = svc.col.Find(query, options).All(&result)
@@ -48,44 +79,115 @@ func (svc *ModelServiceV2[T]) GetMany(query bson.M, options *mongo.FindOptions)
4879
return result, nil
4980
}
5081

82+
func (svc *ModelServiceV2[T]) GetManyContext(ctx context.Context, query bson.M, opts *mongo.FindOptions) (models []T, err error) {
83+
var result []T
84+
_opts := &options.FindOptions{}
85+
if opts != nil {
86+
if opts.Skip != 0 {
87+
skipInt64 := int64(opts.Skip)
88+
_opts.Skip = &skipInt64
89+
}
90+
if opts.Limit != 0 {
91+
limitInt64 := int64(opts.Limit)
92+
_opts.Limit = &limitInt64
93+
}
94+
if opts.Sort != nil {
95+
_opts.Sort = opts.Sort
96+
}
97+
}
98+
cur, err := svc.col.GetCollection().Find(ctx, query, _opts)
99+
if err != nil {
100+
return nil, err
101+
}
102+
defer cur.Close(ctx)
103+
for cur.Next(ctx) {
104+
var model T
105+
if err := cur.Decode(&model); err != nil {
106+
return nil, err
107+
}
108+
result = append(result, model)
109+
}
110+
return result, nil
111+
}
112+
51113
func (svc *ModelServiceV2[T]) DeleteById(id primitive.ObjectID) (err error) {
52114
return svc.col.DeleteId(id)
53115
}
54116

117+
func (svc *ModelServiceV2[T]) DeleteByIdContext(ctx context.Context, id primitive.ObjectID) (err error) {
118+
_, err = svc.col.GetCollection().DeleteOne(ctx, bson.M{"_id": id})
119+
return err
120+
}
121+
55122
func (svc *ModelServiceV2[T]) DeleteOne(query bson.M) (err error) {
56123
_, err = svc.col.GetCollection().DeleteOne(svc.col.GetContext(), query)
57124
return err
58125
}
59126

127+
func (svc *ModelServiceV2[T]) DeleteOneContext(ctx context.Context, query bson.M) (err error) {
128+
_, err = svc.col.GetCollection().DeleteOne(ctx, query)
129+
return err
130+
}
131+
60132
func (svc *ModelServiceV2[T]) DeleteMany(query bson.M) (err error) {
61133
_, err = svc.col.GetCollection().DeleteMany(svc.col.GetContext(), query, nil)
62134
return err
63135
}
64136

137+
func (svc *ModelServiceV2[T]) DeleteManyContext(ctx context.Context, query bson.M) (err error) {
138+
_, err = svc.col.GetCollection().DeleteMany(ctx, query, nil)
139+
return err
140+
}
141+
65142
func (svc *ModelServiceV2[T]) UpdateById(id primitive.ObjectID, update bson.M) (err error) {
66143
return svc.col.UpdateId(id, update)
67144
}
68145

146+
func (svc *ModelServiceV2[T]) UpdateByIdContext(ctx context.Context, id primitive.ObjectID, update bson.M) (err error) {
147+
_, err = svc.col.GetCollection().UpdateOne(ctx, bson.M{"_id": id}, update)
148+
return err
149+
}
150+
69151
func (svc *ModelServiceV2[T]) UpdateOne(query bson.M, update bson.M) (err error) {
70152
_, err = svc.col.GetCollection().UpdateOne(svc.col.GetContext(), query, update)
71153
return err
72154
}
73155

156+
func (svc *ModelServiceV2[T]) UpdateOneContext(ctx context.Context, query bson.M, update bson.M) (err error) {
157+
_, err = svc.col.GetCollection().UpdateOne(ctx, query, update)
158+
return err
159+
}
160+
74161
func (svc *ModelServiceV2[T]) UpdateMany(query bson.M, update bson.M) (err error) {
75162
_, err = svc.col.GetCollection().UpdateMany(svc.col.GetContext(), query, update)
76163
return err
77164
}
78165

166+
func (svc *ModelServiceV2[T]) UpdateManyContext(ctx context.Context, query bson.M, update bson.M) (err error) {
167+
_, err = svc.col.GetCollection().UpdateMany(ctx, query, update)
168+
return err
169+
}
170+
79171
func (svc *ModelServiceV2[T]) ReplaceById(id primitive.ObjectID, model T) (err error) {
80172
_, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), bson.M{"_id": id}, model)
81173
return err
82174
}
83175

176+
func (svc *ModelServiceV2[T]) ReplaceByIdContext(ctx context.Context, id primitive.ObjectID, model T) (err error) {
177+
_, err = svc.col.GetCollection().ReplaceOne(ctx, bson.M{"_id": id}, model)
178+
return err
179+
}
180+
84181
func (svc *ModelServiceV2[T]) ReplaceOne(query bson.M, model T) (err error) {
85182
_, err = svc.col.GetCollection().ReplaceOne(svc.col.GetContext(), query, model)
86183
return err
87184
}
88185

186+
func (svc *ModelServiceV2[T]) ReplaceOneContext(ctx context.Context, query bson.M, model T) (err error) {
187+
_, err = svc.col.GetCollection().ReplaceOne(ctx, query, model)
188+
return err
189+
}
190+
89191
func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err error) {
90192
m := any(&model).(interfaces.Model)
91193
if m.GetId().IsZero() {
@@ -98,6 +200,18 @@ func (svc *ModelServiceV2[T]) InsertOne(model T) (id primitive.ObjectID, err err
98200
return res.InsertedID.(primitive.ObjectID), nil
99201
}
100202

203+
func (svc *ModelServiceV2[T]) InsertOneContext(ctx context.Context, model T) (id primitive.ObjectID, err error) {
204+
m := any(&model).(interfaces.Model)
205+
if m.GetId().IsZero() {
206+
m.SetId(primitive.NewObjectID())
207+
}
208+
res, err := svc.col.GetCollection().InsertOne(ctx, m)
209+
if err != nil {
210+
return primitive.NilObjectID, err
211+
}
212+
return res.InsertedID.(primitive.ObjectID), nil
213+
}
214+
101215
func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID, err error) {
102216
var _models []any
103217
for _, model := range models {
@@ -117,6 +231,25 @@ func (svc *ModelServiceV2[T]) InsertMany(models []T) (ids []primitive.ObjectID,
117231
return ids, nil
118232
}
119233

234+
func (svc *ModelServiceV2[T]) InsertManyContext(ctx context.Context, models []T) (ids []primitive.ObjectID, err error) {
235+
var _models []any
236+
for _, model := range models {
237+
m := any(&model).(interfaces.Model)
238+
if m.GetId().IsZero() {
239+
m.SetId(primitive.NewObjectID())
240+
}
241+
_models = append(_models, m)
242+
}
243+
res, err := svc.col.GetCollection().InsertMany(ctx, _models)
244+
if err != nil {
245+
return nil, err
246+
}
247+
for _, v := range res.InsertedIDs {
248+
ids = append(ids, v.(primitive.ObjectID))
249+
}
250+
return ids, nil
251+
}
252+
120253
func (svc *ModelServiceV2[T]) Count(query bson.M) (total int, err error) {
121254
return svc.col.Count(query)
122255
}

0 commit comments

Comments
 (0)