Skip to content

Commit 6e046d5

Browse files
invis-bitflyinvis-bitfly
committed
add(monitoring): use constants, add v2 exporter monitoring, monitor caller
1 parent 2e7483f commit 6e046d5

26 files changed

+189
-67
lines changed

‎backend/cmd/exporter/main.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/gobitfly/beaconchain/pkg/commons/version"
2020
"github.com/gobitfly/beaconchain/pkg/exporter/modules"
2121
"github.com/gobitfly/beaconchain/pkg/exporter/services"
22+
"github.com/gobitfly/beaconchain/pkg/monitoring"
2223
)
2324

2425
func Run() {
@@ -136,6 +137,10 @@ func Run() {
136137

137138
wg.Wait()
138139

140+
// enable light-weight db connection monitoring
141+
monitoring.Init(false)
142+
monitoring.Start()
143+
139144
if utils.Config.TieredCacheProvider != "redis" {
140145
log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0)
141146
}

‎backend/go.mod‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ require (
5252
github.com/juliangruber/go-intersect v1.1.0
5353
github.com/jung-kurt/gofpdf v1.16.2
5454
github.com/k3a/html2text v1.2.1
55-
github.com/kelseyhightower/envconfig v1.4.0
5655
github.com/klauspost/compress v1.17.6
5756
github.com/klauspost/pgzip v1.2.6
5857
github.com/lib/pq v1.10.9
@@ -66,6 +65,7 @@ require (
6665
github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388
6766
github.com/rocket-pool/rocketpool-go v1.8.4-0.20241009143357-7b6894d57365
6867
github.com/rocket-pool/smartnode v1.14.1
68+
github.com/sethvargo/go-envconfig v1.1.0
6969
github.com/shopspring/decimal v1.4.0
7070
github.com/sirupsen/logrus v1.9.3
7171
github.com/stretchr/testify v1.9.0
@@ -234,7 +234,6 @@ require (
234234
github.com/sanity-io/litter v1.5.5 // indirect
235235
github.com/segmentio/asm v1.2.0 // indirect
236236
github.com/sergi/go-diff v1.2.0 // indirect
237-
github.com/sethvargo/go-envconfig v1.1.0 // indirect
238237
github.com/sethvargo/go-retry v0.2.4 // indirect
239238
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
240239
github.com/spaolacci/murmur3 v1.1.0 // indirect

‎backend/go.sum‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -601,8 +601,6 @@ github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY=
601601
github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
602602
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
603603
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
604-
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
605-
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
606604
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
607605
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
608606
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=

‎backend/pkg/api/data_access/healthz.go‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types.
111111
response.Reports = make(map[string][]types.HealthzResult)
112112
response.ReportingUUID = utils.GetUUID()
113113
response.DeploymentType = utils.Config.DeploymentType
114-
err := db.ClickHouseReader.SelectContext(ctx, &results, query, ch.Named("deployment_type", utils.Config.DeploymentType), ch.Named("clean_shutdown_event_id", constants.CleanShutdownEvent))
114+
err := db.ClickHouseReader.SelectContext(ctx, &results, query, ch.Named("deployment_type", utils.Config.DeploymentType), ch.Named("clean_shutdown_event_id", string(constants.Event_MonitoringCleanShutdown)))
115115
if err != nil {
116116
response.Reports["response_error"] = []types.HealthzResult{
117117
{
@@ -125,25 +125,12 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types.
125125
return response
126126
}
127127

128-
mustExist := []string{
129-
"ch_rolling_1h",
130-
"ch_rolling_24h",
131-
"ch_rolling_7d",
132-
"ch_rolling_30d",
133-
"ch_rolling_90d",
134-
"ch_rolling_total",
135-
"ch_dashboard_epoch",
136-
"api_service_avg_efficiency",
137-
"api_service_validator_mapping",
138-
"api_service_slot_viz",
139-
"monitoring_timeouts",
140-
}
141128
for _, result := range results {
142129
response.Reports[result.EventId] = append(response.Reports[result.EventId], result)
143130
}
144-
for _, id := range mustExist {
145-
if _, ok := response.Reports[id]; !ok {
146-
response.Reports[id] = []types.HealthzResult{
131+
for _, id := range constants.RequiredEvents {
132+
if _, ok := response.Reports[string(id)]; !ok {
133+
response.Reports[string(id)] = []types.HealthzResult{
147134
{
148135
Status: constants.Failure,
149136
Result: []map[string]string{

‎backend/pkg/api/services/service_average_network_efficiency.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func (s *Services) startEfficiencyDataService(wg *sync.WaitGroup) {
2727
for {
2828
startTime := time.Now()
2929
delay := time.Duration(utils.Config.Chain.ClConfig.SlotsPerEpoch*utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second
30-
r := services.NewStatusReport("api_service_avg_efficiency", constants.Default, delay)
30+
r := services.NewStatusReport(constants.Event_ApiServiceAvgEfficiency, constants.Default, delay)
3131
r(constants.Running, nil)
3232
err := s.updateEfficiencyData() // TODO: only update data if something has changed (new head epoch)
3333
if err != nil {

‎backend/pkg/api/services/service_slot_viz.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ func (s *Services) startSlotVizDataService(wg *sync.WaitGroup) {
3232
for {
3333
startTime := time.Now()
3434
delay := time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second
35-
r := services.NewStatusReport("api_service_slot_viz", constants.Default, delay)
35+
r := services.NewStatusReport(constants.Event_ApiServiceSlotViz, constants.Default, delay)
3636
r(constants.Running, nil)
3737
err := s.updateSlotVizData() // TODO: only update data if something has changed (new head slot or new head epoch)
3838
if err != nil {

‎backend/pkg/api/services/service_validator_mapping.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (s *Services) startIndexMappingService(wg *sync.WaitGroup) {
4141
startTime := time.Now()
4242
delay := time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second
4343
err = nil // clear error
44-
r := services.NewStatusReport("api_service_validator_mapping", constants.Default, delay)
44+
r := services.NewStatusReport(constants.Event_ApiServiceValidatorMapping, constants.Default, delay)
4545
r(constants.Running, nil)
4646
latestEpoch := cache.LatestEpoch.Get()
4747
if currentValidatorMapping.Load() == nil || latestEpoch != lastEpochUpdate {

‎backend/pkg/exporter/modules/base.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@ import (
1111
"github.com/gobitfly/beaconchain/pkg/commons/utils"
1212
"github.com/gobitfly/beaconchain/pkg/consapi"
1313
"github.com/gobitfly/beaconchain/pkg/consapi/types"
14+
"github.com/gobitfly/beaconchain/pkg/monitoring/constants"
15+
"github.com/gobitfly/beaconchain/pkg/monitoring/services"
1416
"github.com/pkg/errors"
1517
"golang.org/x/sync/errgroup"
1618
)
1719

1820
type ModuleInterface interface {
1921
Init() error
2022
GetName() string // Used for logging
23+
GetMonitoringEventId() constants.Event
2124

2225
OnHead(*types.StandardEventHeadResponse) error // !Do not block in this functions for an extended period of time!
2326

@@ -63,7 +66,6 @@ func StartAll(context ModuleContext, modules []ModuleInterface, justV2 bool) {
6366
log.Error(err, "beacon-node seems to be unavailable", 0)
6467
time.Sleep(time.Second * 10)
6568
}
66-
6769
// start subscription modules
6870
startSubscriptionModules(&context, modules)
6971
}
@@ -146,15 +148,20 @@ func notifyAllModules(goPool *errgroup.Group, modules []ModuleInterface, f func(
146148
for _, module := range modules {
147149
module := module
148150
goPool.Go(func() error {
151+
start := time.Now()
152+
r := services.NewStatusReport(module.GetMonitoringEventId(), constants.Default, constants.Default)
153+
r(constants.Running, nil)
149154
err := f(module)
150155
if err != nil {
151156
log.Error(err, fmt.Sprintf("error in module %s", module.GetName()), 0)
157+
r(constants.Failure, map[string]string{"error": err.Error()})
158+
return nil // return never gets caught anywhere? lets not risk a memory leak and instead return nil
152159
}
160+
r(constants.Success, map[string]string{"took_raw": fmt.Sprintf("%v", time.Since(start).Milliseconds())})
153161
return nil
154162
})
155163
}
156164
}
157-
158165
func GetModuleContext() (ModuleContext, error) {
159166
cl := consapi.NewClient("http://" + utils.Config.Indexer.Node.Host + ":" + utils.Config.Indexer.Node.Port)
160167

‎backend/pkg/exporter/modules/dashboard_data.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"github.com/gobitfly/beaconchain/pkg/consapi/network"
2020
constypes "github.com/gobitfly/beaconchain/pkg/consapi/types"
2121
edb "github.com/gobitfly/beaconchain/pkg/exporter/db"
22+
"github.com/gobitfly/beaconchain/pkg/monitoring/constants"
2223
"github.com/pkg/errors"
2324
"github.com/prysmaticlabs/go-bitfield"
2425
"golang.org/x/sync/errgroup"
@@ -992,6 +993,10 @@ func (d *dashboardData) GetName() string {
992993
return "Dashboard-Data"
993994
}
994995

996+
func (d *dashboardData) GetMonitoringEventId() constants.Event {
997+
return constants.Event_ExporterModuleDashboardData
998+
}
999+
9951000
func (d *dashboardData) OnHead(event *constypes.StandardEventHeadResponse) error {
9961001
return nil
9971002
}

‎backend/pkg/exporter/modules/execution_deposits_exporter.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/gobitfly/beaconchain/pkg/commons/types"
3232
"github.com/gobitfly/beaconchain/pkg/commons/utils"
3333
constypes "github.com/gobitfly/beaconchain/pkg/consapi/types"
34+
"github.com/gobitfly/beaconchain/pkg/monitoring/constants"
3435
)
3536

3637
// if we ever end up in a situation where we possibly have gaps in the data remember: the merkletree_index is unique.
@@ -155,6 +156,10 @@ func (d *executionDepositsExporter) GetName() string {
155156
return "ExecutionDeposits-Exporter"
156157
}
157158

159+
func (d *executionDepositsExporter) GetMonitoringEventId() constants.Event {
160+
return constants.Event_ExporterModuleELDepositsExporter
161+
}
162+
158163
func (d *executionDepositsExporter) OnChainReorg(event *constypes.StandardEventChainReorg) (err error) {
159164
return nil // nop
160165
}

0 commit comments

Comments
 (0)