CBG-4465: [3.2.3 backport] Add Database Init and Online fatal error stats (#7381)

Co-authored-by: Ben Brooks <ben.brooks@couchbase.com>
Co-authored-by: Tor Colvin <tor.colvin@couchbase.com>

Author: 3 people

base/stats.go

@@ -88,6 +88,7 @@ const (
 	StatAddedVersion3dot2dot0     = "3.2.0"
 	StatAddedVersion3dot2dot1     = "3.2.1"
 	StatAddedVersion3dot2dot2     = "3.2.2"
+	StatAddedVersion3dot2dot3     = "3.2.3"
 	StatAddedVersion3dot3dot0     = "3.3.0"
 
 	StatDeprecatedVersionNotDeprecated = ""
@@ -662,6 +663,11 @@ type DatabaseStats struct {
 	// prior to Hydrogen release. These are not exported as part of prometheus and only exposed through expvars
 	CacheFeedMapStats  *ExpVarMapWrapper `json:"cache_feed"`
 	ImportFeedMapStats *ExpVarMapWrapper `json:"import_feed"`
+
+	// The total number of errors that occurred that prevented the database from being initialized.
+	TotalInitFatalErrors *SgwIntStat `json:"total_init_fatal_errors"`
+	// The total number of errors that occurred that prevented the database from being brought online.
+	TotalOnlineFatalErrors *SgwIntStat `json:"total_online_fatal_errors"`
 }
 
 // This wrapper ensures that an expvar.Map type can be marshalled into JSON. The expvar.Map has no method to go direct to
@@ -1782,6 +1788,14 @@ func (d *DbStats) initDatabaseStats() error {
 	if err != nil {
 		return err
 	}
+	resUtil.TotalInitFatalErrors, err = NewIntStat(SubsystemDatabaseKey, "total_init_fatal_errors", StatUnitNoUnits, TotalInitFatalErrorsDesc, StatAddedVersion3dot2dot3, StatDeprecatedVersionNotDeprecated, StatStabilityVolatile, labelKeys, labelVals, prometheus.CounterValue, 0)
+	if err != nil {
+		return err
+	}
+	resUtil.TotalOnlineFatalErrors, err = NewIntStat(SubsystemDatabaseKey, "total_online_fatal_errors", StatUnitNoUnits, TotalOnlineFatalErrorsDesc, StatAddedVersion3dot2dot3, StatDeprecatedVersionNotDeprecated, StatStabilityVolatile, labelKeys, labelVals, prometheus.CounterValue, 0)
+	if err != nil {
+		return err
+	}
 	resUtil.ImportFeedMapStats = &ExpVarMapWrapper{new(expvar.Map).Init()}
 
 	resUtil.CacheFeedMapStats = &ExpVarMapWrapper{new(expvar.Map).Init()}
@@ -1830,6 +1844,8 @@ func (d *DbStats) unregisterDatabaseStats() {
 	prometheus.Unregister(d.DatabaseStats.NumPublicRestRequests)
 	prometheus.Unregister(d.DatabaseStats.TotalSyncTime)
 	prometheus.Unregister(d.DatabaseStats.PublicRestBytesRead)
+	prometheus.Unregister(d.DatabaseStats.TotalInitFatalErrors)
+	prometheus.Unregister(d.DatabaseStats.TotalOnlineFatalErrors)
 }
 
 func (d *DbStats) CollectionStat(scopeName, collectionName string) (*CollectionStats, error) {

base/stats_descriptions.go

@@ -316,6 +316,9 @@ const (
 
 	NumIdleKvOpsDesc    = "The total number of idle kv operations."
 	NumIdleQueryOpsDesc = "The total number of idle query operations."
+
+	TotalInitFatalErrorsDesc   = "The total number of errors that occurred that prevented the database from being initialized."
+	TotalOnlineFatalErrorsDesc = "The total number of errors that occurred that prevented the database from being brought online."
 )
 
 // Delta Sync stats descriptions

rest/server_context.go

@@ -964,6 +964,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config
 		if dbInitDoneChan != nil {
 			initError := <-dbInitDoneChan
 			if initError != nil {
+				dbcontext.DbStats.DatabaseStats.TotalInitFatalErrors.Add(1)
 				// report error in building/creating indexes
 				dbcontext.DatabaseStartupError = db.NewDatabaseError(db.DatabaseInitializationIndexError)
 				atomic.StoreUint32(&dbcontext.State, db.DBOffline)
@@ -973,6 +974,7 @@ func (sc *ServerContext) _getOrAddDatabaseFromConfig(ctx context.Context, config
 		}
 		base.InfofCtx(ctx, base.KeyAll, "Database init completed, starting online processes")
 		if err := dbcontext.StartOnlineProcesses(ctx); err != nil {
+			dbcontext.DbStats.DatabaseStats.TotalOnlineFatalErrors.Add(1)
 			atomic.StoreUint32(&dbcontext.State, db.DBOffline)
 			_ = dbcontext.EventMgr.RaiseDBStateChangeEvent(ctx, dbName, "offline", dbLoadedStateChangeMsg, &sc.Config.API.AdminInterface)
 			return nil, err
@@ -999,6 +1001,7 @@ func (sc *ServerContext) asyncDatabaseOnline(nonCancelCtx base.NonCancellableCon
 	if doneChan != nil {
 		initError := <-doneChan
 		if initError != nil {
+			dbc.DbStats.DatabaseStats.TotalInitFatalErrors.Add(1)
 			base.WarnfCtx(ctx, "Async database init returned error: %v", initError)
 			dbc.DatabaseStartupError = db.NewDatabaseError(db.DatabaseInitializationIndexError)
 			atomic.CompareAndSwapUint32(&dbc.State, db.DBStarting, db.DBOffline)
@@ -1017,6 +1020,7 @@ func (sc *ServerContext) asyncDatabaseOnline(nonCancelCtx base.NonCancellableCon
 	base.InfofCtx(ctx, base.KeyAll, "Async database initialization complete, starting online processes...")
 	err := dbc.StartOnlineProcesses(ctx)
 	if err != nil {
+		dbc.DbStats.DatabaseStats.TotalOnlineFatalErrors.Add(1)
 		base.ErrorfCtx(ctx, "Error starting online processes after async initialization: %v", err)
 		atomic.CompareAndSwapUint32(&dbc.State, db.DBStarting, db.DBOffline)
 		return