feat(auditor): add metrics instrumentation to SetStatus, GetStatus and GetTokenRequest

Audit(), Append() and Release() already recorded duration histograms and
error counters, but the three query methods SetStatus, GetStatus and
GetTokenRequest had no instrumentation at all. In production these are
high-frequency calls on the audit path and without metrics operators
have no way to detect slow queries or elevated error rates.

Add duration histograms and error counters for all three methods using
the same pattern already established in the same file, and update the
internal metrics tests to reflect the new counter and histogram counts.

Signed-off-by: Rama542 <Rama542@users.noreply.github.com>

Author: Rama542

token/services/auditor/auditor.go

@@ -187,18 +187,44 @@ func (a *Service) Release(ctx context.Context, tx Transaction) {
 
 // SetStatus sets the status of the audit records with the passed transaction id to the passed status
 func (a *Service) SetStatus(ctx context.Context, txID string, status storage.TxStatus, message string) error {
-	return a.auditDB.SetStatus(ctx, txID, status, message)
+	start := time.Now()
+	defer func() { a.metrics.SetStatusDuration.Observe(time.Since(start).Seconds()) }()
+	if err := a.auditDB.SetStatus(ctx, txID, status, message); err != nil {
+		a.metrics.SetStatusErrors.Add(1)
+
+		return err
+	}
+
+	return nil
 }
 
 // GetStatus return the status of the given transaction id.
 // It returns an error if no transaction with that id is found
 func (a *Service) GetStatus(ctx context.Context, txID string) (TxStatus, string, error) {
-	return a.auditDB.GetStatus(ctx, txID)
+	start := time.Now()
+	defer func() { a.metrics.GetStatusDuration.Observe(time.Since(start).Seconds()) }()
+	status, message, err := a.auditDB.GetStatus(ctx, txID)
+	if err != nil {
+		a.metrics.GetStatusErrors.Add(1)
+
+		return status, message, err
+	}
+
+	return status, message, nil
 }
 
 // GetTokenRequest returns the token request bound to the passed transaction id, if available.
 func (a *Service) GetTokenRequest(ctx context.Context, txID string) ([]byte, error) {
-	return a.auditDB.GetTokenRequest(ctx, txID)
+	start := time.Now()
+	defer func() { a.metrics.GetTokenRequestDuration.Observe(time.Since(start).Seconds()) }()
+	data, err := a.auditDB.GetTokenRequest(ctx, txID)
+	if err != nil {
+		a.metrics.GetTokenRequestErrors.Add(1)
+
+		return nil, err
+	}
+
+	return data, nil
 }
 
 func (a *Service) Check(ctx context.Context) ([]string, error) {

token/services/auditor/auditor_internal_test.go

@@ -50,6 +50,12 @@ func TestNewMetrics_NilProvider(t *testing.T) {
 	assert.NotNil(t, m.AppendDuration)
 	assert.NotNil(t, m.AppendErrors)
 	assert.NotNil(t, m.ReleasesTotal)
+	assert.NotNil(t, m.SetStatusDuration)
+	assert.NotNil(t, m.SetStatusErrors)
+	assert.NotNil(t, m.GetStatusDuration)
+	assert.NotNil(t, m.GetStatusErrors)
+	assert.NotNil(t, m.GetTokenRequestDuration)
+	assert.NotNil(t, m.GetTokenRequestErrors)
 }
 
 func TestNewMetrics_WithProvider(t *testing.T) {
@@ -60,10 +66,12 @@ func TestNewMetrics_WithProvider(t *testing.T) {
 
 	m := newMetrics(mp)
 	require.NotNil(t, m)
-	// AuditLockConflicts, AppendErrors, ReleasesTotal = 3 counters
-	assert.Equal(t, 3, mp.NewCounterCallCount())
-	// AuditDuration, AppendDuration = 2 histograms
-	assert.Equal(t, 2, mp.NewHistogramCallCount())
+	// AuditLockConflicts, AppendErrors, ReleasesTotal,
+	// SetStatusErrors, GetStatusErrors, GetTokenRequestErrors = 6 counters
+	assert.Equal(t, 6, mp.NewCounterCallCount())
+	// AuditDuration, AppendDuration,
+	// SetStatusDuration, GetStatusDuration, GetTokenRequestDuration = 5 histograms
+	assert.Equal(t, 5, mp.NewHistogramCallCount())
 }
 
 func TestNoopCounter_With_ReturnsSelf(t *testing.T) {
@@ -182,9 +190,15 @@ func TestMetricsProviderCall(t *testing.T) {
 		m.AuditLockConflicts.Add(1)
 		m.AppendErrors.Add(1)
 		m.ReleasesTotal.Add(1)
+		m.SetStatusErrors.Add(1)
+		m.GetStatusErrors.Add(1)
+		m.GetTokenRequestErrors.Add(1)
 
 		m.AuditDuration.Observe(1.0)
 		m.AppendDuration.Observe(1.0)
+		m.SetStatusDuration.Observe(1.0)
+		m.GetStatusDuration.Observe(1.0)
+		m.GetTokenRequestDuration.Observe(1.0)
 	})
 
 	nc := &noopCounter{}

token/services/auditor/metrics.go

@@ -31,6 +31,27 @@ type Metrics struct {
 	// ReleasesTotal counts all calls to Release(), whether invoked explicitly
 	// or via the defer inside Append().
 	ReleasesTotal metrics.Counter
+
+	// SetStatusDuration is a histogram of the wall-clock time for each
+	// SetStatus() invocation, in seconds.
+	SetStatusDuration metrics.Histogram
+
+	// SetStatusErrors counts calls to SetStatus() that returned an error.
+	SetStatusErrors metrics.Counter
+
+	// GetStatusDuration is a histogram of the wall-clock time for each
+	// GetStatus() invocation, in seconds.
+	GetStatusDuration metrics.Histogram
+
+	// GetStatusErrors counts calls to GetStatus() that returned an error.
+	GetStatusErrors metrics.Counter
+
+	// GetTokenRequestDuration is a histogram of the wall-clock time for each
+	// GetTokenRequest() invocation, in seconds.
+	GetTokenRequestDuration metrics.Histogram
+
+	// GetTokenRequestErrors counts calls to GetTokenRequest() that returned an error.
+	GetTokenRequestErrors metrics.Counter
 }
 
 func newMetrics(p metrics.Provider) *Metrics {
@@ -61,6 +82,33 @@ func newMetrics(p metrics.Provider) *Metrics {
 			Name: "auditor_releases_total",
 			Help: "Total number of Release() calls (explicit and deferred)",
 		}),
+		SetStatusDuration: p.NewHistogram(metrics.HistogramOpts{
+			Name:    "auditor_set_status_duration_seconds",
+			Help:    "Histogram of SetStatus() processing time per call, in seconds",
+			Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
+		}),
+		SetStatusErrors: p.NewCounter(metrics.CounterOpts{
+			Name: "auditor_set_status_errors_total",
+			Help: "Total number of SetStatus() calls that returned an error",
+		}),
+		GetStatusDuration: p.NewHistogram(metrics.HistogramOpts{
+			Name:    "auditor_get_status_duration_seconds",
+			Help:    "Histogram of GetStatus() processing time per call, in seconds",
+			Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
+		}),
+		GetStatusErrors: p.NewCounter(metrics.CounterOpts{
+			Name: "auditor_get_status_errors_total",
+			Help: "Total number of GetStatus() calls that returned an error",
+		}),
+		GetTokenRequestDuration: p.NewHistogram(metrics.HistogramOpts{
+			Name:    "auditor_get_token_request_duration_seconds",
+			Help:    "Histogram of GetTokenRequest() processing time per call, in seconds",
+			Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10},
+		}),
+		GetTokenRequestErrors: p.NewCounter(metrics.CounterOpts{
+			Name: "auditor_get_token_request_errors_total",
+			Help: "Total number of GetTokenRequest() calls that returned an error",
+		}),
 	}
 }