admission: per-store work queue metrics

This patch splits store work queue metrics to be per-store. It also does
some general clean up around the `storeGrantCoordinators` code to
centralize initialization of requesters and metrics.

Fixes #131562

Release note: None

Author: aadityasondhi

pkg/kv/kvserver/stores.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/limit"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/protoutil"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/errors"
@@ -326,3 +327,11 @@ func (ls *Stores) CloseDiskMonitors() {
 		return nil
 	})
 }
+
+// GetStoreMetricRegistry returns the metric registry of the provided store ID.
+func (ls *Stores) GetStoreMetricRegistry(storeID roachpb.StoreID) *metric.Registry {
+	if s, ok := ls.storeMap.Load(storeID); ok {
+		return s.Registry()
+	}
+	return nil
+}

pkg/server/node.go

@@ -1357,6 +1357,10 @@ func (n *Node) registerEnginesForDiskStatsMap(
 	return pmp, nil
 }
 
+func (n *Node) makeStoreRegistryProvider() admission.MetricsRegistryProvider {
+	return &storeMetricsRegistryProvider{n: n}
+}
+
 type nodePebbleMetricsProvider struct {
 	n            *Node
 	diskStatsMap diskStatsMap
@@ -1382,7 +1386,8 @@ func (pmp *nodePebbleMetricsProvider) GetPebbleMetrics() []admission.StoreMetric
 			StoreID:         store.StoreID(),
 			Metrics:         m.Metrics,
 			WriteStallCount: m.WriteStallCount,
-			DiskStats:       diskStats})
+			DiskStats:       diskStats,
+		})
 		return nil
 	})
 	return metrics
@@ -1393,6 +1398,17 @@ func (pmp *nodePebbleMetricsProvider) Close() {
 	pmp.diskStatsMap.closeDiskMonitors()
 }
 
+type storeMetricsRegistryProvider struct {
+	n *Node
+}
+
+// GetMetricsRegistry implements admission.MetricsRegistryProvider.
+func (mrp *storeMetricsRegistryProvider) GetMetricsRegistry(
+	storeID roachpb.StoreID,
+) *metric.Registry {
+	return mrp.n.stores.GetStoreMetricRegistry(storeID)
+}
+
 // GetTenantWeights implements kvserver.TenantWeightProvider.
 func (n *Node) GetTenantWeights() kvadmission.TenantWeights {
 	weights := kvadmission.TenantWeights{

pkg/server/server.go

@@ -1954,6 +1954,10 @@ func (s *topLevelServer) PreStart(ctx context.Context) error {
 		return errors.Wrapf(err, "failed to register engines for the disk stats map")
 	}
 
+	// Set up a store metrics registry provider to register AC store-level
+	// metrics.
+	mrp := s.node.makeStoreRegistryProvider()
+
 	// Stores have been initialized, so Node can now provide Pebble metrics.
 	//
 	// Note that all existing stores will be operational before Pebble-level
@@ -1962,7 +1966,7 @@ func (s *topLevelServer) PreStart(ctx context.Context) error {
 	// existing stores shouldn’t be able to acquire leases yet. Although, below
 	// Raft commands like log application and snapshot application may be able
 	// to bypass admission control.
-	s.storeGrantCoords.SetPebbleMetricsProvider(ctx, pmp, s.node)
+	s.storeGrantCoords.SetPebbleMetricsProvider(ctx, pmp, mrp, s.node)
 
 	// Once all stores are initialized, check if offline storage recovery
 	// was done prior to start and record any actions appropriately.

pkg/ui/workspaces/db-console/src/views/cluster/containers/nodeGraphs/dashboards/overload.tsx

@@ -9,11 +9,8 @@ import React from "react";
 import LineGraph from "src/views/cluster/components/linegraph";
 import { Metric, Axis } from "src/views/shared/components/metricQuery";
 
-import {
-  GraphDashboardProps,
-  nodeDisplayName,
-  storeIDsForNode,
-} from "./dashboardUtils";
+import { GraphDashboardProps, nodeDisplayName } from "./dashboardUtils";
+import { storeMetrics } from "./storeUtils";
 
 export default function (props: GraphDashboardProps) {
   const {
@@ -66,36 +63,32 @@ export default function (props: GraphDashboardProps) {
 
     <LineGraph
       title="Admission IO Tokens Exhausted Duration Per Second"
-      sources={nodeSources}
+      sources={storeSources}
       tenantSource={tenantSource}
       showMetricsInTooltip={true}
       tooltip={`Relative time the node had exhausted IO tokens for all IO-bound work per second of wall time, measured in microseconds/second. Increased IO token exhausted duration indicates IO resource exhaustion.`}
     >
       <Axis label="Duration (micros/sec)">
-        {nodeIDs.map(nid => (
-          <>
-            <Metric
-              key={nid}
-              name="cr.node.admission.granter.io_tokens_exhausted_duration.kv"
-              title={
-                "Regular (Foreground) " +
-                nodeDisplayName(nodeDisplayNameByID, nid)
-              }
-              sources={[nid]}
-              nonNegativeRate
-            />
-            <Metric
-              key={nid}
-              name="cr.node.admission.granter.elastic_io_tokens_exhausted_duration.kv"
-              title={
-                "Elastic (Background) " +
-                nodeDisplayName(nodeDisplayNameByID, nid)
-              }
-              sources={[nid]}
-              nonNegativeRate
-            />
-          </>
-        ))}
+        {storeMetrics(
+          {
+            name: "cr.store.admission.granter.io_tokens_exhausted_duration.kv",
+            nonNegativeRate: true,
+            aggregateMax: true,
+          },
+          nodeIDs,
+          storeIDsByNodeID,
+          "regular (foreground)",
+        )}
+        {storeMetrics(
+          {
+            name: "cr.store.admission.granter.elastic_io_tokens_exhausted_duration.kv",
+            nonNegativeRate: true,
+            aggregateMax: true,
+          },
+          nodeIDs,
+          storeIDsByNodeID,
+          "elastic (background)",
+        )}
       </Axis>
     </LineGraph>,
 
@@ -107,16 +100,14 @@ export default function (props: GraphDashboardProps) {
       showMetricsInTooltip={true}
     >
       <Axis label="Score">
-        {nodeIDs.map(nid => (
-          <>
-            <Metric
-              key={nid}
-              name="cr.store.admission.io.overload"
-              title={nodeDisplayName(nodeDisplayNameByID, nid)}
-              sources={storeIDsForNode(storeIDsByNodeID, nid)}
-            />
-          </>
-        ))}
+        {storeMetrics(
+          {
+            name: "cr.store.admission.io.overload",
+            aggregateMax: true,
+          },
+          nodeIDs,
+          storeIDsByNodeID,
+        )}
       </Axis>
     </LineGraph>,
 
@@ -178,30 +169,30 @@ export default function (props: GraphDashboardProps) {
 
     <LineGraph
       title="Admission Queueing Delay p99 – Store"
-      sources={nodeSources}
+      sources={storeSources}
       tenantSource={tenantSource}
       showMetricsInTooltip={true}
       tooltip={`The 99th percentile latency of requests waiting in the Admission Control store queue.`}
     >
       <Axis units={AxisUnits.Duration} label="Write Delay Duration">
-        {nodeIDs.map(nid => (
-          <>
-            <Metric
-              key={nid}
-              name="cr.node.admission.wait_durations.kv-stores-p99"
-              title={"KV " + nodeDisplayName(nodeDisplayNameByID, nid)}
-              sources={[nid]}
-              downsampleMax
-            />
-            <Metric
-              key={nid}
-              name="cr.node.admission.wait_durations.elastic-stores-p99"
-              title={"Elastic " + nodeDisplayName(nodeDisplayNameByID, nid)}
-              sources={[nid]}
-              downsampleMax
-            />
-          </>
-        ))}
+        {storeMetrics(
+          {
+            name: "cr.store.admission.wait_durations.kv-stores-p99",
+            aggregateMax: true,
+          },
+          nodeIDs,
+          storeIDsByNodeID,
+          "KV",
+        )}
+        {storeMetrics(
+          {
+            name: "cr.store.admission.wait_durations.elastic-stores-p99",
+            aggregateMax: true,
+          },
+          nodeIDs,
+          storeIDsByNodeID,
+          "elastic",
+        )}
       </Axis>
     </LineGraph>,
 
@@ -420,16 +411,14 @@ export default function (props: GraphDashboardProps) {
       showMetricsInTooltip={true}
     >
       <Axis label="Count">
-        {nodeIDs.map(nid => (
-          <>
-            <Metric
-              key={nid}
-              name="cr.store.storage.l0-sublevels"
-              title={nodeDisplayName(nodeDisplayNameByID, nid)}
-              sources={storeIDsForNode(storeIDsByNodeID, nid)}
-            />
-          </>
-        ))}
+        {storeMetrics(
+          {
+            name: "cr.store.storage.l0-sublevels",
+            aggregateMax: true,
+          },
+          nodeIDs,
+          storeIDsByNodeID,
+        )}
       </Axis>
     </LineGraph>,
   ];