From 30a3e1779de9669fdb21c1bd921792ab0644ade1 Mon Sep 17 00:00:00 2001 From: Roberto Alvarez Date: Tue, 17 Mar 2026 15:30:26 -0500 Subject: [PATCH 1/3] SD-11577: add support for kv metrics --- README.md | 3 ++ cloudflare.go | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 1 + prometheus.go | 54 ++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+) diff --git a/README.md b/README.md index ceca244..987aeec 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Required authentication scopes: - `Zone/Firewall Services:Read` is required to fetch zone rule name for `cloudflare_zone_firewall_events_count` metric - `Account/Account Rulesets:Read` is required to fetch account rule name for `cloudflare_zone_firewall_events_count` metric - `Account:Load Balancing: Monitors and Pools:Read` is required to fetch pools origin health status `cloudflare_pool_origin_health_status` metric +- `Account/Workers KV Storage:Read` is required for KV metrics - `Cloudflare Tunnel Read` is required to fetch Cloudflare Tunnel (Cloudflare Zero Trust) metrics To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`) @@ -127,6 +128,8 @@ Note: `ZONE_` configuration is not supported as flag. # HELP cloudflare_r2_operation_count Number of operations performed by R2 # HELP cloudflare_r2_storage_bytes Storage used by R2 # HELP cloudflare_r2_storage_total_bytes Total storage used by R2 +# HELP cloudflare_kv_requests_count Number of KV operations by namespace and action type +# HELP cloudflare_kv_latency KV operation latency quantiles (milliseconds) ``` ## Helm chart repository diff --git a/cloudflare.go b/cloudflare.go index ea5d23b..69b9818 100644 --- a/cloudflare.go +++ b/cloudflare.go @@ -7,6 +7,7 @@ import ( cf "github.com/cloudflare/cloudflare-go/v4" cfaccounts "github.com/cloudflare/cloudflare-go/v4/accounts" + cfkv "github.com/cloudflare/cloudflare-go/v4/kv" cfload_balancers "github.com/cloudflare/cloudflare-go/v4/load_balancers" cfpagination "github.com/cloudflare/cloudflare-go/v4/packages/pagination" cfrulesets "github.com/cloudflare/cloudflare-go/v4/rulesets" @@ -299,6 +300,30 @@ type lbResp struct { ZoneTag string `json:"zoneTag"` } +type cloudflareResponseKV struct { + Viewer struct { + Accounts []kvAccountResp `json:"accounts"` + } `json:"viewer"` +} + +type kvAccountResp struct { + KvOperationsAdaptiveGroups []struct { + Dimensions struct { + NamespaceID string `json:"namespaceId"` + ActionType string `json:"actionType"` + } `json:"dimensions"` + Sum struct { + Requests uint64 `json:"requests"` + } `json:"sum"` + Quantiles struct { + LatencyMsP50 float32 `json:"latencyMsP50"` + LatencyMsP75 float32 `json:"latencyMsP75"` + LatencyMsP99 float32 `json:"latencyMsP99"` + LatencyMsP999 float32 `json:"latencyMsP999"` + } `json:"quantiles"` + } `json:"kvOperationsAdaptiveGroups"` +} + type cloudflareResponseDNSFirewall struct { Viewer struct { Accounts []dnsFirewallAccountResp `json:"accounts"` @@ -1085,6 +1110,80 @@ func filterNonFreePlanZones(zones []cfzones.Zone) (filteredZones []cfzones.Zone) return } +func fetchKVNamespaces(accountID string) (map[string]string, error) { + namespaceMap := make(map[string]string) + ctx, cancel := context.WithTimeout(context.Background(), cftimeout) + defer cancel() + page := cfclient.KV.Namespaces.ListAutoPaging(ctx, cfkv.NamespaceListParams{ + AccountID: cf.F(accountID), + }) + if page.Err() != nil { + return nil, page.Err() + } + + seenIDs := make(map[string]struct{}) + for page.Next() { + if page.Err() != nil { + log.Errorf("error during paging KV namespaces: %v", page.Err()) + break + } + ns := page.Current() + if _, exists := seenIDs[ns.ID]; exists { + log.Errorf("fetchKVNamespaces: duplicate namespace ID detected (%s), breaking loop", ns.ID) + break + } + seenIDs[ns.ID] = struct{}{} + namespaceMap[ns.ID] = ns.Title + } + + return namespaceMap, nil +} + +func fetchKVOperations(accountID string) (*cloudflareResponseKV, error) { + request := graphql.NewRequest(` + query ($accountID: String!, $mintime: Time!, $maxtime: Time!, $limit: Int!) { + viewer { + accounts(filter: {accountTag: $accountID}) { + kvOperationsAdaptiveGroups(limit: $limit, filter: {datetime_geq: $mintime, datetime_lt: $maxtime}) { + dimensions { + namespaceId + actionType + } + sum { + requests + } + quantiles { + latencyMsP50 + latencyMsP75 + latencyMsP99 + latencyMsP999 + } + } + } + } + }`) + + now, now1mAgo := GetTimeRange() + request.Var("limit", gqlQueryLimit) + request.Var("maxtime", now) + request.Var("mintime", now1mAgo) + request.Var("accountID", accountID) + + gql.Mu.RLock() + defer gql.Mu.RUnlock() + + ctx, cancel := context.WithTimeout(context.Background(), cftimeout) + defer cancel() + + var resp cloudflareResponseKV + if err := gql.Client.Run(ctx, request, &resp); err != nil { + log.Errorf("error fetching KV operations, err:%v", err) + return nil, err + } + + return &resp, nil +} + func fetchDNSFirewallTotals(accountID string) (*cloudflareResponseDNSFirewall, error) { request := graphql.NewRequest(` query ($accountID: string, $mintime: Time!, $maxtime: Time!, $limit: Int!) { diff --git a/main.go b/main.go index 5e9bcc6..1298180 100644 --- a/main.go +++ b/main.go @@ -110,6 +110,7 @@ func fetchMetrics(deniedMetricsSet MetricsSet) { for _, a := range accounts { go fetchWorkerAnalytics(a, &wg) + go fetchKVAnalytics(a, &wg) go fetchLogpushAnalyticsForAccount(a, &wg) go fetchR2StorageForAccount(a, &wg) go fetchLoadblancerPoolsHealth(a, &wg) diff --git a/prometheus.go b/prometheus.go index 1f8395a..82f4820 100644 --- a/prometheus.go +++ b/prometheus.go @@ -64,6 +64,8 @@ const ( tunnelConnectorInfoMetricName MetricName = "cloudflare_tunnel_connector_info" tunnelConnectorActiveConnectionsMetricName MetricName = "cloudflare_tunnel_connector_active_connections" dnsFirewallQueryCountMetricName MetricName = "cloudflare_dns_firewall_query_count" + kvRequestsMetricName MetricName = "cloudflare_kv_requests_count" + kvLatencyMetricName MetricName = "cloudflare_kv_latency" ) type MetricsSet map[MetricName]struct{} @@ -336,6 +338,16 @@ var ( Help: "Reports number of active connections for a Cloudflare Tunnel connector", }, []string{"account", "tunnel_id", "client_id"}) + kvRequests = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: kvRequestsMetricName.String(), + Help: "Number of KV operations by namespace and action type", + }, []string{"namespace_name", "action_type", "account"}) + + kvLatency = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: kvLatencyMetricName.String(), + Help: "KV operation latency quantiles (milliseconds)", + }, []string{"namespace_name", "action_type", "account", "quantile"}) + dnsFirewallQueryCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: dnsFirewallQueryCountMetricName.String(), Help: "DNS Firewall query count by query type and response code", @@ -388,6 +400,8 @@ func buildAllMetricsSet() MetricsSet { allMetricsSet.Add(tunnelConnectorInfoMetricName) allMetricsSet.Add(tunnelConnectorActiveConnectionsMetricName) allMetricsSet.Add(dnsFirewallQueryCountMetricName) + allMetricsSet.Add(kvRequestsMetricName) + allMetricsSet.Add(kvLatencyMetricName) return allMetricsSet } @@ -537,6 +551,12 @@ func mustRegisterMetrics(deniedMetrics MetricsSet) { if !deniedMetrics.Has(dnsFirewallQueryCountMetricName) { prometheus.MustRegister(dnsFirewallQueryCount) } + if !deniedMetrics.Has(kvRequestsMetricName) { + prometheus.MustRegister(kvRequests) + } + if !deniedMetrics.Has(kvLatencyMetricName) { + prometheus.MustRegister(kvLatency) + } } func fetchLoadblancerPoolsHealth(account cfaccounts.Account, wg *sync.WaitGroup) { @@ -607,6 +627,40 @@ func fetchWorkerAnalytics(account cfaccounts.Account, wg *sync.WaitGroup) { } } +func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup) { + wg.Add(1) + defer wg.Done() + + namespaceMap, err := fetchKVNamespaces(account.ID) + if err != nil { + log.Error("failed to fetch KV namespaces for account ", account.ID, ": ", err) + return + } + + r, err := fetchKVOperations(account.ID) + if err != nil { + log.Error("failed to fetch KV operations for account ", account.ID, ": ", err) + return + } + + accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-")) + + for _, a := range r.Viewer.Accounts { + for _, kv := range a.KvOperationsAdaptiveGroups { + namespaceName := namespaceMap[kv.Dimensions.NamespaceID] + if namespaceName == "" { + namespaceName = kv.Dimensions.NamespaceID + } + + kvRequests.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName}).Set(float64(kv.Sum.Requests)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999)) + } + } +} + func fetchLogpushAnalyticsForAccount(account cfaccounts.Account, wg *sync.WaitGroup) { wg.Add(1) defer wg.Done() From ddc5fc5288f9a9c58425dd7eb6ce554f83cc9d09 Mon Sep 17 00:00:00 2001 From: Roberto Alvarez Date: Tue, 17 Mar 2026 15:42:43 -0500 Subject: [PATCH 2/3] SD-11577: code review from agent --- main.go | 2 +- prometheus.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 1298180..3bb6ddb 100644 --- a/main.go +++ b/main.go @@ -110,7 +110,7 @@ func fetchMetrics(deniedMetricsSet MetricsSet) { for _, a := range accounts { go fetchWorkerAnalytics(a, &wg) - go fetchKVAnalytics(a, &wg) + go fetchKVAnalytics(a, &wg, deniedMetricsSet) go fetchLogpushAnalyticsForAccount(a, &wg) go fetchR2StorageForAccount(a, &wg) go fetchLoadblancerPoolsHealth(a, &wg) diff --git a/prometheus.go b/prometheus.go index 82f4820..d102a2c 100644 --- a/prometheus.go +++ b/prometheus.go @@ -627,7 +627,7 @@ func fetchWorkerAnalytics(account cfaccounts.Account, wg *sync.WaitGroup) { } } -func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup) { +func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) { wg.Add(1) defer wg.Done() @@ -652,11 +652,15 @@ func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup) { namespaceName = kv.Dimensions.NamespaceID } - kvRequests.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName}).Set(float64(kv.Sum.Requests)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999)) + if !deniedMetricsSet.Has(kvRequestsMetricName) { + kvRequests.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName}).Set(float64(kv.Sum.Requests)) + } + if !deniedMetricsSet.Has(kvLatencyMetricName) { + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99)) + kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999)) + } } } } From e3fb3b0c2d00c2c9ebaf4e02a0a119d3831f1b2a Mon Sep 17 00:00:00 2001 From: Roberto Alvarez Date: Wed, 18 Mar 2026 15:46:01 -0500 Subject: [PATCH 3/3] SD-11577: add support for bucketing metrics based on args --- README.md | 3 ++- cloudflare.go | 30 ------------------------------ main.go | 20 ++++++++++++++++++++ prometheus.go | 26 ++++++++++---------------- 4 files changed, 32 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 987aeec..6517068 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ Required authentication scopes: - `Zone/Firewall Services:Read` is required to fetch zone rule name for `cloudflare_zone_firewall_events_count` metric - `Account/Account Rulesets:Read` is required to fetch account rule name for `cloudflare_zone_firewall_events_count` metric - `Account:Load Balancing: Monitors and Pools:Read` is required to fetch pools origin health status `cloudflare_pool_origin_health_status` metric -- `Account/Workers KV Storage:Read` is required for KV metrics - `Cloudflare Tunnel Read` is required to fetch Cloudflare Tunnel (Cloudflare Zero Trust) metrics To authenticate this way, only set `CF_API_TOKEN` (omit `CF_API_EMAIL` and `CF_API_KEY`) @@ -67,6 +66,7 @@ The exporter can be configured using env variables or command flags. | `SCRAPE_DELAY` | scrape delay in seconds, default `300` | | `SCRAPE_INTERVAL` | scrape interval in seconds (will query cloudflare every SCRAPE_INTERVAL seconds), default `60` | | `METRICS_DENYLIST` | (Optional) cloudflare-exporter metrics to not export, comma delimited list of cloudflare-exporter metrics. If not set, all metrics are exported | +| `KV_NAMESPACE_IDS` | (Optional) KV namespace IDs to track individually, comma delimited. Unlisted namespaces are aggregated as `other` | | `ENABLE_PPROF` | (Optional) enable pprof profiling endpoints at `/debug/pprof/`. Accepts `true` or `false`, default `false`. **Warning**: Only enable in development/debugging environments | | `ZONE_` | `DEPRECATED since 0.0.5` (optional) Zone ID. Add zones you want to scrape by adding env vars in this format. You can find the zone ids in Cloudflare dashboards. | | `LOG_LEVEL` | Set loglevel. Options are error, warn, info, debug. default `error` | @@ -85,6 +85,7 @@ Corresponding flags: -metrics_path="/metrics": path for metrics, default /metrics -scrape_delay=300: scrape delay in seconds, defaults to 300 -scrape_interval=60: scrape interval in seconds, defaults to 60 + -kv_namespace_ids="": KV namespace IDs to track individually, comma delimited -metrics_denylist="": cloudflare-exporter metrics to not export, comma delimited list -enable_pprof=false: enable pprof profiling endpoints at /debug/pprof/ -log_level="error": log level(error,warn,info,debug) diff --git a/cloudflare.go b/cloudflare.go index 69b9818..b4f1b20 100644 --- a/cloudflare.go +++ b/cloudflare.go @@ -7,7 +7,6 @@ import ( cf "github.com/cloudflare/cloudflare-go/v4" cfaccounts "github.com/cloudflare/cloudflare-go/v4/accounts" - cfkv "github.com/cloudflare/cloudflare-go/v4/kv" cfload_balancers "github.com/cloudflare/cloudflare-go/v4/load_balancers" cfpagination "github.com/cloudflare/cloudflare-go/v4/packages/pagination" cfrulesets "github.com/cloudflare/cloudflare-go/v4/rulesets" @@ -1110,35 +1109,6 @@ func filterNonFreePlanZones(zones []cfzones.Zone) (filteredZones []cfzones.Zone) return } -func fetchKVNamespaces(accountID string) (map[string]string, error) { - namespaceMap := make(map[string]string) - ctx, cancel := context.WithTimeout(context.Background(), cftimeout) - defer cancel() - page := cfclient.KV.Namespaces.ListAutoPaging(ctx, cfkv.NamespaceListParams{ - AccountID: cf.F(accountID), - }) - if page.Err() != nil { - return nil, page.Err() - } - - seenIDs := make(map[string]struct{}) - for page.Next() { - if page.Err() != nil { - log.Errorf("error during paging KV namespaces: %v", page.Err()) - break - } - ns := page.Current() - if _, exists := seenIDs[ns.ID]; exists { - log.Errorf("fetchKVNamespaces: duplicate namespace ID detected (%s), breaking loop", ns.ID) - break - } - seenIDs[ns.ID] = struct{}{} - namespaceMap[ns.ID] = ns.Title - } - - return namespaceMap, nil -} - func fetchKVOperations(accountID string) (*cloudflareResponseKV, error) { request := graphql.NewRequest(` query ($accountID: String!, $mintime: Time!, $maxtime: Time!, $limit: Int!) { diff --git a/main.go b/main.go index 3bb6ddb..202e5de 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,10 @@ var ( cftimeout time.Duration gql *GraphQL log = logrus.New() + + // kvTrackedNamespaces is the set of KV namespace IDs that get their own + // namespace_id label. All other namespaces are aggregated under "other". + kvTrackedNamespaces map[string]struct{} ) // var ( @@ -174,6 +178,18 @@ func runExporter() { log.Debugf("Metrics set: %v", metricsSet) mustRegisterMetrics(metricsSet) + // Build tracked KV namespace set from config. + kvTrackedNamespaces = make(map[string]struct{}) + if ids := viper.GetString("kv_namespace_ids"); ids != "" { + for _, id := range strings.Split(ids, ",") { + id = strings.TrimSpace(id) + if id != "" { + kvTrackedNamespaces[id] = struct{}{} + } + } + } + log.Infof("Tracking %d KV namespace IDs", len(kvTrackedNamespaces)) + scrapeInterval := time.Duration(viper.GetInt("scrape_interval")) * time.Second log.Info("Scrape interval set to ", scrapeInterval) @@ -257,6 +273,10 @@ func main() { viper.BindEnv("cf_timeout") viper.SetDefault("cf_timeout", 10*time.Second) + flags.String("kv_namespace_ids", "", "KV namespace IDs to track individually, comma delimited. Unlisted namespaces are aggregated as 'other'") + viper.BindEnv("kv_namespace_ids") + viper.SetDefault("kv_namespace_ids", "") + flags.String("metrics_denylist", "", "metrics to not expose, comma delimited list") viper.BindEnv("metrics_denylist") viper.SetDefault("metrics_denylist", "") diff --git a/prometheus.go b/prometheus.go index d102a2c..48dff94 100644 --- a/prometheus.go +++ b/prometheus.go @@ -341,12 +341,12 @@ var ( kvRequests = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: kvRequestsMetricName.String(), Help: "Number of KV operations by namespace and action type", - }, []string{"namespace_name", "action_type", "account"}) + }, []string{"namespace_id", "action_type", "account"}) kvLatency = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: kvLatencyMetricName.String(), Help: "KV operation latency quantiles (milliseconds)", - }, []string{"namespace_name", "action_type", "account", "quantile"}) + }, []string{"namespace_id", "action_type", "account", "quantile"}) dnsFirewallQueryCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: dnsFirewallQueryCountMetricName.String(), @@ -631,12 +631,6 @@ func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetr wg.Add(1) defer wg.Done() - namespaceMap, err := fetchKVNamespaces(account.ID) - if err != nil { - log.Error("failed to fetch KV namespaces for account ", account.ID, ": ", err) - return - } - r, err := fetchKVOperations(account.ID) if err != nil { log.Error("failed to fetch KV operations for account ", account.ID, ": ", err) @@ -647,19 +641,19 @@ func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetr for _, a := range r.Viewer.Accounts { for _, kv := range a.KvOperationsAdaptiveGroups { - namespaceName := namespaceMap[kv.Dimensions.NamespaceID] - if namespaceName == "" { - namespaceName = kv.Dimensions.NamespaceID + nsID := kv.Dimensions.NamespaceID + if _, tracked := kvTrackedNamespaces[nsID]; !tracked { + nsID = "other" } if !deniedMetricsSet.Has(kvRequestsMetricName) { - kvRequests.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName}).Set(float64(kv.Sum.Requests)) + kvRequests.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName}).Add(float64(kv.Sum.Requests)) } if !deniedMetricsSet.Has(kvLatencyMetricName) { - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99)) - kvLatency.With(prometheus.Labels{"namespace_name": namespaceName, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999)) + kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50)) + kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75)) + kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99)) + kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999)) } } }