prometheus.go

package main

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"

	"github.com/biter777/countries"
	cfaccounts "github.com/cloudflare/cloudflare-go/v4/accounts"
	cfzones "github.com/cloudflare/cloudflare-go/v4/zones"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/spf13/viper"
)

type MetricName string

func (mn MetricName) String() string {
	return string(mn)
}

const (
	zoneRequestTotalMetricName                   MetricName = "cloudflare_zone_requests_total"
	zoneRequestCachedMetricName                  MetricName = "cloudflare_zone_requests_cached"
	zoneRequestSSLEncryptedMetricName            MetricName = "cloudflare_zone_requests_ssl_encrypted"
	zoneRequestContentTypeMetricName             MetricName = "cloudflare_zone_requests_content_type"
	zoneRequestCountryMetricName                 MetricName = "cloudflare_zone_requests_country"
	zoneRequestHTTPStatusMetricName              MetricName = "cloudflare_zone_requests_status"
	zoneRequestStatus5mRatioMetricName           MetricName = "cloudflare_zone_requests_status_5m_ratio"
	zoneRequestEdgeTTFB5mQuantilesMetricName     MetricName = "cloudflare_zone_requests_edge_ttfb_5m_quantiles"
	zoneRequestBrowserMapMetricName              MetricName = "cloudflare_zone_requests_browser_map_page_views_count"
	zoneRequestOriginStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_origin_status_country_host"
	zoneRequestStatusCountryHostMetricName       MetricName = "cloudflare_zone_requests_status_country_host"
	zoneBandwidthTotalMetricName                 MetricName = "cloudflare_zone_bandwidth_total"
	zoneBandwidthCachedMetricName                MetricName = "cloudflare_zone_bandwidth_cached"
	zoneBandwidthSSLEncryptedMetricName          MetricName = "cloudflare_zone_bandwidth_ssl_encrypted"
	zoneBandwidthContentTypeMetricName           MetricName = "cloudflare_zone_bandwidth_content_type"
	zoneBandwidthCountryMetricName               MetricName = "cloudflare_zone_bandwidth_country"
	zoneThreatsTotalMetricName                   MetricName = "cloudflare_zone_threats_total"
	zoneThreatsCountryMetricName                 MetricName = "cloudflare_zone_threats_country"
	zoneThreatsTypeMetricName                    MetricName = "cloudflare_zone_threats_type"
	zonePageviewsTotalMetricName                 MetricName = "cloudflare_zone_pageviews_total"
	zoneUniquesTotalMetricName                   MetricName = "cloudflare_zone_uniques_total"
	zoneColocationVisitsMetricName               MetricName = "cloudflare_zone_colocation_visits"
	zoneColocationEdgeResponseBytesMetricName    MetricName = "cloudflare_zone_colocation_edge_response_bytes"
	zoneColocationRequestsTotalMetricName        MetricName = "cloudflare_zone_colocation_requests_total"
	zoneFirewallEventsCountMetricName            MetricName = "cloudflare_zone_firewall_events_count"
	zoneHealthCheckEventsOriginCountMetricName   MetricName = "cloudflare_zone_health_check_events_origin_count"
	workerRequestsMetricName                     MetricName = "cloudflare_worker_requests_count"
	workerErrorsMetricName                       MetricName = "cloudflare_worker_errors_count"
	workerCPUTimeMetricName                      MetricName = "cloudflare_worker_cpu_time"
	workerDurationMetricName                     MetricName = "cloudflare_worker_duration"
	workerWallTimeMetricName                     MetricName = "cloudflare_worker_wall_time"
	poolHealthStatusMetricName                   MetricName = "cloudflare_zone_pool_health_status"
	poolRequestsTotalMetricName                  MetricName = "cloudflare_zone_pool_requests_total"
	poolOriginHealthStatusMetricName             MetricName = "cloudflare_pool_origin_health_status"
	logpushFailedJobsAccountMetricName           MetricName = "cloudflare_logpush_failed_jobs_account_count"
	logpushFailedJobsZoneMetricName              MetricName = "cloudflare_logpush_failed_jobs_zone_count"
	r2StorageTotalMetricName                     MetricName = "cloudflare_r2_storage_total_bytes"
	r2StorageMetricName                          MetricName = "cloudflare_r2_storage_bytes"
	r2OperationMetricName                        MetricName = "cloudflare_r2_operation_count"
	tunnelInfoMetricName                         MetricName = "cloudflare_tunnel_info"
	tunnelHealthStatusMetricName                 MetricName = "cloudflare_tunnel_health_status"
	tunnelConnectorInfoMetricName                MetricName = "cloudflare_tunnel_connector_info"
	tunnelConnectorActiveConnectionsMetricName   MetricName = "cloudflare_tunnel_connector_active_connections"
	dnsFirewallQueryCountMetricName              MetricName = "cloudflare_dns_firewall_query_count"
	kvRequestsMetricName                         MetricName = "cloudflare_kv_requests_count"
	kvLatencyMetricName                          MetricName = "cloudflare_kv_latency"
	workerSubrequestsCountMetricName             MetricName = "cloudflare_worker_subrequests_count"
	workerSubrequestTimeMetricName               MetricName = "cloudflare_worker_subrequest_time"
	queueBacklogMessagesMetricName               MetricName = "cloudflare_queue_backlog_messages"
	queueBacklogBytesMetricName                  MetricName = "cloudflare_queue_backlog_bytes"
	queueConsumerConcurrencyMetricName           MetricName = "cloudflare_queue_consumer_concurrency"
	queueOperationsMetricName                    MetricName = "cloudflare_queue_operations_count"
	queueOperationBytesMetricName                MetricName = "cloudflare_queue_operations_bytes"
	queueOperationLagTimeMetricName              MetricName = "cloudflare_queue_operations_lag_time"
	queueOperationRetryCountMetricName           MetricName = "cloudflare_queue_operations_retry_count"
	workerOperationDurationMetricName            MetricName = "cloudflare_worker_operation_duration_seconds"
	workerOperationCountMetricName               MetricName = "cloudflare_worker_operation_count"
)

type MetricsSet map[MetricName]struct{}

func (ms MetricsSet) Has(mn MetricName) bool {
	_, exists := ms[mn]
	return exists
}

func (ms MetricsSet) Add(mn MetricName) {
	ms[mn] = struct{}{}
}

var (
	// Requests
	zoneRequestTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestTotalMetricName.String(),
		Help: "Number of requests for zone",
	}, []string{"zone", "account"},
	)

	zoneRequestCached = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestCachedMetricName.String(),
		Help: "Number of cached requests for zone",
	}, []string{"zone", "account"},
	)

	zoneRequestSSLEncrypted = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestSSLEncryptedMetricName.String(),
		Help: "Number of encrypted requests for zone",
	}, []string{"zone", "account"},
	)

	zoneRequestContentType = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestContentTypeMetricName.String(),
		Help: "Number of request for zone per content type",
	}, []string{"zone", "account", "content_type"},
	)

	zoneRequestCountry = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestCountryMetricName.String(),
		Help: "Number of request for zone per country",
	}, []string{"zone", "account", "country", "region"},
	)

	zoneRequestHTTPStatus = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestHTTPStatusMetricName.String(),
		Help: "Number of request for zone per HTTP status",
	}, []string{"zone", "account", "status"},
	)

	zoneRequestStatus5mRatio = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: zoneRequestStatus5mRatioMetricName.String(),
		Help: "Ratio of 4xx and 5xx responses over the past 5 minutes",
	}, []string{"zone", "account", "origin_ip", "status_group"},
	)

	zoneRequestEdgeTTFB5mQuantiles = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: zoneRequestEdgeTTFB5mQuantilesMetricName.String(),
		Help: "Time to first byte (TTFB) quantiles over the past 5 minutes",
	}, []string{"zone", "account", "origin_ip", "quantile"},
	)

	zoneRequestBrowserMap = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestBrowserMapMetricName.String(),
		Help: "Number of successful requests for HTML pages per zone",
	}, []string{"zone", "account", "family"},
	)

	zoneRequestOriginStatusCountryHost = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestOriginStatusCountryHostMetricName.String(),
		Help: "Count of not cached requests for zone per origin HTTP status per country per host",
	}, []string{"zone", "account", "status", "country", "host"},
	)

	zoneRequestStatusCountryHost = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneRequestStatusCountryHostMetricName.String(),
		Help: "Count of requests for zone per edge HTTP status per country per host",
	}, []string{"zone", "account", "status", "country", "host"},
	)

	zoneBandwidthTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneBandwidthTotalMetricName.String(),
		Help: "Total bandwidth per zone in bytes",
	}, []string{"zone", "account"},
	)

	zoneBandwidthCached = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneBandwidthCachedMetricName.String(),
		Help: "Cached bandwidth per zone in bytes",
	}, []string{"zone", "account"},
	)

	zoneBandwidthSSLEncrypted = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneBandwidthSSLEncryptedMetricName.String(),
		Help: "Encrypted bandwidth per zone in bytes",
	}, []string{"zone", "account"},
	)

	zoneBandwidthContentType = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneBandwidthContentTypeMetricName.String(),
		Help: "Bandwidth per zone per content type",
	}, []string{"zone", "account", "content_type"},
	)

	zoneBandwidthCountry = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneBandwidthCountryMetricName.String(),
		Help: "Bandwidth per country per zone",
	}, []string{"zone", "account", "country", "region"},
	)

	zoneThreatsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneThreatsTotalMetricName.String(),
		Help: "Threats per zone",
	}, []string{"zone", "account"},
	)

	zoneThreatsCountry = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneThreatsCountryMetricName.String(),
		Help: "Threats per zone per country",
	}, []string{"zone", "account", "country", "region"},
	)

	zoneThreatsType = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneThreatsTypeMetricName.String(),
		Help: "Threats per zone per type",
	}, []string{"zone", "account", "type"},
	)

	zonePageviewsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zonePageviewsTotalMetricName.String(),
		Help: "Pageviews per zone",
	}, []string{"zone", "account"},
	)

	zoneUniquesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneUniquesTotalMetricName.String(),
		Help: "Uniques per zone",
	}, []string{"zone", "account"},
	)

	zoneColocationVisits = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneColocationVisitsMetricName.String(),
		Help: "Total visits per colocation",
	}, []string{"zone", "account", "colocation", "host"},
	)

	zoneColocationEdgeResponseBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneColocationEdgeResponseBytesMetricName.String(),
		Help: "Edge response bytes per colocation",
	}, []string{"zone", "account", "colocation", "host"},
	)

	zoneColocationRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneColocationRequestsTotalMetricName.String(),
		Help: "Total requests per colocation",
	}, []string{"zone", "account", "colocation", "host"},
	)

	zoneFirewallEventsCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneFirewallEventsCountMetricName.String(),
		Help: "Count of Firewall events",
	}, []string{"zone", "account", "action", "source", "rule", "host", "country"},
	)

	zoneHealthCheckEventsOriginCount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: zoneHealthCheckEventsOriginCountMetricName.String(),
		Help: "Number of Heath check events per region per origin",
	}, []string{"zone", "account", "health_status", "origin_ip", "region", "fqdn"},
	)

	workerRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: workerRequestsMetricName.String(),
		Help: "Number of requests sent to worker by script name",
	}, []string{"script_name", "dispatch_namespace", "account", "status"},
	)

	workerErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: workerErrorsMetricName.String(),
		Help: "Number of errors by script name",
	}, []string{"script_name", "dispatch_namespace", "account", "status"},
	)

	workerCPUTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerCPUTimeMetricName.String(),
		Help: "CPU time quantiles by script name",
	}, []string{"script_name", "dispatch_namespace", "account", "status", "quantile"},
	)

	workerDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerDurationMetricName.String(),
		Help: "Duration quantiles by script name (GB*s)",
	}, []string{"script_name", "dispatch_namespace", "account", "status", "quantile"},
	)

	workerWallTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerWallTimeMetricName.String(),
		Help: "Wall time quantiles by script name (microseconds)",
	}, []string{"script_name", "dispatch_namespace", "account", "status", "quantile"},
	)

	poolHealthStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: poolHealthStatusMetricName.String(),
		Help: "Reports the health of a pool, 1 for healthy, 0 for unhealthy.",
	},
		[]string{"zone", "account", "load_balancer_name", "pool_name"},
	)

	poolOriginHealthStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: poolOriginHealthStatusMetricName.String(),
		Help: "Reports the origin health of a pool, 1 for healthy, 0 for unhealthy.",
	},
		[]string{"account", "pool_name", "origin_name", "ip"},
	)

	poolRequestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: poolRequestsTotalMetricName.String(),
		Help: "Requests per pool",
	},
		[]string{"zone", "account", "load_balancer_name", "pool_name", "origin_name"},
	)

	// TODO: Update this to counter vec and use counts from the query to add
	logpushFailedJobsAccount = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: logpushFailedJobsAccountMetricName.String(),
		Help: "Number of failed logpush jobs on the account level",
	},
		[]string{"account", "destination", "job_id", "final"},
	)

	logpushFailedJobsZone = prometheus.NewCounterVec(prometheus.CounterOpts{
		Name: logpushFailedJobsZoneMetricName.String(),
		Help: "Number of failed logpush jobs on the zone level",
	},
		[]string{"destination", "job_id", "final"},
	)

	r2StorageTotal = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: r2StorageTotalMetricName.String(),
		Help: "Total storage used by R2",
	}, []string{"account"})

	r2Storage = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: r2StorageMetricName.String(),
		Help: "Storage used by R2",
	}, []string{"account", "bucket"})

	r2Operation = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: r2OperationMetricName.String(),
		Help: "Number of operations performed by R2",
	}, []string{"account", "bucket", "operation"})

	tunnelInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: tunnelInfoMetricName.String(),
		Help: "Reports Cloudflare Tunnel details",
	}, []string{"account", "tunnel_id", "tunnel_name", "tunnel_type"})

	tunnelHealthStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: tunnelHealthStatusMetricName.String(),
		Help: "Reports the health of a Cloudflare Tunnel, 0 for unhealthy, 1 for healthy, 2 for degraded, 3 for inactive",
	}, []string{"account", "tunnel_id"})

	tunnelConnectorInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: tunnelConnectorInfoMetricName.String(),
		Help: "Reports Cloudflare Tunnel connector details",
	}, []string{"account", "tunnel_id", "client_id", "version", "arch", "origin_ip"})

	tunnelConnectorActiveConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: tunnelConnectorActiveConnectionsMetricName.String(),
		Help: "Reports number of active connections for a Cloudflare Tunnel connector",
	}, []string{"account", "tunnel_id", "client_id"})

	kvRequests = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: kvRequestsMetricName.String(),
		Help: "Number of KV operations by namespace and action type",
	}, []string{"namespace_id", "action_type", "account"})

	kvLatency = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: kvLatencyMetricName.String(),
		Help: "KV operation latency quantiles (milliseconds)",
	}, []string{"namespace_id", "action_type", "account", "quantile"})

	workerSubrequestsCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerSubrequestsCountMetricName.String(),
		Help: "Number of subrequests by script name",
	}, []string{"script_name", "account"})

	workerSubrequestTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerSubrequestTimeMetricName.String(),
		Help: "Subrequest response time quantiles (microseconds)",
	}, []string{"script_name", "account", "quantile"})

	queueBacklogMessages = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueBacklogMessagesMetricName.String(),
		Help: "Average number of messages in queue backlog",
	}, []string{"queue_name", "account"})

	queueBacklogBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueBacklogBytesMetricName.String(),
		Help: "Average backlog size in bytes",
	}, []string{"queue_name", "account"})

	queueConsumerConcurrency = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueConsumerConcurrencyMetricName.String(),
		Help: "Average number of concurrent queue consumers",
	}, []string{"queue_name", "account"})

	queueOperations = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueOperationsMetricName.String(),
		Help: "Number of queue message operations",
	}, []string{"queue_name", "account", "action_type", "outcome"})

	queueOperationBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueOperationBytesMetricName.String(),
		Help: "Total bytes processed by queue message operations",
	}, []string{"queue_name", "account", "action_type", "outcome"})

	queueOperationLagTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueOperationLagTimeMetricName.String(),
		Help: "Average lag time between write and read/delete (milliseconds)",
	}, []string{"queue_name", "account", "action_type", "outcome"})

	queueOperationRetryCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: queueOperationRetryCountMetricName.String(),
		Help: "Average retry count for queue message operations",
	}, []string{"queue_name", "account", "action_type", "outcome"})

	dnsFirewallQueryCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: dnsFirewallQueryCountMetricName.String(),
		Help: "DNS Firewall query count by query type and response code",
	}, []string{"account_id", "account_name", "dns_firewall_id", "query_type", "response_code"},
	)

	workerOperationDuration = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerOperationDurationMetricName.String(),
		Help: "Worker operation duration quantiles in seconds, sliced by metric_type (worker-defined event type), label (worker-defined slice), and quantile (P50, P75, P99, P999)",
	}, []string{"script_name", "account", "metric_type", "label", "quantile"})

	workerOperationCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: workerOperationCountMetricName.String(),
		Help: "Worker operation count in the last 5 minutes, sliced by metric_type (worker-defined event type) and label (worker-defined slice). Windowed sum, not a cumulative counter.",
	}, []string{"script_name", "account", "metric_type", "label"})
)

func buildAllMetricsSet() MetricsSet {
	allMetricsSet := MetricsSet{}
	allMetricsSet.Add(zoneRequestTotalMetricName)
	allMetricsSet.Add(zoneRequestCachedMetricName)
	allMetricsSet.Add(zoneRequestSSLEncryptedMetricName)
	allMetricsSet.Add(zoneRequestContentTypeMetricName)
	allMetricsSet.Add(zoneRequestCountryMetricName)
	allMetricsSet.Add(zoneRequestHTTPStatusMetricName)
	allMetricsSet.Add(zoneRequestStatus5mRatioMetricName)
	allMetricsSet.Add(zoneRequestEdgeTTFB5mQuantilesMetricName)
	allMetricsSet.Add(zoneRequestBrowserMapMetricName)
	allMetricsSet.Add(zoneRequestOriginStatusCountryHostMetricName)
	allMetricsSet.Add(zoneRequestStatusCountryHostMetricName)
	allMetricsSet.Add(zoneBandwidthTotalMetricName)
	allMetricsSet.Add(zoneBandwidthCachedMetricName)
	allMetricsSet.Add(zoneBandwidthSSLEncryptedMetricName)
	allMetricsSet.Add(zoneBandwidthContentTypeMetricName)
	allMetricsSet.Add(zoneBandwidthCountryMetricName)
	allMetricsSet.Add(zoneThreatsTotalMetricName)
	allMetricsSet.Add(zoneThreatsCountryMetricName)
	allMetricsSet.Add(zoneThreatsTypeMetricName)
	allMetricsSet.Add(zonePageviewsTotalMetricName)
	allMetricsSet.Add(zoneUniquesTotalMetricName)
	allMetricsSet.Add(zoneColocationVisitsMetricName)
	allMetricsSet.Add(zoneColocationEdgeResponseBytesMetricName)
	allMetricsSet.Add(zoneColocationRequestsTotalMetricName)
	allMetricsSet.Add(zoneFirewallEventsCountMetricName)
	allMetricsSet.Add(zoneHealthCheckEventsOriginCountMetricName)
	allMetricsSet.Add(workerRequestsMetricName)
	allMetricsSet.Add(workerErrorsMetricName)
	allMetricsSet.Add(workerCPUTimeMetricName)
	allMetricsSet.Add(workerDurationMetricName)
	allMetricsSet.Add(workerWallTimeMetricName)
	allMetricsSet.Add(poolHealthStatusMetricName)
	allMetricsSet.Add(poolOriginHealthStatusMetricName)
	allMetricsSet.Add(poolRequestsTotalMetricName)
	allMetricsSet.Add(logpushFailedJobsAccountMetricName)
	allMetricsSet.Add(logpushFailedJobsZoneMetricName)
	allMetricsSet.Add(r2StorageTotalMetricName)
	allMetricsSet.Add(r2OperationMetricName)
	allMetricsSet.Add(tunnelInfoMetricName)
	allMetricsSet.Add(tunnelHealthStatusMetricName)
	allMetricsSet.Add(tunnelConnectorInfoMetricName)
	allMetricsSet.Add(tunnelConnectorActiveConnectionsMetricName)
	allMetricsSet.Add(dnsFirewallQueryCountMetricName)
	allMetricsSet.Add(kvRequestsMetricName)
	allMetricsSet.Add(kvLatencyMetricName)
	allMetricsSet.Add(workerSubrequestsCountMetricName)
	allMetricsSet.Add(workerSubrequestTimeMetricName)
	allMetricsSet.Add(queueBacklogMessagesMetricName)
	allMetricsSet.Add(queueBacklogBytesMetricName)
	allMetricsSet.Add(queueConsumerConcurrencyMetricName)
	allMetricsSet.Add(queueOperationsMetricName)
	allMetricsSet.Add(queueOperationBytesMetricName)
	allMetricsSet.Add(queueOperationLagTimeMetricName)
	allMetricsSet.Add(queueOperationRetryCountMetricName)
	allMetricsSet.Add(workerOperationDurationMetricName)
	allMetricsSet.Add(workerOperationCountMetricName)
	return allMetricsSet
}

func buildFilteredMetricsSet(metricsDenylist []string) (MetricsSet, error) {
	deniedMetricsSet := MetricsSet{}
	allMetricsSet := buildAllMetricsSet()

	for _, metric := range metricsDenylist {
		if !allMetricsSet.Has(MetricName(metric)) {
			return nil, fmt.Errorf("metric %s doesn't exists", metric)
		}
		deniedMetricsSet.Add(MetricName(metric))
	}
	return deniedMetricsSet, nil
}

func mustRegisterMetrics(deniedMetrics MetricsSet) {
	if !deniedMetrics.Has(zoneRequestTotalMetricName) {
		prometheus.MustRegister(zoneRequestTotal)
	}
	if !deniedMetrics.Has(zoneRequestCachedMetricName) {
		prometheus.MustRegister(zoneRequestCached)
	}
	if !deniedMetrics.Has(zoneRequestSSLEncryptedMetricName) {
		prometheus.MustRegister(zoneRequestSSLEncrypted)
	}
	if !deniedMetrics.Has(zoneRequestContentTypeMetricName) {
		prometheus.MustRegister(zoneRequestContentType)
	}
	if !deniedMetrics.Has(zoneRequestCountryMetricName) {
		prometheus.MustRegister(zoneRequestCountry)
	}
	if !deniedMetrics.Has(zoneRequestHTTPStatusMetricName) {
		prometheus.MustRegister(zoneRequestHTTPStatus)
	}
	if !deniedMetrics.Has(zoneRequestStatus5mRatioMetricName) {
		prometheus.MustRegister(zoneRequestStatus5mRatio)
	}
	if !deniedMetrics.Has(zoneRequestEdgeTTFB5mQuantilesMetricName) {
		prometheus.MustRegister(zoneRequestEdgeTTFB5mQuantiles)
	}
	if !deniedMetrics.Has(zoneRequestBrowserMapMetricName) {
		prometheus.MustRegister(zoneRequestBrowserMap)
	}
	if !deniedMetrics.Has(zoneRequestOriginStatusCountryHostMetricName) {
		prometheus.MustRegister(zoneRequestOriginStatusCountryHost)
	}
	if !deniedMetrics.Has(zoneRequestStatusCountryHostMetricName) {
		prometheus.MustRegister(zoneRequestStatusCountryHost)
	}
	if !deniedMetrics.Has(zoneBandwidthTotalMetricName) {
		prometheus.MustRegister(zoneBandwidthTotal)
	}
	if !deniedMetrics.Has(zoneBandwidthCachedMetricName) {
		prometheus.MustRegister(zoneBandwidthCached)
	}
	if !deniedMetrics.Has(zoneBandwidthSSLEncryptedMetricName) {
		prometheus.MustRegister(zoneBandwidthSSLEncrypted)
	}
	if !deniedMetrics.Has(zoneBandwidthContentTypeMetricName) {
		prometheus.MustRegister(zoneBandwidthContentType)
	}
	if !deniedMetrics.Has(zoneBandwidthCountryMetricName) {
		prometheus.MustRegister(zoneBandwidthCountry)
	}
	if !deniedMetrics.Has(zoneThreatsTotalMetricName) {
		prometheus.MustRegister(zoneThreatsTotal)
	}
	if !deniedMetrics.Has(zoneThreatsCountryMetricName) {
		prometheus.MustRegister(zoneThreatsCountry)
	}
	if !deniedMetrics.Has(zoneThreatsTypeMetricName) {
		prometheus.MustRegister(zoneThreatsType)
	}
	if !deniedMetrics.Has(zonePageviewsTotalMetricName) {
		prometheus.MustRegister(zonePageviewsTotal)
	}
	if !deniedMetrics.Has(zoneUniquesTotalMetricName) {
		prometheus.MustRegister(zoneUniquesTotal)
	}
	if !deniedMetrics.Has(zoneColocationVisitsMetricName) {
		prometheus.MustRegister(zoneColocationVisits)
	}
	if !deniedMetrics.Has(zoneColocationEdgeResponseBytesMetricName) {
		prometheus.MustRegister(zoneColocationEdgeResponseBytes)
	}
	if !deniedMetrics.Has(zoneColocationRequestsTotalMetricName) {
		prometheus.MustRegister(zoneColocationRequestsTotal)
	}
	if !deniedMetrics.Has(zoneFirewallEventsCountMetricName) {
		prometheus.MustRegister(zoneFirewallEventsCount)
	}
	if !deniedMetrics.Has(zoneHealthCheckEventsOriginCountMetricName) {
		prometheus.MustRegister(zoneHealthCheckEventsOriginCount)
	}
	if !deniedMetrics.Has(workerRequestsMetricName) {
		prometheus.MustRegister(workerRequests)
	}
	if !deniedMetrics.Has(workerErrorsMetricName) {
		prometheus.MustRegister(workerErrors)
	}
	if !deniedMetrics.Has(workerCPUTimeMetricName) {
		prometheus.MustRegister(workerCPUTime)
	}
	if !deniedMetrics.Has(workerDurationMetricName) {
		prometheus.MustRegister(workerDuration)
	}
	if !deniedMetrics.Has(workerWallTimeMetricName) {
		prometheus.MustRegister(workerWallTime)
	}
	if !deniedMetrics.Has(poolHealthStatusMetricName) {
		prometheus.MustRegister(poolHealthStatus)
	}
	if !deniedMetrics.Has(poolOriginHealthStatusMetricName) {
		prometheus.MustRegister(poolOriginHealthStatus)
	}
	if !deniedMetrics.Has(poolRequestsTotalMetricName) {
		prometheus.MustRegister(poolRequestsTotal)
	}
	if !deniedMetrics.Has(logpushFailedJobsAccountMetricName) {
		prometheus.MustRegister(logpushFailedJobsAccount)
	}
	if !deniedMetrics.Has(logpushFailedJobsZoneMetricName) {
		prometheus.MustRegister(logpushFailedJobsZone)
	}
	if !deniedMetrics.Has(r2StorageTotalMetricName) {
		prometheus.MustRegister(r2StorageTotal)
	}
	if !deniedMetrics.Has(r2StorageMetricName) {
		prometheus.MustRegister(r2Storage)
	}
	if !deniedMetrics.Has(r2OperationMetricName) {
		prometheus.MustRegister(r2Operation)
	}
	if !deniedMetrics.Has(tunnelInfoMetricName) {
		prometheus.MustRegister(tunnelInfo)
	}
	if !deniedMetrics.Has(tunnelHealthStatusMetricName) {
		prometheus.MustRegister(tunnelHealthStatus)
	}
	if !deniedMetrics.Has(tunnelConnectorInfoMetricName) {
		prometheus.MustRegister(tunnelConnectorInfo)
	}
	if !deniedMetrics.Has(tunnelConnectorActiveConnectionsMetricName) {
		prometheus.MustRegister(tunnelConnectorActiveConnections)
	}
	if !deniedMetrics.Has(dnsFirewallQueryCountMetricName) {
		prometheus.MustRegister(dnsFirewallQueryCount)
	}
	if !deniedMetrics.Has(kvRequestsMetricName) {
		prometheus.MustRegister(kvRequests)
	}
	if !deniedMetrics.Has(kvLatencyMetricName) {
		prometheus.MustRegister(kvLatency)
	}
	if !deniedMetrics.Has(workerSubrequestsCountMetricName) {
		prometheus.MustRegister(workerSubrequestsCount)
	}
	if !deniedMetrics.Has(workerSubrequestTimeMetricName) {
		prometheus.MustRegister(workerSubrequestTime)
	}
	if !deniedMetrics.Has(queueBacklogMessagesMetricName) {
		prometheus.MustRegister(queueBacklogMessages)
	}
	if !deniedMetrics.Has(queueBacklogBytesMetricName) {
		prometheus.MustRegister(queueBacklogBytes)
	}
	if !deniedMetrics.Has(queueConsumerConcurrencyMetricName) {
		prometheus.MustRegister(queueConsumerConcurrency)
	}
	if !deniedMetrics.Has(queueOperationsMetricName) {
		prometheus.MustRegister(queueOperations)
	}
	if !deniedMetrics.Has(queueOperationBytesMetricName) {
		prometheus.MustRegister(queueOperationBytes)
	}
	if !deniedMetrics.Has(queueOperationLagTimeMetricName) {
		prometheus.MustRegister(queueOperationLagTime)
	}
	if !deniedMetrics.Has(queueOperationRetryCountMetricName) {
		prometheus.MustRegister(queueOperationRetryCount)
	}
	if !deniedMetrics.Has(workerOperationDurationMetricName) {
		prometheus.MustRegister(workerOperationDuration)
	}
	if !deniedMetrics.Has(workerOperationCountMetricName) {
		prometheus.MustRegister(workerOperationCount)
	}
}

func fetchLoadblancerPoolsHealth(account cfaccounts.Account, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	pools := fetchLoadblancerPools(account)
	if pools == nil {
		return
	}

	for _, pool := range pools {
		if !pool.Enabled { // not enabled, no health values
			continue
		}
		if pool.Monitor == "" { // No monitor, no health values
			continue
		}
		for _, o := range pool.Origins {
			if !o.Enabled { // not enabled, no health values
				continue
			}
			healthy := 1 // Assume healthy
			if o.JSON.ExtraFields["healthy"].Raw() == "false" {
				healthy = 0 // Unhealthy
			}
			poolOriginHealthStatus.With(
				prometheus.Labels{
					"account":     account.Name,
					"pool_name":   pool.Name,
					"origin_name": o.Name,
					"ip":          o.Address,
				}).Set(float64(healthy))
		}
	}
}

func fetchWorkerAnalytics(account cfaccounts.Account, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	r, err := fetchWorkerTotals(account.ID)
	if err != nil {
		log.Error("failed to fetch worker analytics for account ", account.ID, ": ", err)
		return
	}

	// Replace spaces with hyphens and convert to lowercase
	accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-"))

	for _, a := range r.Viewer.Accounts {
		for _, w := range a.WorkersInvocationsAdaptive {
			workerRequests.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status}).Add(float64(w.Sum.Requests))
			workerErrors.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status}).Add(float64(w.Sum.Errors))
			workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P50"}).Set(float64(w.Quantiles.CPUTimeP50))
			workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P75"}).Set(float64(w.Quantiles.CPUTimeP75))
			workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P99"}).Set(float64(w.Quantiles.CPUTimeP99))
			workerCPUTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P999"}).Set(float64(w.Quantiles.CPUTimeP999))
			workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P50"}).Set(float64(w.Quantiles.DurationP50))
			workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P75"}).Set(float64(w.Quantiles.DurationP75))
			workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P99"}).Set(float64(w.Quantiles.DurationP99))
			workerDuration.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P999"}).Set(float64(w.Quantiles.DurationP999))
			workerWallTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P50"}).Set(float64(w.Quantiles.WallTimeP50))
			workerWallTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P75"}).Set(float64(w.Quantiles.WallTimeP75))
			workerWallTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P99"}).Set(float64(w.Quantiles.WallTimeP99))
			workerWallTime.With(prometheus.Labels{"script_name": w.Dimensions.ScriptName, "dispatch_namespace": w.Dimensions.DispatchNamespaceName, "account": accountName, "status": w.Dimensions.Status, "quantile": "P999"}).Set(float64(w.Quantiles.WallTimeP999))
		}
	}
}

func fetchKVAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	r, err := fetchKVOperations(account.ID)
	if err != nil {
		log.Error("failed to fetch KV operations for account ", account.ID, ": ", err)
		return
	}

	accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-"))

	for _, a := range r.Viewer.Accounts {
		for _, kv := range a.KvOperationsAdaptiveGroups {
			nsID := kv.Dimensions.NamespaceID
			if _, tracked := kvTrackedNamespaces[nsID]; !tracked {
				nsID = "other"
			}

			if !deniedMetricsSet.Has(kvRequestsMetricName) {
				kvRequests.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName}).Add(float64(kv.Sum.Requests))
			}
			if !deniedMetricsSet.Has(kvLatencyMetricName) {
				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P50"}).Set(float64(kv.Quantiles.LatencyMsP50))
				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P75"}).Set(float64(kv.Quantiles.LatencyMsP75))
				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P99"}).Set(float64(kv.Quantiles.LatencyMsP99))
				kvLatency.With(prometheus.Labels{"namespace_id": nsID, "action_type": kv.Dimensions.ActionType, "account": accountName, "quantile": "P999"}).Set(float64(kv.Quantiles.LatencyMsP999))
			}
		}
	}
}

func fetchWorkerSubrequestAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	r, err := fetchWorkerSubrequests(account.ID)
	if err != nil {
		log.Error("failed to fetch worker subrequests for account ", account.ID, ": ", err)
		return
	}

	accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-"))

	for _, a := range r.Viewer.Accounts {
		for _, sr := range a.WorkersSubrequestsAdaptiveGroups {
			if !deniedMetricsSet.Has(workerSubrequestsCountMetricName) {
				workerSubrequestsCount.With(prometheus.Labels{"script_name": sr.Dimensions.ScriptName, "account": accountName}).Add(float64(sr.Sum.Subrequests))
			}
			if !deniedMetricsSet.Has(workerSubrequestTimeMetricName) {
				workerSubrequestTime.With(prometheus.Labels{"script_name": sr.Dimensions.ScriptName, "account": accountName, "quantile": "P50"}).Set(float64(sr.Quantiles.TimeToResponseUsP50))
				workerSubrequestTime.With(prometheus.Labels{"script_name": sr.Dimensions.ScriptName, "account": accountName, "quantile": "P75"}).Set(float64(sr.Quantiles.TimeToResponseUsP75))
				workerSubrequestTime.With(prometheus.Labels{"script_name": sr.Dimensions.ScriptName, "account": accountName, "quantile": "P99"}).Set(float64(sr.Quantiles.TimeToResponseUsP99))
				workerSubrequestTime.With(prometheus.Labels{"script_name": sr.Dimensions.ScriptName, "account": accountName, "quantile": "P999"}).Set(float64(sr.Quantiles.TimeToResponseUsP999))
			}
		}
	}
}

func fetchQueueAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	names, err := fetchQueueNames(account.ID)
	if err != nil {
		log.Error("failed to fetch queue names for account ", account.ID, ": ", err)
		return
	}

	r, err := fetchQueueMetrics(account.ID)
	if err != nil {
		log.Error("failed to fetch queue metrics for account ", account.ID, ": ", err)
		return
	}

	accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-"))

	resolveQueueName := func(queueID string) string {
		if name, ok := names[queueID]; ok {
			return name
		}
		return queueID
	}

	for _, a := range r.Viewer.Accounts {
		for _, b := range a.QueueBacklogAdaptiveGroups {
			qName := resolveQueueName(b.Dimensions.QueueID)
			if !deniedMetricsSet.Has(queueBacklogMessagesMetricName) {
				queueBacklogMessages.With(prometheus.Labels{"queue_name": qName, "account": accountName}).Set(b.Avg.Messages)
			}
			if !deniedMetricsSet.Has(queueBacklogBytesMetricName) {
				queueBacklogBytes.With(prometheus.Labels{"queue_name": qName, "account": accountName}).Set(b.Avg.Bytes)
			}
		}

		for _, c := range a.QueueConsumerMetricsAdaptiveGroups {
			qName := resolveQueueName(c.Dimensions.QueueID)
			if !deniedMetricsSet.Has(queueConsumerConcurrencyMetricName) {
				queueConsumerConcurrency.With(prometheus.Labels{"queue_name": qName, "account": accountName}).Set(c.Avg.Concurrency)
			}
		}

		for _, op := range a.QueueMessageOperationsAdaptiveGroups {
			qName := resolveQueueName(op.Dimensions.QueueID)
			labels := prometheus.Labels{"queue_name": qName, "account": accountName, "action_type": op.Dimensions.ActionType, "outcome": op.Dimensions.Outcome}
			if !deniedMetricsSet.Has(queueOperationsMetricName) {
				queueOperations.With(labels).Set(float64(op.Sum.BillableOperations))
			}
			if !deniedMetricsSet.Has(queueOperationBytesMetricName) {
				queueOperationBytes.With(labels).Set(float64(op.Sum.Bytes))
			}
			if !deniedMetricsSet.Has(queueOperationLagTimeMetricName) {
				queueOperationLagTime.With(labels).Set(op.Avg.LagTime)
			}
			if !deniedMetricsSet.Has(queueOperationRetryCountMetricName) {
				queueOperationRetryCount.With(labels).Set(op.Avg.RetryCount)
			}
		}
	}
}

func fetchLogpushAnalyticsForAccount(account cfaccounts.Account, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	if viper.GetBool("free_tier") {
		return
	}

	r, err := fetchLogpushAccount(account.ID)

	if err != nil {
		log.Error("failed to fetch logpush analytics for account ", account.ID, ": ", err)
		return
	}

	for _, acc := range r.Viewer.Accounts {
		for _, LogpushHealthAdaptiveGroup := range acc.LogpushHealthAdaptiveGroups {
			logpushFailedJobsAccount.With(prometheus.Labels{"account": account.ID,
				"destination": LogpushHealthAdaptiveGroup.Dimensions.DestinationType,
				"job_id":      strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.JobID),
				"final":       strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.Final)}).Add(float64(LogpushHealthAdaptiveGroup.Count))
		}
	}
}

func fetchR2StorageForAccount(account cfaccounts.Account, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	r, err := fetchR2Account(account.ID)

	if err != nil {
		return
	}
	for _, acc := range r.Viewer.Accounts {
		var totalStorage uint64
		for _, bucket := range acc.R2StorageGroups {
			totalStorage += bucket.Max.PayloadSize
			r2Storage.With(prometheus.Labels{"account": account.Name, "bucket": bucket.Dimensions.BucketName}).Set(float64(bucket.Max.PayloadSize))
		}
		for _, operation := range acc.R2StorageOperations {
			r2Operation.With(prometheus.Labels{"account": account.Name, "bucket": operation.Dimensions.BucketName, "operation": operation.Dimensions.Action}).Set(float64(operation.Sum.Requests))
		}
		r2StorageTotal.With(prometheus.Labels{"account": account.Name}).Set(float64(totalStorage))
	}
}

func fetchLogpushAnalyticsForZone(zones []cfzones.Zone, wg *sync.WaitGroup, _ MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	if viper.GetBool("free_tier") {
		return
	}

	zoneIDs := extractZoneIDs(zones)
	if len(zoneIDs) == 0 {
		return
	}

	r, err := fetchLogpushZone(zoneIDs)

	if err != nil {
		log.Error("failed to fetch logpush analytics for zones: ", err)
		return
	}

	for _, zone := range r.Viewer.Zones {
		for _, LogpushHealthAdaptiveGroup := range zone.LogpushHealthAdaptiveGroups {
			logpushFailedJobsZone.With(prometheus.Labels{"destination": LogpushHealthAdaptiveGroup.Dimensions.DestinationType,
				"job_id": strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.JobID),
				"final":  strconv.Itoa(LogpushHealthAdaptiveGroup.Dimensions.Final)}).Add(float64(LogpushHealthAdaptiveGroup.Count))
		}
	}
}

func fetchDNSFirewallAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	r, err := fetchDNSFirewallTotals(account.ID)
	if err != nil {
		return
	}

	for _, a := range r.Viewer.Accounts {
		for _, d := range a.DNSFirewallAnalyticsAdaptiveGroups {
			if !deniedMetricsSet.Has(dnsFirewallQueryCountMetricName) {
				dnsFirewallQueryCount.WithLabelValues(
					account.ID,
					account.Name,
					d.Dimensions.ClusterTag,
					d.Dimensions.QueryType,
					d.Dimensions.ResponseCode,
				).Set(float64(d.Count))
			}
		}
	}
}

func fetchZoneColocationAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	// Colocation metrics are not available in non-enterprise zones
	if viper.GetBool("free_tier") {
		return
	}

	zoneIDs := extractZoneIDs(zones)
	if len(zoneIDs) == 0 {
		return
	}

	r, err := fetchColoTotals(zoneIDs)
	if err != nil {
		log.Error("failed to fetch colocation analytics for zones: ", err)
		return
	}
	for _, z := range r.Viewer.Zones {
		cg := z.ColoGroups
		name, account := findZoneAccountName(zones, z.ZoneTag)
		for _, c := range cg {
			if !deniedMetricsSet.Has(zoneColocationVisitsMetricName) {
				zoneColocationVisits.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.Visits))
			}
			if !deniedMetricsSet.Has(zoneColocationEdgeResponseBytesMetricName) {
				zoneColocationEdgeResponseBytes.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Sum.EdgeResponseBytes))
			}
			if !deniedMetricsSet.Has(zoneColocationRequestsTotalMetricName) {
				zoneColocationRequestsTotal.With(prometheus.Labels{"zone": name, "account": account, "colocation": c.Dimensions.ColoCode, "host": c.Dimensions.Host}).Add(float64(c.Count))
			}
		}
	}
}

func fetchZoneAdaptiveHTTPStats(zones []cfzones.Zone, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	if deniedMetricsSet.Has(zoneRequestStatus5mRatioMetricName) && deniedMetricsSet.Has(zoneRequestEdgeTTFB5mQuantilesMetricName) {
		return
	}

	zoneIDs := extractZoneIDs(zones)
	if len(zoneIDs) == 0 {
		return
	}

	r, err := fetchZone5mStats(zoneIDs)
	if err != nil {
		log.Error("failed to fetch analyzed zone 5-minute stats: ", err)
		return
	}

	for _, z := range r.Viewer.Zones {
		name, account := findZoneAccountName(zones, z.ZoneTag)

		if !deniedMetricsSet.Has(zoneRequestStatus5mRatioMetricName) {
			for _, g := range z.HTTPRequestsAdaptiveGroups {
				zoneRequestStatus5mRatio.With(
					prometheus.Labels{
						"zone":         name,
						"account":      account,
						"origin_ip":    g.Dimensions.OriginIP,
						"status_group": "4xx",
					}).Add(float64(g.Ratio.Status4xx))
				zoneRequestStatus5mRatio.With(
					prometheus.Labels{
						"zone":         name,
						"account":      account,
						"origin_ip":    g.Dimensions.OriginIP,
						"status_group": "5xx",
					}).Add(float64(g.Ratio.Status5xx))
			}
		}

		if !deniedMetricsSet.Has(zoneRequestEdgeTTFB5mQuantilesMetricName) {
			for _, g := range z.HTTPRequestsAdaptiveGroups {
				zoneRequestEdgeTTFB5mQuantiles.With(
					prometheus.Labels{
						"zone":      name,
						"account":   account,
						"origin_ip": g.Dimensions.OriginIP,
						"quantile":  "P50",
					}).Set(float64(g.Quantiles.EdgeTimeToFirstByteMsP50))
				zoneRequestEdgeTTFB5mQuantiles.With(
					prometheus.Labels{
						"zone":      name,
						"account":   account,
						"origin_ip": g.Dimensions.OriginIP,
						"quantile":  "P95",
					}).Set(float64(g.Quantiles.EdgeTimeToFirstByteMsP95))
				zoneRequestEdgeTTFB5mQuantiles.With(
					prometheus.Labels{
						"zone":      name,
						"account":   account,
						"origin_ip": g.Dimensions.OriginIP,
						"quantile":  "P99",
					}).Set(float64(g.Quantiles.EdgeTimeToFirstByteMsP99))
			}
		}
	}
}

func fetchZoneAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	// None of the below referenced metrics are available in the free tier
	if viper.GetBool("free_tier") {
		return
	}

	zoneIDs := extractZoneIDs(zones)
	if len(zoneIDs) == 0 {
		return
	}

	r, err := fetchZoneTotals(zoneIDs)
	if err != nil {
		log.Error("failed to fetch zone analytics: ", err)
		return
	}

	for _, z := range r.Viewer.Zones {
		name, account := findZoneAccountName(zones, z.ZoneTag)
		z := z

		addHTTPGroups(&z, name, account, deniedMetricsSet)
		addFirewallGroups(&z, name, account, deniedMetricsSet)
		addHealthCheckGroups(&z, name, account, deniedMetricsSet)
		addHTTPAdaptiveGroups(&z, name, account, deniedMetricsSet)
	}
}

func addHTTPGroups(z *zoneResp, name string, account string, deniedMetricsSet MetricsSet) {
	// Nothing to do.
	if len(z.HTTP1mGroups) == 0 {
		return
	}

	zt := z.HTTP1mGroups[0]

	if !deniedMetricsSet.Has(zoneRequestTotalMetricName) {
		zoneRequestTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.Requests))
	}
	if !deniedMetricsSet.Has(zoneRequestCachedMetricName) {
		zoneRequestCached.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.CachedRequests))
	}
	if !deniedMetricsSet.Has(zoneRequestSSLEncryptedMetricName) {
		zoneRequestSSLEncrypted.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.EncryptedRequests))
	}

	for _, ct := range zt.Sum.ContentType {
		if !deniedMetricsSet.Has(zoneRequestContentTypeMetricName) {
			zoneRequestContentType.With(prometheus.Labels{"zone": name, "account": account, "content_type": ct.EdgeResponseContentType}).Add(float64(ct.Requests))
		}
		if !deniedMetricsSet.Has(zoneBandwidthContentTypeMetricName) {
			zoneBandwidthContentType.With(prometheus.Labels{"zone": name, "account": account, "content_type": ct.EdgeResponseContentType}).Add(float64(ct.Bytes))
		}
	}

	for _, country := range zt.Sum.Country {
		c := countries.ByName(country.ClientCountryName)
		region := c.Info().Region.Info().Name

		if !deniedMetricsSet.Has(zoneRequestCountryMetricName) {
			zoneRequestCountry.With(prometheus.Labels{"zone": name, "account": account, "country": country.ClientCountryName, "region": region}).Add(float64(country.Requests))
		}
		if !deniedMetricsSet.Has(zoneBandwidthCountryMetricName) {
			zoneBandwidthCountry.With(prometheus.Labels{"zone": name, "account": account, "country": country.ClientCountryName, "region": region}).Add(float64(country.Bytes))
		}
		if !deniedMetricsSet.Has(zoneThreatsCountryMetricName) {
			zoneThreatsCountry.With(prometheus.Labels{"zone": name, "account": account, "country": country.ClientCountryName, "region": region}).Add(float64(country.Threats))
		}
	}

	if !deniedMetricsSet.Has(zoneRequestHTTPStatusMetricName) {
		for _, status := range zt.Sum.ResponseStatus {
			zoneRequestHTTPStatus.With(prometheus.Labels{"zone": name, "account": account, "status": strconv.Itoa(status.EdgeResponseStatus)}).Add(float64(status.Requests))
		}
	}

	if !deniedMetricsSet.Has(zoneRequestBrowserMapMetricName) {
		for _, browser := range zt.Sum.BrowserMap {
			zoneRequestBrowserMap.With(prometheus.Labels{"zone": name, "account": account, "family": browser.UaBrowserFamily}).Add(float64(browser.PageViews))
		}
	}

	if !deniedMetricsSet.Has(zoneBandwidthTotalMetricName) {
		zoneBandwidthTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.Bytes))
	}
	if !deniedMetricsSet.Has(zoneBandwidthCachedMetricName) {
		zoneBandwidthCached.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.CachedBytes))
	}
	if !deniedMetricsSet.Has(zoneBandwidthSSLEncryptedMetricName) {
		zoneBandwidthSSLEncrypted.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.EncryptedBytes))
	}

	if !deniedMetricsSet.Has(zoneThreatsTotalMetricName) {
		zoneThreatsTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.Threats))
	}

	if !deniedMetricsSet.Has(zoneThreatsTypeMetricName) {
		for _, t := range zt.Sum.ThreatPathing {
			zoneThreatsType.With(prometheus.Labels{"zone": name, "account": account, "type": t.Name}).Add(float64(t.Requests))
		}
	}

	if !deniedMetricsSet.Has(zonePageviewsTotalMetricName) {
		zonePageviewsTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Sum.PageViews))
	}

	if !deniedMetricsSet.Has(zoneUniquesTotalMetricName) {
		// Uniques
		zoneUniquesTotal.With(prometheus.Labels{"zone": name, "account": account}).Add(float64(zt.Unique.Uniques))
	}
}

func addFirewallGroups(z *zoneResp, name string, account string, deniedMetricsSet MetricsSet) {
	// Nothing to do.
	if len(z.FirewallEventsAdaptiveGroups) == 0 {
		return
	}
	rulesMap := fetchFirewallRules(z.ZoneTag)
	if !deniedMetricsSet.Has(zoneFirewallEventsCountMetricName) {
		for _, g := range z.FirewallEventsAdaptiveGroups {
			zoneFirewallEventsCount.With(
				prometheus.Labels{
					"zone":    name,
					"account": account,
					"action":  g.Dimensions.Action,
					"source":  g.Dimensions.Source,
					"rule":    normalizeRuleName(rulesMap[g.Dimensions.RuleID]),
					"host":    g.Dimensions.ClientRequestHTTPHost,
					"country": g.Dimensions.ClientCountryName,
				}).Add(float64(g.Count))
		}
	}
}

func normalizeRuleName(initialText string) string {
	maxLength := 200
	nonSpaceName := strings.ReplaceAll(strings.ToLower(initialText), " ", "_")
	if len(nonSpaceName) > maxLength {
		return nonSpaceName[:maxLength]
	}
	return nonSpaceName
}

func addHealthCheckGroups(z *zoneResp, name string, account string, deniedMetricsSet MetricsSet) {
	if len(z.HealthCheckEventsAdaptiveGroups) == 0 {
		return
	}

	if !deniedMetricsSet.Has(zoneHealthCheckEventsOriginCountMetricName) {
		for _, g := range z.HealthCheckEventsAdaptiveGroups {
			zoneHealthCheckEventsOriginCount.With(
				prometheus.Labels{
					"zone":          name,
					"account":       account,
					"health_status": g.Dimensions.HealthStatus,
					"origin_ip":     g.Dimensions.OriginIP,
					"region":        g.Dimensions.Region,
					"fqdn":          g.Dimensions.Fqdn,
				}).Add(float64(g.Count))
		}
	}
}

func addHTTPAdaptiveGroups(z *zoneResp, name string, account string, deniedMetricsSet MetricsSet) {
	if !deniedMetricsSet.Has(zoneRequestOriginStatusCountryHostMetricName) {
		for _, g := range z.HTTPRequestsAdaptiveGroups {
			zoneRequestOriginStatusCountryHost.With(
				prometheus.Labels{
					"zone":    name,
					"account": account,
					"status":  strconv.Itoa(int(g.Dimensions.OriginResponseStatus)),
					"country": g.Dimensions.ClientCountryName,
					"host":    g.Dimensions.ClientRequestHTTPHost,
				}).Add(float64(g.Count))
		}
	}
	if !deniedMetricsSet.Has(zoneRequestStatusCountryHostMetricName) {
		for _, g := range z.HTTPRequestsEdgeCountryHost {
			zoneRequestStatusCountryHost.With(
				prometheus.Labels{
					"zone":    name,
					"account": account,
					"status":  strconv.Itoa(int(g.Dimensions.EdgeResponseStatus)),
					"country": g.Dimensions.ClientCountryName,
					"host":    g.Dimensions.ClientRequestHTTPHost,
				}).Add(float64(g.Count))
		}
	}
}

func fetchLoadBalancerAnalytics(zones []cfzones.Zone, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	// None of the below referenced metrics are available in the free tier
	if viper.GetBool("free_tier") {
		return
	}

	zoneIDs := extractZoneIDs(zones)
	if len(zoneIDs) == 0 {
		return
	}

	l, err := fetchLoadBalancerTotals(zoneIDs)
	if err != nil {
		log.Error("failed to fetch load balancer analytics: ", err)
		return
	}
	for _, lb := range l.Viewer.Zones {
		name, account := findZoneAccountName(zones, lb.ZoneTag)
		lb := lb
		addLoadBalancingRequestsAdaptive(&lb, name, account, deniedMetricsSet)
		addLoadBalancingRequestsAdaptiveGroups(&lb, name, account, deniedMetricsSet)
	}
}

func addLoadBalancingRequestsAdaptiveGroups(z *lbResp, name string, account string, deniedMetricsSet MetricsSet) {
	if !deniedMetricsSet.Has(poolRequestsTotalMetricName) {
		for _, g := range z.LoadBalancingRequestsAdaptiveGroups {
			poolRequestsTotal.With(
				prometheus.Labels{
					"zone":               name,
					"account":            account,
					"load_balancer_name": g.Dimensions.LbName,
					"pool_name":          g.Dimensions.SelectedPoolName,
					"origin_name":        g.Dimensions.SelectedOriginName,
				}).Add(float64(g.Count))
		}
	}
}

func addLoadBalancingRequestsAdaptive(z *lbResp, name string, account string, deniedMetricsSet MetricsSet) {
	if !deniedMetricsSet.Has(poolHealthStatusMetricName) {
		for _, g := range z.LoadBalancingRequestsAdaptive {
			for _, p := range g.Pools {
				poolHealthStatus.With(
					prometheus.Labels{
						"zone":               name,
						"account":            account,
						"load_balancer_name": g.LbName,
						"pool_name":          p.PoolName,
					}).Set(float64(p.Healthy))
			}
		}
	}
}

func fetchZeroTrustAnalyticsForAccount(account cfaccounts.Account, wg *sync.WaitGroup) {
	wg.Add(1)
	defer wg.Done()

	addCloudflareTunnelStatus(account)
}

func addCloudflareTunnelStatus(account cfaccounts.Account) {
	tunnels := fetchCloudflareTunnels(account)
	for _, t := range tunnels {
		tunnelInfo.With(
			prometheus.Labels{
				"account":     account.Name,
				"tunnel_id":   t.ID,
				"tunnel_name": t.Name,
				"tunnel_type": string(t.TunType),
			}).Set(float64(1))

		tunnelHealthStatus.With(
			prometheus.Labels{
				"account":   account.Name,
				"tunnel_id": t.ID,
			}).Set(float64(getCloudflareTunnelStatusValue(string(t.Status))))

		// Each client/connector can open many connections to the Cloudflare edge,
		// we opt to not expose metrics for each individual connection. We do expose
		// an informational metric for each client/connector however.
		clients := fetchCloudflareTunnelConnectors(account, t.ID)
		for _, c := range clients {
			originIP := ""
			if len(c.Conns) > 0 {
				originIP = c.Conns[0].OriginIP
			}

			tunnelConnectorInfo.With(
				prometheus.Labels{
					"account":   account.Name,
					"tunnel_id": t.ID,
					"client_id": c.ID,
					"version":   c.Version,
					"arch":      c.Arch,
					"origin_ip": originIP,
				}).Set(float64(1))

			tunnelConnectorActiveConnections.With(
				prometheus.Labels{
					"account":   account.Name,
					"tunnel_id": t.ID,
					"client_id": c.ID,
				}).Set(float64(len(c.Conns)))
		}
	}
}

// waeFloat coerces a value from a decoded WAE JSON row into a float64. The SQL
// API returns Float64 columns (e.g. quantiles) as JSON numbers, but renders
// 64-bit integer results such as sum(_sample_interval) as quoted strings to
// preserve precision. Both are expected and handled quietly; anything else is
// logged because it means the data is wrong (bad numeric string, or a column
// missing/renamed), not just a different-but-valid representation.
func waeFloat(column string, v interface{}) float64 {
	switch n := v.(type) {
	case float64:
		return n
	case string:
		f, err := strconv.ParseFloat(n, 64)
		if err != nil {
			log.Errorf("WAE column %q: could not parse %q as float: %v", column, n, err)
		}
		return f
	default:
		log.Warnf("WAE column %q has unexpected type %T", column, v)
		return 0
	}
}

// fetchWorkerWAEAnalytics discovers all WAE datasets in the account that match
// the configured prefix and queries each one for worker metrics. The dataset
// name's suffix (with underscores converted to dashes) is emitted as the
// script_name label so the series are joinable with cloudflare_worker_* metrics.
func fetchWorkerWAEAnalytics(account cfaccounts.Account, wg *sync.WaitGroup, deniedMetricsSet MetricsSet) {
	wg.Add(1)
	defer wg.Done()

	prefix := viper.GetString("wae_dataset_prefix")
	if prefix == "" {
		return
	}

	ctx, cancel := context.WithTimeout(context.Background(), cftimeout)
	defer cancel()

	listResp, err := waeClient.Query(ctx, account.ID, "SHOW TABLES")
	if err != nil {
		log.Error("failed to list WAE datasets for account ", account.ID, ": ", err)
		return
	}

	var datasets []string
	for _, row := range listResp.Data {
		name, _ := row["dataset"].(string)
		if strings.HasPrefix(name, prefix) {
			datasets = append(datasets, name)
		}
	}
	if len(datasets) == 0 {
		log.Debugf("no WAE datasets matched prefix %q in account %s", prefix, account.ID)
		return
	}

	accountName := strings.ToLower(strings.ReplaceAll(account.Name, " ", "-"))

	// Drop this account's previous label combinations so labels that no longer
	// see traffic (or scripts that no longer exist) disappear instead of going
	// stale at their last value.
	workerOperationDuration.DeletePartialMatch(prometheus.Labels{"account": accountName})
	workerOperationCount.DeletePartialMatch(prometheus.Labels{"account": accountName})

	for _, dataset := range datasets {
		scriptName := strings.ReplaceAll(strings.TrimPrefix(dataset, prefix), "_", "-")

		query := fmt.Sprintf(`
			SELECT
				index1 AS metric_type,
				blob1 AS label,
				sum(_sample_interval) AS count,
				quantileWeighted(0.50, double1, _sample_interval) AS p50_ms,
				quantileWeighted(0.75, double1, _sample_interval) AS p75_ms,
				quantileWeighted(0.99, double1, _sample_interval) AS p99_ms,
				quantileWeighted(0.999, double1, _sample_interval) AS p999_ms
			FROM %s
			WHERE timestamp > NOW() - INTERVAL '5' MINUTE
			GROUP BY index1, blob1
			FORMAT JSON`, dataset)

		resp, err := waeClient.Query(ctx, account.ID, query)
		if err != nil {
			log.Errorf("failed to fetch WAE analytics from %s (account %s): %v", dataset, account.ID, err)
			continue
		}

		durationDenied := deniedMetricsSet.Has(workerOperationDurationMetricName)
		countDenied := deniedMetricsSet.Has(workerOperationCountMetricName)
		if durationDenied && countDenied {
			continue
		}

		quantiles := []struct {
			column string
			label  string
		}{
			{"p50_ms", "P50"},
			{"p75_ms", "P75"},
			{"p99_ms", "P99"},
			{"p999_ms", "P999"},
		}

		for _, row := range resp.Data {
			metricType, _ := row["metric_type"].(string)
			label, _ := row["label"].(string)
			if metricType == "" {
				continue
			}

			if !durationDenied {
				for _, q := range quantiles {
					val, _ := row[q.column].(float64)
					workerOperationDuration.With(prometheus.Labels{
						"script_name": scriptName,
						"account":     accountName,
						"metric_type": metricType,
						"label":       label,
						"quantile":    q.label,
					}).Set(val / 1000.0)
				}
			}

			if !countDenied {
				count := waeFloat("count", row["count"])
				workerOperationCount.With(prometheus.Labels{
					"script_name": scriptName,
					"account":     accountName,
					"metric_type": metricType,
					"label":       label,
				}).Set(count)
			}
		}
	}
}

// The status of the tunnel.
// Valid values are:
//   - inactive (tunnel has never been run)
//   - degraded (tunnel is active and able to serve traffic but in an unhealthy state)
//   - healthy (tunnel is active and able to serve traffic)
//   - down (tunnel can not serve traffic as it has no connections to the Cloudflare Edge).
func getCloudflareTunnelStatusValue(status string) uint8 {
	switch status {
	case "inactive":
		return 3
	case "degraded":
		return 2
	case "healthy":
		return 1
	case "down":
		return 0
	default:
		// Undefined status value returned by the API
		return 255
	}
}