metrics: Add metric for k8s resource sync duration

This change adds a new metric to track the sync duration of individual k8s resources. Right now, even using logs, it is almost impossible to determine exactly how long a specific resource takes to sync.

Having this metric is useful because resource syncing can account for a significant portion of the agent startup time. To give an example, at scale, CiliumNode sync is often the reason for hive startup hitting the timeout.

This metric would allow to monitor regressions and provide a clear distribution of sync timings.

This commit introduces the new `kubernetes_resource_sync_duration` gauge metric, labeled by scope, where scope represents the resource name.

Signed-off-by: Illia Kolisnyk <illiakolisnyk@google.com>

Author: ILL1A

Documentation/observability/metrics.rst

@@ -680,6 +680,7 @@ Kubernetes
 =========================================== ================================================== ========== ========================================================
 Name                                        Labels                                             Default    Description
 =========================================== ================================================== ========== ========================================================
+``kubernetes_resource_sync_duration``       ``scope``                                          Enabled    Duration in seconds of a specific Kubernetes resource sync
 ``kubernetes_events_received_total``        ``scope``, ``action``, ``validity``, ``equal``     Enabled    Number of Kubernetes events received
 ``kubernetes_events_total``                 ``scope``, ``action``, ``outcome``                 Enabled    Number of Kubernetes events processed
 ``k8s_cnp_status_completion_seconds``       ``attempts``, ``outcome``                          Enabled    Duration in seconds in how long it took to complete a CNP status update

Documentation/operations/upgrade.rst

@@ -370,6 +370,8 @@ Added Metrics
   proxy redirects that were missing during endpoint policy calculation.
 * ``cilium_endpoint_component_status`` was added and reports the number of endpoints
   tagged by the status (``OK``, ``Warning``, ``Failure``) of each component (``BPF``, ``Policy``).
+* ``cilium_kubernetes_resource_sync_duration`` was added and reports duration in seconds
+  of a specific Kubernetes resource sync.
 
 Changed Metrics
 ###############

pkg/k8s/resource/resource.go

@@ -28,6 +28,7 @@ import (
 	"github.com/cilium/cilium/pkg/lock"
 	"github.com/cilium/cilium/pkg/metrics"
 	"github.com/cilium/cilium/pkg/promise"
+	"github.com/cilium/cilium/pkg/spanstat"
 )
 
 // Resource provides access to a Kubernetes resource through either
@@ -143,6 +144,7 @@ func New[T k8sRuntime.Object](lc cell.Lifecycle, lw cache.ListerWatcher, mp work
 		needed:          make(chan struct{}, 1),
 		lw:              lw,
 		metricsProvider: mp,
+		duration:        spanstat.Start(),
 	}
 	r.opts.sourceObj = func() k8sRuntime.Object {
 		var obj T
@@ -242,6 +244,8 @@ type resource[T k8sRuntime.Object] struct {
 	storeResolver promise.Resolver[Store[T]]
 
 	metricsProvider workqueue.MetricsProvider
+
+	duration *spanstat.SpanStat
 }
 
 var _ Resource[*corev1.Node] = &resource[*corev1.Node]{}
@@ -276,6 +280,7 @@ func (r *resource[T]) metricEventProcessed(eventKind EventKind, status bool) {
 	var action string
 	switch eventKind {
 	case Sync:
+		metrics.KubernetesResourceSyncDuration.WithLabelValues(r.opts.metricScope).Set(r.duration.End(status).Total().Seconds())
 		return
 	case Upsert:
 		action = "update"

pkg/metrics/metrics.go

@@ -436,6 +436,9 @@ var (
 
 	// Kubernetes Events
 
+	// KubernetesResourceSyncDuration is the Kubernetes resource sync duration labeled by scope.
+	KubernetesResourceSyncDuration = NoOpGaugeVec
+
 	// KubernetesEventProcessed is the number of Kubernetes events
 	// processed labeled by scope, action and execution result
 	KubernetesEventProcessed = NoOpCounterVec
@@ -673,6 +676,7 @@ type LegacyMetrics struct {
 	ControllerRuns                          metric.Vec[metric.Counter]
 	ControllerRunsDuration                  metric.Vec[metric.Observer]
 	SubprocessStart                         metric.Vec[metric.Counter]
+	KubernetesResourceSyncDuration          metric.Vec[metric.Gauge]
 	KubernetesEventProcessed                metric.Vec[metric.Counter]
 	KubernetesEventReceived                 metric.Vec[metric.Counter]
 	KubernetesAPIInteractions               metric.Vec[metric.Observer]
@@ -1022,6 +1026,13 @@ func NewLegacyMetrics() *LegacyMetrics {
 			Help:       "Number of times that Cilium has started a subprocess, labeled by subsystem",
 		}, []string{LabelSubsystem}),
 
+		KubernetesResourceSyncDuration: metric.NewGaugeVec(metric.GaugeOpts{
+			ConfigName: Namespace + "_kubernetes_resource_sync_duration",
+			Namespace:  Namespace,
+			Name:       "kubernetes_resource_sync_duration",
+			Help:       "Duration in seconds of a specific Kubernetes resource sync",
+		}, []string{LabelScope}),
+
 		KubernetesEventProcessed: metric.NewCounterVec(metric.CounterOpts{
 			ConfigName: Namespace + "_kubernetes_events_total",
 			Namespace:  Namespace,
@@ -1348,6 +1359,7 @@ func NewLegacyMetrics() *LegacyMetrics {
 	ControllerRuns = lm.ControllerRuns
 	ControllerRunsDuration = lm.ControllerRunsDuration
 	SubprocessStart = lm.SubprocessStart
+	KubernetesResourceSyncDuration = lm.KubernetesResourceSyncDuration
 	KubernetesEventProcessed = lm.KubernetesEventProcessed
 	KubernetesEventReceived = lm.KubernetesEventReceived
 	KubernetesAPIInteractions = lm.KubernetesAPIInteractions