clustermesh: homogenize ClusterMesh metrics based on watch store

Report per cluster metrics using the watch store for endpoints, global
services, MCS service exports like we are doing already for remote
nodes. This doesn't include identities unfortunately which doesn't use
the watch store.

We are no longer attempting to report Global Services and Global Service
Export count. Note that those global count were not accounting the local
cluster which was misleading.

Signed-off-by: Arthur Outhenin-Chalandre <git@mrfreezeex.fr>

Author: MrFreezeex

Documentation/observability/metrics.rst

@@ -371,10 +371,11 @@ Clustermesh
 =============================================== ============================================================ ========== =================================================================
 Name                                            Labels                                                       Default    Description
 =============================================== ============================================================ ========== =================================================================
-``clustermesh_global_services``                 ``source_cluster``, ``source_node_name``                     Enabled    The total number of global services in the cluster mesh
+``clustermesh_remote_cluster_services``         ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The total number of services per remote cluster
+``clustermesh_remote_cluster_endpoints``        ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The total number of endpoints per remote cluster
+``clustermesh_remote_cluster_nodes``            ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The total number of nodes per remote cluster
 ``clustermesh_remote_clusters``                 ``source_cluster``, ``source_node_name``                     Enabled    The total number of remote clusters meshed with the local cluster
 ``clustermesh_remote_cluster_failures``         ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The total number of failures related to the remote cluster
-``clustermesh_remote_cluster_nodes``            ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The total number of nodes in the remote cluster
 ``clustermesh_remote_cluster_last_failure_ts``  ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The timestamp of the last failure of the remote cluster
 ``clustermesh_remote_cluster_readiness_status`` ``source_cluster``, ``source_node_name``, ``target_cluster`` Enabled    The readiness status of the remote cluster
 =============================================== ============================================================ ========== =================================================================
@@ -867,6 +868,16 @@ Name                                                 Labels
 ``serviceimport_info``               ``serviceimport``, ``namespace``                             Enabled    Information about ServiceImport in the local cluster
 ==================================== ============================================================ ========== ===========================================================
 
+Clustermesh
+~~~~~~~~~~~
+
+============================================== ======================================= ========== ==================================================================
+Name                                           Labels                                  Default    Description
+============================================== ======================================= ========== ==================================================================
+``clustermesh_remote_cluster_services``        ``source_cluster``, ``target_cluster``  Enabled    The total number of services per remote cluster
+``clustermesh_remote_cluster_service_exports`` ``source_cluster``, ``target_cluster``  Enabled    The total number of MCS-API service exports per remote cluster
+============================================== ======================================= ========== ==================================================================
+
 
 Hubble
 ------

Documentation/operations/upgrade.rst

@@ -356,6 +356,8 @@ Bugtool Options
 
 Added Metrics
 ~~~~~~~~~~~~~
+* ``cilium_agent_clustermesh_remote_cluster_endpoints`` was added and report
+  the total number of endpoints per remote cluster in a ClusterMesh environment.
 
 Removed Metrics
 ~~~~~~~~~~~~~~~
@@ -381,6 +383,15 @@ As well, any remaining Operator k8s workqueue metrics that use the label ``queue
 
 * ``k8s_client_rate_limiter_duration_seconds`` no longer has labels ``path`` and ``method``.
 
+The following metrics:
+* ``cilium_agent_clustermesh_global_services``
+* ``cilium_operator_clustermesh_global_services``
+* ``cilium_operator_clustermesh_global_service_exports``
+now report per cluster metric instead of a "global" count and were renamed to respectively:
+* ``cilium_agent_clustermesh_remote_cluster_services``
+* ``cilium_operator_clustermesh_remote_cluster_services``
+* ``cilium_operator_clustermesh_remote_cluster_service_exports``
+
 Deprecated Metrics
 ~~~~~~~~~~~~~~~~~~
 

pkg/clustermesh/clustermesh.go

@@ -211,12 +211,14 @@ func (cm *ClusterMesh) NewRemoteCluster(name string, status common.StatusFunc) c
 		),
 		&clusterServiceObserver{serviceMerger: cm.conf.ServiceMerger},
 		store.RWSWithOnSyncCallback(func(ctx context.Context) { close(rc.synced.services) }),
+		store.RWSWithEntriesMetric(cm.conf.Metrics.TotalServices.WithLabelValues(cm.conf.ClusterInfo.Name, cm.nodeName, rc.name)),
 	)
 
 	rc.ipCacheWatcher = ipcache.NewIPIdentityWatcher(
 		cm.conf.Logger,
 		name, cm.conf.IPCache, cm.conf.StoreFactory, source.ClusterMesh,
 		store.RWSWithOnSyncCallback(func(ctx context.Context) { close(rc.synced.ipcache) }),
+		store.RWSWithEntriesMetric(cm.conf.Metrics.TotalEndpoints.WithLabelValues(cm.conf.ClusterInfo.Name, cm.nodeName, rc.name)),
 	)
 	rc.ipCacheWatcherExtraOpts = cm.conf.IPCacheWatcherExtraOpts
 

pkg/clustermesh/common/services.go

@@ -13,7 +13,6 @@ import (
 	"github.com/cilium/cilium/pkg/kvstore/store"
 	"github.com/cilium/cilium/pkg/lock"
 	"github.com/cilium/cilium/pkg/logging/logfields"
-	"github.com/cilium/cilium/pkg/metrics/metric"
 )
 
 type GlobalService struct {
@@ -30,16 +29,12 @@ type GlobalServiceCache struct {
 	logger *slog.Logger
 	mutex  lock.RWMutex
 	byName map[types.NamespacedName]*GlobalService
-
-	// metricTotalGlobalServices is the gauge metric for total of global services
-	metricTotalGlobalServices metric.Gauge
 }
 
-func NewGlobalServiceCache(logger *slog.Logger, metricTotalGlobalServices metric.Gauge) *GlobalServiceCache {
+func NewGlobalServiceCache(logger *slog.Logger) *GlobalServiceCache {
 	return &GlobalServiceCache{
-		logger:                    logger,
-		byName:                    map[types.NamespacedName]*GlobalService{},
-		metricTotalGlobalServices: metricTotalGlobalServices,
+		logger: logger,
+		byName: map[types.NamespacedName]*GlobalService{},
 	}
 }
 
@@ -101,7 +96,6 @@ func (c *GlobalServiceCache) OnUpdate(svc *serviceStore.ClusterService) {
 			logfields.ServiceName, svc,
 			logfields.ClusterName, svc.Cluster,
 		)
-		c.metricTotalGlobalServices.Set(float64(len(c.byName)))
 	}
 
 	c.logger.Debug(
@@ -137,7 +131,6 @@ func (c *GlobalServiceCache) delete(globalService *GlobalService, clusterName st
 			logfields.ClusterName, clusterName,
 		)
 		delete(c.byName, serviceNN)
-		c.metricTotalGlobalServices.Set(float64(len(c.byName)))
 	}
 
 	return true

pkg/clustermesh/common/services_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	serviceStore "github.com/cilium/cilium/pkg/clustermesh/store"
-	"github.com/cilium/cilium/pkg/metrics"
 )
 
 type fakeUpstream struct {
@@ -32,7 +31,7 @@ func TestRemoteServiceObserver(t *testing.T) {
 	}
 	svc1 := serviceStore.ClusterService{Cluster: "remote", Namespace: "namespace", Name: "name", IncludeExternal: false, Shared: true}
 	svc2 := serviceStore.ClusterService{Cluster: "remote", Namespace: "namespace", Name: "name"}
-	cache := NewGlobalServiceCache(hivetest.Logger(t), metrics.NoOpGauge)
+	cache := NewGlobalServiceCache(hivetest.Logger(t))
 
 	var upstream fakeUpstream
 	observer := NewSharedServicesObserver(hivetest.Logger(t), cache, upstream.OnUpdate, upstream.OnDelete)

pkg/clustermesh/endpointslicesync/endpointslice_test.go

@@ -31,7 +31,6 @@ import (
 	slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
 	"github.com/cilium/cilium/pkg/k8s/utils"
 	"github.com/cilium/cilium/pkg/loadbalancer"
-	"github.com/cilium/cilium/pkg/metrics/metric"
 )
 
 const (
@@ -113,7 +112,7 @@ func Test_meshEndpointSlice_Reconcile(t *testing.T) {
 	}
 	defer hive.Stop(tlog, context.Background())
 
-	globalService := common.NewGlobalServiceCache(hivetest.Logger(t), metric.NewGauge(metric.GaugeOpts{}))
+	globalService := common.NewGlobalServiceCache(hivetest.Logger(t))
 	podInformer := newMeshPodInformer(logger, globalService)
 	nodeInformer := newMeshNodeInformer(logger)
 	controller, serviceInformer, endpointsliceInformer := newEndpointSliceMeshController(

pkg/clustermesh/mcsapi/serviceimport_controller_test.go

@@ -26,7 +26,6 @@ import (
 
 	mcsapitypes "github.com/cilium/cilium/pkg/clustermesh/mcsapi/types"
 	"github.com/cilium/cilium/pkg/clustermesh/operator"
-	"github.com/cilium/cilium/pkg/metrics/metric"
 )
 
 const (
@@ -503,7 +502,7 @@ func Test_mcsServiceImport_Reconcile(t *testing.T) {
 		WithStatusSubresource(&mcsapiv1alpha1.ServiceImport{}).
 		WithScheme(testScheme()).
 		Build()
-	globalServiceExports := operator.NewGlobalServiceExportCache(metric.NewGauge(metric.GaugeOpts{}))
+	globalServiceExports := operator.NewGlobalServiceExportCache()
 	remoteClusterServiceSource := &remoteClusterServiceExportSource{Logger: hivetest.Logger(t)}
 	for _, svcExport := range remoteSvcImportTestFixtures {
 		globalServiceExports.OnUpdate(svcExport)

pkg/clustermesh/metrics.go

@@ -9,11 +9,14 @@ import (
 )
 
 type Metrics struct {
-	// TotalNodes tracks the number of total nodes in a remote cluster.
+	// TotalNodes tracks the number of total nodes per remote cluster.
 	TotalNodes metric.Vec[metric.Gauge]
 
-	// TotalGlobalServices tracks the total number of global services.
-	TotalGlobalServices metric.Vec[metric.Gauge]
+	// TotalServices tracks the number of total services per remote cluster.
+	TotalServices metric.Vec[metric.Gauge]
+
+	// TotalEndpoints tracks the number of total IPs per remote cluster.
+	TotalEndpoints metric.Vec[metric.Gauge]
 }
 
 func NewMetrics() Metrics {
@@ -26,12 +29,20 @@ func NewMetrics() Metrics {
 			Help:       "The total number of nodes in the remote cluster",
 		}, []string{metrics.LabelSourceCluster, metrics.LabelSourceNodeName, metrics.LabelTargetCluster}),
 
-		TotalGlobalServices: metric.NewGaugeVec(metric.GaugeOpts{
-			ConfigName: metrics.Namespace + "_" + subsystem + "_global_services",
+		TotalServices: metric.NewGaugeVec(metric.GaugeOpts{
+			ConfigName: metrics.Namespace + "_" + subsystem + "_remote_cluster_services",
+			Namespace:  metrics.Namespace,
+			Subsystem:  subsystem,
+			Name:       "remote_cluster_services",
+			Help:       "The total number of services in the remote cluster",
+		}, []string{metrics.LabelSourceCluster, metrics.LabelSourceNodeName, metrics.LabelTargetCluster}),
+
+		TotalEndpoints: metric.NewGaugeVec(metric.GaugeOpts{
+			ConfigName: metrics.Namespace + "_" + subsystem + "_remote_cluster_endpoints",
 			Namespace:  metrics.Namespace,
 			Subsystem:  subsystem,
-			Name:       "global_services",
-			Help:       "The total number of global services in the cluster mesh",
-		}, []string{metrics.LabelSourceCluster, metrics.LabelSourceNodeName}),
+			Name:       "remote_cluster_endpoints",
+			Help:       "The total number of endpoints in the remote cluster",
+		}, []string{metrics.LabelSourceCluster, metrics.LabelSourceNodeName, metrics.LabelTargetCluster}),
 	}
 }

pkg/clustermesh/operator/clustermesh.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cilium/cilium/pkg/clustermesh/common"
 	mcsapitypes "github.com/cilium/cilium/pkg/clustermesh/mcsapi/types"
 	serviceStore "github.com/cilium/cilium/pkg/clustermesh/store"
+	"github.com/cilium/cilium/pkg/clustermesh/types"
 	"github.com/cilium/cilium/pkg/clustermesh/wait"
 	"github.com/cilium/cilium/pkg/dial"
 	"github.com/cilium/cilium/pkg/kvstore/store"
@@ -29,10 +30,11 @@ type clusterMesh struct {
 	// common implements the common logic to connect to remote clusters.
 	common common.ClusterMesh
 
-	cfg       ClusterMeshConfig
-	cfgMCSAPI MCSAPIConfig
-	logger    *slog.Logger
-	Metrics   Metrics
+	cfg         ClusterMeshConfig
+	cfgMCSAPI   MCSAPIConfig
+	logger      *slog.Logger
+	clusterInfo types.ClusterInfo
+	metrics     Metrics
 
 	// globalServices is a list of all global services. The datastructure
 	// is protected by its own mutex inside the structure.
@@ -97,15 +99,15 @@ func newClusterMesh(lc cell.Lifecycle, params clusterMeshParams) (*clusterMesh,
 	params.Logger.Info("Operator ClusterMesh component enabled")
 
 	cm := clusterMesh{
-		cfg:            params.Cfg,
-		cfgMCSAPI:      params.CfgMCSAPI,
-		logger:         params.Logger,
-		globalServices: common.NewGlobalServiceCache(params.Logger, params.Metrics.TotalGlobalServices.WithLabelValues(params.ClusterInfo.Name)),
-		globalServiceExports: NewGlobalServiceExportCache(
-			params.Metrics.TotalGlobalServiceExports.WithLabelValues(params.ClusterInfo.Name),
-		),
-		storeFactory:      params.StoreFactory,
-		syncTimeoutConfig: params.TimeoutConfig,
+		cfg:                  params.Cfg,
+		cfgMCSAPI:            params.CfgMCSAPI,
+		logger:               params.Logger,
+		clusterInfo:          params.ClusterInfo,
+		metrics:              params.Metrics,
+		globalServices:       common.NewGlobalServiceCache(params.Logger),
+		globalServiceExports: NewGlobalServiceExportCache(),
+		storeFactory:         params.StoreFactory,
+		syncTimeoutConfig:    params.TimeoutConfig,
 	}
 	cm.common = common.NewClusterMesh(common.Configuration{
 		Logger:              params.Logger,
@@ -223,6 +225,7 @@ func (cm *clusterMesh) newRemoteCluster(name string, status common.StatusFunc) c
 			},
 		),
 		store.RWSWithOnSyncCallback(func(ctx context.Context) { rc.synced.services.Stop() }),
+		store.RWSWithEntriesMetric(cm.metrics.TotalServices.WithLabelValues(cm.clusterInfo.Name, rc.name)),
 	)
 
 	rc.remoteServiceExports = cm.storeFactory.NewWatchStore(
@@ -245,6 +248,7 @@ func (cm *clusterMesh) newRemoteCluster(name string, status common.StatusFunc) c
 			},
 		),
 		store.RWSWithOnSyncCallback(func(ctx context.Context) { rc.synced.serviceExports.Stop() }),
+		store.RWSWithEntriesMetric(cm.metrics.TotalServiceExports.WithLabelValues(cm.clusterInfo.Name, name)),
 	)
 
 	return rc

pkg/clustermesh/operator/metrics.go

@@ -9,25 +9,25 @@ import (
 )
 
 type Metrics struct {
-	// TotalGlobalServices tracks the total number of global services.
-	TotalGlobalServices metric.Vec[metric.Gauge]
-	// TotalGlobalServiceExports tracks the total number of global service exports.
-	TotalGlobalServiceExports metric.Vec[metric.Gauge]
+	// TotalServices tracks the number of total global services per remote cluster.
+	TotalServices metric.Vec[metric.Gauge]
+	// TotalServiceExports tracks the number of total MCS-API service exports per remote cluster.
+	TotalServiceExports metric.Vec[metric.Gauge]
 }
 
 func NewMetrics() Metrics {
 	return Metrics{
-		TotalGlobalServices: metric.NewGaugeVec(metric.GaugeOpts{
+		TotalServices: metric.NewGaugeVec(metric.GaugeOpts{
 			Namespace: metrics.CiliumOperatorNamespace,
 			Subsystem: subsystem,
-			Name:      "global_services",
-			Help:      "The total number of global services in the cluster mesh",
-		}, []string{metrics.LabelSourceCluster}),
-		TotalGlobalServiceExports: metric.NewGaugeVec(metric.GaugeOpts{
+			Name:      "remote_cluster_services",
+			Help:      "The total number of services in the remote cluster",
+		}, []string{metrics.LabelSourceCluster, metrics.LabelTargetCluster}),
+		TotalServiceExports: metric.NewGaugeVec(metric.GaugeOpts{
 			Namespace: metrics.CiliumOperatorNamespace,
 			Subsystem: subsystem,
-			Name:      "global_service_exports",
-			Help:      "The total number of MCS-API global service exports in the cluster mesh",
-		}, []string{metrics.LabelSourceCluster}),
+			Name:      "remote_cluster_service_exports",
+			Help:      "The total number of MCS-API service exports in the remote cluster",
+		}, []string{metrics.LabelSourceCluster, metrics.LabelTargetCluster}),
 	}
 }