metrics: introduce metrics based on the NNS (#1414)

This PR introduces a small set of metrics based on the NodeNetworkState,
i.e. `kubernetes_nmstate_network_interfaces` and
`kubernetes_nmstate_routes`. Those track number of network interfaces
and number of routes for every node in the cluster where nmstate handler
pods are running.

This creates a skeleton for k-nmstate to create metrics based on the
effective state of the network on the system and not only based on the
content of NNCP.

In the future it may be used to create statistics about network
topologies of clusters.

Signed-off-by: Mat Kowalski <[email protected]>

Author: mkowalski

cmd/handler/main.go

@@ -83,6 +83,8 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 
 	metrics.Registry.MustRegister(monitoring.AppliedFeatures)
+	metrics.Registry.MustRegister(monitoring.NetworkInterfaces)
+	metrics.Registry.MustRegister(monitoring.NetworkRoutes)
 }
 
 func main() {
@@ -390,6 +392,17 @@ func setupMetricsManager(mgr manager.Manager) error {
 		setupLog.Error(err, "unable to create NodeNetworkConfigurationEnactment metrics controller", "metrics", "NMState")
 		return err
 	}
+
+	setupLog.Info("Creating Metrics NodeNetworkState controller")
+	if err := (&controllersmetrics.NodeNetworkStateReconciler{
+		Client: mgr.GetClient(),
+		Log:    ctrl.Log.WithName("metrics").WithName("NodeNetworkState"),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create NodeNetworkState metrics controller", "metrics", "NMState")
+		return err
+	}
+
 	return nil
 }
 

controllers/metrics/nodenetworkstate_controller.go

@@ -0,0 +1,217 @@
+/*
+Copyright The Kubernetes NMState Authors.
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"context"
+
+	"github.com/go-logr/logr"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+
+	nmstatev1beta1 "github.com/nmstate/kubernetes-nmstate/api/v1beta1"
+	"github.com/nmstate/kubernetes-nmstate/pkg/monitoring"
+	"github.com/nmstate/kubernetes-nmstate/pkg/state"
+)
+
+// NodeNetworkStateReconciler reconciles a NodeNetworkState object for metrics
+type NodeNetworkStateReconciler struct {
+	client.Client
+	Log    logr.Logger
+	Scheme *runtime.Scheme
+	// Track interface types per node to clean up stale metrics
+	oldInterfaceTypes map[string]map[string]struct{} // node name -> set of interface types
+	// Track route keys per node to clean up stale metrics
+	oldRouteKeys map[string]map[state.RouteKey]struct{} // node name -> set of route keys
+}
+
+// Reconcile reads the state of the cluster for a NodeNetworkState object and calculates
+// metrics for network interface counts by type and node.
+func (r *NodeNetworkStateReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
+	log := r.Log.WithValues("metrics.nodenetworkstate", request.NamespacedName)
+	log.Info("Reconcile")
+
+	nodeName := request.Name
+
+	nnsInstance := &nmstatev1beta1.NodeNetworkState{}
+	err := r.Client.Get(ctx, request.NamespacedName, nnsInstance)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			// NNS has been deleted, clean up metrics for this node
+			r.deleteNodeMetrics(nodeName)
+			return ctrl.Result{}, nil
+		}
+		log.Error(err, "Error retrieving NodeNetworkState")
+		return ctrl.Result{}, err
+	}
+
+	// Count interfaces by type for this node
+	counts, err := state.CountInterfacesByType(nnsInstance.Status.CurrentState)
+	if err != nil {
+		log.Error(err, "Failed to count interfaces by type")
+		return ctrl.Result{}, err
+	}
+
+	// Update interface metrics for this node
+	r.updateNodeInterfaceMetrics(nodeName, counts)
+
+	// Count routes by IP stack and type for this node
+	routeCounts, err := state.CountRoutes(nnsInstance.Status.CurrentState)
+	if err != nil {
+		log.Error(err, "Failed to count routes")
+		return ctrl.Result{}, err
+	}
+
+	// Update route metrics for this node
+	r.updateNodeRouteMetrics(nodeName, routeCounts)
+
+	return ctrl.Result{}, nil
+}
+
+func (r *NodeNetworkStateReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	r.oldInterfaceTypes = make(map[string]map[string]struct{})
+	r.oldRouteKeys = make(map[string]map[state.RouteKey]struct{})
+
+	onCreationOrUpdateForThisNNS := predicate.Funcs{
+		CreateFunc: func(createEvent event.CreateEvent) bool {
+			return true
+		},
+		DeleteFunc: func(e event.DeleteEvent) bool {
+			return true
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			oldNNS, ok := e.ObjectOld.(*nmstatev1beta1.NodeNetworkState)
+			if !ok {
+				return false
+			}
+			newNNS, ok := e.ObjectNew.(*nmstatev1beta1.NodeNetworkState)
+			if !ok {
+				return false
+			}
+
+			// Reconcile if the current state has changed
+			return oldNNS.Status.CurrentState.String() != newNNS.Status.CurrentState.String()
+		},
+		GenericFunc: func(event.GenericEvent) bool {
+			return false
+		},
+	}
+
+	err := ctrl.NewControllerManagedBy(mgr).
+		For(&nmstatev1beta1.NodeNetworkState{}).
+		WithEventFilter(onCreationOrUpdateForThisNNS).
+		Complete(r)
+	if err != nil {
+		return errors.Wrap(err, "failed to add controller to NNS metrics Reconciler")
+	}
+
+	return nil
+}
+
+// updateNodeInterfaceMetrics sets the interface count metrics for a specific node
+func (r *NodeNetworkStateReconciler) updateNodeInterfaceMetrics(nodeName string, counts map[string]int) {
+	// Get the old interface types for this node to detect removed types
+	oldTypes := r.oldInterfaceTypes[nodeName]
+	newTypes := make(map[string]struct{})
+
+	// Set metrics for current interface types
+	for ifaceType, count := range counts {
+		monitoring.NetworkInterfaces.With(prometheus.Labels{
+			"type": ifaceType,
+			"node": nodeName,
+		}).Set(float64(count))
+		newTypes[ifaceType] = struct{}{}
+	}
+
+	// Delete metrics for interface types that no longer exist on this node
+	for oldType := range oldTypes {
+		if _, exists := newTypes[oldType]; !exists {
+			monitoring.NetworkInterfaces.Delete(prometheus.Labels{
+				"type": oldType,
+				"node": nodeName,
+			})
+		}
+	}
+
+	// Store current types for next reconcile
+	r.oldInterfaceTypes[nodeName] = newTypes
+}
+
+// updateNodeRouteMetrics sets the route count metrics for a specific node
+func (r *NodeNetworkStateReconciler) updateNodeRouteMetrics(nodeName string, counts map[state.RouteKey]int) {
+	// Get the old route keys for this node to detect removed keys
+	oldKeys := r.oldRouteKeys[nodeName]
+	newKeys := make(map[state.RouteKey]struct{})
+
+	// Set metrics for current route keys
+	for key, count := range counts {
+		monitoring.NetworkRoutes.With(prometheus.Labels{
+			"node":     nodeName,
+			"ip_stack": key.IPStack,
+			"type":     key.Type,
+		}).Set(float64(count))
+		newKeys[key] = struct{}{}
+	}
+
+	// Delete metrics for route keys that no longer exist on this node
+	for oldKey := range oldKeys {
+		if _, exists := newKeys[oldKey]; !exists {
+			monitoring.NetworkRoutes.Delete(prometheus.Labels{
+				"node":     nodeName,
+				"ip_stack": oldKey.IPStack,
+				"type":     oldKey.Type,
+			})
+		}
+	}
+
+	// Store current keys for next reconcile
+	r.oldRouteKeys[nodeName] = newKeys
+}
+
+// deleteNodeMetrics removes all interface and route count metrics for a specific node
+func (r *NodeNetworkStateReconciler) deleteNodeMetrics(nodeName string) {
+	// Delete interface metrics
+	if oldTypes, ok := r.oldInterfaceTypes[nodeName]; ok {
+		for ifaceType := range oldTypes {
+			monitoring.NetworkInterfaces.Delete(prometheus.Labels{
+				"type": ifaceType,
+				"node": nodeName,
+			})
+		}
+		delete(r.oldInterfaceTypes, nodeName)
+	}
+
+	// Delete route metrics
+	if oldKeys, ok := r.oldRouteKeys[nodeName]; ok {
+		for key := range oldKeys {
+			monitoring.NetworkRoutes.Delete(prometheus.Labels{
+				"node":     nodeName,
+				"ip_stack": key.IPStack,
+				"type":     key.Type,
+			})
+		}
+		delete(r.oldRouteKeys, nodeName)
+	}
+}

pkg/monitoring/metrics.go

@@ -29,12 +29,35 @@ var (
 		Help: "Number of nmstate features applied labeled by its name",
 	}
 
+	NetworkInterfacesOpts = prometheus.GaugeOpts{
+		Name: "kubernetes_nmstate_network_interfaces",
+		Help: "Number of network interfaces labeled by its type",
+	}
+
+	NetworkRoutesOpts = prometheus.GaugeOpts{
+		Name: "kubernetes_nmstate_routes",
+		Help: "Number of network routes labeled by node, IP stack and type (static/dynamic)",
+	}
+
 	AppliedFeatures = prometheus.NewGaugeVec(
 		AppliedFeaturesOpts,
 		[]string{"name"},
 	)
+
+	NetworkInterfaces = prometheus.NewGaugeVec(
+		NetworkInterfacesOpts,
+		[]string{"type", "node"},
+	)
+
+	NetworkRoutes = prometheus.NewGaugeVec(
+		NetworkRoutesOpts,
+		[]string{"node", "ip_stack", "type"},
+	)
+
 	gaugeOpts = []prometheus.GaugeOpts{
 		AppliedFeaturesOpts,
+		NetworkInterfacesOpts,
+		NetworkRoutesOpts,
 	}
 )
 

pkg/state/filter.go

@@ -18,6 +18,8 @@ limitations under the License.
 package state
 
 import (
+	"strings"
+
 	"github.com/nmstate/kubernetes-nmstate/api/shared"
 	"github.com/nmstate/kubernetes-nmstate/pkg/environment"
 
@@ -38,6 +40,76 @@ func FilterOut(currentState shared.State) (shared.State, error) {
 	return filterOut(currentState)
 }
 
+// CountInterfacesByType parses the state and returns a map of interface type to count.
+func CountInterfacesByType(currentState shared.State) (map[string]int, error) {
+	var state rootState
+	if err := yaml.Unmarshal(currentState.Raw, &state); err != nil {
+		return nil, err
+	}
+
+	counts := make(map[string]int)
+	for _, iface := range state.Interfaces {
+		if iface.Type != "" {
+			counts[iface.Type]++
+		}
+	}
+	return counts, nil
+}
+
+// RouteKey represents the grouping key for route metrics.
+type RouteKey struct {
+	IPStack string // "ipv4" or "ipv6"
+	Type    string // "static" or "dynamic"
+}
+
+// CountRoutes parses the state and returns a map of RouteKey to count.
+// Routes are categorized by:
+// - IP stack: determined by presence of ":" in destination (ipv6) or not (ipv4)
+// - Type: "static" if route exists in routes.config, "dynamic" if only in routes.running
+func CountRoutes(currentState shared.State) (map[RouteKey]int, error) {
+	var state rootState
+	if err := yaml.Unmarshal(currentState.Raw, &state); err != nil {
+		return nil, err
+	}
+
+	counts := make(map[RouteKey]int)
+	if state.Routes == nil {
+		return counts, nil
+	}
+
+	// Build a set of static route destinations for quick lookup
+	staticRoutes := make(map[string]struct{})
+	for _, route := range state.Routes.Config {
+		staticRoutes[route.Destination] = struct{}{}
+	}
+
+	// Count running routes by IP stack and type
+	for _, route := range state.Routes.Running {
+		ipStack := getIPStack(route.Destination)
+		routeType := "dynamic"
+		if _, isStatic := staticRoutes[route.Destination]; isStatic {
+			routeType = "static"
+		}
+
+		key := RouteKey{
+			IPStack: ipStack,
+			Type:    routeType,
+		}
+		counts[key]++
+	}
+
+	return counts, nil
+}
+
+// getIPStack determines the IP stack from a destination CIDR.
+// Returns "ipv6" if the destination contains ":", otherwise "ipv4".
+func getIPStack(destination string) string {
+	if strings.Contains(destination, ":") {
+		return "ipv6"
+	}
+	return "ipv4"
+}
+
 func filterOutRoutes(routes []routeState, filteredInterfaces []interfaceState) []routeState {
 	filteredRoutes := []routeState{}
 	for _, route := range routes {