Add L4NetLB metrics

Author: TortillaZHawaii

providers/gce/gce_loadbalancer_external.go

@@ -67,6 +67,18 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 		return nil, cloudprovider.ImplementedElsewhere
 	}
 
+	nm := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name}
+	metricsState := L4NetLBServiceState{
+		Status:       StatusError,
+		DenyFirewall: DenyFirewallStatusNone,
+	}
+	if !g.enableL4DenyFirewallRule && g.enableL4DenyFirewallRollbackCleanup {
+		metricsState.DenyFirewall = DenyFirewallStatusDisabled
+	}
+	defer func() {
+		g.metricsCollector.SetL4NetLBService(nm.String(), metricsState)
+	}()
+
 	if hasLoadBalancerClass(apiService, LegacyRegionalExternalLoadBalancerClass) {
 		if apiService.Annotations[ServiceAnnotationLoadBalancerType] == string(LBTypeInternal) {
 			g.eventRecorder.Event(apiService, v1.EventTypeWarning, "ConflictingConfiguration", fmt.Sprintf("loadBalancerClass conflicts with %s: %s annotation. External LoadBalancer Service provisioned.", ServiceAnnotationLoadBalancerType, string(LBTypeInternal)))
@@ -292,6 +304,11 @@ func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string,
 	status := &v1.LoadBalancerStatus{}
 	status.Ingress = []v1.LoadBalancerIngress{{IP: ipAddressToUse}}
 
+	metricsState.Status = StatusSuccess
+	if g.enableL4DenyFirewallRule {
+		metricsState.DenyFirewall = DenyFirewallStatusIPv4
+	}
+
 	syncResult.status = status
 	return syncResult, nil
 }
@@ -409,6 +426,7 @@ func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string,
 		klog.Errorf("Failed to remove finalizer '%s' from service %s - %v", NetLBFinalizerV1, service.Name, err)
 		return err
 	}
+	g.metricsCollector.DeleteL4NetLBService(serviceName.String())
 	return nil
 }
 

providers/gce/gce_loadbalancer_external_test.go

@@ -2530,3 +2530,70 @@ func TestFirewallsEqual(t *testing.T) {
 		})
 	}
 }
+
+func TestEnsureExternalLoadBalancerMetrics(t *testing.T) {
+	// t.Parallel() // Disable parallel to avoid race with global metrics registry
+	
+	vals := DefaultTestClusterValues()
+	gce, err := fakeGCECloud(vals)
+	require.NoError(t, err)
+	
+	lm, ok := gce.metricsCollector.(*LoadBalancerMetrics)
+	require.True(t, ok)
+	
+	svc := fakeLoadbalancerService("")
+	svc, err = gce.client.CoreV1().Services(svc.Namespace).Create(context.TODO(), svc, metav1.CreateOptions{})
+	require.NoError(t, err)
+	
+	nodes, err := createAndInsertNodes(gce, []string{"test-node-1"}, vals.ZoneName)
+	require.NoError(t, err)
+	
+	// Case 1: Success
+	_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)
+	assert.NoError(t, err)
+
+	// We expect 1 success, and deny firewall None (default)
+	lm.exportNetLBMetrics()
+	verifyL4NetLBMetric(t, 1, StatusSuccess, DenyFirewallStatusNone)
+
+	// Case 2: Enable deny firewall cleanup
+	gce.enableL4DenyFirewallRollbackCleanup = true
+	_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)
+	assert.NoError(t, err)
+
+	// We expect 1 success, and deny firewall Disabled
+	lm.exportNetLBMetrics()
+	verifyL4NetLBMetric(t, 1, StatusSuccess, DenyFirewallStatusDisabled)
+
+	// Case 3: Enable deny firewall
+	gce.enableL4DenyFirewallRule = true
+	_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)
+	assert.NoError(t, err)
+
+	// We expect 1 success, and deny firewall IPv4
+	lm.exportNetLBMetrics()
+	verifyL4NetLBMetric(t, 1, StatusSuccess, DenyFirewallStatusIPv4)
+
+	// Case 4: Error on fetch
+	mockGCE := gce.Compute().(*cloud.MockGCE)
+	mockGCE.MockFirewalls.GetHook = func(ctx context.Context, key *meta.Key, m *cloud.MockFirewalls, options ...cloud.Option) (bool, *compute.Firewall, error) {
+		return true, nil, fmt.Errorf("error on fetch")
+	}
+	_, err = gce.ensureExternalLoadBalancer(vals.ClusterName, vals.ClusterID, svc, nil, nodes)
+	assert.Error(t, err)
+
+	// We expect 1 error, and deny firewall IPv4
+	lm.exportNetLBMetrics()
+	verifyL4NetLBMetric(t, 1, StatusError, DenyFirewallStatusNone)
+
+	// Clear mock
+	mockGCE.MockFirewalls.GetHook = nil
+
+	// Case 5: Delete
+	err = gce.ensureExternalLoadBalancerDeleted(vals.ClusterName, vals.ClusterID, svc)
+	assert.NoError(t, err)
+	
+	// Now verify success count is 0 (since we deleted the success service)
+	lm.exportNetLBMetrics()
+	verifyL4NetLBMetric(t, 0, StatusError, DenyFirewallStatusNone)
+}

providers/gce/gce_loadbalancer_metrics.go

@@ -43,19 +43,29 @@ var (
 		},
 		[]string{label},
 	)
+	l4NetLBCount = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Name: "number_of_l4_netlbs",
+			Help: "Metric containing the number of NetLBs that can be filtered by feature labels and status",
+		},
+		[]string{"status", "deny_firewall"},
+	)
 )
 
 // init registers L4 internal loadbalancer usage metrics.
 func init() {
 	klog.V(3).Infof("Registering Service Controller loadbalancer usage metrics %v", l4ILBCount)
 	legacyregistry.MustRegister(l4ILBCount)
+	klog.V(3).Infof("Registering Service Controller loadbalancer usage metrics %v", l4NetLBCount)
+	legacyregistry.MustRegister(l4NetLBCount)
 }
 
 // LoadBalancerMetrics is a cache that contains loadbalancer service resource
 // states for computing usage metrics.
 type LoadBalancerMetrics struct {
 	// l4ILBServiceMap is a map of service key and L4 ILB service state.
 	l4ILBServiceMap map[string]L4ILBServiceState
+	l4NetLBMap      map[string]L4NetLBServiceState
 
 	sync.Mutex
 }
@@ -97,13 +107,18 @@ type loadbalancerMetricsCollector interface {
 	SetL4ILBService(svcKey string, state L4ILBServiceState)
 	// DeleteL4ILBService removes the given L4 ILB service key.
 	DeleteL4ILBService(svcKey string)
+	// SetL4NetLBService adds/updates L4 NetLB service state for given service key.
+	SetL4NetLBService(svcKey string, state L4NetLBServiceState)
+	// DeleteL4NetLBService removes the given L4 NetLB service key.
+	DeleteL4NetLBService(svcKey string)
 }
 
 // newLoadBalancerMetrics initializes LoadBalancerMetrics and starts a goroutine
 // to compute and export metrics periodically.
 func newLoadBalancerMetrics() loadbalancerMetricsCollector {
 	return &LoadBalancerMetrics{
 		l4ILBServiceMap: make(map[string]L4ILBServiceState),
+		l4NetLBMap:      make(map[string]L4NetLBServiceState),
 	}
 }
 
@@ -140,6 +155,11 @@ func (lm *LoadBalancerMetrics) DeleteL4ILBService(svcKey string) {
 
 // export computes and exports loadbalancer usage metrics.
 func (lm *LoadBalancerMetrics) export() {
+	lm.exportILBMetrics()
+	lm.exportNetLBMetrics()
+}
+
+func (lm *LoadBalancerMetrics) exportILBMetrics() {
 	ilbCount := lm.computeL4ILBMetrics()
 	klog.V(5).Infof("Exporting L4 ILB usage metrics: %#v", ilbCount)
 	for feature, count := range ilbCount {
@@ -180,3 +200,60 @@ func (lm *LoadBalancerMetrics) computeL4ILBMetrics() map[feature]int {
 	klog.V(4).Info("L4 ILB usage metrics computed.")
 	return counts
 }
+
+// L4ServiceStatus denotes the status of the service
+type L4ServiceStatus string
+
+// L4ServiceStatus denotes the status of the service
+const (
+	StatusSuccess         = L4ServiceStatus("Success")
+	StatusUserError       = L4ServiceStatus("UserError")
+	StatusError           = L4ServiceStatus("Error")
+	StatusPersistentError = L4ServiceStatus("PersistentError")
+)
+
+// DenyFirewallStatus represents IP stack used when the deny firewalls are provisioned.
+type DenyFirewallStatus string
+
+// DenyFirewallStatus represents IP stack used when the deny firewalls are provisioned.
+const (
+	DenyFirewallStatusUnknown  = DenyFirewallStatus("UNKNOWN")  // Shouldn't happen, but if it does something is wrong.
+	DenyFirewallStatusNone     = DenyFirewallStatus("")         // Case when no firewalls have been provisioned yet or when the feature has not been enabled explicitly
+	DenyFirewallStatusDisabled = DenyFirewallStatus("DISABLED") // Case to mark when the feature has been enabled then explicitly disabled - for example when the feature is rolled back
+	DenyFirewallStatusIPv4     = DenyFirewallStatus("IPv4")
+)
+
+type L4NetLBServiceState struct {
+	Status       L4ServiceStatus
+	DenyFirewall DenyFirewallStatus
+}
+
+// SetL4NetLBService patches information about L4 NetLB
+func (lm *LoadBalancerMetrics) SetL4NetLBService(svcKey string, state L4NetLBServiceState) {
+	lm.Lock()
+	defer lm.Unlock()
+
+	lm.l4NetLBMap[svcKey] = state
+}
+
+// DeleteL4NetLBService removes the given L4 NetLB service key.
+func (lm *LoadBalancerMetrics) DeleteL4NetLBService(svcKey string) {
+	lm.Lock()
+	defer lm.Unlock()
+
+	delete(lm.l4NetLBMap, svcKey)
+}
+
+// exportNetLBMetrics computes and exports loadbalancer usage metrics.
+func (lm *LoadBalancerMetrics) exportNetLBMetrics() {
+	lm.Lock()
+	defer lm.Unlock()
+
+	klog.Info("Exporting L4 NetLB usage metrics for services", "serviceCount", len(lm.l4NetLBMap))
+
+	l4NetLBCount.Reset()
+	for _, svcState := range lm.l4NetLBMap {
+		l4NetLBCount.WithLabelValues(string(svcState.Status), string(svcState.DenyFirewall)).Inc()
+	}
+	klog.Info("L4 NetLB usage metrics exported")
+}

providers/gce/gce_loadbalancer_metrics_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"k8s.io/component-base/metrics/testutil"
 )
 
 func TestComputeL4ILBMetrics(t *testing.T) {
@@ -168,3 +169,63 @@ func newL4ILBServiceState(globalAccess, customSubnet, inSuccess bool) L4ILBServi
 		InSuccess:           inSuccess,
 	}
 }
+
+func TestL4NetLBMetrics(t *testing.T) {
+	metrics := newLoadBalancerMetrics()
+	// Cast to *LoadBalancerMetrics to access methods
+	lbMetrics, ok := metrics.(*LoadBalancerMetrics)
+	if !ok {
+		t.Fatalf("Failed to cast loadbalancerMetricsCollector to *LoadBalancerMetrics")
+	}
+
+	lbMetrics.SetL4NetLBService("svc-success-ipv4", L4NetLBServiceState{
+		Status:       StatusSuccess,
+		DenyFirewall: DenyFirewallStatusIPv4,
+	})
+	lbMetrics.SetL4NetLBService("svc-success-ipv4-2", L4NetLBServiceState{
+		Status:       StatusSuccess,
+		DenyFirewall: DenyFirewallStatusIPv4,
+	})
+	lbMetrics.SetL4NetLBService("svc-success-disabled", L4NetLBServiceState{
+		Status:       StatusSuccess,
+		DenyFirewall: DenyFirewallStatusDisabled,
+	})
+	lbMetrics.SetL4NetLBService("svc-error-none", L4NetLBServiceState{
+		Status:       StatusError,
+		DenyFirewall: DenyFirewallStatusNone,
+	})
+	lbMetrics.SetL4NetLBService("svc-user-error-none", L4NetLBServiceState{
+		Status:       StatusUserError,
+		DenyFirewall: DenyFirewallStatusNone,
+	})
+	lbMetrics.SetL4NetLBService("svc-persistent-error-none", L4NetLBServiceState{
+		Status:       StatusPersistentError,
+		DenyFirewall: DenyFirewallStatusNone,
+	})
+
+	// Add keys to be checked for deletion
+	lbMetrics.SetL4NetLBService("svc-to-delete", L4NetLBServiceState{
+		Status:       StatusSuccess,
+		DenyFirewall: DenyFirewallStatusNone,
+	})
+	lbMetrics.DeleteL4NetLBService("svc-to-delete")
+
+	lbMetrics.exportNetLBMetrics()
+
+	verifyL4NetLBMetric(t, 2, StatusSuccess, DenyFirewallStatusIPv4)
+	verifyL4NetLBMetric(t, 1, StatusSuccess, DenyFirewallStatusDisabled)
+	verifyL4NetLBMetric(t, 1, StatusError, DenyFirewallStatusNone)
+	verifyL4NetLBMetric(t, 1, StatusUserError, DenyFirewallStatusNone)
+	verifyL4NetLBMetric(t, 1, StatusPersistentError, DenyFirewallStatusNone)
+}
+
+func verifyL4NetLBMetric(t *testing.T, expectedCount int, status L4ServiceStatus, denyFirewall DenyFirewallStatus) {
+	t.Helper()
+	val, err := testutil.GetGaugeMetricValue(l4NetLBCount.WithLabelValues(string(status), string(denyFirewall)))
+	if err != nil {
+		t.Errorf("Failed to get metric value: %v", err)
+	}
+	if int(val) != expectedCount {
+		t.Errorf("Expected count %d but got %d for status %s, denyFirewall %s", expectedCount, int(val), status, denyFirewall)
+	}
+}