Fix LoadBalancer hang by cleaning up orphaned NIC on quota failures

When VM provisioning fails due to Azure quota exhaustion, CAPZ leaves
the NIC orphaned (without VirtualMachine.ID). This causes cloud-provider-azure's
LoadBalancer reconciler to fail cluster-wide when querying NICs.

This change implements defensive cleanup of orphaned NICs after a 1-minute
grace period when quota errors are detected. The Machine is then marked as
permanently Failed to avoid recreation loops.

Changes:
- Add quota error detection using azcore.ResponseError with multiple error codes
- Track quota failures via VMRunningCondition (uses existing CAPI conditions API)
- Delete orphaned NIC if quota error persists after 1-minute grace period
- Mark Machine as Failed with FailureReason="QuotaExhausted"
- Add unit tests for condition-based grace period and quota detection

Fixes cluster-wide LoadBalancer failures during quota exhaustion while
avoiding recreation loops by marking Machines as permanently Failed.

AI assisted code

Signed-off-by: Chetan Giradkar <cgiradka@redhat.com>

Author: cgiradkar

controllers/azuremachine_controller.go

@@ -19,7 +19,11 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"strings"
+	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -41,6 +45,8 @@ import (
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/scope"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/networkinterfaces"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus"
 	"sigs.k8s.io/cluster-api-provider-azure/pkg/coalescing"
 	"sigs.k8s.io/cluster-api-provider-azure/util/reconciler"
 	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
@@ -235,6 +241,119 @@ func (amr *AzureMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	return amr.reconcileNormal(ctx, machineScope, clusterScope)
 }
 
+// quotaExceededErrorCodes contains Azure error codes that indicate quota exhaustion.
+// Reference: https://learn.microsoft.com/en-us/azure/azure-resource-manager/troubleshooting/error-resource-quota
+var quotaExceededErrorCodes = []string{
+	"QuotaExceeded",
+	"OperationNotAllowed",
+	"ResourceQuotaExceeded",
+	"DeploymentQuotaExceeded",
+}
+
+// isQuotaExceededError checks if error is due to Azure quota exhaustion.
+func isQuotaExceededError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	var respErr *azcore.ResponseError
+	if errors.As(err, &respErr) {
+		for _, code := range quotaExceededErrorCodes {
+			if respErr.ErrorCode == code {
+				return true
+			}
+		}
+
+		if respErr.RawResponse != nil {
+			errMsg := strings.ToLower(respErr.Error())
+			return strings.Contains(errMsg, "quota") && strings.Contains(errMsg, "exceeded")
+		}
+	}
+
+	return false
+}
+
+// shouldCleanupNIC checks if grace period has elapsed since VMRunningCondition
+// was set to False with QuotaExhausted reason.
+func shouldCleanupNIC(machineScope *scope.MachineScope) bool {
+	condition := v1beta1conditions.Get(machineScope.AzureMachine, infrav1.VMRunningCondition)
+	if condition == nil {
+		return false
+	}
+
+	if condition.Status != corev1.ConditionFalse || condition.Reason != "QuotaExhausted" {
+		return false
+	}
+
+	gracePeriod := 1 * time.Minute
+	return time.Since(condition.LastTransitionTime.Time) >= gracePeriod
+}
+
+// markQuotaFailureCondition sets VMRunningCondition to False with QuotaExhausted reason.
+func markQuotaFailureCondition(machineScope *scope.MachineScope, err error) {
+	v1beta1conditions.MarkFalse(
+		machineScope.AzureMachine,
+		infrav1.VMRunningCondition,
+		"QuotaExhausted",
+		clusterv1beta1.ConditionSeverityWarning,
+		"VM creation failed due to quota exhaustion: %s", err.Error(),
+	)
+}
+
+// cleanupOrphanedNIC deletes the NIC when it's orphaned due to VM creation failure.
+func (amr *AzureMachineReconciler) cleanupOrphanedNIC(ctx context.Context, machineScope *scope.MachineScope) error {
+	ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureMachineReconciler.cleanupOrphanedNIC")
+	defer done()
+
+	skuCache, err := resourceskus.GetCache(machineScope, machineScope.Location())
+	if err != nil {
+		return errors.Wrap(err, "failed to get resource SKU cache")
+	}
+
+	nicSvc, err := networkinterfaces.New(machineScope, skuCache)
+	if err != nil {
+		return errors.Wrap(err, "failed to create networkinterfaces service")
+	}
+
+	if err := nicSvc.Delete(ctx); err != nil {
+		var respErr *azcore.ResponseError
+		if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound {
+			log.Info("NIC already deleted")
+			return nil
+		}
+		return errors.Wrap(err, "failed to delete orphaned NIC")
+	}
+
+	log.Info("Orphaned NIC deleted successfully")
+	return nil
+}
+
+/*
+// getRetryCount retrieves retry count from Machine annotations.
+// Used by exponential backoff retry mechanism (Option 1).
+func getRetryCount(machineScope *scope.MachineScope) int {
+	countStr, exists := machineScope.AzureMachine.Annotations["azure.infrastructure.cluster.x-k8s.io/quota-retry-count"]
+	if !exists {
+		return 0
+	}
+
+	count, err := strconv.Atoi(countStr)
+	if err != nil {
+		return 0
+	}
+	return count
+}
+
+// setRetryCount stores retry count in Machine annotations.
+// Used by exponential backoff retry mechanism (Option 1).
+func setRetryCount(machineScope *scope.MachineScope, count int) {
+	if machineScope.AzureMachine.Annotations == nil {
+		machineScope.AzureMachine.Annotations = make(map[string]string)
+	}
+	machineScope.AzureMachine.Annotations["azure.infrastructure.cluster.x-k8s.io/quota-retry-count"] = strconv.Itoa(count)
+}.
+*/
+
 func (amr *AzureMachineReconciler) reconcileNormal(ctx context.Context, machineScope *scope.MachineScope, clusterScope *scope.ClusterScope) (reconcile.Result, error) {
 	ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.AzureMachineReconciler.reconcileNormal")
 	defer done()
@@ -312,6 +431,48 @@ func (amr *AzureMachineReconciler) reconcileNormal(ctx context.Context, machineS
 			return reconcile.Result{}, errors.Wrap(err, "failed to reconcile AzureMachine")
 		}
 
+		// Check for quota errors and cleanup orphaned NIC if grace period expired
+		if isQuotaExceededError(err) {
+			if shouldCleanupNIC(machineScope) {
+				log.Info("Grace period expired, cleaning up orphaned NIC", "error", err.Error())
+				if nicErr := amr.cleanupOrphanedNIC(ctx, machineScope); nicErr != nil {
+					log.Error(nicErr, "Failed to delete orphaned NIC during quota failure cleanup")
+					return ctrl.Result{}, nicErr
+				}
+
+				// Mark Machine as permanently Failed
+				machineScope.SetFailureReason("QuotaExhausted")
+				machineScope.SetFailureMessage(fmt.Errorf("VM creation failed due to quota exhaustion"))
+				machineScope.SetNotReady()
+
+				// Option 2: Fail permanently (ACTIVE)
+				log.Info("Machine marked as Failed due to persistent quota exhaustion")
+				return ctrl.Result{}, nil
+
+				// Option 1: Retry with exponential backoff (COMMENTED OUT)
+				// Uncomment this block and comment out the above return to enable retries
+				//nolint:gocritic // Intentionally commented code for alternative retry strategy
+				/*
+					// Calculate retry count from condition observation
+					retryCount := getRetryCount(machineScope)
+					if retryCount >= 3 {
+						log.Info("Max retries exceeded, failing permanently")
+						return ctrl.Result{}, nil
+					}
+
+					// Exponential backoff: 15min, 30min, 60min
+					backoffDuration := time.Duration(math.Pow(2, float64(retryCount))) * 15 * time.Minute
+					log.Info("Will retry VM creation", "attempt", retryCount+1, "backoff", backoffDuration)
+					setRetryCount(machineScope, retryCount+1)
+					return ctrl.Result{RequeueAfter: backoffDuration}, nil
+				*/
+			}
+
+			markQuotaFailureCondition(machineScope, err)
+			log.Info("Quota failure detected, will cleanup NIC on next reconcile if still failing",
+				"gracePeriod", "1m", "error", err.Error())
+		}
+
 		// Handle transient and terminal errors
 		if errors.As(err, &reconcileError) {
 			if reconcileError.IsTerminal() {

controllers/azuremachine_controller_test.go

@@ -18,9 +18,11 @@ package controllers
 
 import (
 	"context"
+	"net/http"
 	"testing"
 	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -31,6 +33,7 @@ import (
 	"k8s.io/utils/ptr"
 	clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
+	v1beta1conditions "sigs.k8s.io/cluster-api/util/deprecated/v1beta1/conditions"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -831,3 +834,168 @@ func conditionsMatch(i, j clusterv1beta1.Condition) bool {
 		i.Reason == j.Reason &&
 		i.Severity == j.Severity
 }
+
+func TestIsQuotaExceededError(t *testing.T) {
+	g := NewWithT(t)
+
+	tests := []struct {
+		name     string
+		err      error
+		expected bool
+	}{
+		{
+			name:     "nil error",
+			err:      nil,
+			expected: false,
+		},
+		{
+			name:     "non-azcore error",
+			err:      errors.New("some random error"),
+			expected: false,
+		},
+		{
+			name:     "azcore error with QuotaExceeded code",
+			err:      &azcore.ResponseError{ErrorCode: "QuotaExceeded"},
+			expected: true,
+		},
+		{
+			name:     "azcore error with OperationNotAllowed code",
+			err:      &azcore.ResponseError{ErrorCode: "OperationNotAllowed"},
+			expected: true,
+		},
+		{
+			name:     "azcore error with ResourceQuotaExceeded code",
+			err:      &azcore.ResponseError{ErrorCode: "ResourceQuotaExceeded"},
+			expected: true,
+		},
+		{
+			name:     "azcore error with DeploymentQuotaExceeded code",
+			err:      &azcore.ResponseError{ErrorCode: "DeploymentQuotaExceeded"},
+			expected: true,
+		},
+		{
+			name:     "azcore error with different code",
+			err:      &azcore.ResponseError{ErrorCode: "NetworkTimeout"},
+			expected: false,
+		},
+		{
+			name: "azcore error with quota in message",
+			err: &azcore.ResponseError{
+				RawResponse: &http.Response{StatusCode: http.StatusBadRequest},
+			},
+			expected: false, // No message set, so won't match
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := isQuotaExceededError(tt.err)
+			g.Expect(result).To(Equal(tt.expected))
+		})
+	}
+}
+
+func TestShouldCleanupNIC(t *testing.T) {
+	g := NewWithT(t)
+
+	tests := []struct {
+		name          string
+		condition     *clusterv1beta1.Condition
+		shouldCleanup bool
+	}{
+		{
+			name:          "No condition",
+			condition:     nil,
+			shouldCleanup: false,
+		},
+		{
+			name: "Condition not QuotaExhausted",
+			condition: &clusterv1beta1.Condition{
+				Type:               infrav1.VMRunningCondition,
+				Status:             corev1.ConditionFalse,
+				Reason:             "OtherReason",
+				LastTransitionTime: metav1.Time{Time: time.Now().Add(-2 * time.Minute)},
+			},
+			shouldCleanup: false,
+		},
+		{
+			name: "Grace period not elapsed",
+			condition: &clusterv1beta1.Condition{
+				Type:               infrav1.VMRunningCondition,
+				Status:             corev1.ConditionFalse,
+				Reason:             "QuotaExhausted",
+				LastTransitionTime: metav1.Time{Time: time.Now().Add(-30 * time.Second)},
+			},
+			shouldCleanup: false,
+		},
+		{
+			name: "Grace period elapsed",
+			condition: &clusterv1beta1.Condition{
+				Type:               infrav1.VMRunningCondition,
+				Status:             corev1.ConditionFalse,
+				Reason:             "QuotaExhausted",
+				LastTransitionTime: metav1.Time{Time: time.Now().Add(-2 * time.Minute)},
+			},
+			shouldCleanup: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			azureMachine := &infrav1.AzureMachine{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-machine",
+					Namespace: "default",
+				},
+			}
+			if tt.condition != nil {
+				v1beta1conditions.Set(azureMachine, tt.condition)
+			}
+			machineScope := &scope.MachineScope{
+				AzureMachine: azureMachine,
+			}
+
+			result := shouldCleanupNIC(machineScope)
+			g.Expect(result).To(Equal(tt.shouldCleanup))
+		})
+	}
+}
+
+func TestMarkQuotaFailureCondition(t *testing.T) {
+	g := NewWithT(t)
+
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{
+			name: "marks condition with quota error",
+			err:  errors.New("quota exceeded for resource"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			azureMachine := &infrav1.AzureMachine{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-machine",
+					Namespace: "default",
+				},
+			}
+
+			machineScope := &scope.MachineScope{
+				AzureMachine: azureMachine,
+			}
+
+			markQuotaFailureCondition(machineScope, tt.err)
+
+			condition := v1beta1conditions.Get(machineScope.AzureMachine, infrav1.VMRunningCondition)
+			g.Expect(condition).NotTo(BeNil())
+			g.Expect(condition.Status).To(Equal(corev1.ConditionFalse))
+			g.Expect(condition.Reason).To(Equal("QuotaExhausted"))
+			g.Expect(condition.Severity).To(Equal(clusterv1beta1.ConditionSeverityWarning))
+			g.Expect(condition.Message).To(ContainSubstring("VM creation failed due to quota exhaustion"))
+			g.Expect(condition.Message).To(ContainSubstring(tt.err.Error()))
+		})
+	}
+}