feat: ca: do not backoff scale up on specified errors

cluster-autoscaler/FAQ.md

@@ -1104,6 +1104,7 @@ The following startup parameters are supported for cluster autoscaler:
 | `scale-down-unready-enabled` | Should CA scale down unready nodes of the cluster | true |
 | `scale-down-unready-time` | How long an unready node should be unneeded before it is eligible for scale down | 20m0s |
 | `scale-down-utilization-threshold` | The maximum value between the sum of cpu requests and sum of memory requests of all pods running on the node divided by node's corresponding allocatable resource, below which a node can be considered for scale down | 0.5 |
+| `scale-up-ignore-backoff-errors` | Cloud provider error message parts that should not trigger a scaleup backoff |  |
 | `scale-up-from-zero` | Should CA scale up when there are 0 ready nodes. | true |
 | `scan-interval` | How often cluster is reevaluated for scale up or down | 10s |
 | `scheduler-config-file` | scheduler-config allows changing configuration of in-tree scheduler plugins acting on PreFilter and Filter extension points |  |

cluster-autoscaler/config/autoscaling_options.go

@@ -142,6 +142,8 @@ type AutoscalingOptions struct {
 	OkTotalUnreadyCount int
 	// ScaleUpFromZero defines if CA should scale up when there 0 ready nodes.
 	ScaleUpFromZero bool
+	// ScaleUpIgnoreBackoffErrors defines cloud provider error message parts that should not trigger a nodegroup scaleup backoff
+	ScaleUpIgnoreBackoffErrors []string
 	// ParallelScaleUp defines whether CA can scale up node groups in parallel.
 	ParallelScaleUp bool
 	// CloudConfig is the path to the cloud provider configuration file. Empty string for no configuration file.

cluster-autoscaler/config/flags/flags.go

@@ -119,13 +119,16 @@ var (
 	maxBulkSoftTaintTime       = flag.Duration("max-bulk-soft-taint-time", 3*time.Second, "Maximum duration of tainting/untainting nodes as PreferNoSchedule at the same time.")
 	maxGracefulTerminationFlag = flag.Int("max-graceful-termination-sec", 10*60, "Maximum number of seconds CA waits for pod termination when trying to scale down a node. "+
 		"This flag is mutually exclusion with drain-priority-config flag which allows more configuration options.")
-	maxTotalUnreadyPercentage = flag.Float64("max-total-unready-percentage", 45, "Maximum percentage of unready nodes in the cluster.  After this is exceeded, CA halts operations")
-	okTotalUnreadyCount       = flag.Int("ok-total-unready-count", 3, "Number of allowed unready nodes, irrespective of max-total-unready-percentage")
-	scaleUpFromZero           = flag.Bool("scale-up-from-zero", true, "Should CA scale up when there are 0 ready nodes.")
-	parallelScaleUp           = flag.Bool("parallel-scale-up", false, "Whether to allow parallel node groups scale up. Experimental: may not work on some cloud providers, enable at your own risk.")
-	maxNodeProvisionTime      = flag.Duration("max-node-provision-time", 15*time.Minute, "The default maximum time CA waits for node to be provisioned - the value can be overridden per node group")
-	maxPodEvictionTime        = flag.Duration("max-pod-eviction-time", 2*time.Minute, "Maximum time CA tries to evict a pod before giving up")
-	nodeGroupsFlag            = multiStringFlag(
+	maxTotalUnreadyPercentage  = flag.Float64("max-total-unready-percentage", 45, "Maximum percentage of unready nodes in the cluster.  After this is exceeded, CA halts operations")
+	okTotalUnreadyCount        = flag.Int("ok-total-unready-count", 3, "Number of allowed unready nodes, irrespective of max-total-unready-percentage")
+	scaleUpFromZero            = flag.Bool("scale-up-from-zero", true, "Should CA scale up when there are 0 ready nodes.")
+	scaleUpIgnoreBackoffErrors = multiStringFlag(
+		"scale-up-ignore-backoff-errors",
+		"Cloud provider error message parts that should not trigger a scaleup backoff. Can be used multiple times.")
+	parallelScaleUp      = flag.Bool("parallel-scale-up", false, "Whether to allow parallel node groups scale up. Experimental: may not work on some cloud providers, enable at your own risk.")
+	maxNodeProvisionTime = flag.Duration("max-node-provision-time", 15*time.Minute, "The default maximum time CA waits for node to be provisioned - the value can be overridden per node group")
+	maxPodEvictionTime   = flag.Duration("max-pod-eviction-time", 2*time.Minute, "Maximum time CA tries to evict a pod before giving up")
+	nodeGroupsFlag       = multiStringFlag(
 		"nodes",
 		"sets min,max size and other configuration data for a node group in a format accepted by cloud provider. Can be used multiple times. Format: <min>:<max>:<other...>")
 	nodeGroupAutoDiscoveryFlag = multiStringFlag(
@@ -296,6 +299,7 @@ func createAutoscalingOptions() config.AutoscalingOptions {
 		MaxTotalUnreadyPercentage:        *maxTotalUnreadyPercentage,
 		OkTotalUnreadyCount:              *okTotalUnreadyCount,
 		ScaleUpFromZero:                  *scaleUpFromZero,
+		ScaleUpIgnoreBackoffErrors:       *scaleUpIgnoreBackoffErrors,
 		ParallelScaleUp:                  *parallelScaleUp,
 		EstimatorName:                    *estimatorFlag,
 		ExpanderNames:                    *expanderFlag,

cluster-autoscaler/core/scaleup/orchestrator/executor.go

@@ -18,6 +18,7 @@ package orchestrator
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -166,10 +167,22 @@ func (e *scaleUpExecutor) executeScaleUp(
 		"Scale-up: setting group %s size to %d instead of %d (max: %d)", info.Group.Id(), info.NewSize, info.CurrentSize, info.MaxSize)
 	increase := info.NewSize - info.CurrentSize
 	if err := e.increaseSize(info.Group, increase, atomic); err != nil {
-		e.autoscalingContext.LogRecorder.Eventf(apiv1.EventTypeWarning, "FailedToScaleUpGroup", "Scale-up failed for group %s: %v", info.Group.Id(), err)
 		aerr := errors.ToAutoscalerError(errors.CloudProviderError, err).AddPrefix("failed to increase node group size: ")
-		e.scaleStateNotifier.RegisterFailedScaleUp(info.Group, string(aerr.Type()), aerr.Error(), gpuResourceName, gpuType, now)
-		return aerr
+
+		backoff := true
+		for _, part := range e.autoscalingContext.AutoscalingOptions.ScaleUpIgnoreBackoffErrors {
+			if strings.Contains(err.Error(), part) {
+				e.autoscalingContext.LogRecorder.Eventf(apiv1.EventTypeWarning, "ScaledUpGroup", "Scale-up: retriable error %s", aerr.Error())
+				backoff = false
+				break
+			}
+		}
+
+		if backoff {
+			e.autoscalingContext.LogRecorder.Eventf(apiv1.EventTypeWarning, "FailedToScaleUpGroup", "Scale-up failed for group %s: %v", info.Group.Id(), err)
+			e.scaleStateNotifier.RegisterFailedScaleUp(info.Group, string(aerr.Type()), aerr.Error(), gpuResourceName, gpuType, now)
+			return aerr
+		}
 	}
 	if increase < 0 {
 		return errors.NewAutoscalerError(errors.InternalError, fmt.Sprintf("increase in number of nodes cannot be negative, got: %v", increase))