Merge pull request #315 from uselagoon/build-pod-cluster-autoscaler-eviction

feat: default label to prevent autoscaler evictions on pods

Author: shreddedbacon

cmd/main.go

@@ -199,6 +199,8 @@ func main() {
 	var dockerHostNamespace string
 	var dockerHostReuseType string
 
+	var clusterAutoscalerEvict bool
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true,
@@ -430,6 +432,10 @@ func main() {
 		`The resource type (namespace, project, organization) to use when assigning a docker-host to a build to preference an already used dockerhost
 		eg. If project is defined, all builds from a project will prefer to build on the same docker-host where possible`)
 
+	// Flag to control the setting for the label cluster-autoscaler.kubernetes.io/safe-to-evict on build pods, defaults to false to avoid evicting pods
+	flag.BoolVar(&clusterAutoscalerEvict, "enable-cluster-autoscaler-eviction", false,
+		"Flag to enable cluster autoscaler eviction on build pods, defaults to false to avoid evicting running builds")
+
 	flag.Parse()
 
 	// get overrides from environment variables
@@ -990,6 +996,7 @@ func main() {
 		DockerHost:              dockerhosts,
 		QueueCache:              buildsQueueCache,
 		BuildCache:              buildsCache,
+		ClusterAutoscalerEvict:  clusterAutoscalerEvict,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "LagoonBuild")
 		os.Exit(1)
@@ -1018,11 +1025,12 @@ func main() {
 			HTTPSProxy: httpsProxy,
 			NoProxy:    noProxy,
 		},
-		LFFTaskQoSEnabled: lffTaskQoSEnabled,
-		TaskQoS:           taskQoSConfigv1beta2,
-		ImagePullPolicy:   tipp,
-		QueueCache:        tasksQueueCache,
-		TasksCache:        tasksCache,
+		LFFTaskQoSEnabled:      lffTaskQoSEnabled,
+		TaskQoS:                taskQoSConfigv1beta2,
+		ImagePullPolicy:        tipp,
+		QueueCache:             tasksQueueCache,
+		TasksCache:             tasksCache,
+		ClusterAutoscalerEvict: clusterAutoscalerEvict,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "LagoonTask")
 		os.Exit(1)

internal/controllers/v1beta2/build_controller.go

@@ -84,6 +84,7 @@ type LagoonBuildReconciler struct {
 	DockerHost                       *dockerhost.DockerHost
 	QueueCache                       *lru.Cache[string, string]
 	BuildCache                       *lru.Cache[string, string]
+	ClusterAutoscalerEvict           bool
 }
 
 // BackupConfig holds all the backup configuration settings

internal/controllers/v1beta2/build_helpers.go

@@ -799,6 +799,11 @@ func (r *LagoonBuildReconciler) processBuild(ctx context.Context, opLog logr.Log
 		newPod.Labels["organization.lagoon.sh/name"] = lagoonBuild.Spec.Project.Organization.Name
 	}
 
+	if !r.ClusterAutoscalerEvict {
+		// try to prevent build pods from being evicted by cluster autoscaler
+		newPod.Labels["cluster-autoscaler.kubernetes.io/safe-to-evict"] = "false"
+	}
+
 	// set the pod security context, if defined to a non-default value
 	if r.BuildPodRunAsUser != 0 || r.BuildPodRunAsGroup != 0 ||
 		r.BuildPodFSGroup != 0 {

internal/controllers/v1beta2/task_controller.go

@@ -58,6 +58,7 @@ type LagoonTaskReconciler struct {
 	ImagePullPolicy        corev1.PullPolicy
 	QueueCache             *lru.Cache[string, string]
 	TasksCache             *lru.Cache[string, string]
+	ClusterAutoscalerEvict bool
 }
 
 var (
@@ -323,6 +324,11 @@ func (r *LagoonTaskReconciler) getTaskPodDeployment(ctx context.Context, lagoonT
 					taskPod.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID)
 					taskPod.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name
 				}
+
+				if !r.ClusterAutoscalerEvict {
+					// try to prevent build pods from being evicted by cluster autoscaler
+					taskPod.Labels["cluster-autoscaler.kubernetes.io/safe-to-evict"] = "false"
+				}
 				return taskPod, nil
 			}
 		}
@@ -662,6 +668,10 @@ func (r *LagoonTaskReconciler) createAdvancedTask(ctx context.Context, lagoonTas
 			newPod.Labels["organization.lagoon.sh/id"] = fmt.Sprintf("%d", *lagoonTask.Spec.Project.Organization.ID)
 			newPod.Labels["organization.lagoon.sh/name"] = lagoonTask.Spec.Project.Organization.Name
 		}
+		if !r.ClusterAutoscalerEvict {
+			// try to prevent build pods from being evicted by cluster autoscaler
+			newPod.Labels["cluster-autoscaler.kubernetes.io/safe-to-evict"] = "false"
+		}
 		if lagoonTask.Spec.AdvancedTask.DeployerToken {
 			// start this with the serviceaccount so that it gets the token mounted into it
 			newPod.Spec.ServiceAccountName = "lagoon-deployer"