Merge pull request #188 from alexander-demicev/maxsurge

Add maxsurge option for control plane upgrade

Author: richardcase

controlplane/api/v1alpha1/rke2controlplane_types.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
@@ -71,6 +72,11 @@ type RKE2ControlPlaneSpec struct {
 	// the registration type is "address". Its for scenarios where a load-balancer or VIP is used.
 	// +optional
 	RegistrationAddress string `json:"registrationAddress,omitempty"`
+
+	// The RolloutStrategy to use to replace control plane machines with new ones.
+	// +optional
+	// +kubebuilder:default={type: "RollingUpdate", rollingUpdate: {maxSurge: 1}}
+	RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"`
 }
 
 // RKE2ServerConfig specifies configuration for the agent nodes.
@@ -343,6 +349,40 @@ const (
 	MetricsServer DisabledPluginComponent = "rke2-metrics-server"
 )
 
+// RolloutStrategy describes how to replace existing machines
+// with new ones.
+type RolloutStrategy struct {
+	// Type of rollout. Currently the only supported strategy is "RollingUpdate".
+	// Default is RollingUpdate.
+	// +optional
+	Type RolloutStrategyType `json:"type,omitempty"`
+
+	// Rolling update config params. Present only if RolloutStrategyType = RollingUpdate.
+	// +optional
+	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+}
+
+// RollingUpdate is used to control the desired behavior of rolling update.
+type RollingUpdate struct {
+	// The maximum number of control planes that can be scheduled above or under the
+	// desired number of control planes.
+	// Value can be an absolute number 1 or 0.
+	// Defaults to 1.
+	// Example: when this is set to 1, the control plane can be scaled
+	// up immediately when the rolling update starts.
+	// +optional
+	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
+}
+
+// RolloutStrategyType defines the rollout strategies for a RKE2ControlPlane.
+type RolloutStrategyType string
+
+const (
+	// RollingUpdateStrategyType replaces the old control planes by new one using rolling update
+	// i.e. gradually scale up or down the old control planes and scale up or down the new one.
+	RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate"
+)
+
 func init() { //nolint:gochecknoinits
 	SchemeBuilder.Register(&RKE2ControlPlane{}, &RKE2ControlPlaneList{})
 }

controlplane/api/v1alpha1/zz_generated.deepcopy.go

@@ -605,6 +605,34 @@ spec:
                 description: Replicas is the number of replicas for the Control Plane.
                 format: int32
                 type: integer
+              rolloutStrategy:
+                default:
+                  rollingUpdate:
+                    maxSurge: 1
+                  type: RollingUpdate
+                description: The RolloutStrategy to use to replace control plane machines
+                  with new ones.
+                properties:
+                  rollingUpdate:
+                    description: Rolling update config params. Present only if RolloutStrategyType
+                      = RollingUpdate.
+                    properties:
+                      maxSurge:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        description: 'The maximum number of control planes that can
+                          be scheduled above or under the desired number of control
+                          planes. Value can be an absolute number 1 or 0. Defaults
+                          to 1. Example: when this is set to 1, the control plane
+                          can be scaled up immediately when the rolling update starts.'
+                        x-kubernetes-int-or-string: true
+                    type: object
+                  type:
+                    description: Type of rollout. Currently the only supported strategy
+                      is "RollingUpdate". Default is RollingUpdate.
+                    type: string
+                type: object
               serverConfig:
                 description: ServerConfig specifies configuration for the agent nodes.
                 properties:

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_rke2controlplanes.yaml

@@ -217,6 +217,7 @@ func patchRKE2ControlPlane(ctx context.Context, patchHelper *patch.Helper, rcp *
 func (r *RKE2ControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	c, err := ctrl.NewControllerManagedBy(mgr).
 		For(&controlplanev1.RKE2ControlPlane{}).
+		Owns(&clusterv1.Machine{}).
 		Build(r)
 	if err != nil {
 		return errors.Wrap(err, "failed setting up with a controller manager")
@@ -742,14 +743,22 @@ func (r *RKE2ControlPlaneReconciler) upgradeControlPlane(
 		return ctrl.Result{}, err
 	}
 
-	status := workloadCluster.ClusterStatus()
+	switch rcp.Spec.RolloutStrategy.Type {
+	case controlplanev1.RollingUpdateStrategyType:
+		// RolloutStrategy is currently defaulted and validated to be RollingUpdate.
+		maxNodes := *rcp.Spec.Replicas + int32(rcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntValue())
+		if int32(controlPlane.Machines.Len()) < maxNodes {
+			// scaleUpControlPlane ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
+			return r.scaleUpControlPlane(ctx, cluster, rcp, controlPlane)
+		}
 
-	if status.Nodes <= *rcp.Spec.Replicas {
-		// scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
-		return r.scaleUpControlPlane(ctx, cluster, rcp, controlPlane)
-	}
+		return r.scaleDownControlPlane(ctx, cluster, rcp, controlPlane, machinesRequireUpgrade)
+	default:
+		err := fmt.Errorf("unknown rollout strategy type %q", rcp.Spec.RolloutStrategy.Type)
+		logger.Error(err, "RolloutStrategy type is not set to RollingUpdateStrategyType, unable to determine the strategy for rolling out machines")
 
-	return r.scaleDownControlPlane(ctx, cluster, rcp, controlPlane, machinesRequireUpgrade)
+		return ctrl.Result{}, nil
+	}
 }
 
 // ClusterToRKE2ControlPlane is a handler.ToRequestsFunc to be used to enqueue requests for reconciliation

controlplane/internal/controllers/rke2controlplane_controller.go

@@ -40,11 +40,11 @@ import (
 const (
 	KubernetesVersionManagement     = "KUBERNETES_VERSION_MANAGEMENT"
 	KubernetesVersion               = "KUBERNETES_VERSION"
-	KubernetesVersionUpgradeFrom    = "KUBERNETES_VERSION_UPGRADE_FROM"
 	KubernetesVersionUpgradeTo      = "KUBERNETES_VERSION_UPGRADE_TO"
 	CPMachineTemplateUpgradeTo      = "CONTROL_PLANE_MACHINE_TEMPLATE_UPGRADE_TO"
 	WorkersMachineTemplateUpgradeTo = "WORKERS_MACHINE_TEMPLATE_UPGRADE_TO"
 	IPFamily                        = "IP_FAMILY"
+	KindImageVersion                = "KIND_IMAGE_VERSION"
 )
 
 func Byf(format string, a ...interface{}) {

test/e2e/common.go

@@ -69,13 +69,13 @@ providers:
             new: "--leader-elect=false"
 
 variables:
-  KUBERNETES_VERSION_MANAGEMENT: "v1.24.4"
-  KUBERNETES_VERSION: "v1.24.4"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.28.0"
+  KUBERNETES_VERSION: "v1.28.1"
+  KIND_IMAGE_VERSION: "v1.28.0"
   NODE_DRAIN_TIMEOUT: "60s"
   CONFORMANCE_WORKER_MACHINE_COUNT: "2"
   CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
-  KUBERNETES_VERSION_UPGRADE_TO: "v1.24.2"
-  KUBERNETES_VERSION_UPGRADE_FROM: "v1.23.8"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.28.2"
   KUBERNETES_UPGRADE_OCI_IMAGE_ID: "${KUBERNETES_UPGRADE_OCI_IMAGE_ID}"
   IP_FAMILY: "IPv4"
   EXP_CLUSTER_RESOURCE_SET: "true"

test/e2e/config/e2e_conf.yaml

@@ -48,7 +48,8 @@ metadata:
   name:  "${CLUSTER_NAME}-control-plane"
 spec:
   template:
-    spec: {}
+    spec:
+      customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -80,7 +81,8 @@ metadata:
   name: ${CLUSTER_NAME}-md-0
 spec:
   template:
-    spec: {}
+    spec:
+      customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1alpha1
 kind: RKE2ConfigTemplate
@@ -92,4 +94,4 @@ spec:
       agentConfig:
         version: ${KUBERNETES_VERSION}+rke2r1
         nodeAnnotations:
-          test: "true"
+          test: "true"

test/e2e/data/infrastructure/cluster-template-docker.yaml

@@ -31,6 +31,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
 
 	bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1"
 	controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1"
@@ -104,6 +105,8 @@ func init() {
 func TestE2E(t *testing.T) {
 	RegisterFailHandler(Fail)
 
+	ctrl.SetLogger(klog.Background())
+
 	RunSpecs(t, "caprke2-e2e")
 }
 

test/e2e/e2e_suite_test.go

@@ -148,6 +148,38 @@ var _ = Describe("Workload cluster creation", func() {
 				ControlPlane: result.ControlPlane,
 			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
 
+			By("Upgrading control plane and worker machines")
+			ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
+				ClusterProxy: bootstrapClusterProxy,
+				ConfigCluster: clusterctl.ConfigClusterInput{
+					LogFolder:                clusterctlLogFolder,
+					ClusterctlConfigPath:     clusterctlConfigPath,
+					KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
+					InfrastructureProvider:   "docker",
+					Flavor:                   "docker",
+					Namespace:                namespace.Name,
+					ClusterName:              clusterName,
+					KubernetesVersion:        e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
+					ControlPlaneMachineCount: pointer.Int64Ptr(3),
+					WorkerMachineCount:       pointer.Int64Ptr(3),
+				},
+				WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
+				WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
+				WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
+			}, result)
+
+			WaitForClusterToUpgrade(ctx, WaitForClusterToUpgradeInput{
+				Lister:              bootstrapClusterProxy.GetClient(),
+				ControlPlane:        result.ControlPlane,
+				MachineDeployments:  result.MachineDeployments,
+				VersionAfterUpgrade: e2eConfig.GetVariable(KubernetesVersionUpgradeTo),
+			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
+
+			WaitForControlPlaneToBeReady(ctx, WaitForControlPlaneToBeReadyInput{
+				Getter:       bootstrapClusterProxy.GetClient(),
+				ControlPlane: result.ControlPlane,
+			}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
+
 			// TODO: this can be uncommented when control plane scaling down is working
 
 			// By("Scaling control plane nodes to 1")

test/e2e/e2e_test.go

@@ -20,6 +20,7 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -251,6 +252,43 @@ func WaitForMachineConditions(ctx context.Context, input WaitForMachineCondition
 	}, intervals...).Should(BeTrue(), framework.PrettyPrint(input.Machine)+"\n")
 }
 
+// WaitForClusterToUpgradeInput is the input for WaitForClusterToUpgrade.
+type WaitForClusterToUpgradeInput struct {
+	Lister              framework.Lister
+	ControlPlane        *controlplanev1.RKE2ControlPlane
+	MachineDeployments  []*clusterv1.MachineDeployment
+	VersionAfterUpgrade string
+}
+
+// WaitForClusterToUpgrade will wait for a cluster to be upgraded.
+func WaitForClusterToUpgrade(ctx context.Context, input WaitForClusterToUpgradeInput, intervals ...interface{}) {
+	By("Waiting for machines to update")
+
+	totallMachineCount := *input.ControlPlane.Spec.Replicas
+	for _, md := range input.MachineDeployments {
+		totallMachineCount += *md.Spec.Replicas
+	}
+
+	Eventually(func() (bool, error) {
+		machineList := &clusterv1.MachineList{}
+		if err := input.Lister.List(ctx, machineList); err != nil {
+			return false, fmt.Errorf("failed to list machines: %w", err)
+		}
+
+		if len(machineList.Items) != int(totallMachineCount) { // not all replicas are created
+			return false, nil
+		}
+
+		for _, machine := range machineList.Items {
+			if machine.Spec.Version != nil && *machine.Spec.Version != input.VersionAfterUpgrade {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	}, intervals...).Should(BeTrue(), framework.PrettyPrint(input.ControlPlane)+"\n")
+}
+
 func setDefaults(input *ApplyClusterTemplateAndWaitInput) {
 	if input.WaitForControlPlaneInitialized == nil {
 		input.WaitForControlPlaneInitialized = func(ctx context.Context, input ApplyClusterTemplateAndWaitInput, result *ApplyClusterTemplateAndWaitResult) {