rke2controlplane_controller.go

/*
Copyright 2022 SUSE.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/go-logr/logr"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	storagev1 "k8s.io/api/storage/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	kerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog/v2"
	"k8s.io/utils/ptr"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/source"

	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
	"sigs.k8s.io/cluster-api/controllers/clustercache"
	"sigs.k8s.io/cluster-api/controllers/remote"
	runtimeclient "sigs.k8s.io/cluster-api/exp/runtime/client"
	"sigs.k8s.io/cluster-api/feature"
	"sigs.k8s.io/cluster-api/util"
	"sigs.k8s.io/cluster-api/util/annotations"
	"sigs.k8s.io/cluster-api/util/cache"
	"sigs.k8s.io/cluster-api/util/certs"
	"sigs.k8s.io/cluster-api/util/collections"
	"sigs.k8s.io/cluster-api/util/conditions"
	capikubeconfig "sigs.k8s.io/cluster-api/util/kubeconfig"
	"sigs.k8s.io/cluster-api/util/patch"

	bootstrapv1 "github.com/rancher/cluster-api-provider-rke2/bootstrap/api/v1beta2"
	controlplanev1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta2"
	"github.com/rancher/cluster-api-provider-rke2/controlplane/internal/contract"
	"github.com/rancher/cluster-api-provider-rke2/controlplane/internal/util/ssa"
	"github.com/rancher/cluster-api-provider-rke2/pkg/capi/inplace"
	"github.com/rancher/cluster-api-provider-rke2/pkg/kubeconfig"
	"github.com/rancher/cluster-api-provider-rke2/pkg/rke2"
	"github.com/rancher/cluster-api-provider-rke2/pkg/secret"
	rke2util "github.com/rancher/cluster-api-provider-rke2/pkg/util"
)

const (
	// rke2ManagerName is the SSA field manager used for the main RKE2 control-plane
	// objects (Machine spec, InfraMachine, RKE2Config).
	rke2ManagerName = "rke2controlplane"

	// rke2MetadataManagerName is a separate SSA field manager used for the
	// labels and annotations only patches written every reconcile by syncMachines.
	rke2MetadataManagerName = "rke2controlplane-metadata"

	// rke2ControlPlaneKind is the kind of the RKE2 control plane.
	rke2ControlPlaneKind = "RKE2ControlPlane"

	// dependentCertRequeueAfter is how long to wait before checking again to see if
	// dependent certificates have been created.
	dependentCertRequeueAfter = 30 * time.Second

	// DefaultRequeueTime is the default requeue time for the controller.
	DefaultRequeueTime = 20 * time.Second

	// certCacheTtl is the default TTL for cached certificates.
	certCacheTtl = 24 * time.Hour
)

// RKE2ControlPlaneReconciler reconciles a RKE2ControlPlane object.
type RKE2ControlPlaneReconciler struct {
	client.Client

	Log    logr.Logger
	Scheme *runtime.Scheme

	SecretCachingClient client.Client

	// RuntimeClient is the runtime client to interact with extensions.
	RuntimeClient runtimeclient.Client

	// WatchFilterValue is the label value used to filter events prior to reconciliation.
	WatchFilterValue string

	managementClusterUncached rke2.ManagementCluster
	managementCluster         rke2.ManagementCluster
	recorder                  record.EventRecorder
	controller                controller.Controller
	ssaCache                  ssa.Cache
}

//nolint:lll
//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=rke2controlplanes,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=rke2controlplanes/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=rke2controlplanes/finalizers,verbs=update
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machinesets;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="",resources=secrets;events;configmaps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups="bootstrap.cluster.x-k8s.io",resources=rke2configs,verbs=get;list;watch;create;patch;delete
// +kubebuilder:rbac:groups="infrastructure.cluster.x-k8s.io",resources=*,verbs=get;list;watch;create;patch;delete
// +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch
// +kubebuilder:rbac:groups=runtime.cluster.x-k8s.io,resources=extensionconfigs,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
func (r *RKE2ControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) {
	logger := log.FromContext(ctx)
	r.Log = logger
	rcp := &controlplanev1.RKE2ControlPlane{}

	if err := r.Get(ctx, req.NamespacedName, rcp); err != nil {
		if apierrors.IsNotFound(err) {
			return ctrl.Result{}, err
		}

		return ctrl.Result{}, err
	}

	// Fetch the Cluster.
	cluster, err := util.GetOwnerCluster(ctx, r.Client, rcp.ObjectMeta)
	if err != nil {
		logger.Error(err, "Failed to retrieve owner Cluster from the API Server")

		return ctrl.Result{}, err
	}

	if cluster == nil {
		logger.Info("Cluster Controller has not yet set OwnerRef")

		return ctrl.Result{Requeue: true}, nil
	}

	logger = logger.WithValues("cluster", cluster.Name)

	if annotations.IsPaused(cluster, rcp) {
		logger.Info("Reconciliation is paused for this object")

		return ctrl.Result{}, nil
	}

	// Initialize the patch helper.
	patchHelper, err := patch.NewHelper(rcp, r.Client)
	if err != nil {
		logger.Error(err, "Failed to configure the patch helper")

		return ctrl.Result{Requeue: true}, nil
	}

	// Add finalizer first if not exist to avoid the race condition between init and delete
	if !controllerutil.ContainsFinalizer(rcp, controlplanev1.RKE2ControlPlaneFinalizer) {
		controllerutil.AddFinalizer(rcp, controlplanev1.RKE2ControlPlaneFinalizer)

		// patch and return right away instead of reusing the main defer,
		// because the main defer may take too much time to get cluster status
		// Patch ObservedGeneration only if the reconciliation completed successfully
		patchOpts := []patch.Option{patch.WithStatusObservedGeneration{}}
		if err := patchHelper.Patch(ctx, rcp, patchOpts...); err != nil {
			return ctrl.Result{}, fmt.Errorf("failed to add finalizer: %w", err)
		}

		return ctrl.Result{Requeue: true}, nil
	}

	defer func() {
		// Always attempt to update status.
		if err := r.updateStatus(ctx, rcp, cluster); err != nil {
			var connFailure *rke2.RemoteClusterConnectionError
			if errors.As(err, &connFailure) {
				logger.Info("Could not connect to workload cluster to fetch status", "err", err.Error())
			} else {
				logger.Error(err, "Failed to update RKE2ControlPlane Status")
				reterr = kerrors.NewAggregate([]error{reterr, err})
			}
		}

		// Always attempt to Patch the RKE2ControlPlane object and status after each reconciliation.
		if err := patchRKE2ControlPlane(ctx, patchHelper, rcp); err != nil {
			reterr = kerrors.NewAggregate([]error{reterr, err})
		}

		// Make rcp to requeue in case status is not ready, so we can check for node
		// status without waiting for a full resync (by default 10 minutes).
		// Only requeue if we are not going in exponential backoff due to error,
		// or if we are not already re-queueing, or if the object has a deletion timestamp.
		if reterr == nil && res.RequeueAfter <= 0 && rcp.DeletionTimestamp.IsZero() {
			if !ptr.Deref(rcp.Status.Initialization.ControlPlaneInitialized, false) {
				res = ctrl.Result{RequeueAfter: DefaultRequeueTime}
			}
		}
	}()

	if !rcp.DeletionTimestamp.IsZero() {
		// Handle deletion reconciliation loop.
		res, err = r.reconcileDelete(ctx, cluster, rcp)

		return res, err
	}

	updated := false

	if updated {
		if err := patchHelper.Patch(ctx, rcp); err != nil {
			logger.Error(err, "Failed to patch RKE2ControlPlane during backfill")
			// If patching fails, we return an error to avoid re-queuing the object.
			return ctrl.Result{}, err
		}
		// Log the backfill operation.
		logger.Info("Backfilled missing RKE2ControlPlane fields from legacy format", "rcp", klog.KObj(rcp))
		// Requeue to ensure the controller reprocesses the object with the updated fields.
		return ctrl.Result{Requeue: true}, nil
	}

	// Handle normal reconciliation loop.
	res, err = r.reconcileNormal(ctx, cluster, rcp)

	return res, err
}

func patchRKE2ControlPlane(ctx context.Context, patchHelper *patch.Helper, rcp *controlplanev1.RKE2ControlPlane) error {
	// Patch the object, ignoring conflicts on the conditions owned by this controller.
	return patchHelper.Patch(
		ctx,
		rcp,
		patch.WithOwnedConditions{Conditions: []string{
			clusterv1.PausedCondition,
			controlplanev1.RKE2ControlPlaneAvailableCondition,
			controlplanev1.RKE2ControlPlaneInitializedCondition,
			controlplanev1.RKE2ControlPlaneCertificatesAvailableCondition,
			controlplanev1.RKE2ControlPlaneEtcdClusterHealthyCondition,
			controlplanev1.RKE2ControlPlaneControlPlaneComponentsHealthyCondition,
			controlplanev1.RKE2ControlPlaneMachinesReadyCondition,
			controlplanev1.RKE2ControlPlaneMachinesUpToDateCondition,
			controlplanev1.RKE2ControlPlaneRollingOutCondition,
			controlplanev1.RKE2ControlPlaneScalingUpCondition,
			controlplanev1.RKE2ControlPlaneScalingDownCondition,
			controlplanev1.RKE2ControlPlaneRemediatingCondition,
			controlplanev1.RKE2ControlPlaneDeletingCondition,
		}},
		patch.WithStatusObservedGeneration{},
	)
}

// SetupWithManager sets up the controller with the Manager.
func (r *RKE2ControlPlaneReconciler) SetupWithManager(
	ctx context.Context, mgr ctrl.Manager, clientQPS float32,
	clientBurst, clusterCacheConcurrency, concurrency int,
) error {
	c, err := ctrl.NewControllerManagedBy(mgr).
		For(&controlplanev1.RKE2ControlPlane{}).
		Owns(&clusterv1.Machine{}).
		WithOptions(controller.Options{
			MaxConcurrentReconciles: concurrency,
		}).
		Build(r)
	if err != nil {
		return fmt.Errorf("failed setting up with a controller manager: %w", err)
	}

	err = c.Watch(
		source.Kind[client.Object](mgr.GetCache(), &clusterv1.Cluster{},
			handler.EnqueueRequestsFromMapFunc((r.ClusterToRKE2ControlPlane(ctx))),
		),
	)
	if err != nil {
		return fmt.Errorf("failed adding Watch for Clusters to controller manager: %w", err)
	}

	r.controller = c
	r.recorder = mgr.GetEventRecorderFor("rke2-control-plane-controller")
	r.ssaCache = ssa.NewCache("rke2-control-plane")

	// Set up a clusterCache to provide to controllers
	// requiring a connection to a remote cluster
	clusterCache, err := clustercache.SetupWithManager(ctx, mgr, clustercache.Options{
		SecretClient: r.SecretCachingClient,
		Cache: clustercache.CacheOptions{
			Indexes: []clustercache.CacheOptionsIndex{clustercache.NodeProviderIDIndex},
		},
		Client: clustercache.ClientOptions{
			QPS:       clientQPS,
			Burst:     clientBurst,
			UserAgent: remote.DefaultClusterAPIUserAgent("rke2-control-plane-controller"),
			Cache: clustercache.ClientCacheOptions{
				DisableFor: []client.Object{
					// Don't cache ConfigMaps & Secrets.
					&corev1.ConfigMap{},
					&corev1.Secret{},
					// Don't cache Pods & DaemonSets (we get/list them e.g. during drain).
					&corev1.Pod{},
					&appsv1.DaemonSet{},
					// Don't cache PersistentVolumes and VolumeAttachments (we get/list them e.g. during wait for volumes to detach)
					&storagev1.VolumeAttachment{},
					&corev1.PersistentVolume{},
				},
			},
		},
	}, controller.Options{
		MaxConcurrentReconciles: clusterCacheConcurrency,
	})
	if err != nil {
		return fmt.Errorf("unable to create cluster cache tracker: %w", err)
	}

	if r.managementCluster == nil {
		r.managementCluster = &rke2.Management{
			Client:              r.Client,
			SecretCachingClient: r.SecretCachingClient,
			ClusterCache:        clusterCache,
			ClientCertCache:     cache.New[rke2.ClientCertEntry](certCacheTtl),
		}
	}

	if r.managementClusterUncached == nil {
		r.managementClusterUncached = &rke2.Management{Client: mgr.GetClient()}
	}

	return nil
}

// ClusterToRKE2ControlPlane is a handler.ToRequestsFunc to be used to enqueue requests for reconciliation
// for RKE2ControlPlane based on updates to a Cluster.
func (r *RKE2ControlPlaneReconciler) ClusterToRKE2ControlPlane(ctx context.Context) handler.MapFunc {
	log := log.FromContext(ctx)

	return func(_ context.Context, o client.Object) []ctrl.Request {
		c, ok := o.(*clusterv1.Cluster)
		if !ok {
			log.Error(nil, fmt.Sprintf("Expected a Cluster but got a %T", o))

			return nil
		}

		controlPlaneRef := c.Spec.ControlPlaneRef
		if controlPlaneRef.IsDefined() && controlPlaneRef.Kind == "RKE2ControlPlane" {
			return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: c.Namespace, Name: controlPlaneRef.Name}}}
		}

		return nil
	}
}

func (r *RKE2ControlPlaneReconciler) reconcileNormal(
	ctx context.Context,
	cluster *clusterv1.Cluster,
	rcp *controlplanev1.RKE2ControlPlane,
) (ctrl.Result, error) {
	logger := log.FromContext(ctx)
	logger.Info("Reconcile RKE2 Control Plane")

	// Wait for the cluster infrastructure to be ready before creating machines
	if !ptr.Deref(cluster.Status.Initialization.InfrastructureProvisioned, false) {
		conditions.Set(rcp, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneEtcdClusterHealthyCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  controlplanev1.RKE2ControlPlaneEtcdClusterInspectionFailedReason,
			Message: "Waiting for Cluster status.infrastructureReady to be true",
		})

		conditions.Set(rcp, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneControlPlaneComponentsHealthyCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  controlplanev1.RKE2ControlPlaneControlPlaneComponentsInspectionFailedReason,
			Message: "Waiting for Cluster status.infrastructureReady to be true",
		})

		logger.Info("Cluster infrastructure is not ready yet")

		return ctrl.Result{}, nil
	}

	certificates := secret.NewCertificatesForInitialControlPlane()
	if _, found := rcp.Annotations[controlplanev1.LegacyRKE2ControlPlane]; found {
		certificates = secret.NewCertificatesForLegacyControlPlane()
	}

	controllerRef := metav1.NewControllerRef(rcp, controlplanev1.GroupVersion.WithKind("RKE2ControlPlane"))

	if err := certificates.LookupOrGenerate(ctx, r.Client, util.ObjectKey(cluster), *controllerRef); err != nil {
		logger.Error(err, "unable to lookup or create cluster certificates")
		conditions.Set(rcp, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneCertificatesAvailableCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  controlplanev1.RKE2ControlPlaneCertificatesInternalErrorReason,
			Message: "Please check controller logs for errors",
		})

		return ctrl.Result{}, err
	}

	conditions.Set(rcp, metav1.Condition{
		Type:   controlplanev1.RKE2ControlPlaneCertificatesAvailableCondition,
		Status: metav1.ConditionTrue,
		Reason: controlplanev1.RKE2ControlPlaneCertificatesAvailableReason,
	})

	// If ControlPlaneEndpoint is not set, return early
	if !cluster.Spec.ControlPlaneEndpoint.IsValid() {
		conditions.Set(rcp, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneEtcdClusterHealthyCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  controlplanev1.RKE2ControlPlaneEtcdClusterInspectionFailedReason,
			Message: "Waiting for Cluster spec.controlPlaneEndpoint to be set",
		})

		conditions.Set(rcp, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneControlPlaneComponentsHealthyCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  controlplanev1.RKE2ControlPlaneControlPlaneComponentsInspectionFailedReason,
			Message: "Waiting for Cluster spec.controlPlaneEndpoint to be set",
		})

		logger.Info("Cluster does not yet have a ControlPlaneEndpoint defined")

		return ctrl.Result{}, nil
	}

	// Generate Cluster Kubeconfig if needed
	if result, err := r.reconcileKubeconfig(
		ctx,
		util.ObjectKey(cluster),
		cluster.Spec.ControlPlaneEndpoint,
		rcp); err != nil {
		logger.Error(err, "failed to reconcile Kubeconfig")

		return result, err
	}

	controlPlaneMachines, err := r.managementClusterUncached.GetMachinesForCluster(
		ctx,
		cluster,
		collections.ControlPlaneMachines(cluster.Name))
	if err != nil {
		logger.Error(err, "failed to retrieve control plane machines for cluster")

		return ctrl.Result{}, err
	}

	ownedMachines := controlPlaneMachines.Filter(collections.OwnedMachines(rcp, controlplanev1.GroupVersion.WithKind("RKE2ControlPlane").GroupKind()))
	if len(ownedMachines) != len(controlPlaneMachines) {
		logger.Info("Not all control plane machines are owned by this RKE2ControlPlane, refusing to operate in mixed management mode") //nolint:lll

		return ctrl.Result{}, nil
	}

	controlPlane, err := rke2.NewControlPlane(ctx, r.managementCluster, r.Client, cluster, rcp, ownedMachines)
	if err != nil {
		logger.Error(err, "failed to initialize control plane")

		return ctrl.Result{}, err
	}

	if err := controlPlane.ReconcileExternalReference(ctx, r.Client); err != nil {
		logger.Error(err, "Could not reconcile external reference")

		return ctrl.Result{}, fmt.Errorf("reconciling external reference: %w", err)
	}

	if err := r.syncMachines(ctx, controlPlane); err != nil {
		return ctrl.Result{}, fmt.Errorf("failed to sync Machines: %w", err)
	}

	// Updates conditions reporting the status of static pods and the status of the etcd cluster.
	// NOTE: Conditions reporting RCP operation progress like e.g. Resized or SpecUpToDate are inlined with the rest of the execution.
	if result, err := r.reconcileControlPlaneConditions(ctx, controlPlane); err != nil || !result.IsZero() {
		logger.Error(err, "failed to reconcile Control Plane conditions")

		return result, err
	}

	if result, err := r.reconcileLifecycleHooks(ctx, controlPlane); err != nil || !result.IsZero() {
		return result, err
	}

	// Complete triggering in-place update if necessary (reentrancy).
	// This handles the case where a previous triggerInPlaceUpdate call partially completed
	// (e.g. UpdateInProgressAnnotation was set, but the SSA patches or PendingHooksAnnotation were not written).
	if machines := controlPlane.MachinesToCompleteTriggerInPlaceUpdate(); len(machines) > 0 {
		_, machinesUpToDateResults := controlPlane.NotUpToDateMachines()
		for _, m := range machines {
			if err := r.triggerInPlaceUpdate(ctx, m, machinesUpToDateResults[m.Name]); err != nil {
				return ctrl.Result{}, err
			}
		}

		return ctrl.Result{}, nil
	}

	// Reconcile unhealthy machines by triggering deletion and requeue if it is considered safe to remediate,
	// otherwise continue with the other RCP operations.
	if result, err := r.reconcileUnhealthyMachines(ctx, controlPlane); err != nil || !result.IsZero() {
		return result, err
	}

	// Wait for in-place update to complete.
	// Note: If a Machine becomes unhealthy during in-place update reconcileUnhealthyMachines above remediates it.
	// Note: We have to wait here even if there are no more Machines that need rollout (in-place update in
	//       progress is not counted as needs rollout).
	if machines := controlPlane.MachinesToCompleteInPlaceUpdate(); machines.Len() > 0 {
		for _, machine := range machines {
			logger.Info(fmt.Sprintf("Waiting for in-place update of Machine %s to complete", machine.Name), "Machine", klog.KObj(machine))
		}

		return ctrl.Result{}, nil
	}

	// Control plane machines rollout due to configuration changes (e.g. upgrades) takes precedence over other operations.
	needRollout, machinesUpToDateResults := controlPlane.MachinesNeedingRollout()

	switch {
	case len(needRollout) > 0:
		logger.Info("Rolling out Control Plane machines", "needRollout", needRollout.Names())
		conditions.Set(controlPlane.RCP, metav1.Condition{
			Type:   controlplanev1.RKE2ControlPlaneRollingOutCondition,
			Status: metav1.ConditionTrue,
			Reason: controlplanev1.RKE2ControlPlaneRollingOutReason,
			Message: fmt.Sprintf("Rolling %d replicas with outdated spec"+
				"(%d replicas up to date)", len(needRollout), len(controlPlane.Machines)-len(needRollout)),
		})

		return r.upgradeControlPlane(ctx, cluster, rcp, controlPlane, needRollout, machinesUpToDateResults)
	default:
		if conditions.Has(controlPlane.RCP, controlplanev1.RKE2ControlPlaneRollingOutCondition) {
			conditions.Set(controlPlane.RCP, metav1.Condition{
				Type:   controlplanev1.RKE2ControlPlaneRollingOutCondition,
				Status: metav1.ConditionFalse,
				Reason: controlplanev1.RKE2ControlPlaneNotRollingOutReason,
			})
		}
	}

	// If we've made it this far, we can assume that all ownedMachines are up to date
	numMachines := len(ownedMachines)

	desiredReplicas := int(*rcp.Spec.Replicas)

	switch {
	// We are creating the first replica
	case numMachines < desiredReplicas && numMachines == 0:
		// Create new Machine w/ init
		logger.Info("Initializing control plane", "Desired", desiredReplicas, "Existing", numMachines)

		return r.initializeControlPlane(ctx, cluster, rcp, controlPlane)
	// We are scaling up
	case numMachines < desiredReplicas && numMachines > 0:
		// Create a new Machine w/ join
		logger.Info("Scaling up control plane", "Desired", desiredReplicas, "Existing", numMachines)

		return r.scaleUpControlPlane(ctx, cluster, rcp, controlPlane)

	// We are scaling down
	case numMachines > desiredReplicas:
		logger.Info("Scaling down control plane", "Desired", desiredReplicas, "Existing", numMachines)
		// The last parameter (i.e. machines needing to be rolled out) should always be empty here.
		return r.scaleDownControlPlane(ctx, cluster, rcp, controlPlane, collections.Machines{})
	}

	return ctrl.Result{}, nil
}

func (r *RKE2ControlPlaneReconciler) reconcileDelete(ctx context.Context,
	cluster *clusterv1.Cluster,
	rcp *controlplanev1.RKE2ControlPlane,
) (res ctrl.Result, err error) {
	logger := log.FromContext(ctx)

	// Gets all machines, not just control plane machines.
	allMachines, err := r.managementCluster.GetMachinesForCluster(ctx, cluster)
	if err != nil {
		return ctrl.Result{}, err
	}

	ownedMachines := allMachines.Filter(collections.OwnedMachines(rcp, controlplanev1.GroupVersion.WithKind("RKE2ControlPlane").GroupKind()))

	// If no control plane machines remain, remove the finalizer
	if len(ownedMachines) == 0 {
		// If the legacy finalizer is present, remove it.
		if controllerutil.ContainsFinalizer(rcp, controlplanev1.RKE2ControlPlaneLegacyFinalizer) {
			controllerutil.RemoveFinalizer(rcp, controlplanev1.RKE2ControlPlaneLegacyFinalizer)
		}

		controllerutil.RemoveFinalizer(rcp, controlplanev1.RKE2ControlPlaneFinalizer)

		conditions.Set(rcp, metav1.Condition{
			Type:   controlplanev1.RKE2ControlPlaneDeletingCondition,
			Status: metav1.ConditionTrue,
			Reason: controlplanev1.RKE2ControlPlaneDeletingDeletionCompletedReason,
		})

		return ctrl.Result{}, nil
	}

	controlPlane, err := rke2.NewControlPlane(ctx, r.managementCluster, r.Client, cluster, rcp, ownedMachines)
	if err != nil {
		logger.Error(err, "failed to initialize control plane")

		return ctrl.Result{}, err
	}

	// Updates conditions reporting the status of static pods and the status of the etcd cluster.
	// NOTE: Ignoring failures given that we are deleting
	if _, err := r.reconcileControlPlaneConditions(ctx, controlPlane); err != nil {
		logger.Info("failed to reconcile conditions", "error", err.Error())
	}

	// Verify that only control plane machines remain
	if len(allMachines) != len(ownedMachines) {
		logger.Info("Waiting for worker nodes to be deleted first")

		conditions.Set(rcp, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneDeletingCondition,
			Status:  metav1.ConditionTrue,
			Reason:  controlplanev1.RKE2ControlPlaneDeletingWaitingForWorkersDeletionReason,
			Message: "Waiting for worker nodes to be deleted first",
		})

		return ctrl.Result{RequeueAfter: deleteRequeueAfter}, nil
	}

	// Delete control plane machines in parallel
	machinesToDelete := ownedMachines

	var errs []error

	for i := range machinesToDelete {
		m := machinesToDelete[i]
		logger := logger.WithValues("machine", m)

		// During RKE2CP deletion we don't care about forwarding etcd leadership or removing etcd members.
		// So we are removing the pre-terminate hook.
		// This is important because when deleting RKE2CP we will delete all members of etcd and it's not possible
		// to forward etcd leadership without any member left after we went through the Machine deletion.
		// Also in this case the reconcileDelete code of the Machine controller won't execute Node drain
		// and wait for volume detach.
		if err := r.removePreTerminateHookAnnotationFromMachine(ctx, m); err != nil {
			errs = append(errs, err)

			continue
		}

		if err := r.removeHookAnnotationFromMachine(ctx, m, controlplanev1.PreDrainLoadbalancerExclusionAnnotation); err != nil {
			errs = append(errs, err)

			continue
		}

		if !m.DeletionTimestamp.IsZero() {
			// Nothing to do, Machine already has deletionTimestamp set.
			continue
		}

		if err := r.Delete(ctx, machinesToDelete[i]); err != nil && !apierrors.IsNotFound(err) {
			logger.Error(err, "Failed to cleanup owned machine")
			errs = append(errs, err)
		}
	}

	if len(errs) > 0 {
		err := kerrors.NewAggregate(errs)
		r.recorder.Eventf(rcp, corev1.EventTypeWarning, "FailedDelete",
			"Failed to delete control plane Machines for cluster %s/%s control plane: %v", cluster.Namespace, cluster.Name, err)

		return ctrl.Result{}, err
	}

	logger.Info("Waiting for control plane Machines to not exist anymore")

	conditions.Set(rcp, metav1.Condition{
		Type:    controlplanev1.RKE2ControlPlaneDeletingCondition,
		Status:  metav1.ConditionTrue,
		Reason:  controlplanev1.RKE2ControlPlaneDeletingWaitingForMachineDeletionReason,
		Message: "Waiting for control plane Machines to be deleted",
	})

	return ctrl.Result{RequeueAfter: deleteRequeueAfter}, nil
}

func (r *RKE2ControlPlaneReconciler) reconcileKubeconfig(
	ctx context.Context,
	clusterName client.ObjectKey,
	endpoint clusterv1.APIEndpoint,
	rcp *controlplanev1.RKE2ControlPlane,
) (ctrl.Result, error) {
	logger := ctrl.LoggerFrom(ctx)
	if endpoint.IsZero() {
		logger.V(5).Info("API Endpoint not yet known")

		return ctrl.Result{RequeueAfter: DefaultRequeueTime}, nil
	}

	controllerOwnerRef := *metav1.NewControllerRef(rcp, controlplanev1.GroupVersion.WithKind("RKE2ControlPlane"))
	configSecret, err := secret.GetFromNamespacedName(ctx, r.Client, clusterName, secret.Kubeconfig)

	switch {
	case apierrors.IsNotFound(err):
		logger.Info("Kubeconfig Secret not found, creating a new one")

		createErr := kubeconfig.CreateSecretWithOwner(
			ctx,
			r.Client,
			clusterName,
			endpoint.String(),
			controllerOwnerRef,
		)

		if errors.Is(createErr, kubeconfig.ErrDependentCertificateNotFound) {
			logger.Error(createErr, "Could not find Secret CA to create Kubeconfig Secret, requeuing...")

			return ctrl.Result{RequeueAfter: dependentCertRequeueAfter}, nil
		}
		// always return if we have just created in order to skip rotation checks
		return ctrl.Result{}, createErr

	case err != nil:
		return ctrl.Result{}, fmt.Errorf("failed to retrieve kubeconfig Secret: %w", err)
	}

	// only do rotation on owned secrets
	if !util.IsControlledBy(configSecret, rcp, controlplanev1.GroupVersion.WithKind("RKE2ControlPlane").GroupKind()) {
		logger.Info("Kubeconfig Secret not controlled by RKE2ControlPlane, nothing to do")

		return ctrl.Result{}, nil
	}

	needsRotation, err := capikubeconfig.NeedsClientCertRotation(configSecret, certs.ClientCertificateRenewalDuration)
	if err != nil {
		return ctrl.Result{}, err
	}

	if needsRotation {
		logger.Info("Rotating kubeconfig secret")

		if err := kubeconfig.UpdateSecret(ctx, r.Client, clusterName, endpoint.String(), configSecret); err != nil {
			return ctrl.Result{}, fmt.Errorf("failed to regenerate kubeconfig: %w", err)
		}
	}

	return ctrl.Result{}, nil
}

// reconcileControlPlaneConditions is responsible of reconciling conditions reporting the status of static pods and
// the status of the etcd cluster.
func (r *RKE2ControlPlaneReconciler) reconcileControlPlaneConditions(
	ctx context.Context, controlPlane *rke2.ControlPlane,
) (res ctrl.Result, retErr error) {
	logger := log.FromContext(ctx)

	// If the control plane is being deleted, we don't need to reconcile conditions. The DeletingCondition is set directly in reconcileDelete.
	if !controlPlane.RCP.DeletionTimestamp.IsZero() {
		return ctrl.Result{}, nil
	}

	// If the cluster is not yet initialized, there is no way to connect to the workload cluster and fetch information
	// for updating conditions. Return early.
	// We additionally check for the ControlPlaneInitialized condition. The ControlPlaneInitialized condition is set at the same time
	// as .status.initialization.controlPlaneInitialized and is never changed to false again. Below we'll need the transition time of the
	// ControlPlaneInitialized condition to check if the remote conditions grace period is already reached.
	controlPlaneInitialized := conditions.Get(controlPlane.RCP, controlplanev1.RKE2ControlPlaneInitializedCondition)
	if !ptr.Deref(controlPlane.RCP.Status.Initialization.ControlPlaneInitialized, false) ||
		controlPlaneInitialized == nil || controlPlaneInitialized.Status != metav1.ConditionTrue {
		setConditionsToUnknown(setConditionsToUnknownInput{
			ControlPlane:                        controlPlane,
			Overwrite:                           true,
			EtcdClusterHealthyReason:            controlplanev1.RKE2ControlPlaneEtcdClusterInspectionFailedReason,
			ControlPlaneComponentsHealthyReason: controlplanev1.RKE2ControlPlaneControlPlaneComponentsInspectionFailedReason,
			StaticPodReason:                     controlplanev1.RKE2ControlPlaneMachinePodInspectionFailedReason,
			EtcdMemberHealthyReason:             controlplanev1.RKE2ControlPlaneMachineEtcdMemberInspectionFailedReason,
			Message:                             "Waiting for Cluster control plane to be initialized",
		})

		return ctrl.Result{}, nil
	}

	readyCPMachines := controlPlane.Machines.Filter(collections.IsReady())

	if readyCPMachines.Len() == 0 {
		controlPlane.RCP.Status.ReadyReplicas = ptr.To(int32(0))
		controlPlane.RCP.Status.AvailableServerIPs = nil

		conditions.Set(controlPlane.RCP, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneAvailableCondition,
			Status:  metav1.ConditionFalse,
			Reason:  controlplanev1.RKE2ControlPlaneWaitingForRKE2ServerReason,
			Message: "Waiting for at least one machine to be ready for control plane " + controlPlane.RCP.Name,
		})

		conditions.Set(controlPlane.RCP, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneMachinesReadyCondition,
			Status:  metav1.ConditionFalse,
			Reason:  controlplanev1.RKE2ControlPlaneWaitingForRKE2ServerReason,
			Message: "Waiting for control plane machines ",
		})
	}

	// If the cluster is not yet initialized, there is no way to connect to the workload cluster and fetch information
	// for updating conditions. Return early.
	if !ptr.Deref(controlPlane.RCP.Status.Initialization.ControlPlaneInitialized, false) {
		return ctrl.Result{}, nil
	}

	workloadCluster, err := controlPlane.GetWorkloadCluster(ctx)
	if err != nil {
		logger.Error(err, "Failed to get remote client for workload cluster", "cluster key", util.ObjectKey(controlPlane.Cluster))

		return ctrl.Result{}, fmt.Errorf("getting workload cluster: %w", err)
	}

	defer func() {
		// Always attempt to Patch the Machine conditions after each reconcile.
		if err := controlPlane.PatchMachines(ctx); err != nil {
			retErr = kerrors.NewAggregate([]error{retErr, err})
		}
	}()

	// Always reconcile machine's UpToDate condition.
	reconcileMachineUpToDateCondition(controlPlane)

	if err := workloadCluster.InitWorkload(ctx, controlPlane); err != nil {
		logger.Error(err, "Unable to initialize workload cluster")

		return ctrl.Result{}, err
	}

	// Update conditions status
	workloadCluster.UpdateAgentConditions(controlPlane)
	workloadCluster.UpdateEtcdConditions(controlPlane)

	// Patch nodes metadata
	if err := workloadCluster.UpdateNodeMetadata(ctx, controlPlane); err != nil {
		logger.Error(err, "Unable to update node metadata")

		return ctrl.Result{}, err
	}

	// RCP will be patched at the end of Reconcile to reflect updated conditions, so we can return now.
	return ctrl.Result{}, nil
}

func (r *RKE2ControlPlaneReconciler) upgradeControlPlane(
	ctx context.Context,
	cluster *clusterv1.Cluster,
	rcp *controlplanev1.RKE2ControlPlane,
	controlPlane *rke2.ControlPlane,
	machinesRequireUpgrade collections.Machines,
	machinesUpToDateResults map[string]rke2.UpToDateResult,
) (ctrl.Result, error) {
	logger := controlPlane.Logger()

	// If the cluster is not yet initialized, there is no way to connect to the workload cluster and fetch information
	// for updating conditions. Return early.
	if !ptr.Deref(rcp.Status.Initialization.ControlPlaneInitialized, false) {
		logger.Info("ControlPlane not yet initialized")

		return ctrl.Result{}, nil
	}

	workloadCluster, err := controlPlane.GetWorkloadCluster(ctx)
	if err != nil {
		logger.Error(err, "Failed to get remote client for workload cluster", "cluster key", util.ObjectKey(cluster))

		return ctrl.Result{}, fmt.Errorf("getting workload cluster: %w", err)
	}

	if err := workloadCluster.InitWorkload(ctx, controlPlane); err != nil {
		return ctrl.Result{}, err
	}

	switch rcp.Spec.RolloutStrategy.Type {
	case controlplanev1.RollingUpdateStrategyType:
		// RolloutStrategy is currently defaulted and validated to be RollingUpdate.
		// Defaulted to 1 if not specified
		maxSurge := intstr.FromInt(1)
		if rcp.Spec.RolloutStrategy.RollingUpdate != nil && rcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge != nil {
			maxSurge = *rcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge
		}

		maxNodes := *rcp.Spec.Replicas + rke2util.SafeInt32(maxSurge.IntValue())
		if rke2util.SafeInt32(controlPlane.Machines.Len()) < maxNodes {
			// scaleUpControlPlane ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
			return r.scaleUpControlPlane(ctx, cluster, rcp, controlPlane)
		}

		// Pick a machine for in-place update or scale down.
		machineToUpdate, err := selectMachineForInPlaceUpdateOrScaleDown(ctx, controlPlane, machinesRequireUpgrade)
		if err != nil {
			return ctrl.Result{}, err
		}

		machineUpToDateResult, ok := machinesUpToDateResults[machineToUpdate.Name]
		if !ok {
			return ctrl.Result{}, fmt.Errorf("failed to check if Machine %s is UpToDate", machineToUpdate.Name)
		}

		// Try in-place update if eligible.
		// Note: To be safe we only try an in-place update when we would otherwise delete a Machine. This ensures we could
		// afford if the in-place update fails and the Machine becomes unavailable (and eventually MHC kicks in and the Machine is recreated).
		currentUpToDateReplicas := int32(controlPlane.UpToDateMachines().Len())
		if feature.Gates.Enabled(feature.InPlaceUpdates) &&
			machineUpToDateResult.EligibleForInPlaceUpdate &&
			currentUpToDateReplicas < *rcp.Spec.Replicas {
			fallbackToScaleDown, res, err := r.tryInPlaceUpdate(ctx, controlPlane, machineToUpdate, machineUpToDateResult)
			if err != nil {
				return ctrl.Result{}, err
			}

			if !res.IsZero() {
				return res, nil
			}

			if fallbackToScaleDown {
				return r.scaleDownControlPlane(ctx, cluster, rcp, controlPlane, collections.Machines{machineToUpdate.Name: machineToUpdate})
			}

			return ctrl.Result{}, nil
		}

		return r.scaleDownControlPlane(ctx, cluster, rcp, controlPlane, machinesRequireUpgrade)
	default:
		err := fmt.Errorf("unknown rollout strategy type %q", rcp.Spec.RolloutStrategy.Type)
		logger.Error(err, "RolloutStrategy type is not set to RollingUpdateStrategyType, unable to determine the strategy for rolling out machines")

		return ctrl.Result{}, nil
	}
}

// syncMachines updates Machines, InfrastructureMachines and Rke2Configs to propagate in-place mutable fields from RKE2ControlPlane.
// Note: For InfrastructureMachines and Rke2Configs it also drops ownership of "metadata.labels" and
// "metadata.annotations" from "manager" so that "rke2controlplane" can own these fields and can work with SSA.
// Otherwise, fields would be co-owned by our "old" "manager" and "rke2controlplane" and then we would not be
// able to e.g. drop labels and annotations.
func (r *RKE2ControlPlaneReconciler) syncMachines(ctx context.Context, controlPlane *rke2.ControlPlane) error {
	patchHelpers := map[string]*patch.Helper{}

	for machineName := range controlPlane.Machines {
		m := controlPlane.Machines[machineName]
		// If the Machine is already being deleted, we only need to sync
		// the subset of fields that impact tearing down the Machine.
		if !m.DeletionTimestamp.IsZero() {
			patchHelper, err := patch.NewHelper(m, r.Client)
			if err != nil {
				return err
			}

			// Set all other in-place mutable fields that impact the ability to tear down existing machines.
			m.Spec.Deletion = clusterv1.MachineDeletionSpec{
				NodeDrainTimeoutSeconds:        controlPlane.RCP.Spec.MachineTemplate.Spec.Deletion.NodeDrainTimeoutSeconds,
				NodeDeletionTimeoutSeconds:     controlPlane.RCP.Spec.MachineTemplate.Spec.Deletion.NodeDeletionTimeoutSeconds,
				NodeVolumeDetachTimeoutSeconds: controlPlane.RCP.Spec.MachineTemplate.Spec.Deletion.NodeVolumeDetachTimeoutSeconds,
			}

			if err := patchHelper.Patch(ctx, m); err != nil {
				return err
			}

			controlPlane.Machines[machineName] = m

			patchHelper, err = patch.NewHelper(m, r.Client)
			if err != nil { //nolint:wsl
				return err
			}

			patchHelpers[machineName] = patchHelper

			continue
		}

		// Cleanup managed fields of all Machines.
		if err := ssa.CleanUpManagedFieldsForSSAAdoption(ctx, r.Client, m, rke2ManagerName); err != nil {
			return fmt.Errorf("failed to update Machine: failed to adjust the managedFields of the Machine %v: %w", klog.KObj(m), err)
		}
		// Update Machine to propagate in-place mutable fields from RCP.
		updatedMachine, err := r.UpdateMachine(ctx, m, controlPlane.RCP, controlPlane.Cluster)
		if err != nil {
			return fmt.Errorf("failed to update Machine: %v: %w", klog.KObj(m), err)
		}

		controlPlane.Machines[machineName] = updatedMachine
		// Since the machine is updated, re-create the patch helper so that any subsequent
		// Patch calls use the correct base machine object to calculate the diffs.
		// Example: reconcileControlPlaneConditions patches the machine objects in a subsequent call
		// and, it should use the updated machine to calculate the diff.
		// Note: If the patchHelpers are not re-computed based on the new updated machines, subsequent
		// Patch calls will fail because the patch will be calculated based on an outdated machine and will error
		// because of outdated resourceVersion.
		patchHelper, err := patch.NewHelper(updatedMachine, r.Client)
		if err != nil {
			return fmt.Errorf("failed to create patch helper for Machine %v: %w", klog.KObj(updatedMachine), err)
		}

		patchHelpers[machineName] = patchHelper

		labelsAndAnnotationsManagedFieldPaths := []contract.Path{
			{"f:metadata", "f:annotations"},
			{"f:metadata", "f:labels"},
		}
		infraMachine, infraMachineFound := controlPlane.InfraResources[machineName]
		// Only update the InfraMachine if it is already found, otherwise just skip it.
		// This could happen e.g. if the cache is not up-to-date yet.
		if infraMachineFound {
			// Cleanup managed fields of all InfrastructureMachines to drop ownership of labels and annotations
			// from "manager". We do this so that InfrastructureMachines that are created using the Create method
			// can also work with SSA. Otherwise, labels and annotations would be co-owned by our "old" "manager"
			// and "rke2-controlplane" and then we would not be able to e.g. drop labels and annotations.
			if err := ssa.DropManagedFields(ctx, r.Client, infraMachine, rke2ManagerName, labelsAndAnnotationsManagedFieldPaths); err != nil {
				return fmt.Errorf("failed to clean up managedFields of InfrastructureMachine %v: %w", klog.KObj(infraMachine), err)
			}
			// Update in-place mutating fields on InfrastructureMachine.
			if err := r.UpdateExternalObject(ctx, infraMachine, controlPlane.RCP, controlPlane.Cluster); err != nil {
				return fmt.Errorf("failed to update InfrastructureMachine %v: %w", klog.KObj(infraMachine), err)
			}
		}

		rke2Config, rke2ConfigFound := controlPlane.Rke2Configs[machineName]
		// Only update the RKE2Config if it is already found, otherwise just skip it.
		// This could happen e.g. if the cache is not up-to-date yet.
		if rke2ConfigFound {
			// Note: Set the GroupVersionKind because updateExternalObject depends on it.
			rke2Config.SetGroupVersionKind(bootstrapv1.GroupVersion.WithKind("RKE2Config"))
			// Cleanup managed fields of all RKE2Configs to drop ownership of labels and annotations
			// from "manager". We do this so that RKE2Configs that are created using the Create method
			// can also work with SSA. Otherwise, labels and annotations would be co-owned by our "old" "manager"
			// and "rke2-controlplane" and then we would not be able to e.g. drop labels and annotations.
			if err := ssa.DropManagedFields(ctx, r.Client, rke2Config, rke2ManagerName, labelsAndAnnotationsManagedFieldPaths); err != nil {
				return fmt.Errorf("failed to clean up managedFields of RKE2Config %v: %w", klog.KObj(rke2Config), err)
			}
			// Update in-place mutating fields on BootstrapConfig.
			if err := r.UpdateExternalObject(ctx, rke2Config, controlPlane.RCP, controlPlane.Cluster); err != nil {
				return fmt.Errorf("failed to update RKE2Config %v: %w", klog.KObj(rke2Config), err)
			}
		}
	}
	// Update the patch helpers.
	controlPlane.SetPatchHelpers(patchHelpers)

	return nil
}

func reconcileMachineUpToDateCondition(controlPlane *rke2.ControlPlane) {
	machinesNotUptoDate, machinesUpToDateResults := controlPlane.NotUpToDateMachines()
	machinesNotUptoDateNames := sets.New(machinesNotUptoDate.Names()...)

	for _, machine := range controlPlane.Machines {
		if machinesNotUptoDateNames.Has(machine.Name) {
			// Note: the code computing the message for RCP's RolloutOut condition is making assumptions on the format/content of this message.
			message := ""

			if result, ok := machinesUpToDateResults[machine.Name]; ok {
				reasons := make([]string, len(result.ConditionMessages))
				copy(reasons, result.ConditionMessages)

				for i := range reasons {
					reasons[i] = "* " + reasons[i]
				}

				message = strings.Join(reasons, "\n")
			}

			conditions.Set(machine, metav1.Condition{
				Type:    clusterv1.MachineUpToDateCondition,
				Status:  metav1.ConditionFalse,
				Reason:  clusterv1.MachineNotUpToDateReason,
				Message: message,
			})

			continue
		}

		// Check for in-place update in progress.
		if inplace.IsUpdateInProgress(machine) {
			msg := "* In-place update in progress"
			if c := conditions.Get(machine, clusterv1.MachineUpdatingCondition); c != nil && c.Status == metav1.ConditionTrue && c.Message != "" {
				msg = "* " + c.Message
			}

			conditions.Set(machine, metav1.Condition{
				Type:    clusterv1.MachineUpToDateCondition,
				Status:  metav1.ConditionFalse,
				Reason:  clusterv1.MachineUpToDateUpdatingReason,
				Message: msg,
			})

			continue
		}

		conditions.Set(machine, metav1.Condition{
			Type:   clusterv1.MachineUpToDateCondition,
			Status: metav1.ConditionTrue,
			Reason: clusterv1.MachineUpToDateReason,
		})
	}
}

type setConditionsToUnknownInput struct {
	ControlPlane                        *rke2.ControlPlane
	Overwrite                           bool
	EtcdClusterHealthyReason            string
	ControlPlaneComponentsHealthyReason string
	StaticPodReason                     string
	EtcdMemberHealthyReason             string
	Message                             string
}

func setConditionsToUnknown(input setConditionsToUnknownInput) {
	// Note: We are not checking if conditions on the Machines are already set, we just check the RCP conditions instead.
	// This means if Overwrite is set to false, we only set the EtcdMemberHealthy condition if the EtcdClusterHealthy condition is not set.
	// The same applies to ControlPlaneComponentsHealthy and the control plane component conditions on the Machines.
	etcdClusterHealthySet := conditions.Has(input.ControlPlane.RCP, controlplanev1.RKE2ControlPlaneEtcdClusterHealthyCondition)
	controlPlaneComponentsHealthySet := conditions.Has(input.ControlPlane.RCP, controlplanev1.RKE2ControlPlaneControlPlaneComponentsHealthyCondition)

	if input.Overwrite || !etcdClusterHealthySet {
		conditions.Set(input.ControlPlane.RCP, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneEtcdClusterHealthyCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  input.EtcdClusterHealthyReason,
			Message: input.Message,
		})

		for _, machine := range input.ControlPlane.Machines {
			if input.ControlPlane.IsEtcdManaged() {
				conditions.Set(machine, metav1.Condition{
					Type:    controlplanev1.RKE2ControlPlaneMachineEtcdMemberHealthyCondition,
					Status:  metav1.ConditionUnknown,
					Reason:  input.EtcdMemberHealthyReason,
					Message: input.Message,
				})
			}
		}
	}

	if input.Overwrite || !controlPlaneComponentsHealthySet {
		conditions.Set(input.ControlPlane.RCP, metav1.Condition{
			Type:    controlplanev1.RKE2ControlPlaneControlPlaneComponentsHealthyCondition,
			Status:  metav1.ConditionUnknown,
			Reason:  input.ControlPlaneComponentsHealthyReason,
			Message: input.Message,
		})
	}
}