feat: execute k8s and os plans sequentially node by node

Assemble two plans (k8s and os) to make them execute sequentially in a
node-by-node fashion by manipulating node labels.

Signed-off-by: Zespre Chang <[email protected]>

Author: starbops

api/v1beta1/upgradeplan_types.go

@@ -115,6 +115,10 @@ type UpgradePlanSpec struct {
 	// +kubebuilder:default:=automatic
 	// +kubebuilder:validation:Enum:=automatic;interactive
 	Mode *string `json:"mode,omitempty"`
+
+	// skipOSUpgrade indicates whether to skip the operating system upgrade. Default to "false" (OS will be upgraded).
+	// +optional
+	SkipOSUpgrade *bool `json:"skipOSUpgrade,omitempty"`
 }
 
 // UpgradePlanStatus defines the observed state of UpgradePlan.

api/v1beta1/zz_generated.deepcopy.go

@@ -70,6 +70,10 @@ spec:
                 - automatic
                 - interactive
                 type: string
+              skipOSUpgrade:
+                description: skipOSUpgrade indicates whether to skip the operating
+                  system upgrade. Default to "false" (OS will be upgraded).
+                type: boolean
               upgrade:
                 description: |-
                   upgrade can be specified to opt for any other specific upgrade image. If not provided, the version resource name is used.

config/crd/bases/management.harvesterhci.io_upgradeplans.yaml

@@ -11,6 +11,8 @@ rules:
   verbs:
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - ""

config/rbac/role.yaml

@@ -48,6 +48,7 @@ type reconcileFuncs func(context.Context, *batchv1.Job) error
 
 // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch
 // +kubebuilder:rbac:groups=batch,resources=jobs/status,verbs=get
+// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=management.harvesterhci.io,resources=upgradeplans/status,verbs=get;update;patch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
@@ -60,7 +61,7 @@ type reconcileFuncs func(context.Context, *batchv1.Job) error
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	r.Log.V(1).Info("reconciling job")
+	r.Log.V(2).Info("reconciling job")
 
 	var job batchv1.Job
 	if err := r.Get(ctx, req.NamespacedName, &job); err != nil {
@@ -78,7 +79,10 @@ func (r *JobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
 		return ctrl.Result{}, nil
 	}
 
-	reconcilers := []reconcileFuncs{r.nodeUpgradeStatusUpdate}
+	reconcilers := []reconcileFuncs{
+		r.nodeUpgradeStatusUpdate,
+		r.nodeLabelUpdate,
+	}
 
 	for _, reconciler := range reconcilers {
 		if err := reconciler(ctx, jobCopy); err != nil {
@@ -138,6 +142,55 @@ func (r *JobReconciler) nodeUpgradeStatusUpdate(ctx context.Context, job *batchv
 	return nil
 }
 
+func (r *JobReconciler) nodeLabelUpdate(ctx context.Context, job *batchv1.Job) error {
+	r.Log.V(1).Info("node label update")
+
+	upgradePlanName, ok := job.Labels[upgradeplan.HarvesterUpgradePlanLabel]
+	if !ok {
+		return fmt.Errorf("label %s not found", upgradeplan.HarvesterUpgradePlanLabel)
+	}
+	upgradeComponent, ok := job.Labels[upgradeplan.HarvesterUpgradeComponentLabel]
+	if !ok {
+		return fmt.Errorf("label %s not found", upgradeplan.HarvesterUpgradeComponentLabel)
+	}
+	nodeName, ok := job.Labels[nodeLabel]
+	if !ok {
+		return fmt.Errorf("label %s not found", nodeLabel)
+	}
+
+	if upgradeComponent != upgradeplan.NodeComponent {
+		return nil
+	}
+
+	finished, success := isJobFinished(job)
+
+	// Job still running or failed
+	if !finished || !success {
+		return nil
+	}
+
+	var node corev1.Node
+	if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
+		return err
+	}
+
+	var upgradePlan managementv1beta1.UpgradePlan
+	if err := r.Get(ctx, types.NamespacedName{Name: upgradePlanName}, &upgradePlan); err != nil {
+		return err
+	}
+
+	// Nudge the node to the next upgrade state once the Job has finished successfully
+	if err := updateNodeLabel(&node, &upgradePlan); err != nil {
+		return err
+	}
+
+	if err := r.Update(ctx, &node); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func isHarvesterUpgradePlanJobs(job *batchv1.Job) bool {
 	if job.Labels == nil {
 		return false
@@ -158,43 +211,47 @@ func isHarvesterUpgradePlanJobs(job *batchv1.Job) bool {
 	return true
 }
 
-func defaultStateFor(component string) string {
-	switch component {
-	case upgradeplan.PrepareComponent:
+func defaultStateFor(component, t string) string {
+	switch {
+	case component == upgradeplan.PrepareComponent:
 		return managementv1beta1.NodeStateImagePreloading
-	case upgradeplan.NodeComponent:
+	case component == upgradeplan.NodeComponent && t == upgradeplan.NodeUpgradeTypeKubernetes:
 		return managementv1beta1.NodeStateKubernetesUpgrading
+	case component == upgradeplan.NodeComponent && t == upgradeplan.NodeUpgradeTypeOS:
+		return managementv1beta1.NodeStateOSUpgrading
 	default:
 		return ""
 	}
 }
 
-func successStateFor(component string) string {
-	switch component {
-	case upgradeplan.PrepareComponent:
+func successStateFor(component, t string) string {
+	switch {
+	case component == upgradeplan.PrepareComponent:
 		return managementv1beta1.NodeStateImagePreloaded
-	case upgradeplan.NodeComponent:
+	case component == upgradeplan.NodeComponent && t == upgradeplan.NodeUpgradeTypeKubernetes:
 		return managementv1beta1.NodeStateKubernetesUpgraded
+	case component == upgradeplan.NodeComponent && t == upgradeplan.NodeUpgradeTypeOS:
+		return managementv1beta1.NodeStateOSUpgraded
 	default:
 		return ""
 	}
 }
 
-func failureStateFor(component string) string {
-	switch component {
-	case upgradeplan.PrepareComponent:
+func failureStateFor(component, t string) string {
+	switch {
+	case component == upgradeplan.PrepareComponent:
 		return managementv1beta1.NodeStateImagePreloadFailed
-	case upgradeplan.NodeComponent:
+	case component == upgradeplan.NodeComponent && t == upgradeplan.NodeUpgradeTypeKubernetes:
 		return managementv1beta1.NodeStateKubernetesUpgradeFailed
+	case component == upgradeplan.NodeComponent && t == upgradeplan.NodeUpgradeTypeOS:
+		return managementv1beta1.NodeStateOSUpgradeFailed
 	default:
 		return ""
 	}
 }
 
 func buildNodeUpgradeStatus(job *batchv1.Job, upgradeComponent string) managementv1beta1.NodeUpgradeStatus {
-	status := managementv1beta1.NodeUpgradeStatus{
-		State: defaultStateFor(upgradeComponent),
-	}
+	jobType := job.Labels[upgradeplan.HarvesterNodeUpgradeTypeLabel]
 
 	for _, condition := range job.Status.Conditions {
 		if condition.Status != corev1.ConditionTrue {
@@ -203,16 +260,80 @@ func buildNodeUpgradeStatus(job *batchv1.Job, upgradeComponent string) managemen
 		switch condition.Type {
 		case batchv1.JobComplete:
 			return managementv1beta1.NodeUpgradeStatus{
-				State: successStateFor(upgradeComponent),
+				State: successStateFor(upgradeComponent, jobType),
 			}
 		case batchv1.JobFailed:
 			return managementv1beta1.NodeUpgradeStatus{
-				State:   failureStateFor(upgradeComponent),
+				State:   failureStateFor(upgradeComponent, jobType),
 				Reason:  condition.Reason,
 				Message: condition.Message,
 			}
 		}
 	}
 
-	return status
+	return managementv1beta1.NodeUpgradeStatus{
+		State: defaultStateFor(upgradeComponent, jobType),
+	}
+}
+
+func isJobFinished(job *batchv1.Job) (finished, success bool) {
+	for _, condition := range job.Status.Conditions {
+		if condition.Type == batchv1.JobComplete && condition.Status == corev1.ConditionTrue {
+			finished, success = true, true
+			return
+		}
+		if condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue {
+			finished, success = true, false
+			return
+		}
+	}
+	return
+}
+
+func updateNodeLabel(node *corev1.Node, upgradePlan *managementv1beta1.UpgradePlan) error {
+	if node.Labels == nil {
+		node.Labels = make(map[string]string)
+	}
+
+	nodeUpgradeDesiredStateLabelKey := fmt.Sprintf("%s/%s", upgradeplan.LabelPrefix, upgradePlan.Name)
+	desiredState, ok := node.Labels[nodeUpgradeDesiredStateLabelKey]
+	if !ok {
+		return nil
+	}
+
+	switch desiredState {
+	case upgradeplan.KubernetesUpgradeState:
+		// If the desired upgrade state for the node is KubernetesUpgradeState and the node is already at
+		// NodeStateKubernetesUpgraded, nudge the node to enter OSUpgradeState.
+		currentStatus, ok := upgradePlan.Status.NodeUpgradeStatuses[node.Name]
+		if !ok {
+			return fmt.Errorf("node %s not found in the node upgrade statuses map", node.Name)
+		}
+		if currentStatus.State != managementv1beta1.NodeStateKubernetesUpgraded {
+			return nil
+		}
+
+		if upgradePlan.Spec.SkipOSUpgrade != nil && *upgradePlan.Spec.SkipOSUpgrade {
+			delete(node.Labels, nodeUpgradeDesiredStateLabelKey)
+			return nil
+		}
+
+		node.Labels[nodeUpgradeDesiredStateLabelKey] = upgradeplan.OSUpgradeState
+	case upgradeplan.OSUpgradeState:
+		// If the desired upgrade state for the node is OSUpgradeState and the node is already at
+		// NodeStateOSUpgraded, the node is considered fully upgraded.
+		currentStatus, ok := upgradePlan.Status.NodeUpgradeStatuses[node.Name]
+		if !ok {
+			return fmt.Errorf("node %s not found in the node upgrade statuses map", node.Name)
+		}
+		if currentStatus.State != managementv1beta1.NodeStateOSUpgraded {
+			return nil
+		}
+
+		delete(node.Labels, nodeUpgradeDesiredStateLabelKey)
+	default:
+		return fmt.Errorf("unrecognized %s label value: %s", nodeUpgradeDesiredStateLabelKey, desiredState)
+	}
+
+	return nil
 }

internal/controller/job_controller.go

@@ -68,7 +68,7 @@ type UpgradePlanReconciler struct {
 // For more details, check Reconcile and its Result here:
 // - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *UpgradePlanReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	r.Log.V(1).Info("reconciling upgradeplan")
+	r.Log.V(2).Info("reconciling upgradeplan")
 
 	var upgradePlan managementv1beta1.UpgradePlan
 	if err := r.Get(ctx, req.NamespacedName, &upgradePlan); err != nil {

internal/controller/upgradeplan_controller.go

@@ -984,6 +984,10 @@ pause_all_charts() {
   done
 }
 
+disable_cluster_version_management() {
+  kubectl annotate clusters.management local rancher.io/imported-cluster-version-management="false" --overwrite=true
+}
+
 skip_restart_rancher_system_agent() {
   # to prevent rke2-server/agent from restarting during the rancher upgrade.
   # by adding an env var to temporarily make rancher-system-agent on each node skip restarting rke2-server/agent.
@@ -1243,6 +1247,7 @@ detect_repo
 detect_upgrade
 pre_upgrade_manifest
 pause_all_charts
+disable_cluster_version_management
 skip_restart_rancher_system_agent
 upgrade_rancher
 patch_local_cluster_details