Merge pull request #40 from FrankYang0529/HARV-6584

fix: add patch local provisioing cluster status for rancher v2.9.x

Author: bk201

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.7.0
 	golang.org/x/crypto v0.14.0
+	golang.org/x/mod v0.8.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.3

go.sum

@@ -2172,6 +2172,7 @@ golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/plan/bootstrap.go

@@ -3,9 +3,9 @@ package plan
 import (
 	"context"
 	"fmt"
-	"strings"
 
 	"github.com/rancher/system-agent/pkg/applyinator"
+	"golang.org/x/mod/semver"
 
 	"github.com/rancher/rancherd/pkg/cacerts"
 	"github.com/rancher/rancherd/pkg/config"
@@ -131,17 +131,31 @@ func (p *plan) addInstructions(cfg *config.Config, dataDir string) error {
 		return err
 	}
 
+	// Above Rancher v2.9.x, we cannot patch provisioing cluster with empty rkeConfig,
+	// so we need to delete the webhook validation configuration.
+	if semver.Compare(cfg.RancherVersion, "v2.9.0") >= 0 {
+		if err := p.addInstruction(rancher.ToDeleteRancherWebhookValidationConfiguration(k8sVersion)); err != nil {
+			return err
+		}
+	}
+
 	if err := p.addInstruction(resources.ToInstruction(cfg.RancherInstallerImage, cfg.SystemDefaultRegistry, k8sVersion, dataDir)); err != nil {
 		return err
 	}
 
-	// currently instruction is only needed for v2.8.x
-	if strings.HasPrefix(cfg.RancherVersion, "v2.8") {
+	// currently instruction is needed for version above v2.8.x
+	if semver.Compare(cfg.RancherVersion, "v2.8.0") >= 0 {
 		if err := p.addInstruction(rancher.PatchLocalProvisioningClusterStatus(cfg.RancherInstallerImage, cfg.SystemDefaultRegistry, k8sVersion)); err != nil {
 			return err
 		}
 	}
 
+	if semver.Compare(cfg.RancherVersion, "v2.9.0") >= 0 {
+		if err := p.addInstruction(rancher.ToRestartRancherWebhookInstruction(k8sVersion)); err != nil {
+			return err
+		}
+	}
+
 	if err := p.addInstruction(rancher.ToWaitSUCInstruction(cfg.RancherInstallerImage, cfg.SystemDefaultRegistry, k8sVersion)); err != nil {
 		return err
 	}

pkg/rancher/wait.go

@@ -124,6 +124,34 @@ func ToScaleUpFleetControllerInstruction(_, _, k8sVersion string) (*applyinator.
 	}, nil
 }
 
+func ToDeleteRancherWebhookValidationConfiguration(k8sVersion string) (*applyinator.Instruction, error) {
+	cmd, err := self.Self()
+	if err != nil {
+		return nil, fmt.Errorf("resolving location of %s: %w", os.Args[0], err)
+	}
+	return &applyinator.Instruction{
+		Name:       "delete-rancher-webhook-validation-configuration",
+		SaveOutput: true,
+		Args:       []string{"retry", kubectl.Command(k8sVersion), "delete", "validatingwebhookconfiguration", "rancher.cattle.io"},
+		Env:        kubectl.Env(k8sVersion),
+		Command:    cmd,
+	}, nil
+}
+
+func ToRestartRancherWebhookInstruction(k8sVersion string) (*applyinator.Instruction, error) {
+	cmd, err := self.Self()
+	if err != nil {
+		return nil, fmt.Errorf("resolving location of %s: %w", os.Args[0], err)
+	}
+	return &applyinator.Instruction{
+		Name:       "wait-rancher-webhook",
+		SaveOutput: true,
+		Args:       []string{"retry", kubectl.Command(k8sVersion), "-n", "cattle-system", "rollout", "restart", "deploy/rancher-webhook"},
+		Env:        kubectl.Env(k8sVersion),
+		Command:    cmd,
+	}, nil
+}
+
 // Needs to patch status subresource
 // k patch cluster.provisioning local -n fleet-local --subresource=status --type=merge --patch '{"status":{"fleetWorkspaceName": "fleet-local"}}'
 func PatchLocalProvisioningClusterStatus(_, _, k8sVersion string) (*applyinator.Instruction, error) {
@@ -132,7 +160,7 @@ func PatchLocalProvisioningClusterStatus(_, _, k8sVersion string) (*applyinator.
 		return nil, fmt.Errorf("resolving location of %s: %w", os.Args[0], err)
 	}
 	return &applyinator.Instruction{
-		Name:       "wait-suc-plan-resolved",
+		Name:       "patch-provisioning-cluster-status",
 		SaveOutput: true,
 		Args:       []string{"retry", kubectl.Command(k8sVersion), "-n", "fleet-local", "patch", "cluster.provisioning", "local", "--subresource=status", "--type=merge", "--patch", "{\"status\":{\"fleetWorkspaceName\": \"fleet-local\"}}"},
 		Env:        kubectl.Env(k8sVersion),

pkg/resources/resources.go

@@ -107,9 +107,8 @@ func ToBootstrapFile(config *config.Config, path string) (*applyinator.File, err
 			},
 			"spec": map[string]interface{}{
 				"kubernetesVersion": k8sVersion,
-				"rkeConfig": map[string]interface{}{
-					"controlPlaneConfig": config.ConfigValues,
-				},
+				// Rancher needs a non-null rkeConfig to apply system-upgrade-controller managed chart.
+				"rkeConfig": map[string]interface{}{},
 			},
 		},
 	}, v1.GenericMap{