Merge pull request #147 from richardcase/136_registration_method

feat: support different registration methods

Author: richardcase

Makefile

@@ -99,7 +99,7 @@ GOLANGCI_LINT_VER := v1.49.0
 GOLANGCI_LINT_BIN := golangci-lint
 GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN))
 
-GINKGO_VER := v2.9.1
+GINKGO_VER := v2.9.4
 GINKGO_BIN := ginkgo
 GINKGO := $(abspath $(TOOLS_BIN_DIR)/$(GINKGO_BIN)-$(GINKGO_VER))
 GINKGO_PKG := github.com/onsi/ginkgo/v2/ginkgo

bootstrap/api/v1alpha1/rke2config_webhook.go

@@ -23,11 +23,16 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
-var cannotUseWithIgnition = fmt.Sprintf("not supported when spec.format is set to %q", Ignition)
+var (
+	cannotUseWithIgnition = fmt.Sprintf("not supported when spec.format is set to %q", Ignition)
+	rke2configlog         = logf.Log.WithName("rke2config-resource")
+)
 
 // SetupWebhookWithManager sets up and registers the webhook with the manager.
 func (r *RKE2Config) SetupWebhookWithManager(mgr ctrl.Manager) error {
@@ -58,12 +63,32 @@ var _ webhook.Validator = &RKE2Config{}
 
 // ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
 func (r *RKE2Config) ValidateCreate() error {
-	return ValidateRKE2ConfigSpec(r.Name, &r.Spec)
+	rke2configlog.Info("RKE2Config validate create", "rke2config", klog.KObj(r))
+
+	var allErrs field.ErrorList
+
+	allErrs = append(allErrs, ValidateRKE2ConfigSpec(r.Name, &r.Spec)...)
+
+	if len(allErrs) == 0 {
+		return nil
+	}
+
+	return apierrors.NewInvalid(GroupVersion.WithKind("RKE2Config").GroupKind(), r.Name, allErrs)
 }
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
 func (r *RKE2Config) ValidateUpdate(_ runtime.Object) error {
-	return ValidateRKE2ConfigSpec(r.Name, &r.Spec)
+	rke2configlog.Info("RKE2Config validate update", "rke2config", klog.KObj(r))
+
+	var allErrs field.ErrorList
+
+	allErrs = append(allErrs, ValidateRKE2ConfigSpec(r.Name, &r.Spec)...)
+
+	if len(allErrs) == 0 {
+		return nil
+	}
+
+	return apierrors.NewInvalid(GroupVersion.WithKind("RKE2Config").GroupKind(), r.Name, allErrs)
 }
 
 // ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
@@ -72,14 +97,14 @@ func (r *RKE2Config) ValidateDelete() error {
 }
 
 // ValidateRKE2ConfigSpec validates the RKE2ConfigSpec.
-func ValidateRKE2ConfigSpec(name string, spec *RKE2ConfigSpec) error {
+func ValidateRKE2ConfigSpec(name string, spec *RKE2ConfigSpec) field.ErrorList {
 	allErrs := spec.validate(field.NewPath("spec"))
 
 	if len(allErrs) == 0 {
 		return nil
 	}
 
-	return apierrors.NewInvalid(GroupVersion.WithKind("RKE2Config").GroupKind(), name, allErrs)
+	return allErrs
 }
 
 func (s *RKE2ConfigSpec) validate(pathPrefix *field.Path) field.ErrorList {

controlplane/api/v1alpha1/rke2controlplane_types.go

@@ -60,6 +60,17 @@ type RKE2ControlPlaneSpec struct {
 	// NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`
 	// +optional
 	NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"`
+
+	// RegistrationMethod is the method to use for registering nodes into the RKE2 cluster.
+	// +kubebuilder:validation:Enum=internal-first;internal-only-ips;external-only-ips;address
+	// +kubebuilder:default=internal-first
+	// +optional
+	RegistrationMethod RegistrationMethod `json:"registrationMethod"`
+
+	// RegistrationAddress is an explicit address to use when registering a node. This is required if
+	// the registration type is "address". Its for scenarios where a load-balancer or VIP is used.
+	// +optional
+	RegistrationAddress string `json:"registrationAddress,omitempty"`
 }
 
 // RKE2ServerConfig specifies configuration for the agent nodes.

controlplane/api/v1alpha1/rke2controlplane_webhook.go

@@ -17,9 +17,12 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"errors"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -44,6 +47,16 @@ var _ webhook.Defaulter = &RKE2ControlPlane{}
 // Default implements webhook.Defaulter so a webhook will be registered for the type.
 func (r *RKE2ControlPlane) Default() {
 	bootstrapv1.DefaultRKE2ConfigSpec(&r.Spec.RKE2ConfigSpec)
+
+	if r.Spec.RegistrationMethod == RegistrationMethodAddress {
+		if r.Spec.ServerConfig.AdvertiseAddress == "" {
+			rke2controlplanelog.Info("setting advertise address from registration address",
+				"rke2-control-plane", klog.KObj(r),
+				"address", r.Spec.RegistrationAddress)
+
+			r.Spec.ServerConfig.AdvertiseAddress = r.Spec.RegistrationAddress
+		}
+	}
 }
 
 //+kubebuilder:webhook:path=/validate-controlplane-cluster-x-k8s-io-v1alpha1-rke2controlplane,mutating=false,failurePolicy=fail,sideEffects=None,groups=controlplane.cluster.x-k8s.io,resources=rke2controlplanes,verbs=create;update,versions=v1alpha1,name=vrke2controlplane.kb.io,admissionReviewVersions=v1
@@ -52,20 +65,46 @@ var _ webhook.Validator = &RKE2ControlPlane{}
 
 // ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
 func (r *RKE2ControlPlane) ValidateCreate() error {
-	if bootstrapv1.ValidateRKE2ConfigSpec(r.Name, &r.Spec.RKE2ConfigSpec) != nil {
-		return bootstrapv1.ValidateRKE2ConfigSpec(r.Name, &r.Spec.RKE2ConfigSpec)
+	rke2controlplanelog.Info("RKE2ControlPlane validate create", "control-plane", klog.KObj(r))
+
+	var allErrs field.ErrorList
+
+	allErrs = append(allErrs, bootstrapv1.ValidateRKE2ConfigSpec(r.Name, &r.Spec.RKE2ConfigSpec)...)
+	allErrs = append(allErrs, r.validateCNI()...)
+	allErrs = append(allErrs, r.validateRegistrationMethod()...)
+
+	if len(allErrs) == 0 {
+		return nil
 	}
 
-	return ValidateRKE2ControlPlaneSpec(r.Name, &r.Spec)
+	return apierrors.NewInvalid(GroupVersion.WithKind("RKE2ControlPlane").GroupKind(), r.Name, allErrs)
 }
 
 // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
 func (r *RKE2ControlPlane) ValidateUpdate(old runtime.Object) error {
-	if bootstrapv1.ValidateRKE2ConfigSpec(r.Name, &r.Spec.RKE2ConfigSpec) != nil {
-		return bootstrapv1.ValidateRKE2ConfigSpec(r.Name, &r.Spec.RKE2ConfigSpec)
+	oldControlplane, ok := old.(*RKE2ControlPlane)
+	if !ok {
+		return apierrors.NewInvalid(GroupVersion.WithKind("RKE2ControlPlane").GroupKind(), r.Name, field.ErrorList{
+			field.InternalError(nil, errors.New("failed to convert old RKE2ControlPlane to object")),
+		})
 	}
 
-	return ValidateRKE2ControlPlaneSpec(r.Name, &r.Spec)
+	var allErrs field.ErrorList
+
+	allErrs = append(allErrs, bootstrapv1.ValidateRKE2ConfigSpec(r.Name, &r.Spec.RKE2ConfigSpec)...)
+	allErrs = append(allErrs, r.validateCNI()...)
+
+	if r.Spec.RegistrationMethod != oldControlplane.Spec.RegistrationMethod {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec", "registrationMethod"), r.Spec.RegistrationMethod, "field is immutable"),
+		)
+	}
+
+	if len(allErrs) == 0 {
+		return nil
+	}
+
+	return apierrors.NewInvalid(GroupVersion.WithKind("RKE2ControlPlane").GroupKind(), r.Name, allErrs)
 }
 
 // ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
@@ -75,24 +114,27 @@ func (r *RKE2ControlPlane) ValidateDelete() error {
 	return nil
 }
 
-// ValidateRKE2ControlPlaneSpec validates the RKE2ControlPlaneSpec Object.
-func ValidateRKE2ControlPlaneSpec(name string, spec *RKE2ControlPlaneSpec) error {
-	allErrs := spec.validate()
-	if len(allErrs) == 0 {
-		return nil
+func (r *RKE2ControlPlane) validateCNI() field.ErrorList {
+	var allErrs field.ErrorList
+
+	if r.Spec.ServerConfig.CNIMultusEnable && r.Spec.ServerConfig.CNI == "" {
+		allErrs = append(allErrs,
+			field.Invalid(field.NewPath("spec", "serverConfig", "cni"),
+				r.Spec.ServerConfig.CNI, "must be specified when cniMultusEnable is true"))
 	}
 
-	return apierrors.NewInvalid(GroupVersion.WithKind("RKE2ControlPlane").GroupKind(), name, allErrs)
+	return allErrs
 }
 
-// validate validates the RKE2ControlPlaneSpec Object.
-func (s *RKE2ControlPlaneSpec) validate() field.ErrorList {
+func (r *RKE2ControlPlane) validateRegistrationMethod() field.ErrorList {
 	var allErrs field.ErrorList
 
-	if s.ServerConfig.CNIMultusEnable && s.ServerConfig.CNI == "" {
-		allErrs = append(allErrs,
-			field.Invalid(field.NewPath("spec", "serverConfig", "cni"),
-				s.ServerConfig.CNI, "must be specified when cniMultusEnable is true"))
+	if r.Spec.RegistrationMethod == RegistrationMethodAddress {
+		if r.Spec.RegistrationAddress == "" {
+			allErrs = append(allErrs,
+				field.Invalid(field.NewPath("spec.registrationAddress"),
+					r.Spec.RegistrationAddress, "registrationAddress must be supplied when using registration method 'address'"))
+		}
 	}
 
 	return allErrs

controlplane/api/v1alpha1/types.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2023 SUSE.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+// RegistrationMethod defines the methods to use for registering a new node in a cluster.
+type RegistrationMethod string
+
+var (
+	// RegistrationMethodFavourInternalIPs is a registration method where the IP address of the control plane
+	// machines are used for registration. For each machine it will check if there is an internal IP address
+	// and will use that. If there is no internal IP address it will use the external IP address if there is one.
+	RegistrationMethodFavourInternalIPs = RegistrationMethod("internal-first")
+	// RegistrationMethodInternalIPs is a registration method where the internal IP address of the control plane
+	// machines are used for registration.
+	RegistrationMethodInternalIPs = RegistrationMethod("internal-only-ips")
+	// RegistrationMethodExternalIPs is a registration method where the external IP address of the control plane
+	// machines are used for registration.
+	RegistrationMethodExternalIPs = RegistrationMethod("external-only-ips")
+	// RegistrationMethodAddress is a registration method where an explicit address supplied at cluster creation
+	// time is used for registration. This is for use in LB or VIP scenarios.
+	RegistrationMethodAddress = RegistrationMethod("address")
+)

controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_rke2controlplanes.yaml

@@ -567,6 +567,21 @@ spec:
                     description: Mirrors are namespace to mirror mapping for all namespaces.
                     type: object
                 type: object
+              registrationAddress:
+                description: RegistrationAddress is an explicit address to use when
+                  registering a node. This is required if the registration type is
+                  "address". Its for scenarios where a load-balancer or VIP is used.
+                type: string
+              registrationMethod:
+                default: internal-first
+                description: RegistrationMethod is the method to use for registering
+                  nodes into the RKE2 cluster.
+                enum:
+                - internal-first
+                - internal-only-ips
+                - external-only-ips
+                - address
+                type: string
               replicas:
                 description: Replicas is the number of replicas for the Control Plane.
                 format: int32

controlplane/internal/controllers/rke2controlplane_controller.go

@@ -47,6 +47,7 @@ import (
 
 	controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1"
 	"github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/kubeconfig"
+	"github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/registration"
 	"github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/rke2"
 	"github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/secret"
 )
@@ -352,15 +353,14 @@ func (r *RKE2ControlPlaneReconciler) updateStatus(ctx context.Context, rcp *cont
 
 	availableCPMachines := readyMachines
 
-	validIPAddresses := []string{}
-
-	for _, machine := range availableCPMachines {
-		ipAddress, err := getIPAddress(*machine)
-		if err != nil {
-			break
-		}
+	registrationmethod, err := registration.NewRegistrationMethod(string(rcp.Spec.RegistrationMethod))
+	if err != nil {
+		return fmt.Errorf("getting node registration method: %w", err)
+	}
 
-		validIPAddresses = append(validIPAddresses, ipAddress)
+	validIPAddresses, err := registrationmethod(rcp, availableCPMachines)
+	if err != nil {
+		return fmt.Errorf("getting registration addresses: %w", err)
 	}
 
 	rcp.Status.AvailableServerIPs = validIPAddresses
@@ -751,24 +751,3 @@ func (r *RKE2ControlPlaneReconciler) ClusterToRKE2ControlPlane(o client.Object)
 
 	return nil
 }
-
-func getIPAddress(machine clusterv1.Machine) (ip string, err error) {
-	for _, address := range machine.Status.Addresses {
-		switch address.Type {
-		case clusterv1.MachineInternalIP:
-			if address.Address != "" {
-				return address.Address, nil
-			}
-		case clusterv1.MachineExternalIP:
-			if address.Address != "" {
-				ip = address.Address
-			}
-		}
-	}
-
-	if ip == "" {
-		err = fmt.Errorf("no IP Address found for machine: %s", machine.Name)
-	}
-
-	return
-}

docs/registration-methods.md

@@ -0,0 +1,58 @@
+# Node Registration Methods
+
+The provider supports multiple methods for registering a new node into the cluster.
+
+## Usage
+
+The method to use is specified on the **RKEControlPlane** within the **spec**. If no method is supplied then the default method of **internal-first** will be used.
+
+> You cannot change the registration method after creation.
+
+An example of using a different method:
+
+```yaml
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
+kind: RKE2ControlPlane
+metadata:
+  name: test1-control-plane
+  namespace: default
+spec:
+  agentConfig:
+    version: v1.26.4+rke2r1
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachineTemplate
+    name: controlplane
+  nodeDrainTimeout: 2m
+  replicas: 3
+  serverConfig:
+    cni: calico
+  registrationMethod: "address"
+  registrationAddress: "172.19.0.3"
+```
+
+## Registration Methods
+
+### internal-first
+
+For each CAPI `Machine` that is used for the control plane,  we take the **internal** ip address from `Machine.status.addresses` if it exists. If there is no **internal** ip for a machine then we will use an **external** address instead. For the ip address found for a machine then we add it to  `RKEControlPlane.status.availableServerIPs`.
+
+The first IP address listed in `RKEControlPlane.status.availableServerIPs` is then used for the join.
+
+### internal-only-ips
+
+For each CAPI `Machine` that is used for the control plane,  we take the **internal** ip address from `Machine.status.addresses` if it exists and then we add it to  `RKEControlPlane.status.availableServerIPs`.
+
+The first IP address listed in `RKEControlPlane.status.availableServerIPs` is then used for the join.
+
+### external-only-ips
+
+For each CAPI `Machine` that is used for the control plane,  we take the **external** ip address from `Machine.status.addresses` if it exists and then we add it to  `RKEControlPlane.status.availableServerIPs`.
+
+The first IP address listed in `RKEControlPlane.status.availableServerIPs` is then used for the join.
+
+### address
+
+For this method you must supply an address in the control plane spec (i.e. `RKE2ControlPlane.spec.registrationAddress`). This address is then used for the join.
+
+With this method its expected that you have a load balancer / VIP solution sitting in front of all the control plane machines and all the join requests will be routed via this.