Add support for externally managed members (#1214)

Author: CaptainIRS

api/core/v1alpha1/constants.go

@@ -39,9 +39,6 @@ const (
 	// This value will only be effective if etcd-druid is not configured with auto-reconciliation of Etcd resource specification via
 	// --enable-etcd-spec-auto-reconcile CLI flag.
 	DruidOperationReconcile = "reconcile"
-	// DisableEtcdRuntimeComponentCreationAnnotation is an annotation set by an operator to disable the creation and management of
-	// runtime components of the etcd cluster such as pods, PVCs, leases, RBAC resources, PDBs, services, etc.
-	DisableEtcdRuntimeComponentCreationAnnotation = "druid.gardener.cloud/disable-etcd-runtime-component-creation"
 )
 
 // Compaction Job/Pod reasons that are used to set the reason for a pod condition in the status of an Etcd resource.

api/core/v1alpha1/crds/druid.gardener.cloud_etcds.yaml

@@ -668,13 +668,22 @@ spec:
                   snapshotCount:
                     description: |-
                       SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction.
-                      More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention
+                      More info: https://etcd.io/docs/v3.5/op-guide/maintenance/#raft-log-retention
                     format: int64
                     type: integer
                   wrapperPort:
                     format: int32
                     type: integer
                 type: object
+              externallyManagedMemberAddresses:
+                description: |-
+                  ExternallyManagedMemberAddresses defines the list of addresses of externally managed etcd members. Specifying this
+                  will disable components that are involved in management of etcd members like Pods, Services and PDBs.
+                  Allowed values include: IPv4/IPv6 addresses and hostnames. Protocol or port shall not be specified.
+                items:
+                  type: string
+                type: array
+                x-kubernetes-list-type: set
               labels:
                 additionalProperties:
                   type: string
@@ -1901,10 +1910,18 @@ spec:
             - replicas
             type: object
             x-kubernetes-validations:
-            - message: etcd.spec.storageClass is an immutable field.
+            - message: etcd.spec.storageClass field cannot be added or removed dynamically.
               rule: has(oldSelf.storageClass) ==  has(self.storageClass)
-            - message: etcd.spec.volumeClaimTemplate is an immutable field.
+            - message: etcd.spec.volumeClaimTemplate field cannot be added or removed
+                dynamically.
               rule: has(oldSelf.volumeClaimTemplate) == has(self.volumeClaimTemplate)
+            - message: etcd.spec.replicas must be equal to length of etcd.spec.externallyManagedMemberAddresses
+                when etcd.spec.externallyManagedMemberAddresses is specified.
+              rule: 'has(self.externallyManagedMemberAddresses) ? self.replicas ==
+                self.externallyManagedMemberAddresses.size() : true'
+            - message: etcd.spec.externallyManagedMemberAddresses field cannot be
+                added or removed dynamically.
+              rule: has(oldSelf.externallyManagedMemberAddresses) == has(self.externallyManagedMemberAddresses)
           status:
             description: EtcdStatus defines the observed state of Etcd.
             properties:

api/core/v1alpha1/crds/druid.gardener.cloud_etcds_without_cel.yaml

@@ -601,13 +601,22 @@ spec:
                     snapshotCount:
                       description: |-
                         SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction.
-                        More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention
+                        More info: https://etcd.io/docs/v3.5/op-guide/maintenance/#raft-log-retention
                       format: int64
                       type: integer
                     wrapperPort:
                       format: int32
                       type: integer
                   type: object
+                externallyManagedMemberAddresses:
+                  description: |-
+                    ExternallyManagedMemberAddresses defines the list of addresses of externally managed etcd members. Specifying this
+                    will disable components that are involved in management of etcd members like Pods, Services and PDBs.
+                    Allowed values include: IPv4/IPv6 addresses and hostnames. Protocol or port shall not be specified.
+                  items:
+                    type: string
+                  type: array
+                  x-kubernetes-list-type: set
                 labels:
                   additionalProperties:
                     type: string

api/core/v1alpha1/etcd.go

@@ -220,7 +220,7 @@ type EtcdConfig struct {
 	// +optional
 	Quota *resource.Quantity `json:"quota,omitempty"`
 	// SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction.
-	// More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention
+	// More info: https://etcd.io/docs/v3.5/op-guide/maintenance/#raft-log-retention
 	// +optional
 	SnapshotCount *int64 `json:"snapshotCount,omitempty"`
 	// EnableGRPCGateway enables the gRPC-Gateway proxy for etcd.
@@ -310,8 +310,10 @@ type SchedulingConstraints struct {
 }
 
 // EtcdSpec defines the desired state of Etcd
-// +kubebuilder:validation:XValidation:message="etcd.spec.storageClass is an immutable field.",rule="has(oldSelf.storageClass) ==  has(self.storageClass)"
-// +kubebuilder:validation:XValidation:message="etcd.spec.volumeClaimTemplate is an immutable field.",rule="has(oldSelf.volumeClaimTemplate) == has(self.volumeClaimTemplate)"
+// +kubebuilder:validation:XValidation:message="etcd.spec.storageClass field cannot be added or removed dynamically.",rule="has(oldSelf.storageClass) ==  has(self.storageClass)"
+// +kubebuilder:validation:XValidation:message="etcd.spec.volumeClaimTemplate field cannot be added or removed dynamically.",rule="has(oldSelf.volumeClaimTemplate) == has(self.volumeClaimTemplate)"
+// +kubebuilder:validation:XValidation:message="etcd.spec.replicas must be equal to length of etcd.spec.externallyManagedMemberAddresses when etcd.spec.externallyManagedMemberAddresses is specified.",rule="has(self.externallyManagedMemberAddresses) ? self.replicas == self.externallyManagedMemberAddresses.size() : true"
+// +kubebuilder:validation:XValidation:message="etcd.spec.externallyManagedMemberAddresses field cannot be added or removed dynamically.",rule="has(oldSelf.externallyManagedMemberAddresses) == has(self.externallyManagedMemberAddresses)"
 type EtcdSpec struct {
 	// selector is a label query over pods that should match the replica count.
 	// It must match the pod template's labels.
@@ -358,6 +360,12 @@ type EtcdSpec struct {
 	// run as root. By default, they run as non-root with user 'nobody'.
 	// +optional
 	RunAsRoot *bool `json:"runAsRoot,omitempty"`
+	// ExternallyManagedMemberAddresses defines the list of addresses of externally managed etcd members. Specifying this
+	// will disable components that are involved in management of etcd members like Pods, Services and PDBs.
+	// Allowed values include: IPv4/IPv6 addresses and hostnames. Protocol or port shall not be specified.
+	// +optional
+	// +listType=set
+	ExternallyManagedMemberAddresses []string `json:"externallyManagedMemberAddresses,omitempty"`
 }
 
 // CrossVersionObjectReference contains enough information to let you identify the referred resource.

api/core/v1alpha1/helper.go

@@ -5,7 +5,9 @@
 package v1alpha1
 
 import (
+	"crypto/rand"
 	"fmt"
+	"math/big"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -24,6 +26,22 @@ func GetClientServiceName(etcdObjMeta metav1.ObjectMeta) string {
 	return fmt.Sprintf("%s-client", etcdObjMeta.Name)
 }
 
+// GetClientHostname returns the hostname of the client endpoint for the Etcd cluster. This is the client service hostname when the Etcd members
+// are managed by etcd-druid, else it is a randomly selected member address out of the externally managed member addresses.
+func GetClientHostname(etcd *Etcd) string {
+	if ArePodsManagedByEtcdDruid(etcd) {
+		return fmt.Sprintf("%s.%s.svc", GetClientServiceName(etcd.ObjectMeta), etcd.Namespace)
+	} else {
+		n, err := rand.Int(rand.Reader, big.NewInt(int64(len(etcd.Spec.ExternallyManagedMemberAddresses))))
+		if err != nil {
+			// Fallback to first member address in case of an error
+			return etcd.Spec.ExternallyManagedMemberAddresses[0]
+		}
+		randomIndex := int(n.Int64())
+		return etcd.Spec.ExternallyManagedMemberAddresses[randomIndex]
+	}
+}
+
 // GetServiceAccountName returns the service account name for the Etcd.
 func GetServiceAccountName(etcdObjMeta metav1.ObjectMeta) string {
 	return etcdObjMeta.Name
@@ -44,6 +62,11 @@ func GetOrdinalPodName(etcdObjMeta metav1.ObjectMeta, ordinal int) string {
 	return fmt.Sprintf("%s-%d", etcdObjMeta.Name, ordinal)
 }
 
+// GetMemberNameFromAddress returns the name of the etcd member based on the address.
+func GetMemberNameFromAddress(etcdObjMeta metav1.ObjectMeta, memberAddress string) string {
+	return fmt.Sprintf("%s-%s", etcdObjMeta.Name, memberAddress)
+}
+
 // GetAllPodNames returns the names of all pods for the Etcd.
 func GetAllPodNames(etcdObjMeta metav1.ObjectMeta, replicas int32) []string {
 	podNames := make([]string, replicas)
@@ -54,12 +77,17 @@ func GetAllPodNames(etcdObjMeta metav1.ObjectMeta, replicas int32) []string {
 }
 
 // GetMemberLeaseNames returns the name of member leases for the Etcd.
-func GetMemberLeaseNames(etcdObjMeta metav1.ObjectMeta, replicas int32) []string {
-	leaseNames := make([]string, replicas)
-	for i := range int(replicas) {
-		leaseNames[i] = fmt.Sprintf("%s-%d", etcdObjMeta.Name, i)
+func GetMemberLeaseNames(etcd *Etcd) []string {
+	if ArePodsManagedByEtcdDruid(etcd) {
+		return GetAllPodNames(etcd.ObjectMeta, etcd.Spec.Replicas)
+	} else {
+		memberAddresses := etcd.Spec.ExternallyManagedMemberAddresses
+		memberNames := make([]string, len(memberAddresses))
+		for i, memberAddress := range memberAddresses {
+			memberNames[i] = GetMemberNameFromAddress(etcd.ObjectMeta, memberAddress)
+		}
+		return memberNames
 	}
-	return leaseNames
 }
 
 // GetPodDisruptionBudgetName returns the name of the pod disruption budget for the Etcd.
@@ -166,7 +194,7 @@ func RemoveOperationAnnotation(etcdObjMeta metav1.ObjectMeta) {
 	delete(etcdObjMeta.Annotations, GardenerOperationAnnotation)
 }
 
-// IsEtcdRuntimeComponentCreationEnabled checks if the creation of runtime components is enabled for an Etcd resource.
-func IsEtcdRuntimeComponentCreationEnabled(etcdObjMeta metav1.ObjectMeta) bool {
-	return !metav1.HasAnnotation(etcdObjMeta, DisableEtcdRuntimeComponentCreationAnnotation)
+// ArePodsManagedByEtcdDruid checks if the management of pods is handled by etcd-druid for an Etcd resource.
+func ArePodsManagedByEtcdDruid(etcd *Etcd) bool {
+	return len(etcd.Spec.ExternallyManagedMemberAddresses) == 0
 }