feat: support more container/pod fields overrides, enchance default security context and probes

Makefile

@@ -397,7 +397,8 @@ check-crd-compat: crd-schema-checker ## Check CRD backward compatibility against
 			rm -f "$$BASELINE"; \
 			continue; \
 		fi; \
-		if ! $(CRD_SCHEMA_CHECKER) check-manifests --existing-crd-filename="$$BASELINE" --new-crd-filename="$$crd"; then \
+		if ! $(CRD_SCHEMA_CHECKER) check-manifests --existing-crd-filename="$$BASELINE" --new-crd-filename="$$crd" \
+			--disabled-validators NoBools,NoMaps; then ## TODO remove it after merge of the violating k8s-owned objects\
 			FAILED=1; \
 		fi; \
 		rm -f "$$BASELINE"; \

api/v1alpha1/common.go

@@ -213,6 +213,14 @@ type PodTemplateSpec struct {
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 
+	// PriorityClassName is the name of the PriorityClass to use for the pod.
+	// +optional
+	PriorityClassName *string `json:"priorityClassName,omitempty"`
+
+	// RuntimeClassName is the name of the RuntimeClass to use for the pod.
+	// +optional
+	RuntimeClassName *string `json:"runtimeClassName,omitempty"`
+
 	// Volumes defines the list of volumes that can be mounted by containers belonging to the pod.
 	// More info: https://kubernetes.io/docs/concepts/storage/volumes
 	// Merged with operator defaults by name; a user volume replaces any operator volume with the same name.
@@ -241,6 +249,16 @@ type PodTemplateSpec struct {
 	// Recommended to be set to "kubernetes.io/hostname"
 	// +optional
 	NodeHostnameKey *string `json:"nodeHostnameKey,omitempty"`
+
+	// InitContainers is the list of init containers to run before the main server container starts.
+	// Merged with operator defaults by name.
+	// with the same name.
+	// +optional
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	// +listType=map
+	// +listMapKey=name
+	InitContainers []corev1.Container `json:"initContainers,omitempty" patchMergeKey:"name" patchStrategy:"merge"`
 }
 
 // ContainerTemplateSpec describes the container configuration overrides for the cluster's containers.
@@ -283,6 +301,14 @@ type ContainerTemplateSpec struct {
 	// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
 	// +optional
 	SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
+
+	// LivenessProbe overrides the operator's default liveness probe.
+	// +optional
+	LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"`
+
+	// ReadinessProbe overrides the operator's default readiness probe.
+	// +optional
+	ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"`
 }
 
 // ClusterTLSSpec defines cluster TLS configuration.

docs/api_reference.md

@@ -156,6 +156,8 @@ ContainerTemplateSpec describes the container configuration overrides for the cl
 | `volumeMounts` | [VolumeMount](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#volumemount-v1-core) array | VolumeMounts is the list of volume mounts for the container.<br />Concatenated with operator-generated mounts. Entries sharing a `mountPath` with an operator<br />mount are merged into a projected volume. | false |  |
 | `env` | [EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#envvar-v1-core) array | Env is the list of environment variables to set in the container.<br />Merged with operator defaults by name. | false |  |
 | `securityContext` | [SecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#securitycontext-v1-core) | SecurityContext defines the security options the container should be run with.<br />Deep-merged with operator defaults via SMP. When nil, operator defaults are preserved.<br />More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ | false |  |
+| `livenessProbe` | [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core) | LivenessProbe overrides the operator's default liveness probe. | false |  |
+| `readinessProbe` | [Probe](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#probe-v1-core) | ReadinessProbe overrides the operator's default readiness probe. | false |  |
 
 Appears in:
 - [ClickHouseClusterSpec](#clickhouseclusterspec)
@@ -330,10 +332,13 @@ PodTemplateSpec describes the pod configuration overrides for the cluster's pods
 | `tolerations` | [Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#toleration-v1-core) array | If specified, the pod's Tolerations. | false |  |
 | `schedulerName` | string | If specified, the pod will be dispatched by specified scheduler.<br />If not specified, the pod will be dispatched by default scheduler. | false |  |
 | `serviceAccountName` | string | ServiceAccountName is the name of the ServiceAccount to use to run this pod.<br />More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | false |  |
+| `priorityClassName` | string | PriorityClassName is the name of the PriorityClass to use for the pod. | false |  |
+| `runtimeClassName` | string | RuntimeClassName is the name of the RuntimeClass to use for the pod. | false |  |
 | `volumes` | [Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#volume-v1-core) array | Volumes defines the list of volumes that can be mounted by containers belonging to the pod.<br />More info: https://kubernetes.io/docs/concepts/storage/volumes<br />Merged with operator defaults by name; a user volume replaces any operator volume with the same name. | false |  |
 | `securityContext` | [PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podsecuritycontext-v1-core) | SecurityContext holds pod-level security attributes and common container settings.<br />Deep-merged with operator defaults via SMP. When nil, operator defaults are preserved. | false |  |
 | `topologyZoneKey` | string | TopologyZoneKey is the key of node labels.<br />Nodes that have a label with this key and identical values are considered to be in the same topology zone.<br />Set it to enable default TopologySpreadConstraints and Affinity rules to spread pods across zones.<br />Recommended to be set to "topology.kubernetes.io/zone" | false |  |
 | `nodeHostnameKey` | string | NodeHostnameKey is the key of node labels.<br />Nodes that have a label with this key and identical values are considered to be on the same node.<br />Set it to enable default AntiAffinity rules to spread replicas from the different shards across nodes.<br />Recommended to be set to "kubernetes.io/hostname" | false |  |
+| `initContainers` | [Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#container-v1-core) array | InitContainers is the list of init containers to run before the main server container starts.<br />Merged with operator defaults by name.<br />with the same name. | false |  |
 
 Appears in:
 - [ClickHouseClusterSpec](#clickhouseclusterspec)

docs/styles/config/vocabularies/ClickHouse/accept.txt

@@ -15,3 +15,4 @@ Env
 ANDed
 boolean
 lts
+liveness

internal/controller/clickhouse/templates.go

@@ -3,7 +3,6 @@ package clickhouse
 import (
 	"fmt"
 	"maps"
-	"net"
 	"path"
 	"strconv"
 
@@ -288,7 +287,7 @@ func generateConfigForSingleReplica(r *clickhouseReconciler, id v1.ClickHouseRep
 func templatePodSpec(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (corev1.PodSpec, error) {
 	cr := r.Cluster
 
-	container, err := templateContainer(r, id)
+	container, err := templateContainer(r)
 	if err != nil {
 		return corev1.PodSpec{}, fmt.Errorf("template container: %w", err)
 	}
@@ -301,6 +300,11 @@ func templatePodSpec(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (corev1
 		DNSPolicy:     corev1.DNSClusterFirst,
 		Volumes:       volumes,
 		Containers:    []corev1.Container{container},
+		SecurityContext: &corev1.PodSecurityContext{
+			FSGroup:    new(controller.DefaultUser),
+			RunAsUser:  new(controller.DefaultUser),
+			RunAsGroup: new(controller.DefaultUser),
+		},
 	}
 
 	if cr.Spec.PodTemplate.TopologyZoneKey != nil && *cr.Spec.PodTemplate.TopologyZoneKey != "" {
@@ -389,35 +393,27 @@ func templatePodSpec(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (corev1
 	return podSpec, nil
 }
 
-func templateContainer(r *clickhouseReconciler, id v1.ClickHouseReplicaID) (corev1.Container, error) {
+func templateContainer(r *clickhouseReconciler) (corev1.Container, error) {
 	cr := r.Cluster
 	protocols := buildProtocols(cr)
 
-	var probeCommand []string
-	if proto, ok := protocols["http"]; ok && proto.Port != 0 {
-		probeCommand = []string{"/bin/bash", "-c", fmt.Sprintf(
-			"wget -qO- http://%s | grep -o Ok.",
-			net.JoinHostPort("127.0.0.1", strconv.Itoa(PortHTTP)),
-		)}
-	} else {
-		probeCommand = []string{"/bin/bash", "-c", fmt.Sprintf(
-			"wget --ca-certificate=%s -qO- https://%s | grep -o Ok.",
-			path.Join(TLSConfigPath, CABundleFilename),
-			net.JoinHostPort(cr.HostnameByID(id), strconv.Itoa(PortHTTPSecure)),
-		)}
-	}
-
 	livenessProbe := controller.DefaultLivenessProbeSettings
 	livenessProbe.ProbeHandler = corev1.ProbeHandler{
-		Exec: &corev1.ExecAction{
-			Command: probeCommand,
+		TCPSocket: &corev1.TCPSocketAction{
+			Port: intstr.FromInt32(PortNative),
 		},
 	}
 
+	if cr.Spec.Settings.TLS.Enabled && cr.Spec.Settings.TLS.Required {
+		livenessProbe.TCPSocket.Port.IntVal = PortNativeSecure
+	}
+
 	readinessProbe := controller.DefaultReadinessProbeSettings
 	readinessProbe.ProbeHandler = corev1.ProbeHandler{
-		Exec: &corev1.ExecAction{
-			Command: probeCommand,
+		HTTPGet: &corev1.HTTPGetAction{
+			Path:   "/ping",
+			Port:   intstr.FromInt32(PortInterserver),
+			Scheme: corev1.URISchemeHTTP,
 		},
 	}
 

internal/controller/constants.go

@@ -10,6 +10,7 @@ const (
 	RequeueOnRefreshTimeout       = time.Second
 	LoadReplicaStateTimeout       = 10 * time.Second
 	TLSFileMode             int32 = 0444
+	DefaultUser             int64 = 101
 )
 
 var (
@@ -20,6 +21,7 @@ var (
 		TimeoutSeconds:      10,
 		PeriodSeconds:       5,
 		FailureThreshold:    10,
+		SuccessThreshold:    1,
 	}
 
 	// DefaultReadinessProbeSettings defines default settings for Kubernetes liveness probes.

internal/controller/keeper/templates.go

@@ -2,7 +2,6 @@ package keeper
 
 import (
 	"fmt"
-	"net"
 	"path"
 	"slices"
 	"strconv"
@@ -407,6 +406,11 @@ func templatePodSpec(cr *v1.KeeperCluster, id v1.KeeperReplicaID) (corev1.PodSpe
 		DNSPolicy:     corev1.DNSClusterFirst,
 		Volumes:       volumes,
 		Containers:    []corev1.Container{container},
+		SecurityContext: &corev1.PodSecurityContext{
+			FSGroup:    new(controller.DefaultUser),
+			RunAsUser:  new(controller.DefaultUser),
+			RunAsGroup: new(controller.DefaultUser),
+		},
 	}
 
 	podTemplate := cr.Spec.PodTemplate
@@ -487,22 +491,24 @@ func templatePodSpec(cr *v1.KeeperCluster, id v1.KeeperReplicaID) (corev1.PodSpe
 }
 
 func templateContainer(cr *v1.KeeperCluster) (corev1.Container, error) {
-	probeAction := corev1.ExecAction{
-		Command: []string{"/bin/bash", "-c",
-			fmt.Sprintf("wget -qO- http://%s/ready | grep -o '\"status\":\"ok\"'",
-				net.JoinHostPort("127.0.0.1", strconv.Itoa(PortHTTPControl)),
-			),
+	livenessProbe := controller.DefaultLivenessProbeSettings
+	livenessProbe.ProbeHandler = corev1.ProbeHandler{
+		TCPSocket: &corev1.TCPSocketAction{
+			Port: intstr.FromInt32(PortNative),
 		},
 	}
 
-	livenessProbe := controller.DefaultLivenessProbeSettings
-	livenessProbe.ProbeHandler = corev1.ProbeHandler{
-		Exec: &probeAction,
+	if cr.Spec.Settings.TLS.Enabled && cr.Spec.Settings.TLS.Required {
+		livenessProbe.TCPSocket.Port.IntVal = PortNativeSecure
 	}
 
 	readinessProbe := controller.DefaultReadinessProbeSettings
 	readinessProbe.ProbeHandler = corev1.ProbeHandler{
-		Exec: &probeAction,
+		HTTPGet: &corev1.HTTPGetAction{
+			Path:   "/ready",
+			Port:   intstr.FromInt32(PortHTTPControl),
+			Scheme: corev1.URISchemeHTTP,
+		},
 	}
 
 	container := corev1.Container{
@@ -524,6 +530,11 @@ func templateContainer(cr *v1.KeeperCluster) (corev1.Container, error) {
 				Name:          "prometheus",
 				ContainerPort: PortPrometheusScrape,
 			},
+			{
+				Protocol:      corev1.ProtocolTCP,
+				Name:          "http-control",
+				ContainerPort: PortHTTPControl,
+			},
 		},
 		VolumeMounts:             buildMounts(cr),
 		LivenessProbe:            &livenessProbe,

internal/controller/overrides.go

@@ -154,6 +154,8 @@ func ApplyContainerTemplateOverrides(container *corev1.Container, t *v1.Containe
 		// VolumeMounts are handled separately
 		Env:             t.Env,
 		SecurityContext: t.SecurityContext,
+		// LivenessProbe is handled manually
+		// ReadinessProbe is handled manually
 	}
 
 	patchJSON, err := json.Marshal(patchContainer)
@@ -178,5 +180,13 @@ func ApplyContainerTemplateOverrides(container *corev1.Container, t *v1.Containe
 
 	mergedContainer.VolumeMounts = append(mergedContainer.VolumeMounts, t.VolumeMounts...)
 
+	if t.LivenessProbe != nil {
+		mergedContainer.LivenessProbe = t.LivenessProbe.DeepCopy()
+	}
+
+	if t.ReadinessProbe != nil {
+		mergedContainer.ReadinessProbe = t.ReadinessProbe.DeepCopy()
+	}
+
 	return mergedContainer, nil
 }