K8SPS-471 leaf certrotation causes no restarts

e2e-tests/functions

@@ -1116,13 +1116,21 @@ get_mysql_pod_names() {
 		--sort-by=.metadata.name -o jsonpath='{.items[*].metadata.name}'
 }
 
-get_mysql_pod_uids() {
+get_component_pod_uids() {
 	local cluster_name=$1
+	local component=$2
 
-	kubectl get pod -n "${NAMESPACE}" --selector="$(mysql_pod_selector "${cluster_name}")" \
+	kubectl get pod -n "${NAMESPACE}" \
+		--selector="app.kubernetes.io/instance=${cluster_name},app.kubernetes.io/name=${component}" \
 		--sort-by=.metadata.name -o jsonpath='{.items[*].metadata.uid}'
 }
 
+get_mysql_pod_uids() {
+	local cluster_name=$1
+
+	get_component_pod_uids "${cluster_name}" mysql
+}
+
 # Prints the global value of a MySQL variable from every database pod of the
 # given cluster, in pod name order, space separated. Pods that cannot be queried
 # print "unavailable" so that callers can keep polling through a rolling restart.
@@ -1172,18 +1180,27 @@ wait_for_mysql_variable() {
 	done
 }
 
-# Waits until every database pod of the given cluster has been replaced,
-# comparing against a UID list captured with get_mysql_pod_uids before the
-# change that should restart them.
 wait_for_mysql_pods_restart() {
 	local cluster_name=$1
 	local uids_before=$2
 	local timeout=${3:-600}
+
+	wait_for_component_pods_restart "${cluster_name}" mysql "${uids_before}" "${timeout}"
+}
+
+# Waits until every pod of the given component has been replaced, comparing
+# against a UID list captured with get_component_pod_uids before the change that
+# should restart them.
+wait_for_component_pods_restart() {
+	local cluster_name=$1
+	local component=$2
+	local uids_before=$3
+	local timeout=${4:-600}
 	local elapsed=0
 	local expected_count=$(echo "${uids_before}" | wc -w)
 
 	while :; do
-		local uids_now=$(get_mysql_pod_uids "${cluster_name}")
+		local uids_now=$(get_component_pod_uids "${cluster_name}" "${component}")
 		local replaced=true
 		local uid
 
@@ -1201,17 +1218,73 @@ wait_for_mysql_pods_restart() {
 			return 0
 		fi
 		if ((elapsed >= timeout)); then
-			echo "Timeout (${timeout}s) exceeded while waiting for ${cluster_name} mysql pods to be replaced"
+			echo "Timeout (${timeout}s) exceeded while waiting for ${cluster_name} ${component} pods to be replaced"
 			echo "uids before: ${uids_before}"
 			echo "uids now:    ${uids_now}"
 			return 1
 		fi
-		echo "waiting for ${cluster_name} mysql pods to be replaced"
+		echo "waiting for ${cluster_name} ${component} pods to be replaced"
 		sleep 10
 		elapsed=$((elapsed + 10))
 	done
 }
 
+# Prints the notBefore date of the certificate mysqld currently serves from every
+# database pod, in pod name order. Spaces inside a date become underscores so
+# that every pod stays one word; unreachable pods print "unavailable".
+get_mysql_cert_not_before_on_pods() {
+	local cluster_name=$1
+	local values=()
+
+	for pod in $(get_mysql_pod_names "${cluster_name}"); do
+		values+=("$(kubectl -n "${NAMESPACE}" exec "${pod}" -c mysql -- bash -c \
+			'mysql -uroot -p"$(cat /etc/mysql/mysql-users-secret/root)" -NB -e "SHOW GLOBAL STATUS LIKE \"Ssl_server_not_before\"" | cut -f2 | tr " " "_"' \
+			2>/dev/null || echo 'unavailable')")
+	done
+
+	echo "${values[*]}"
+}
+
+# Waits until every database pod serves a certificate other than the one in the
+# list captured with get_mysql_cert_not_before_on_pods before the rotation.
+wait_for_mysql_cert_reload() {
+	local cluster_name=$1
+	local not_before_before=$2
+	local timeout=${3:-300}
+	local elapsed=0
+
+	local before=(${not_before_before})
+
+	while :; do
+		local now=($(get_mysql_cert_not_before_on_pods "${cluster_name}"))
+		local reloaded=true
+		local i
+
+		if [[ ${#now[@]} -ne ${#before[@]} ]]; then
+			reloaded=false
+		else
+			for i in "${!now[@]}"; do
+				if [[ ${now[$i]} == "unavailable" || ${now[$i]} == "${before[$i]}" ]]; then
+					reloaded=false
+				fi
+			done
+		fi
+
+		if [[ ${reloaded} == true ]]; then
+			return 0
+		fi
+		if ((elapsed >= timeout)); then
+			echo "Timeout (${timeout}s) exceeded while waiting for ${cluster_name} mysql pods to reload their certificate"
+			echo "not before, before: ${before[*]}"
+			echo "not before, now:    ${now[*]}"
+			return 1
+		fi
+		echo "waiting for ${cluster_name} mysql pods to reload their certificate"
+		sleep 5
+		elapsed=$((elapsed + 5))
+	done
+}
+
 get_mysql_users() {
 	local host=$1
 	local user="${2:--uroot -p'$(get_user_pass root)'}"

e2e-tests/tests/gr-tls-cert-manager/06-assert.yaml

@@ -2,6 +2,15 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 400
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: 06-renew-certs
+data:
+  reloaded: "true"
+  mysql_recycled: "false"
+  router_recycled: "true"
+---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:

e2e-tests/tests/gr-tls-cert-manager/06-renew-certs.yaml

@@ -1,11 +1,34 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
-timeout: 150
+timeout: 1200
 commands:
   - script: |-
       set -o errexit
       set -o xtrace
 
       source ../../functions
 
+      cluster=$(get_cluster_name)
+      mysql_uids_before=$(get_mysql_pod_uids "${cluster}")
+      router_uids_before=$(get_component_pod_uids "${cluster}" router)
+      not_before=$(get_mysql_cert_not_before_on_pods "${cluster}")
+
       renew_certificate "gr-tls-cert-manager-ssl"
+
+      reloaded="true"
+      wait_for_mysql_cert_reload "${cluster}" "${not_before}" 300 || reloaded="false"
+
+      # Router reads its certificates once at startup, so unlike mysqld it still
+      # has to be restarted to pick up the new ones.
+      router_recycled="true"
+      wait_for_component_pods_restart "${cluster}" router "${router_uids_before}" 600 || router_recycled="false"
+
+      # The renewed leaf is signed by the same CA, so no database pod may be
+      # replaced - not even while everything around it rolls.
+      mysql_recycled="false"
+      [[ $(get_mysql_pod_uids "${cluster}") == "${mysql_uids_before}" ]] || mysql_recycled="true"
+
+      kubectl -n "${NAMESPACE}" create configmap 06-renew-certs \
+          --from-literal=reloaded="${reloaded}" \
+          --from-literal=mysql_recycled="${mysql_recycled}" \
+          --from-literal=router_recycled="${router_recycled}"

e2e-tests/tests/tls-cert-manager/06-assert.yaml

@@ -2,6 +2,15 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 500
 ---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: 06-renew-certs
+data:
+  reloaded: "true"
+  mysql_recycled: "false"
+  proxies_recycled: "true"
+---
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:

e2e-tests/tests/tls-cert-manager/06-renew-certs.yaml

@@ -1,11 +1,37 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - timeout: 30
+  - timeout: 1200
     script: |-
       set -o errexit
       set -o xtrace
 
       source ../../functions
 
+      cluster=$(get_cluster_name)
+      mysql_uids_before=$(get_mysql_pod_uids "${cluster}")
+      orc_uids_before=$(get_component_pod_uids "${cluster}" orchestrator)
+      haproxy_uids_before=$(get_component_pod_uids "${cluster}" haproxy)
+      not_before=$(get_mysql_cert_not_before_on_pods "${cluster}")
+
       renew_certificate "tls-cert-manager-ssl"
+
+      reloaded="true"
+      wait_for_mysql_cert_reload "${cluster}" "${not_before}" 300 || reloaded="false"
+
+      # Orchestrator connects to MySQL with a client certificate it reads once at
+      # startup, so unlike mysqld it still has to be restarted to pick up the new
+      # one. HAProxy shares that hash today.
+      proxies_recycled="true"
+      wait_for_component_pods_restart "${cluster}" orchestrator "${orc_uids_before}" 600 || proxies_recycled="false"
+      wait_for_component_pods_restart "${cluster}" haproxy "${haproxy_uids_before}" 600 || proxies_recycled="false"
+
+      # The renewed leaf is signed by the same CA, so no database pod may be
+      # replaced - not even while everything around it rolls.
+      mysql_recycled="false"
+      [[ $(get_mysql_pod_uids "${cluster}") == "${mysql_uids_before}" ]] || mysql_recycled="true"
+
+      kubectl -n "${NAMESPACE}" create configmap 06-renew-certs \
+          --from-literal=reloaded="${reloaded}" \
+          --from-literal=mysql_recycled="${mysql_recycled}" \
+          --from-literal=proxies_recycled="${proxies_recycled}"

pkg/controller/ps/controller.go

@@ -985,6 +985,10 @@ func (r *PerconaServerMySQLReconciler) reconcileDatabase(ctx context.Context, cr
 		return errors.Wrap(err, "reconcile MySQL config")
 	}
 
+	if err := r.reconcileTLSReload(ctx, cr, sts); err != nil {
+		return errors.Wrap(err, "reconcile TLS reload")
+	}
+
 	return nil
 }
 

pkg/controller/ps/tls_reload.go

@@ -0,0 +1,131 @@
+package ps
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"path/filepath"
+	"slices"
+
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+
+	apiv1 "github.com/percona/percona-server-mysql-operator/api/v1"
+	"github.com/percona/percona-server-mysql-operator/pkg/db"
+	"github.com/percona/percona-server-mysql-operator/pkg/k8s"
+	"github.com/percona/percona-server-mysql-operator/pkg/mysql"
+	"github.com/percona/percona-server-mysql-operator/pkg/naming"
+)
+
+// reconcileTLSReload picks up a rotated leaf certificate on the running MySQL pods.
+func (r *PerconaServerMySQLReconciler) reconcileTLSReload(ctx context.Context, cr *apiv1.PerconaServerMySQL, sts *appsv1.StatefulSet) error {
+	if cr.CompareVersion("1.3.0") < 0 {
+		return nil
+	}
+
+	log := logf.FromContext(ctx).WithName("reconcileTLSReload")
+
+	secret := new(corev1.Secret)
+	if err := r.Get(ctx, types.NamespacedName{
+		Name:      cr.Spec.SSLSecretName,
+		Namespace: cr.Namespace,
+	}, secret); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrap(err, "get TLS secret")
+	}
+
+	certHash := tlsCertHash(secret)
+	if certHash == "" {
+		return nil
+	}
+
+	writeAnnotation := func() error {
+		if err := k8s.AnnotateObject(ctx, r.Client, sts, map[naming.AnnotationKey]string{
+			naming.AnnotationLastReloadedTLS: certHash,
+		}); err != nil {
+			return errors.Wrap(err, "annotate object")
+		}
+		return nil
+	}
+
+	lastReloaded, ok := sts.Annotations[naming.AnnotationLastReloadedTLS.String()]
+
+	// Pods read the certificates on startup, so a cluster with no hash recorded
+	// yet only needs something to compare against on the next rotation.
+	if !ok || cr.Status.State == apiv1.StateNew {
+		return writeAnnotation()
+	}
+
+	if lastReloaded == certHash {
+		return nil
+	}
+
+	pods, err := k8s.RunningPods(ctx, r.Client, mysql.MatchLabels(cr), cr.Namespace)
+	if err != nil {
+		return errors.Wrap(err, "get running pods")
+	}
+
+	if cr.Spec.Pause || len(pods) < int(cr.Spec.MySQL.Size) {
+		log.Info("Not all pods are running, defer reloading TLS certificates", "running", len(pods), "desired", cr.Spec.MySQL.Size)
+		return nil
+	}
+
+	// kubelet refreshes a mounted secret on its own schedule, so a pod can still
+	// hold the previous certificate, and reloading it now would re-read the old file.
+	for _, pod := range pods {
+		onDisk, err := r.tlsCertHashOnPod(ctx, &pod)
+		if err != nil {
+			return errors.Wrapf(err, "get certificate from pod %s", pod.Name)
+		}
+		if onDisk != certHash {
+			// Certificate is not propagated to the pod yet, defer reloading TLS certificates
+			return nil
+		}
+	}
+
+	operatorPass, err := k8s.UserPassword(ctx, r.Client, cr, apiv1.UserOperator)
+	if err != nil {
+		return errors.Wrap(err, "get operator password")
+	}
+
+	for _, pod := range pods {
+		mgr := db.NewAdminManager(&pod, r.ClientCmd, apiv1.UserOperator, operatorPass, mysql.PodFQDN(cr, &pod))
+		if err := mgr.ReloadTLS(ctx); err != nil {
+			return errors.Wrapf(err, "reload TLS on pod %s", pod.Name)
+		}
+	}
+
+	log.Info("Reloaded TLS certificates without restarting pods", "pods", len(pods))
+
+	return writeAnnotation()
+}
+
+func tlsCertHash(secret *corev1.Secret) string {
+	cert, key := secret.Data[naming.TLSCertKey], secret.Data[naming.TLSKeyKey]
+	if len(cert) == 0 || len(key) == 0 {
+		return ""
+	}
+	return fmt.Sprintf("%x", md5.Sum(slices.Concat(cert, key)))
+}
+
+func (r *PerconaServerMySQLReconciler) tlsCertHashOnPod(ctx context.Context, pod *corev1.Pod) (string, error) {
+	cmd := []string{
+		"cat",
+		filepath.Join(mysql.TLSMountPath, naming.TLSCertKey),
+		filepath.Join(mysql.TLSMountPath, naming.TLSKeyKey),
+	}
+
+	var stdout, stderr bytes.Buffer
+	if err := r.ClientCmd.Exec(ctx, pod, "mysql", cmd, nil, &stdout, &stderr, false); err != nil {
+		return "", errors.Wrapf(err, "stderr: %s", stderr.String())
+	}
+
+	return fmt.Sprintf("%x", md5.Sum(stdout.Bytes())), nil
+}