feat(spec.tls.additionalTrustedCAs)

build/crd/percona/generated/pgv2.percona.com_perconapgclusters.yaml

@@ -16678,6 +16678,15 @@ spec:
                 type: object
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:

config/crd/bases/pgv2.percona.com_perconapgclusters.yaml

@@ -17092,6 +17092,15 @@ spec:
                 type: object
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:

config/crd/bases/upstream.pgv2.percona.com_postgresclusters.yaml

@@ -14352,6 +14352,15 @@ spec:
                 type: array
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:

deploy/bundle.yaml

@@ -17293,6 +17293,15 @@ spec:
                 type: object
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:
@@ -36005,6 +36014,15 @@ spec:
                 type: array
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:

deploy/cr.yaml

@@ -67,6 +67,11 @@ spec:
 #      name: some-selfsigned-issuer
 #      kind: ClusterIssuer
 #      group: cert-manager.io
+#    # Extra CA bundles to trust, each a Secret with a PEM bundle in key `ca.crt`.
+#    # Required when the issuer returns no CA certificate of its own, as ACME
+#    # issuers do: without it there is nothing to verify peers against.
+#    additionalTrustedCAs:
+#      - name: my-ca-bundle
 #  standby:
 #    enabled: true
 #    host: "<primary-ip>"

deploy/crd.yaml

@@ -17293,6 +17293,15 @@ spec:
                 type: object
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:
@@ -36005,6 +36014,15 @@ spec:
                 type: array
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:

deploy/cw-bundle.yaml

@@ -17293,6 +17293,15 @@ spec:
                 type: object
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:
@@ -36005,6 +36014,15 @@ spec:
                 type: array
               tls:
                 properties:
+                  additionalTrustedCAs:
+                    items:
+                      properties:
+                        name:
+                          default: ""
+                          type: string
+                      type: object
+                      x-kubernetes-map-type: atomic
+                    type: array
                   caValidityDuration:
                     type: string
                   certManagementPolicy:

e2e-tests/tests/cert-manager-tls/29-verify-additional-trusted-cas.yaml

@@ -0,0 +1,96 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      cluster="cert-manager-tls"
+
+      # A CA the operator knows nothing about and that no issuer here signs
+      # with. spec.tls.additionalTrustedCAs is what a user sets when the
+      # configured issuer returns no ca.crt of its own, so the anchor has to
+      # reach every trust file on its own merits - not because some issuer
+      # happened to put it there.
+      openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
+        -keyout "${TEMP_DIR}/cluster-extra-ca.key" \
+        -out "${TEMP_DIR}/cluster-extra-ca.crt" \
+        -subj "/CN=cluster-extra-trusted-ca"
+
+      kubectl -n "${NAMESPACE}" create secret generic cluster-extra-ca \
+        --from-file=ca.crt="${TEMP_DIR}/cluster-extra-ca.crt"
+
+      kubectl -n "${NAMESPACE}" patch perconapgcluster "${cluster}" --type=merge \
+        -p '{"spec":{"tls":{"additionalTrustedCAs":[{"name":"cluster-extra-ca"}]}}}'
+
+      # The merged bundle lives in a Secret the operator owns; the
+      # cert-manager-issued ones are rebuilt on every issuance and must not be
+      # written to.
+      bundle_has_two_cas() {
+        [[ $(kubectl -n "${NAMESPACE}" get secret "${cluster}-ca-bundle" \
+          -o jsonpath='{.data.ca\.crt}' 2>/dev/null \
+          | base64 -d | grep -c 'BEGIN CERTIFICATE') -eq 2 ]]
+      }
+      retry 30 5 bundle_has_two_cas
+
+      # Adding a source to the certificate volume rolls the instances.
+      wait_cluster_consistency "${cluster}"
+
+      # Inspect the PEM locally so the assertions do not depend on which tools
+      # each container image happens to ship.
+      check_bundle() {
+        local what=$1 file=$2
+
+        if [[ $(grep -c 'BEGIN CERTIFICATE' "${file}") -ne 2 ]]; then
+            echo "${what} should hold the issuer CA and the additional one:"
+            openssl crl2pkcs7 -nocrl -certfile "${file}" | openssl pkcs7 -print_certs -noout
+            exit 1
+        fi
+
+        # Both anchors, specifically: the issuer's, so everything that verified
+        # before still does, and the added one.
+        local subjects
+        subjects=$(openssl crl2pkcs7 -nocrl -certfile "${file}" | openssl pkcs7 -print_certs -noout)
+        if ! grep -q 'cluster-extra-trusted-ca' <<<"${subjects}"; then
+            echo "${what} is missing the additional CA: ${subjects}"
+            exit 1
+        fi
+        if ! grep -q 'cert-manager-tls-ca' <<<"${subjects}"; then
+            echo "${what} lost the issuer CA: ${subjects}"
+            exit 1
+        fi
+      }
+
+      instance=$(kubectl -n "${NAMESPACE}" get pod \
+        -l postgres-operator.crunchydata.com/cluster="${cluster}",postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+      instance_set=$(kubectl -n "${NAMESPACE}" get pod "${instance}" \
+        -o jsonpath='{.metadata.labels.postgres-operator\.crunchydata\.com/instance}')
+
+      # ssl_ca_file, which verifies the _crunchyrepl client certificate.
+      kubectl -n "${NAMESPACE}" exec "${instance}" -c database -- \
+        cat /pgconf/tls/ca.crt >"${TEMP_DIR}/mounted-ca.crt"
+      check_bundle "/pgconf/tls/ca.crt" "${TEMP_DIR}/mounted-ca.crt"
+
+      # sslrootcert for replication and pg_rewind.
+      kubectl -n "${NAMESPACE}" exec "${instance}" -c database -- \
+        cat /pgconf/tls/replication/ca.crt >"${TEMP_DIR}/mounted-repl-ca.crt"
+      check_bundle "/pgconf/tls/replication/ca.crt" "${TEMP_DIR}/mounted-repl-ca.crt"
+
+      # tls-server-ca-file and pgN-/repoN-host-ca-file are the same file.
+      kubectl -n "${NAMESPACE}" exec "${instance}" -c pgbackrest -- \
+        cat /etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt >"${TEMP_DIR}/mounted-pgbackrest-ca.crt"
+      check_bundle "pgbackrest.ca-roots" "${TEMP_DIR}/mounted-pgbackrest-ca.crt"
+
+      # Patroni's restapi.cafile and ctl.cacert.
+      kubectl -n "${NAMESPACE}" get secret "${instance_set}-certs" \
+        -o jsonpath='{.data.patroni\.ca-roots}' | base64 -d >"${TEMP_DIR}/mounted-patroni-ca.crt"
+      check_bundle "patroni.ca-roots" "${TEMP_DIR}/mounted-patroni-ca.crt"
+
+      # PgBouncer's client_tls_ca_file.
+      kubectl -n "${NAMESPACE}" get secret "${cluster}-pgbouncer" \
+        -o jsonpath='{.data.pgbouncer-frontend\.ca-roots}' | base64 -d >"${TEMP_DIR}/mounted-pgbouncer-ca.crt"
+      check_bundle "pgbouncer-frontend.ca-roots" "${TEMP_DIR}/mounted-pgbouncer-ca.crt"
+    timeout: 420

e2e-tests/tests/cert-manager-tls/30-verify-tls-after-additional-cas.yaml

@@ -0,0 +1,47 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      cluster="cert-manager-tls"
+
+      # Widening a trust bundle must not narrow what already verified. These
+      # repeat the checks from steps 25-28 with the additional CA in place.
+      instance=$(kubectl -n "${NAMESPACE}" get pod \
+        -l postgres-operator.crunchydata.com/cluster="${cluster}",postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      certificate_data=$(run_comand_on_pod "openssl s_client -connect ${cluster}-primary:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -subject -issuer")
+      if [[ "$certificate_data" != *"issuer=CN=${cluster}-ca"* ]]; then
+          echo "Unexpected PostgreSQL certificate issuer. Expected CN=${cluster}-ca"
+          echo "Got: $certificate_data"
+          exit 1
+      fi
+
+      # mTLS: the _crunchyrepl client certificate is verified against
+      # ssl_ca_file, which now carries two anchors.
+      repl_ssl_count=$(run_psql_local \
+        "SELECT count(*) FROM pg_stat_ssl s JOIN pg_stat_replication r ON s.pid = r.pid WHERE s.ssl = true;" \
+        "postgres:$(get_psql_user_pass ${cluster}-pguser-postgres)@${cluster}-primary")
+      repl_ssl_count=$(echo "$repl_ssl_count" | tr -d '[:space:]')
+
+      if [[ "$repl_ssl_count" -lt 1 ]]; then
+          echo "No SSL replication connections found after adding a trusted CA, got: $repl_ssl_count"
+          exit 1
+      fi
+
+      # pgBackRest verifies its client certificate against the same merged file.
+      kubectl -n "${NAMESPACE}" exec "${instance}" -c pgbackrest -- pgbackrest info
+
+      # PgBouncer still serves and reaches the backend.
+      pgbouncer_certificate_data=$(run_comand_on_pod "openssl s_client -connect ${cluster}-pgbouncer:5432 -starttls postgres <<< '' 2>/dev/null | openssl x509 -noout -subject -issuer")
+      if [[ "$pgbouncer_certificate_data" != *"issuer=CN=${cluster}-ca"* ]]; then
+          echo "Unexpected PgBouncer certificate issuer. Expected CN=${cluster}-ca"
+          echo "Got: $pgbouncer_certificate_data"
+          exit 1
+      fi
+    timeout: 120

internal/controller/postgrescluster/controller.go

@@ -201,6 +201,7 @@ func (r *Reconciler) Reconcile(
 		instances                    *observedInstances
 		patroniLeaderService         *corev1.Service
 		primaryCertificate           *corev1.SecretProjection
+		caBundle                     *corev1.SecretProjection
 		primaryService               *corev1.Service
 		replicaService               *corev1.Service
 		rootCA                       *pki.RootCertificateAuthority
@@ -443,6 +444,11 @@ func (r *Reconciler) Reconcile(
 	if err == nil {
 		primaryCertificate, err = r.reconcileClusterCertificate(ctx, rootCA, cluster, primaryService, replicaService)
 	}
+	if err == nil {
+		// After the cluster and replication certificates, whose ca.crt this
+		// merges with the CAs from spec.tls.additionalTrustedCAs.
+		caBundle, err = r.reconcileCABundleSecret(ctx, cluster)
+	}
 	if err == nil {
 		err = r.reconcilePatroniDistributedConfiguration(ctx, cluster)
 	}
@@ -462,7 +468,7 @@ func (r *Reconciler) Reconcile(
 		err = r.reconcileInstanceSets(
 			ctx, cluster, clusterConfigMap, clusterReplicationSecret, rootCA,
 			clusterPodService, instanceServiceAccount, instances, patroniLeaderService,
-			primaryCertificate, clusterVolumes, exporterQueriesConfig, exporterWebConfig,
+			primaryCertificate, caBundle, clusterVolumes, exporterQueriesConfig, exporterWebConfig,
 			backupsSpecFound,
 		)
 	}
@@ -498,7 +504,7 @@ func (r *Reconciler) Reconcile(
 		err = r.reconcileVolumeSnapshots(ctx, cluster, dedicatedSnapshotPVC)
 	}
 	if err == nil {
-		err = r.reconcilePGBouncer(ctx, cluster, instances, primaryCertificate, rootCA)
+		err = r.reconcilePGBouncer(ctx, cluster, instances, primaryCertificate, caBundle, rootCA)
 	}
 	if err == nil {
 		err = r.reconcilePGMonitor(ctx, cluster, instances, monitoringSecret)
@@ -632,6 +638,15 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error {
 		return err
 	}
 
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(),
+		&v1beta1.PostgresCluster{},
+		v1beta1.IndexFieldAdditionalTrustedCASecrets,
+		v1beta1.AdditionalTrustedCASecretsIndexerFunc,
+	); err != nil {
+		return err
+	}
+
 	// K8SPG-712: Allow overriding default configurations
 	configMapPredicate := builder.WithPredicates(predicate.Funcs{
 		UpdateFunc: func(e event.UpdateEvent) bool {
@@ -666,6 +681,7 @@ func (r *Reconciler) SetupWithManager(mgr manager.Manager) error {
 		}))).
 		Watches(&corev1.Pod{}, r.watchPods()).
 		Watches(&corev1.Secret{}, r.watchPGBouncerUserSecrets()).
+		Watches(&corev1.Secret{}, r.watchAdditionalTrustedCASecrets()).
 		Watches(&appsv1.StatefulSet{},
 			r.controllerRefHandlerFuncs()) // watch all StatefulSets