feat(clickhouse): add directory-based TLS certificate loading

.github/workflows/flow.yml

@@ -245,6 +245,21 @@ jobs:
           MINIO_ROOT_PASSWORD: miniosecret
           AWS_EC2_METADATA_DISABLED: true
 
+      - name: Generate ClickHouse TLS certificates
+        run: |
+          mkdir -p ch-certs
+          # CA
+          openssl genrsa -out ch-certs/ca.key 2048
+          openssl req -new -x509 -key ch-certs/ca.key -out ch-certs/ca.crt -days 3650 -subj "/CN=ClickHouse-CA"
+          # Server cert (CN=localhost, SAN for TLS 1.3)
+          openssl genrsa -out ch-certs/server.key 2048
+          openssl req -new -key ch-certs/server.key -out ch-certs/server.csr -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
+          openssl x509 -req -days 3650 -in ch-certs/server.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/server.crt -copy_extensions copyall
+          # Client cert for mTLS (CN=peerdb-client), using cert-manager naming convention
+          openssl genrsa -out ch-certs/tls.key 2048
+          openssl req -new -key ch-certs/tls.key -out ch-certs/client.csr -subj "/CN=peerdb-client"
+          openssl x509 -req -days 3650 -in ch-certs/client.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/tls.crt
+
       - name: create postgres extensions, increase logical replication limits, and setup catalog database
         run: >
           docker exec "${{ job.services.catalog.id }}" apk add --no-cache build-base git &&
@@ -310,16 +325,39 @@ jobs:
                 <access_management>1</access_management>
                 <named_collection_control>1</named_collection_control>
               </default>
+              <peerdb_tls>
+                <ssl_certificates>
+                  <common_name>peerdb-client</common_name>
+                </ssl_certificates>
+                <networks>
+                  <ip>::/0</ip>
+                </networks>
+                <profile>default</profile>
+                <quota>default</quota>
+                <access_management>1</access_management>
+              </peerdb_tls>
             </users>
             <logger><level>none</level></logger>
             <path>var/lib/clickhouse</path>
             <tmp_path>var/lib/clickhouse/tmp</tmp_path>
             <user_files_path>var/lib/clickhouse/user_files</user_files_path>
             <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
             <tcp_port>9000</tcp_port>
+            <tcp_port_secure>9440</tcp_port_secure>
             <http_port remove="1"/>
             <postgresql_port remove="1"/>
             <mysql_port remove="1"/>
+            <openSSL>
+              <server>
+                <certificateFile>../ch-certs/server.crt</certificateFile>
+                <privateKeyFile>../ch-certs/server.key</privateKeyFile>
+                <caConfig>../ch-certs/ca.crt</caConfig>
+                <verificationMode>relaxed</verificationMode>
+                <cacheSessions>true</cacheSessions>
+                <disableProtocols>sslv2,sslv3</disableProtocols>
+                <preferServerCiphers>true</preferServerCiphers>
+              </server>
+            </openSSL>
             <macros>
               <shard>1</shard>
               <replica>1</replica>
@@ -365,16 +403,39 @@ jobs:
                 <access_management>1</access_management>
                 <named_collection_control>1</named_collection_control>
               </default>
+              <peerdb_tls>
+                <ssl_certificates>
+                  <common_name>peerdb-client</common_name>
+                </ssl_certificates>
+                <networks>
+                  <ip>::/0</ip>
+                </networks>
+                <profile>default</profile>
+                <quota>default</quota>
+                <access_management>1</access_management>
+              </peerdb_tls>
             </users>
             <logger><level>none</level></logger>
             <path>var/lib/clickhouse</path>
             <tmp_path>var/lib/clickhouse/tmp</tmp_path>
             <user_files_path>var/lib/clickhouse/user_files</user_files_path>
             <format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
             <tcp_port>9001</tcp_port>
+            <tcp_port_secure>9441</tcp_port_secure>
             <http_port remove="1"/>
             <postgresql_port remove="1"/>
             <mysql_port remove="1"/>
+            <openSSL>
+              <server>
+                <certificateFile>../ch-certs/server.crt</certificateFile>
+                <privateKeyFile>../ch-certs/server.key</privateKeyFile>
+                <caConfig>../ch-certs/ca.crt</caConfig>
+                <verificationMode>relaxed</verificationMode>
+                <cacheSessions>true</cacheSessions>
+                <disableProtocols>sslv2,sslv3</disableProtocols>
+                <preferServerCiphers>true</preferServerCiphers>
+              </server>
+            </openSSL>
             <macros>
               <shard>2</shard>
               <replica>1</replica>
@@ -535,6 +596,9 @@ jobs:
           FLOW_TESTS_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }}
           FLOW_TESTS_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }}
           FLOW_TESTS_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }}
+          # ClickHouse TLS/mTLS test certificates
+          PEERDB_CLICKHOUSE_TLS_PORT: "9440"
+          PEERDB_CLICKHOUSE_TLS_CERT_DIR: ${{ github.workspace }}/ch-certs
 
       - name: Upload peer-flow logs and test results
         if: always()

flow/connectors/clickhouse/clickhouse.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"log/slog"
 	"net/url"
+	"os"
+	"path/filepath"
 	"slices"
 	"strings"
 	"time"
@@ -232,32 +234,119 @@ func (c *ClickHouseConnector) ValidateCheck(ctx context.Context) error {
 	return nil
 }
 
-func Connect(ctx context.Context, env map[string]string, config *protos.ClickhouseConfig) (clickhouse.Conn, error) {
-	var tlsSetting *tls.Config
-	if !config.DisableTls {
-		tlsSetting = &tls.Config{MinVersion: tls.VersionTLS13}
-		if config.Certificate != nil || config.PrivateKey != nil {
-			if config.Certificate == nil || config.PrivateKey == nil {
-				return nil, fmt.Errorf("both certificate and private key must be provided if using certificate-based authentication")
-			}
-			cert, err := tls.X509KeyPair([]byte(*config.Certificate), []byte(*config.PrivateKey))
-			if err != nil {
-				return nil, fmt.Errorf("failed to parse provided certificate: %w", err)
-			}
-			tlsSetting.Certificates = []tls.Certificate{cert}
+// configureDirectoryTLS configures the tls.Config by loading certificate files
+// from a directory. It expects tls.crt and tls.key files, and optionally ca.crt.
+// This is typically used when a Kubernetes Secret is mounted as a volume.
+//
+// Client certificates are loaded via GetClientCertificate so that every TLS
+// handshake re-reads the files from disk, automatically picking up rotated
+// certificates (e.g. renewed by cert-manager) without requiring a reconnect.
+func configureDirectoryTLS(tlsConfig *tls.Config, dir string) error {
+	certPath := filepath.Join(dir, "tls.crt")
+	keyPath := filepath.Join(dir, "tls.key")
+	caPath := filepath.Join(dir, "ca.crt")
+
+	// Verify that the required files are readable at configuration time
+	if _, err := os.ReadFile(certPath); err != nil {
+		return fmt.Errorf("failed to read TLS certificate from %q: %w", certPath, err)
+	}
+	if _, err := os.ReadFile(keyPath); err != nil {
+		return fmt.Errorf("failed to read TLS private key from %q: %w", keyPath, err)
+	}
+
+	// Load CA certificate upfront — RootCAs must be set before the TLS
+	// handshake so the server certificate can be verified.
+	// ca.crt is optional: when absent, the system CA pool is used instead.
+	caCertPEM, err := os.ReadFile(caPath)
+	if errors.Is(err, os.ErrNotExist) {
+		// ca.crt is optional — use system CA pool
+	} else if err != nil {
+		return fmt.Errorf("failed to read CA certificate from %q: %w", caPath, err)
+	} else if len(caCertPEM) > 0 {
+		caPool := x509.NewCertPool()
+		if !caPool.AppendCertsFromPEM(caCertPEM) {
+			return fmt.Errorf("failed to parse CA certificate from %q", caPath)
 		}
-		if config.RootCa != nil {
-			caPool := x509.NewCertPool()
-			if !caPool.AppendCertsFromPEM([]byte(*config.RootCa)) {
-				return nil, fmt.Errorf("failed to parse provided root CA")
-			}
-			tlsSetting.RootCAs = caPool
+		tlsConfig.RootCAs = caPool
+	}
+
+	// Use GetClientCertificate so the cert/key are re-read on every TLS
+	// handshake, ensuring rotated certificates are picked up immediately.
+	tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+		certPEM, err := os.ReadFile(certPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read TLS certificate from %q: %w", certPath, err)
+		}
+		keyPEM, err := os.ReadFile(keyPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read TLS private key from %q: %w", keyPath, err)
+		}
+		cert, err := tls.X509KeyPair(certPEM, keyPEM)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse TLS certificate from directory %q: %w", dir, err)
+		}
+		return &cert, nil
+	}
+
+	return nil
+}
+
+// configureInlineTLS configures the tls.Config using inline certificate/key
+// PEM values from the ClickHouse peer config.
+func configureInlineTLS(tlsConfig *tls.Config, config *protos.ClickhouseConfig) error {
+	if config.Certificate != nil || config.PrivateKey != nil {
+		if config.Certificate == nil || config.PrivateKey == nil {
+			return errors.New("both certificate and private key must be provided if using certificate-based authentication")
+		}
+		cert, err := tls.X509KeyPair([]byte(*config.Certificate), []byte(*config.PrivateKey))
+		if err != nil {
+			return fmt.Errorf("failed to parse provided certificate: %w", err)
+		}
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+	if config.RootCa != nil {
+		caPool := x509.NewCertPool()
+		if !caPool.AppendCertsFromPEM([]byte(*config.RootCa)) {
+			return errors.New("failed to parse provided root CA")
+		}
+		tlsConfig.RootCAs = caPool
+	}
+	return nil
+}
+
+// buildTLSConfig builds the TLS configuration for a ClickHouse connection.
+// When a TLS certificate directory is configured, it loads certificates from the directory.
+// Otherwise, inline certificates are used.
+func buildTLSConfig(config *protos.ClickhouseConfig) (*tls.Config, error) {
+	if config.DisableTls {
+		return nil, nil
+	}
+
+	tlsConfig := &tls.Config{
+		MinVersion: tls.VersionTLS13,
+		ServerName: config.TlsHost,
+	}
+	certDir := config.GetTlsCertificateDirectory()
+
+	if certDir != "" {
+		if err := configureDirectoryTLS(tlsConfig, certDir); err != nil {
+			return nil, err
 		}
-		if config.TlsHost != "" {
-			tlsSetting.ServerName = config.TlsHost
+	} else {
+		if err := configureInlineTLS(tlsConfig, config); err != nil {
+			return nil, err
 		}
 	}
 
+	return tlsConfig, nil
+}
+
+func Connect(ctx context.Context, env map[string]string, config *protos.ClickhouseConfig) (clickhouse.Conn, error) {
+	tlsSetting, err := buildTLSConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
 	settings := clickhouse.Settings{
 		// See: https://clickhouse.com/docs/en/cloud/reference/shared-merge-tree#consistency
 		"select_sequential_consistency": uint64(1),

flow/e2e/clickhouse_tls_test.go

@@ -0,0 +1,46 @@
+package e2e
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+
+	connclickhouse "github.com/PeerDB-io/peerdb/flow/connectors/clickhouse"
+	"github.com/PeerDB-io/peerdb/flow/generated/protos"
+)
+
+// TestClickHouseTLSDirectory verifies that connclickhouse.Connect works
+// with TLS certificates loaded from a directory (simulating a volume-mounted K8s Secret).
+func TestClickHouseTLSDirectory(t *testing.T) {
+	portStr := os.Getenv("PEERDB_CLICKHOUSE_TLS_PORT")
+	if portStr == "" {
+		t.Skip("PEERDB_CLICKHOUSE_TLS_PORT not set, skipping TLS test")
+	}
+	port, err := strconv.ParseUint(portStr, 10, 32)
+	require.NoError(t, err, "invalid PEERDB_CLICKHOUSE_TLS_PORT value: %s", portStr)
+
+	certDir := os.Getenv("PEERDB_CLICKHOUSE_TLS_CERT_DIR")
+	if certDir == "" {
+		t.Skip("PEERDB_CLICKHOUSE_TLS_CERT_DIR not set, skipping TLS test")
+	}
+
+	config := &protos.ClickhouseConfig{
+		Host:                    "localhost",
+		Port:                    uint32(port),
+		User:                    "peerdb_tls",
+		Database:                "default",
+		DisableTls:              false,
+		TlsCertificateDirectory: proto.String(certDir),
+	}
+
+	conn, err := connclickhouse.Connect(t.Context(), nil, config)
+	require.NoError(t, err, "failed to connect to ClickHouse")
+	defer conn.Close()
+
+	var result uint8
+	require.NoError(t, conn.QueryRow(t.Context(), "SELECT 1").Scan(&result), "failed to execute SELECT 1")
+	require.Equal(t, uint8(1), result)
+}

generate-protos.sh

@@ -2,11 +2,11 @@
 set -xeu
 
 # check if buf is installed
-if ! command -v buf &> /dev/null
+if ! command -v buf > /dev/null 2>&1
 then
     echo "buf could not be found"
     echo "Please install buf: https://buf.build/docs/installation"
-    exit
+    exit 1
 fi
 
 buf generate protos

nexus/analyzer/src/lib.rs

@@ -885,6 +885,9 @@ fn parse_db_options(db_type: DbType, with_options: &[SqlOption]) -> anyhow::Resu
                     .map(|s| s.parse::<bool>().unwrap_or_default())
                     .unwrap_or_default(),
                 s3: None,
+                tls_certificate_directory: opts
+                    .get("tls_certificate_directory")
+                    .map(|s| s.to_string()),
             };
             Config::ClickhouseConfig(clickhouse_config)
         }

protos/peers.proto

@@ -186,6 +186,9 @@ message ClickhouseConfig{
   optional S3Config s3 = 16;
   string cluster = 17;
   bool replicated = 18;
+  // Directory path containing TLS certificate files (tls.crt, tls.key, and optionally ca.crt).
+  // Typically a Kubernetes Secret mounted as a volume. When set, takes precedence over inline cert fields.
+  optional string tls_certificate_directory = 19;
 }
 
 message SqlServerConfig {

ui/app/peers/create/[peerType]/helpers/ch.ts

@@ -108,6 +108,24 @@ export const clickhouseSetting: PeerSetting[] = [
     optional: true,
     tips: 'If not provided, host CA roots will be used.',
   },
+  {
+    label: 'TLS Certificate Directory',
+    stateHandler: (value, setter) => {
+      if (!value) {
+        setter((curr) => {
+          const newCurr = { ...curr } as ClickhouseConfig;
+          delete newCurr.tlsCertificateDirectory;
+          return newCurr;
+        });
+      } else
+        setter((curr) => ({
+          ...curr,
+          tlsCertificateDirectory: value as string,
+        }));
+    },
+    optional: true,
+    tips: 'Path to a directory containing TLS certificate files (tls.crt, tls.key, and optionally ca.crt). Typically a Kubernetes Secret mounted as a volume. When set, inline certificate/private key fields are ignored.',
+  },
   {
     label: 'S3 Path',
     stateHandler: (value, setter) =>

ui/app/peers/create/[peerType]/schema.ts

@@ -434,6 +434,12 @@ export function chSchema(hostDomains: string[]) {
       })
       .optional()
       .transform((e) => (e === '' ? undefined : e)),
+    tlsCertificateDirectory: z
+      .string({
+        error: () => 'TLS Certificate Directory must be a string',
+      })
+      .optional()
+      .transform((e) => (e === '' ? undefined : e)),
     tlsHost: z.string(),
   });
 }