Skip to content

Commit c92cd39

Browse files
whites11jkaflikclaude
committed
feat(clickhouse): load TLS certificates from directory for cert-manager support
Add TLS certificate directory support for ClickHouse peers, enabling certificates to be loaded from a mounted Kubernetes Secret (cert-manager). Client certs are re-read on every TLS handshake via GetClientCertificate, so rotated certificates are picked up without requiring a restart. Also fixes generate-protos.sh to use POSIX-compatible shell syntax (the &> bashism silently broke proto generation on cache miss). Co-Authored-By: Kuba Kaflik <kuba.kaflik@clickhouse.com> Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 7ef375c commit c92cd39

10 files changed

Lines changed: 253 additions & 25 deletions

File tree

.github/workflows/flow.yml

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,21 @@ jobs:
245245
MINIO_ROOT_PASSWORD: miniosecret
246246
AWS_EC2_METADATA_DISABLED: true
247247

248+
- name: Generate ClickHouse TLS certificates
249+
run: |
250+
mkdir -p ch-certs
251+
# CA
252+
openssl genrsa -out ch-certs/ca.key 2048
253+
openssl req -new -x509 -key ch-certs/ca.key -out ch-certs/ca.crt -days 3650 -subj "/CN=ClickHouse-CA"
254+
# Server cert (CN=localhost, SAN for TLS 1.3)
255+
openssl genrsa -out ch-certs/server.key 2048
256+
openssl req -new -key ch-certs/server.key -out ch-certs/server.csr -subj "/CN=localhost" -addext "subjectAltName=DNS:localhost,IP:127.0.0.1"
257+
openssl x509 -req -days 3650 -in ch-certs/server.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/server.crt -copy_extensions copyall
258+
# Client cert for mTLS (CN=peerdb-client), using cert-manager naming convention
259+
openssl genrsa -out ch-certs/tls.key 2048
260+
openssl req -new -key ch-certs/tls.key -out ch-certs/client.csr -subj "/CN=peerdb-client"
261+
openssl x509 -req -days 3650 -in ch-certs/client.csr -CA ch-certs/ca.crt -CAkey ch-certs/ca.key -CAcreateserial -out ch-certs/tls.crt
262+
248263
- name: create postgres extensions, increase logical replication limits, and setup catalog database
249264
run: >
250265
docker exec "${{ job.services.catalog.id }}" apk add --no-cache build-base git &&
@@ -310,16 +325,39 @@ jobs:
310325
<access_management>1</access_management>
311326
<named_collection_control>1</named_collection_control>
312327
</default>
328+
<peerdb_tls>
329+
<ssl_certificates>
330+
<common_name>peerdb-client</common_name>
331+
</ssl_certificates>
332+
<networks>
333+
<ip>::/0</ip>
334+
</networks>
335+
<profile>default</profile>
336+
<quota>default</quota>
337+
<access_management>1</access_management>
338+
</peerdb_tls>
313339
</users>
314340
<logger><level>none</level></logger>
315341
<path>var/lib/clickhouse</path>
316342
<tmp_path>var/lib/clickhouse/tmp</tmp_path>
317343
<user_files_path>var/lib/clickhouse/user_files</user_files_path>
318344
<format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
319345
<tcp_port>9000</tcp_port>
346+
<tcp_port_secure>9440</tcp_port_secure>
320347
<http_port remove="1"/>
321348
<postgresql_port remove="1"/>
322349
<mysql_port remove="1"/>
350+
<openSSL>
351+
<server>
352+
<certificateFile>../ch-certs/server.crt</certificateFile>
353+
<privateKeyFile>../ch-certs/server.key</privateKeyFile>
354+
<caConfig>../ch-certs/ca.crt</caConfig>
355+
<verificationMode>relaxed</verificationMode>
356+
<cacheSessions>true</cacheSessions>
357+
<disableProtocols>sslv2,sslv3</disableProtocols>
358+
<preferServerCiphers>true</preferServerCiphers>
359+
</server>
360+
</openSSL>
323361
<macros>
324362
<shard>1</shard>
325363
<replica>1</replica>
@@ -365,16 +403,39 @@ jobs:
365403
<access_management>1</access_management>
366404
<named_collection_control>1</named_collection_control>
367405
</default>
406+
<peerdb_tls>
407+
<ssl_certificates>
408+
<common_name>peerdb-client</common_name>
409+
</ssl_certificates>
410+
<networks>
411+
<ip>::/0</ip>
412+
</networks>
413+
<profile>default</profile>
414+
<quota>default</quota>
415+
<access_management>1</access_management>
416+
</peerdb_tls>
368417
</users>
369418
<logger><level>none</level></logger>
370419
<path>var/lib/clickhouse</path>
371420
<tmp_path>var/lib/clickhouse/tmp</tmp_path>
372421
<user_files_path>var/lib/clickhouse/user_files</user_files_path>
373422
<format_schema_path>var/lib/clickhouse/format_schemas</format_schema_path>
374423
<tcp_port>9001</tcp_port>
424+
<tcp_port_secure>9441</tcp_port_secure>
375425
<http_port remove="1"/>
376426
<postgresql_port remove="1"/>
377427
<mysql_port remove="1"/>
428+
<openSSL>
429+
<server>
430+
<certificateFile>../ch-certs/server.crt</certificateFile>
431+
<privateKeyFile>../ch-certs/server.key</privateKeyFile>
432+
<caConfig>../ch-certs/ca.crt</caConfig>
433+
<verificationMode>relaxed</verificationMode>
434+
<cacheSessions>true</cacheSessions>
435+
<disableProtocols>sslv2,sslv3</disableProtocols>
436+
<preferServerCiphers>true</preferServerCiphers>
437+
</server>
438+
</openSSL>
378439
<macros>
379440
<shard>2</shard>
380441
<replica>1</replica>
@@ -535,6 +596,9 @@ jobs:
535596
FLOW_TESTS_AWS_ACCESS_KEY_ID: ${{ steps.setup-aws.outputs.aws-access-key-id }}
536597
FLOW_TESTS_AWS_SECRET_ACCESS_KEY: ${{ steps.setup-aws.outputs.aws-secret-access-key }}
537598
FLOW_TESTS_AWS_SESSION_TOKEN: ${{ steps.setup-aws.outputs.aws-session-token }}
599+
# ClickHouse TLS/mTLS test certificates
600+
PEERDB_CLICKHOUSE_TLS_PORT: "9440"
601+
PEERDB_CLICKHOUSE_TLS_CERT_DIR: ${{ github.workspace }}/ch-certs
538602

539603
- name: Upload peer-flow logs and test results
540604
if: always()

flow/cmd/handler.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -586,7 +586,9 @@ func (h *FlowRequestHandler) CreatePeer(
586586
req *protos.CreatePeerRequest,
587587
) (*protos.CreatePeerResponse, APIError) {
588588
if !req.DisableValidation {
589-
status, validateErr := h.ValidatePeer(ctx, &protos.ValidatePeerRequest{Peer: req.Peer})
589+
status, validateErr := h.ValidatePeer(ctx, &protos.ValidatePeerRequest{
590+
Peer: req.Peer,
591+
})
590592
if validateErr != nil {
591593
return nil, validateErr
592594
}

flow/connectors/clickhouse/clickhouse.go

Lines changed: 105 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99
"fmt"
1010
"log/slog"
1111
"net/url"
12+
"os"
13+
"path/filepath"
1214
"slices"
1315
"strings"
1416
"time"
@@ -232,32 +234,114 @@ func (c *ClickHouseConnector) ValidateCheck(ctx context.Context) error {
232234
return nil
233235
}
234236

235-
func Connect(ctx context.Context, env map[string]string, config *protos.ClickhouseConfig) (clickhouse.Conn, error) {
236-
var tlsSetting *tls.Config
237-
if !config.DisableTls {
238-
tlsSetting = &tls.Config{MinVersion: tls.VersionTLS13}
239-
if config.Certificate != nil || config.PrivateKey != nil {
240-
if config.Certificate == nil || config.PrivateKey == nil {
241-
return nil, fmt.Errorf("both certificate and private key must be provided if using certificate-based authentication")
242-
}
243-
cert, err := tls.X509KeyPair([]byte(*config.Certificate), []byte(*config.PrivateKey))
244-
if err != nil {
245-
return nil, fmt.Errorf("failed to parse provided certificate: %w", err)
246-
}
247-
tlsSetting.Certificates = []tls.Certificate{cert}
237+
// configureDirectoryTLS configures the tls.Config by loading certificate files
238+
// from a directory. It expects tls.crt and tls.key files, and optionally ca.crt.
239+
// This is typically used when a Kubernetes Secret is mounted as a volume.
240+
//
241+
// Client certificates are loaded via GetClientCertificate so that every TLS
242+
// handshake re-reads the files from disk, automatically picking up rotated
243+
// certificates (e.g. renewed by cert-manager) without requiring a reconnect.
244+
func configureDirectoryTLS(tlsConfig *tls.Config, dir string) error {
245+
certPath := filepath.Join(dir, "tls.crt")
246+
keyPath := filepath.Join(dir, "tls.key")
247+
caPath := filepath.Join(dir, "ca.crt")
248+
249+
// Verify that the required files are readable at configuration time
250+
if _, err := os.ReadFile(certPath); err != nil {
251+
return fmt.Errorf("failed to read TLS certificate from %q: %w", certPath, err)
252+
}
253+
if _, err := os.ReadFile(keyPath); err != nil {
254+
return fmt.Errorf("failed to read TLS private key from %q: %w", keyPath, err)
255+
}
256+
257+
// Load CA certificate upfront — RootCAs must be set before the TLS
258+
// handshake so the server certificate can be verified.
259+
caCertPEM, err := os.ReadFile(caPath)
260+
if err == nil && len(caCertPEM) > 0 {
261+
caPool := x509.NewCertPool()
262+
if !caPool.AppendCertsFromPEM(caCertPEM) {
263+
return fmt.Errorf("failed to parse CA certificate from %q", caPath)
248264
}
249-
if config.RootCa != nil {
250-
caPool := x509.NewCertPool()
251-
if !caPool.AppendCertsFromPEM([]byte(*config.RootCa)) {
252-
return nil, fmt.Errorf("failed to parse provided root CA")
253-
}
254-
tlsSetting.RootCAs = caPool
265+
tlsConfig.RootCAs = caPool
266+
}
267+
268+
// Use GetClientCertificate so the cert/key are re-read on every TLS
269+
// handshake, ensuring rotated certificates are picked up immediately.
270+
tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
271+
certPEM, err := os.ReadFile(certPath)
272+
if err != nil {
273+
return nil, fmt.Errorf("failed to read TLS certificate from %q: %w", certPath, err)
274+
}
275+
keyPEM, err := os.ReadFile(keyPath)
276+
if err != nil {
277+
return nil, fmt.Errorf("failed to read TLS private key from %q: %w", keyPath, err)
278+
}
279+
cert, err := tls.X509KeyPair(certPEM, keyPEM)
280+
if err != nil {
281+
return nil, fmt.Errorf("failed to parse TLS certificate from directory %q: %w", dir, err)
282+
}
283+
return &cert, nil
284+
}
285+
286+
return nil
287+
}
288+
289+
// configureInlineTLS configures the tls.Config using inline certificate/key
290+
// PEM values from the ClickHouse peer config.
291+
func configureInlineTLS(tlsConfig *tls.Config, config *protos.ClickhouseConfig) error {
292+
if config.Certificate != nil || config.PrivateKey != nil {
293+
if config.Certificate == nil || config.PrivateKey == nil {
294+
return errors.New("both certificate and private key must be provided if using certificate-based authentication")
295+
}
296+
cert, err := tls.X509KeyPair([]byte(*config.Certificate), []byte(*config.PrivateKey))
297+
if err != nil {
298+
return fmt.Errorf("failed to parse provided certificate: %w", err)
299+
}
300+
tlsConfig.Certificates = []tls.Certificate{cert}
301+
}
302+
if config.RootCa != nil {
303+
caPool := x509.NewCertPool()
304+
if !caPool.AppendCertsFromPEM([]byte(*config.RootCa)) {
305+
return errors.New("failed to parse provided root CA")
306+
}
307+
tlsConfig.RootCAs = caPool
308+
}
309+
return nil
310+
}
311+
312+
// buildTLSConfig builds the TLS configuration for a ClickHouse connection.
313+
// When a TLS certificate directory is configured, it loads certificates from the directory.
314+
// Otherwise, inline certificates are used.
315+
func buildTLSConfig(config *protos.ClickhouseConfig) (*tls.Config, error) {
316+
if config.DisableTls {
317+
return nil, nil
318+
}
319+
320+
tlsConfig := &tls.Config{
321+
MinVersion: tls.VersionTLS13,
322+
ServerName: config.TlsHost,
323+
}
324+
certDir := config.GetTlsCertificateDirectory()
325+
326+
if certDir != "" {
327+
if err := configureDirectoryTLS(tlsConfig, certDir); err != nil {
328+
return nil, err
255329
}
256-
if config.TlsHost != "" {
257-
tlsSetting.ServerName = config.TlsHost
330+
} else {
331+
if err := configureInlineTLS(tlsConfig, config); err != nil {
332+
return nil, err
258333
}
259334
}
260335

336+
return tlsConfig, nil
337+
}
338+
339+
func Connect(ctx context.Context, env map[string]string, config *protos.ClickhouseConfig) (clickhouse.Conn, error) {
340+
tlsSetting, err := buildTLSConfig(config)
341+
if err != nil {
342+
return nil, err
343+
}
344+
261345
settings := clickhouse.Settings{
262346
// See: https://clickhouse.com/docs/en/cloud/reference/shared-merge-tree#consistency
263347
"select_sequential_consistency": uint64(1),

flow/e2e/clickhouse_tls_test.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package e2e
2+
3+
import (
4+
"os"
5+
"strconv"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
"google.golang.org/protobuf/proto"
10+
11+
connclickhouse "github.com/PeerDB-io/peerdb/flow/connectors/clickhouse"
12+
"github.com/PeerDB-io/peerdb/flow/generated/protos"
13+
)
14+
15+
// TestClickHouseTLSDirectory verifies that connclickhouse.Connect works
16+
// with TLS certificates loaded from a directory (simulating a volume-mounted K8s Secret).
17+
func TestClickHouseTLSDirectory(t *testing.T) {
18+
portStr := os.Getenv("PEERDB_CLICKHOUSE_TLS_PORT")
19+
if portStr == "" {
20+
t.Skip("PEERDB_CLICKHOUSE_TLS_PORT not set, skipping TLS test")
21+
}
22+
port, err := strconv.ParseUint(portStr, 10, 32)
23+
require.NoError(t, err, "invalid PEERDB_CLICKHOUSE_TLS_PORT value: %s", portStr)
24+
25+
certDir := os.Getenv("PEERDB_CLICKHOUSE_TLS_CERT_DIR")
26+
if certDir == "" {
27+
t.Skip("PEERDB_CLICKHOUSE_TLS_CERT_DIR not set, skipping TLS test")
28+
}
29+
30+
config := &protos.ClickhouseConfig{
31+
Host: "localhost",
32+
Port: uint32(port),
33+
User: "peerdb_tls",
34+
Database: "default",
35+
DisableTls: false,
36+
TlsCertificateDirectory: proto.String(certDir),
37+
}
38+
39+
conn, err := connclickhouse.Connect(t.Context(), nil, config)
40+
require.NoError(t, err, "failed to connect to ClickHouse")
41+
defer conn.Close()
42+
43+
var result uint8
44+
require.NoError(t, conn.QueryRow(t.Context(), "SELECT 1").Scan(&result), "failed to execute SELECT 1")
45+
require.Equal(t, uint8(1), result)
46+
}

generate-protos.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
set -xeu
33

44
# check if buf is installed
5-
if ! command -v buf &> /dev/null
5+
if ! command -v buf > /dev/null 2>&1
66
then
77
echo "buf could not be found"
88
echo "Please install buf: https://buf.build/docs/installation"
9-
exit
9+
exit 1
1010
fi
1111

1212
buf generate protos

nexus/analyzer/src/lib.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -885,6 +885,9 @@ fn parse_db_options(db_type: DbType, with_options: &[SqlOption]) -> anyhow::Resu
885885
.map(|s| s.parse::<bool>().unwrap_or_default())
886886
.unwrap_or_default(),
887887
s3: None,
888+
tls_certificate_directory: opts
889+
.get("tls_certificate_directory")
890+
.map(|s| s.to_string()),
888891
};
889892
Config::ClickhouseConfig(clickhouse_config)
890893
}

protos/peers.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,9 @@ message ClickhouseConfig{
186186
optional S3Config s3 = 16;
187187
string cluster = 17;
188188
bool replicated = 18;
189+
// Directory path containing TLS certificate files (tls.crt, tls.key, and optionally ca.crt).
190+
// Typically a Kubernetes Secret mounted as a volume. When set, takes precedence over inline cert fields.
191+
optional string tls_certificate_directory = 19;
189192
}
190193

191194
message SqlServerConfig {

protos/route.proto

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,9 @@ message PostScriptResponse { int32 id = 1; }
6666
message DeleteScriptRequest { int32 id = 1; }
6767
message DeleteScriptResponse {}
6868

69-
message ValidatePeerRequest { peerdb_peers.Peer peer = 1; }
69+
message ValidatePeerRequest {
70+
peerdb_peers.Peer peer = 1;
71+
}
7072

7173
message CreatePeerRequest {
7274
peerdb_peers.Peer peer = 1;

ui/app/peers/create/[peerType]/helpers/ch.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,24 @@ export const clickhouseSetting: PeerSetting[] = [
108108
optional: true,
109109
tips: 'If not provided, host CA roots will be used.',
110110
},
111+
{
112+
label: 'TLS Certificate Directory',
113+
stateHandler: (value, setter) => {
114+
if (!value) {
115+
setter((curr) => {
116+
const newCurr = { ...curr } as ClickhouseConfig;
117+
delete newCurr.tlsCertificateDirectory;
118+
return newCurr;
119+
});
120+
} else
121+
setter((curr) => ({
122+
...curr,
123+
tlsCertificateDirectory: value as string,
124+
}));
125+
},
126+
optional: true,
127+
tips: 'Path to a directory containing TLS certificate files (tls.crt, tls.key, and optionally ca.crt). Typically a Kubernetes Secret mounted as a volume. When set, inline certificate/private key fields are ignored.',
128+
},
111129
{
112130
label: 'S3 Path',
113131
stateHandler: (value, setter) =>

ui/app/peers/create/[peerType]/schema.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,12 @@ export function chSchema(hostDomains: string[]) {
434434
})
435435
.optional()
436436
.transform((e) => (e === '' ? undefined : e)),
437+
tlsCertificateDirectory: z
438+
.string({
439+
error: () => 'TLS Certificate Directory must be a string',
440+
})
441+
.optional()
442+
.transform((e) => (e === '' ? undefined : e)),
437443
tlsHost: z.string(),
438444
});
439445
}

0 commit comments

Comments
 (0)