Skip to content

Commit 7e9c5f8

Browse files
dtunikovdtunikov
committed
Merge branch 'main' of github.com:PeerDB-io/peerdb into fix/dbi-823/json-representation-snapshot-cdc-mismatch
2 parents 564fb88 + cd189a0 commit 7e9c5f8

11 files changed

Lines changed: 243 additions & 18 deletions

File tree

‎.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ CI_CLICKHOUSE_HOST=host.docker.internal
1111
CI_CLICKHOUSE_HTTP_PORT=11123
1212
CI_CLICKHOUSE_NATIVE_PORT=11000
1313

14+
CI_CLICKHOUSE_HTTP_PORT_02=13123
15+
CI_CLICKHOUSE_NATIVE_PORT_02=13000
16+
1417
CI_MYSQL_HOST=host.docker.internal
1518
CI_MYSQL_GTID_PORT=3306
1619
CI_MYSQL_POS_PORT=3307

‎Tiltfile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,13 @@ local_resource(
108108
resource_deps=['clickhouse']
109109
)
110110

111+
local_resource(
112+
'provision-clickhouse-cluster',
113+
cmd='./local_provision_scripts/clickhouse-cluster.sh',
114+
labels=['Ancillary-DB-Provisioning'],
115+
resource_deps=['provision-clickhouse', 'clickhouse-02', 'clickhouse-keeper'],
116+
)
117+
111118
local_resource(
112119
'provision-mysql-gtid',
113120
cmd='./local_provision_scripts/mysql.sh peerdb-mysql-gtid',
@@ -164,6 +171,13 @@ local_resource(
164171
resource_deps=['peerdb', 'provision-clickhouse'],
165172
)
166173

174+
local_resource(
175+
'setup-clickhouse-cluster-peer',
176+
cmd='./local_provision_scripts/setup-clickhouse-cluster-peer.sh',
177+
labels=['Setup-PeerDB-Peers'],
178+
resource_deps=['peerdb', 'provision-clickhouse-cluster'],
179+
)
180+
167181
local_resource(
168182
'setup-mongodb-peer',
169183
cmd='./local_provision_scripts/setup-mongodb-peer.sh',
@@ -209,6 +223,13 @@ dc_resource('clickhouse', labels=['Ancillary-DB'], links=[
209223
link('http://localhost:' + resolve_ancillary_env('CI_CLICKHOUSE_NATIVE_PORT', '9000'), 'ClickHouse TCP'),
210224
], auto_init=False)
211225

226+
dc_resource('clickhouse-keeper', labels=['Ancillary-DB'], auto_init=False)
227+
228+
dc_resource('clickhouse-02', labels=['Ancillary-DB'], links=[
229+
link('http://localhost:' + resolve_ancillary_env('CI_CLICKHOUSE_HTTP_PORT_02', '13123'), 'CH Node 2 HTTP'),
230+
link('http://localhost:' + resolve_ancillary_env('CI_CLICKHOUSE_NATIVE_PORT_02', '13000'), 'CH Node 2 TCP'),
231+
], auto_init=False)
232+
212233
dc_resource('mongodb', labels=['Ancillary-DB'], links=[
213234
link('http://localhost:' + resolve_ancillary_env('CI_MONGO_PORT', '27017'), 'MongoDB'),
214235
], auto_init=False)

‎ancillary-docker-compose.yml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,66 @@ services:
4040

4141
clickhouse:
4242
container_name: peerdb-clickhouse
43+
hostname: clickhouse
4344
image: ${CLICKHOUSE_IMAGE}
4445
restart: unless-stopped
4546
ports:
4647
- "${CI_CLICKHOUSE_HTTP_PORT}:8123"
4748
- "${CI_CLICKHOUSE_NATIVE_PORT}:9000"
4849
volumes:
4950
- clickhouse_data:/var/lib/clickhouse/
51+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
52+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/users.d/default-user.xml:/etc/clickhouse-server/users.d/default-user.xml:ro
53+
- ./local_provision_scripts/clickhouse-cluster-config/node1/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml:ro
54+
ulimits:
55+
nofile:
56+
soft: 262144
57+
hard: 262144
58+
extra_hosts:
59+
- "host.docker.internal:host-gateway"
60+
healthcheck:
61+
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8123/ping"]
62+
interval: 2s
63+
timeout: 10s
64+
retries: 5
65+
66+
clickhouse-keeper:
67+
container_name: peerdb-clickhouse-keeper
68+
hostname: clickhouse-keeper
69+
image: ${CLICKHOUSE_IMAGE}
70+
restart: unless-stopped
71+
command: ["clickhouse", "keeper", "--config-file=/etc/clickhouse-keeper/keeper_config.xml"]
72+
volumes:
73+
- clickhouse_keeper_data:/var/lib/clickhouse/
74+
- ./local_provision_scripts/clickhouse-cluster-config/keeper/keeper_config.xml:/etc/clickhouse-keeper/keeper_config.xml:ro
75+
ulimits:
76+
nofile:
77+
soft: 262144
78+
hard: 262144
79+
extra_hosts:
80+
- "host.docker.internal:host-gateway"
81+
healthcheck:
82+
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9181 && printf ruok >&3 && timeout 2 cat <&3 | grep -q imok"]
83+
interval: 2s
84+
timeout: 10s
85+
retries: 15
86+
87+
clickhouse-02:
88+
container_name: peerdb-clickhouse-02
89+
hostname: clickhouse-02
90+
image: ${CLICKHOUSE_IMAGE}
91+
restart: unless-stopped
92+
ports:
93+
- "${CI_CLICKHOUSE_HTTP_PORT_02}:8123"
94+
- "${CI_CLICKHOUSE_NATIVE_PORT_02}:9000"
95+
volumes:
96+
- clickhouse_02_data:/var/lib/clickhouse/
97+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/config.d/cluster.xml:/etc/clickhouse-server/config.d/cluster.xml:ro
98+
- ./local_provision_scripts/clickhouse-cluster-config/node-common/users.d/default-user.xml:/etc/clickhouse-server/users.d/default-user.xml:ro
99+
- ./local_provision_scripts/clickhouse-cluster-config/node2/config.d/macros.xml:/etc/clickhouse-server/config.d/macros.xml:ro
100+
depends_on:
101+
clickhouse-keeper:
102+
condition: service_healthy
50103
ulimits:
51104
nofile:
52105
soft: 262144
@@ -229,6 +282,8 @@ volumes:
229282
postgres_data:
230283
postgres2_data:
231284
clickhouse_data:
285+
clickhouse_keeper_data:
286+
clickhouse_02_data:
232287
mongo_data:
233288
mysql_gtid_data:
234289
mysql_pos_data:

‎local_provision_scripts/clickhouse-cluster-config/keeper/keeper_config.xml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<clickhouse replace="true">
2+
<logger>
3+
<level>information</level>
4+
<console>1</console>
5+
</logger>
6+
<listen_host>0.0.0.0</listen_host>
7+
<keeper_server>
8+
<tcp_port>9181</tcp_port>
9+
<server_id>1</server_id>
10+
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
11+
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
12+
<coordination_settings>
13+
<operation_timeout_ms>10000</operation_timeout_ms>
14+
<session_timeout_ms>30000</session_timeout_ms>
15+
<raft_logs_level>information</raft_logs_level>
16+
</coordination_settings>
17+
<raft_configuration>
18+
<server>
19+
<id>1</id>
20+
<hostname>clickhouse-keeper</hostname>
21+
<port>9234</port>
22+
</server>
23+
</raft_configuration>
24+
</keeper_server>
25+
</clickhouse>

‎local_provision_scripts/clickhouse-cluster-config/node-common/config.d/cluster.xml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
<clickhouse>
2+
<remote_servers>
3+
<cicluster>
4+
<shard>
5+
<replica>
6+
<host>clickhouse</host>
7+
<port>9000</port>
8+
</replica>
9+
</shard>
10+
<shard>
11+
<replica>
12+
<host>clickhouse-02</host>
13+
<port>9000</port>
14+
</replica>
15+
</shard>
16+
</cicluster>
17+
</remote_servers>
18+
<zookeeper>
19+
<node>
20+
<host>clickhouse-keeper</host>
21+
<port>9181</port>
22+
</node>
23+
</zookeeper>
24+
<distributed_ddl>
25+
<path>/clickhouse/task_queue/ddl</path>
26+
</distributed_ddl>
27+
</clickhouse>

‎local_provision_scripts/clickhouse-cluster-config/node-common/users.d/default-user.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<clickhouse>
2+
<users>
3+
<default>
4+
<password></password>
5+
<networks>
6+
<ip>::/0</ip>
7+
</networks>
8+
<access_management>1</access_management>
9+
<named_collection_control>1</named_collection_control>
10+
</default>
11+
</users>
12+
</clickhouse>

‎local_provision_scripts/clickhouse-cluster-config/node1/config.d/macros.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<clickhouse>
2+
<macros>
3+
<shard>1</shard>
4+
<replica>1</replica>
5+
</macros>
6+
</clickhouse>

‎local_provision_scripts/clickhouse-cluster-config/node2/config.d/macros.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<clickhouse>
2+
<macros>
3+
<shard>2</shard>
4+
<replica>1</replica>
5+
</macros>
6+
</clickhouse>

‎local_provision_scripts/clickhouse-cluster.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/sh
2+
set -Eeu
3+
4+
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
5+
# shellcheck source=../.env
6+
. "$SCRIPT_DIR/../.env"
7+
. "$SCRIPT_DIR/../ancillary.env"
8+
9+
CONTAINER1="peerdb-clickhouse"
10+
CONTAINER2="peerdb-clickhouse-02"
11+
12+
wait_for_node() {
13+
container="$1"
14+
echo "Waiting for $container to be ready..."
15+
i=0
16+
while [ $i -lt 30 ]; do
17+
if docker exec "$container" wget --no-verbose --tries=1 --spider "http://localhost:8123/ping" 2>/dev/null; then
18+
echo "$container is ready"
19+
return 0
20+
fi
21+
sleep 2
22+
i=$((i + 1))
23+
done
24+
echo "Timed out waiting for $container"
25+
return 1
26+
}
27+
28+
wait_for_node "$CONTAINER1"
29+
wait_for_node "$CONTAINER2"
30+
31+
echo "Verifying cicluster is recognized by both nodes..."
32+
shard_count=$(docker exec "$CONTAINER1" clickhouse-client --query \
33+
"SELECT count() FROM system.clusters WHERE cluster='cicluster'" 2>/dev/null || echo "0")
34+
if [ "$shard_count" = "0" ]; then
35+
echo "ERROR: cicluster not found on $CONTAINER1"
36+
exit 1
37+
fi
38+
echo "cicluster found with $shard_count shard(s)"
39+
40+
echo "ClickHouse cluster is ready: node1=${CI_CLICKHOUSE_HOST}:${CI_CLICKHOUSE_NATIVE_PORT}, node2=${CI_CLICKHOUSE_HOST}:${CI_CLICKHOUSE_NATIVE_PORT_02}"

‎local_provision_scripts/clickhouse.sh‎

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,23 +6,6 @@ SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
66
. "$SCRIPT_DIR/../.env"
77
. "$SCRIPT_DIR/../ancillary.env"
88

9-
CONTAINER="peerdb-clickhouse"
10-
11-
echo "Setting empty password for default user"
12-
docker exec "$CONTAINER" bash -c 'cat > /etc/clickhouse-server/users.d/default-user.xml <<EOF
13-
<clickhouse>
14-
<users>
15-
<default>
16-
<password></password>
17-
<networks>
18-
<ip>::/0</ip>
19-
</networks>
20-
<access_management>1</access_management>
21-
<named_collection_control>1</named_collection_control>
22-
</default>
23-
</users>
24-
</clickhouse>
25-
EOF'
26-
docker restart "$CONTAINER"
9+
# Placeholder script for future provisioning steps.
2710

2811
echo "ClickHouse is ready at ${CI_CLICKHOUSE_HOST}:${CI_CLICKHOUSE_NATIVE_PORT}"

0 commit comments

Comments
 (0)