Skip to content

Commit 9b30f8d

Browse files
Daesgargithub-actions[bot]
Daesgar
and
github-actions[bot]
authored
feat: add support for coordinator schemas (#28031)
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
1 parent f7563e3 commit 9b30f8d

20 files changed

+292
-73
lines changed

‎.flox/env/manifest.toml

+3-3
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,10 @@ fi
9191
9292
if [[ -t 0 ]]; then # The block below only runs when in an interactive shell
9393
# Add required entries to /etc/hosts if not present
94-
if ! grep -q "127.0.0.1 kafka clickhouse" /etc/hosts; then
94+
if ! grep -q "127.0.0.1 kafka clickhouse clickhouse-coordinator" /etc/hosts; then
9595
echo
96-
echo "🚨 Amending /etc/hosts to map hostnames 'kafka' and 'clickhouse' to 127.0.0.1..."
97-
echo "127.0.0.1 kafka clickhouse" | sudo tee -a /etc/hosts 1> /dev/null
96+
echo "🚨 Amending /etc/hosts to map hostnames 'kafka', 'clickhouse' and 'clickhouse-coordinator' to 127.0.0.1..."
97+
echo "127.0.0.1 kafka clickhouse clickhouse-coordinator" | sudo tee -a /etc/hosts 1> /dev/null
9898
echo "✅ /etc/hosts amended"
9999
fi
100100

‎docker-compose.dev-full.yml

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ services:
4747
- ./posthog/idl:/idl
4848
- ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
4949
- ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
50+
- ./docker/clickhouse/config.d/default.xml:/etc/clickhouse-server/config.d/default.xml
5051
- ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
5152
- ./docker/clickhouse/user_defined_function.xml:/etc/clickhouse-server/user_defined_function.xml
5253
- ./posthog/user_scripts:/var/lib/clickhouse/user_scripts

‎docker-compose.dev.yml

+19-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ services:
5151
ports:
5252
- '5555:5555'
5353

54-
clickhouse:
54+
clickhouse: &clickhouse
5555
extends:
5656
file: docker-compose.base.yml
5757
service: clickhouse
@@ -69,6 +69,7 @@ services:
6969
- ./posthog/idl:/idl
7070
- ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
7171
- ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
72+
- ./docker/clickhouse/config.d/worker.xml:/etc/clickhouse-server/config.d/worker.xml
7273
- ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
7374
- ./docker/clickhouse/user_defined_function.xml:/etc/clickhouse-server/user_defined_function.xml
7475
- ./posthog/user_scripts:/var/lib/clickhouse/user_scripts
@@ -78,6 +79,23 @@ services:
7879
- kafka
7980
- zookeeper
8081

82+
clickhouse-coordinator:
83+
hostname: clickhouse-coordinator
84+
<<: *clickhouse
85+
volumes:
86+
# this new entrypoint file is to fix a bug detailed here https://github.com/ClickHouse/ClickHouse/pull/59991
87+
# revert this when we upgrade clickhouse
88+
- ./docker/clickhouse/entrypoint.sh:/entrypoint.sh
89+
- ./posthog/idl:/idl
90+
- ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
91+
- ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
92+
- ./docker/clickhouse/config.d/coordinator.xml:/etc/clickhouse-server/config.d/coordinator.xml
93+
- ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
94+
- ./docker/clickhouse/user_defined_function.xml:/etc/clickhouse-server/user_defined_function.xml
95+
- ./posthog/user_scripts:/var/lib/clickhouse/user_scripts
96+
ports:
97+
- '9001:9001'
98+
8199
zookeeper:
82100
extends:
83101
file: docker-compose.base.yml

‎docker-compose.hobby.yml

+1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ services:
4848
- ./posthog/posthog/idl:/idl
4949
- ./posthog/docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
5050
- ./posthog/docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
51+
- ./posthog/docker/clickhouse/config.d/default.xml:/etc/clickhouse-server/config.d/default.xml
5152
- ./posthog/docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml
5253
- clickhouse-data:/var/lib/clickhouse
5354
zookeeper:

‎docker/clickhouse/config.d/coordinator.xml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<clickhouse>
2+
<tcp_port>9001</tcp_port>
3+
<remote_servers>
4+
<posthog>
5+
<shard>
6+
<replica>
7+
<host>clickhouse</host>
8+
<port>9000</port>
9+
</replica>
10+
</shard>
11+
</posthog>
12+
<posthog_single_shard>
13+
<shard>
14+
<replica>
15+
<host>clickhouse</host>
16+
<port>9000</port>
17+
</replica>
18+
</shard>
19+
</posthog_single_shard>
20+
<posthog_migrations>
21+
<shard>
22+
<replica>
23+
<host>clickhouse</host>
24+
<port>9000</port>
25+
</replica>
26+
</shard>
27+
<shard>
28+
<replica>
29+
<host>clickhouse-coordinator</host>
30+
<port>9001</port>
31+
</replica>
32+
</shard>
33+
</posthog_migrations>
34+
</remote_servers>
35+
36+
<macros>
37+
<shard>02</shard>
38+
<replica>coord</replica>
39+
<hostClusterType>online</hostClusterType>
40+
<hostClusterRole>coordinator</hostClusterRole>
41+
</macros>
42+
</clickhouse>

‎docker/clickhouse/config.d/default.xml

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
<clickhouse>
2+
<tcp_port>9000</tcp_port>
3+
4+
<remote_servers>
5+
<posthog>
6+
<shard>
7+
<replica>
8+
<host>localhost</host>
9+
<port>9000</port>
10+
</replica>
11+
</shard>
12+
</posthog>
13+
<posthog_single_shard>
14+
<shard>
15+
<replica>
16+
<host>localhost</host>
17+
<port>9000</port>
18+
</replica>
19+
</shard>
20+
</posthog_single_shard>
21+
<posthog_migrations>
22+
<shard>
23+
<replica>
24+
<host>localhost</host>
25+
<port>9000</port>
26+
</replica>
27+
</shard>
28+
</posthog_migrations>
29+
</remote_servers>
30+
31+
<macros>
32+
<shard>01</shard>
33+
<replica>ch1</replica>
34+
<hostClusterType>online</hostClusterType>
35+
<hostClusterRole>worker</hostClusterRole>
36+
</macros>
37+
</clickhouse>

‎docker/clickhouse/config.d/worker.xml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<clickhouse>
2+
<tcp_port>9000</tcp_port>
3+
<remote_servers>
4+
<posthog>
5+
<shard>
6+
<replica>
7+
<host>clickhouse</host>
8+
<port>9000</port>
9+
</replica>
10+
</shard>
11+
</posthog>
12+
<posthog_single_shard>
13+
<shard>
14+
<replica>
15+
<host>clickhouse</host>
16+
<port>9000</port>
17+
</replica>
18+
</shard>
19+
</posthog_single_shard>
20+
<posthog_migrations>
21+
<shard>
22+
<replica>
23+
<host>clickhouse</host>
24+
<port>9000</port>
25+
</replica>
26+
</shard>
27+
<shard>
28+
<replica>
29+
<host>clickhouse-coordinator</host>
30+
<port>9001</port>
31+
</replica>
32+
</shard>
33+
</posthog_migrations>
34+
</remote_servers>
35+
36+
<macros>
37+
<shard>01</shard>
38+
<replica>ch1</replica>
39+
<hostClusterType>online</hostClusterType>
40+
<hostClusterRole>worker</hostClusterRole>
41+
</macros>
42+
</clickhouse>

‎docker/clickhouse/config.xml

+1-26
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
1313
<size>1000M</size>
1414
<count>10</count>
15+
<console>1</console>
1516
</logger>
1617

1718
<http_port>8123</http_port>
18-
<tcp_port>9000</tcp_port>
1919
<mysql_port>9004</mysql_port>
2020
<postgresql_port>9005</postgresql_port>
2121
<https_port>8443</https_port>
@@ -162,25 +162,6 @@
162162
<!-- Reallocate memory for machine code ("text") using huge pages. Highly experimental. -->
163163
<remap_executable>false</remap_executable>
164164

165-
<remote_servers>
166-
<posthog>
167-
<shard>
168-
<replica>
169-
<host>localhost</host>
170-
<port>9000</port>
171-
</replica>
172-
</shard>
173-
</posthog>
174-
<posthog_single_shard>
175-
<shard>
176-
<replica>
177-
<host>localhost</host>
178-
<port>9000</port>
179-
</replica>
180-
</shard>
181-
</posthog_single_shard>
182-
</remote_servers>
183-
184165
<remote_url_allow_hosts>
185166
<host_regexp>.*</host_regexp>
186167
</remote_url_allow_hosts>
@@ -192,12 +173,6 @@
192173
</node>
193174
</zookeeper>
194175

195-
<macros>
196-
<shard>01</shard>
197-
<replica>ch1</replica>
198-
</macros>
199-
200-
201176
<!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. -->
202177
<builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval>
203178

‎posthog/clickhouse/client/connection.py

+6
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ class Workload(Enum):
1919
OFFLINE = "OFFLINE"
2020

2121

22+
class NodeRole(Enum):
23+
ALL = "ALL"
24+
COORDINATOR = "COORDINATOR"
25+
WORKER = "WORKER"
26+
27+
2228
_default_workload = Workload.ONLINE
2329

2430

‎posthog/clickhouse/client/migration_tools.py

+17-12
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,28 @@
1-
from typing import Union
2-
from collections.abc import Callable
1+
import logging
32

43
from infi.clickhouse_orm import migrations
54

6-
from posthog.clickhouse.client.execute import sync_execute
5+
from posthog.clickhouse.client.connection import NodeRole
6+
from posthog.clickhouse.cluster import get_cluster
7+
from posthog.settings.data_stores import CLICKHOUSE_MIGRATIONS_CLUSTER
78

9+
logger = logging.getLogger("migrations")
810

9-
def run_sql_with_exceptions(sql: Union[str, Callable[[], str]], settings=None):
11+
12+
def run_sql_with_exceptions(sql: str, settings=None, node_role: NodeRole = NodeRole.WORKER):
1013
"""
1114
migrations.RunSQL does not raise exceptions, so we need to wrap it in a function that does.
15+
node_role is set to WORKER by default to keep compatibility with the old migrations.
1216
"""
1317

14-
if settings is None:
15-
settings = {}
18+
cluster = get_cluster(client_settings=settings, cluster=CLICKHOUSE_MIGRATIONS_CLUSTER)
1619

17-
def run_sql(database):
18-
nonlocal sql
19-
if callable(sql):
20-
sql = sql()
21-
sync_execute(sql, settings=settings)
20+
def run_migration():
21+
if node_role == NodeRole.ALL:
22+
logger.info(" Running migration on coordinators and workers")
23+
return cluster.map_all_hosts(lambda client: client.execute(sql)).result()
24+
else:
25+
logger.info(f" Running migration on {node_role.value.lower()}s")
26+
return cluster.map_hosts_by_role(lambda client: client.execute(sql), node_role=node_role).result()
2227

23-
return migrations.RunPython(run_sql)
28+
return migrations.RunPython(lambda _: run_migration())

0 commit comments

Comments
 (0)