There are many errors, see the attachment for details。

[xc1989xc]

This window uses Kafka
log.log

config.yml

docker-compose.yaml

[xc1989xc]

data is inconsistent。This window uses a lightweight。

mysql> select count() from sbtest.sbtest1;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.07 sec)

mysql> select count() from sbtest.sbtest2;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.13 sec)

mysql> select count() from sbtest.sbtest3;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.03 sec)

mysql> select count() from sbtest.sbtest4;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.05 sec)

mysql> select count() from sbtest.sbtest5;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.04 sec)

mysql> select count() from sbtest.sbtest6;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.04 sec)

mysql> select count() from sbtest.sbtest7;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.10 sec)

mysql> select count() from sbtest.sbtest8;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.11 sec)

mysql> select count() from sbtest.sbtest9;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.13 sec)

mysql> select count() from sbtest.sbtest10;
+----------+
| count() |
+----------+
| 100000 |
+----------+
1 row in set (0.12 sec)
++++++++++++++++++++++++++++++++++++++++++++++
select count(*) from sbtest.sbtest1;

SELECT count(*)
FROM sbtest.sbtest1

┌─count()─┐
│ 122528 │
└─────────┘
↓ Progress: 6.00 rows, 4.14 KB (1.47 thousand rows/s., 1.01 MB/s.)
1 rows in set. Elapsed: 0.004 sec.

dgdf :) select count(*) from sbtest.sbtest2;

SELECT count(*)
FROM sbtest.sbtest2

┌─count()─┐
│ 115351 │
└─────────┘
← Progress: 5.00 rows, 4.14 KB (1.32 thousand rows/s., 1.09 MB/s.)
1 rows in set. Elapsed: 0.004 sec.

dgdf :) select count(*) from sbtest.sbtest3;

SELECT count(*)
FROM sbtest.sbtest3

┌─count()─┐
│ 116384 │
└─────────┘
↑ Progress: 3.00 rows, 4.12 KB (831.73 rows/s., 1.14 MB/s.)
1 rows in set. Elapsed: 0.004 sec.

dgdf :) select count(*) from sbtest.sbtest4;

SELECT count(*)
FROM sbtest.sbtest4

┌─count()─┐
│ 112599 │
└─────────┘
→ Progress: 4.00 rows, 4.13 KB (1.22 thousand rows/s., 1.25 MB/s.)
1 rows in set. Elapsed: 0.004 sec.

dgdf :) select count(*) from sbtest.sbtest5;

SELECT count(*)
FROM sbtest.sbtest5

┌─count()─┐
│ 112288 │
└─────────┘
↓ Progress: 5.00 rows, 4.14 KB (1.76 thousand rows/s., 1.46 MB/s.)
1 rows in set. Elapsed: 0.003 sec.

dgdf :) select count(*) from sbtest.sbtest6;

SELECT count(*)
FROM sbtest.sbtest6

┌─count()─┐
│ 120087 │
└─────────┘
← Progress: 7.00 rows, 4.15 KB (1.48 thousand rows/s., 877.30 KB/s.)
1 rows in set. Elapsed: 0.005 sec.

dgdf :) select count(*) from sbtest.sbtest7;

SELECT count(*)
FROM sbtest.sbtest7

┌─count()─┐
│ 122528 │
└─────────┘
↑ Progress: 6.00 rows, 4.14 KB (1.28 thousand rows/s., 881.14 KB/s.)
1 rows in set. Elapsed: 0.005 sec.

dgdf :) select count(*) from sbtest.sbtest8;

SELECT count(*)
FROM sbtest.sbtest8

┌─count()─┐
│ 116384 │
└─────────┘
→ Progress: 6.00 rows, 4.14 KB (1.40 thousand rows/s., 964.33 KB/s.)
1 rows in set. Elapsed: 0.005 sec.

dgdf :) select count(*) from sbtest.sbtest9;

SELECT count(*)
FROM sbtest.sbtest9

┌─count()─┐
│ 104695 │
└─────────┘
↓ Progress: 3.00 rows, 4.12 KB (955.90 rows/s., 1.31 MB/s.)
1 rows in set. Elapsed: 0.004 sec.

dgdf :) select count(*) from sbtest.sbtest10;

SELECT count(*)
FROM sbtest.sbtest10

┌─count()─┐
│ 109559 │
└─────────┘
← Progress: 5.00 rows, 4.14 KB (1.84 thousand rows/s., 1.52 MB/s.)
1 rows in set. Elapsed: 0.003 sec.

dgdf :)
dgdf :)
Bye
++++++++++++++++++++++++++++++++++++++++++++++++++++++++
root@dgdf:~/clickhouse-sink-connector/sink-connector-lightweight/docker# more docker-compose-mysql.yml
version: "3.4"

Ubuntu , set this for redpanda to start

https://sort.veritas.com/public/documents/HSO/2.0/linux/productguides/html/hfo_admin_ubuntu/ch04s03.htm

Clickhouse Table Schema

create table test(id int, message String) ENGINE=MergeTree() PRIMARY KEY id;

services:

mysql-master:

extends:

file: mysql-master-service.yml

service: mysql-master

clickhouse:

extends:

file: clickhouse-service.yml

service: clickhouse

depends_on:

zookeeper:

condition: service_healthy

zookeeper:
extends:
file: zookeeper-service.yml
service: zookeeper

clickhouse-sink-connector-lt:
extends:
file: clickhouse-sink-connector-lt-service.yml
service: clickhouse-sink-connector-lt

MONITORING

prometheus:
extends:
file: prometheus-service.yml
service: prometheus

grafana:
extends:
file: grafana-service.yml
service: grafana
volumes:
- ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
- ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
- ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
depends_on:
- prometheus

## END OF MONITORING ###

++++++++++++++++++++++++++++++++++++++++
root@dgdf:~/clickhouse-sink-connector/sink-connector-lightweight/docker
name: "company-1"
topic.prefix: "sink-connector-1"
database.hostname: "192.168.241.130"
database.port: "3306"
database.user: "test"
database.password: "Aa1234"
database.server.id: "130"
database.server.name: "ER54"
database.include.list: sbtest
table.include.list: ""
clickhouse.server.url: "192.168.241.130"
clickhouse.server.user: "default"
clickhouse.server.password: "123456"
clickhouse.server.port: "8123"
database.allowPublicKeyRetrieval: "true"
snapshot.mode: "initial"
offset.flush.interval.ms: 500
connector.class: "io.debezium.connector.mysql.MySqlConnector"
offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore"
offset.storage.jdbc.table.name: "altinity_sink_connector.replica_source_info"
offset.storage.jdbc.url: "jdbc:clickhouse://192.168.241.130:8123/altinity_sink_connector"
offset.storage.jdbc.user: "default"
offset.storage.jdbc.password: "123456"
offset.storage.jdbc.table.ddl: "CREATE TABLE if not exists %s
(
id String,
offset_key String,
offset_val String,
record_insert_ts DateTime,
record_insert_seq UInt64,
_version UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
)
ENGINE = ReplacingMergeTree(_version)
ORDER BY id
SETTINGS index_granularity = 8192"
offset.storage.jdbc.table.delete: "delete from %s where 1=1"
schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"
schema.history.internal.jdbc.url: "jdbc:clickhouse://192.168.241.130:8123/altinity_sink_connector"
schema.history.internal.jdbc.user: "default"
schema.history.internal.jdbc.password: "123456"
schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s
(id VARCHAR(36) NOT NULL, history_data VARCHAR(65000), history_data_seq INTEGER, record_insert_ts TIMESTAMP NOT NULL, record_insert_seq INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_se
q) order by id"
schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history"
enable.snapshot.ddl: "true"
persist.raw.bytes: "false"
auto.create.tables: "true"
database.connectionTimeZone: "America/Chicago"
errors.max.retries: 3
clickhouse.datetime.timezone: "America/Chicago"
ignore.ddl.regex: "(?i)(ANALYZE PARTITION).*"
clickhouse.jdbc.params: "keepalive.timeout=3,max_buffer_size=1000000,socket_timeout=30000,connection_timeout=30000"
clickhouse.jdbc.settings: "input_format_null_as_default=0,allow_experimental_object_type=1,insert_allow_materialized_columns=1"
metrics.enable: "true"
use.nongraceful.disconnect: "true"
database.keep.alive.interval.ms: "30000"
database.connection.reconnect.backoff.ms: "1000"
database.connection.reconnect.backoff.max.ms: "10000"
database.ssl.mode: "disabled"
non.default.value: "true"
++++++++++++++++++++++++++++++++
log.log
++++++++++++++++++++++++++++++++++
Schema in MSYQL：
mysql> show databases;
+--------------------+
| Database |
+--------------------+
| information_schema |
| mysql |
| performance_schema |
| sbtest |
| sbtest1 |
| sbtest2 |
| sys |
+--------------------+

[xc1989xc]

docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
debezium/connect 3.1.3.Final 034b518c95b2 22 hours ago 1.42GB
docker_debezium latest 034b518c95b2 22 hours ago 1.42GB
grafana/grafana 9.1.1 c7e52e8e9cae 22 hours ago 299MB
docker_grafana latest c7e52e8e9cae 22 hours ago 299MB
docker-grafana latest e62e7e444f56 3 days ago 724MB
mongodb/mongodb-community-server latest da5eb60fccb3 3 days ago 1.34GB
redpandadata/redpanda latest 869ec6677b26 6 days ago 417MB
redpandadata/console latest 106daed4d6f1 9 days ago 132MB
redis latest 9d1fe3a9a889 10 days ago 137MB
hashicorp/consul latest ee6d75ac9539 2 weeks ago 216MB
altinity/clickhouse-sink-connector 2.7.1-lt 67b790ec2740 7 weeks ago 1.19GB
confluentinc/cp-schema-registry latest 0b81a734ce6a 3 months ago 2.01GB
bitnami/mysql 8.0 5a9aec44856c 8 months ago 726MB
hanchuanchuan/goinception latest d0fd80a2747e 10 months ago 125MB
debezium/connect 3.0.0.Final 1ecad018ac0c 10 months ago 1.4GB
debezium/connect 3.0.0.Final 1ecad018ac0c 10 months ago 1.4GB
clickhouse/clickhouse-server 23.8.5 20a15ce58806 22 months ago 1.01GB
actiontech/dtle latest b3023ac479f8 2 years ago 868MB
hhyo/archery v1.9.1 72a11c080e80 2 years ago 1.99GB
bitnami/prometheus 2.36.0 f8df5fc53dac 3 years ago 508MB
bitnami/kafka 2 ec9c62095562 3 years ago 585MB
bitnami/zookeeper 3 b7f8fb55f3cd 4 years ago 468MB
zookeeper 3.6.2 a72350516291 4 years ago 268MB
hlebalbau/kafka-manager latest 2743fc8f2a3e 4 years ago 347MB
sscaling/jmx-prometheus-exporter latest 46303433ca79 5 years ago 131MB

[aadant]

Regarding the results, you are querying a RMT so you should use FINAL or settings final=1