Skip to content

数据库双向增量同步报错 #404

@yangbin1212

Description

@yangbin1212

主--> 从 配置
[extractor]
db_type=mysql
;拉取类型(全量:snapshot,增量:cdc)
extract_type=cdc
binlog_position=173104571
binlog_filename=mysql-bin.000009
server_id=1
;url数据库连接信息
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
heartbeat_interval_secs=1
heartbeat_tb=heartbeat_db.ape_dts_heartbeat

[filter]
;需过滤的库,和 ignore_tbs 取并集
ignore_dbs=
;需同步的库,和 do_tbs 取并集
do_dbs=cgdb,data_exchange,iot_access,iot_core,mortisejointenon,operation,park,parksupervision,tongtongopen,tongtongpay,wukong
do_tbs=
ignore_tbs=
;需同步的事件
do_events=insert,update,delete
;需同步的 ddl,适用于 mysql cdc 任务
do_ddls=create_database,drop_database,alter_database,create_table,drop_table,truncate_table,rename_table,alter_table,create_index,drop_index

[data_marker]
topo_name=topo1
topo_nodes=node1,node2
src_node=node1
dst_node=node2
do_nodes=node1
ignore_nodes=node2
marker=ape_trans_mysql.topo1

[sinker]
db_type=mysql
sink_type=write
;batch_size批量写入数据条数,1 代表串行
batch_size=20
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
;插入数据时,如果已存在于目标库,是否强行替换,适用于 mysql/pg 的全量/增量任务.默认true
replace=false

[router]
;用于库表映射
tb_map=
col_map=
db_map=

[parallelizer]
;并发类型
parallel_type=serial
;并发线程数
parallel_size=8

[pipeline]
;内存中最多缓存数据的条数,数据同步采用多线程 & 批量写入,故须配置此项
buffer_size=100
;任务当前状态(统计数据,同步位点信息等)写入日志的频率,单位:秒
checkpoint_interval_secs=10

[runtime]
log_dir=./logs
log_level=info
log4rs_file=./log4rs.yaml

从--> 主 配置
[extractor]
db_type=mysql
;拉取类型(全量:snapshot,增量:cdc)
extract_type=cdc
binlog_position=221962113
binlog_filename=mysql-bin.000008
server_id=1
;url数据库连接信息
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
heartbeat_interval_secs=1
heartbeat_tb=heartbeat_db.ape_dts_heartbeat

[filter]
;需过滤的库,和 ignore_tbs 取并集
ignore_dbs=
;需同步的库,和 do_tbs 取并集
do_dbs=cgdb,data_exchange,iot_access,iot_core,mortisejointenon,operation,park,parksupervision,tongtongopen,tongtongpay,wukong
do_tbs=
ignore_tbs=
;需同步的事件
do_events=insert,update,delete
;需同步的 ddl,适用于 mysql cdc 任务
do_ddls=create_database,drop_database,alter_database,create_table,drop_table,truncate_table,rename_table,alter_table,create_index,drop_index

[data_marker]
topo_name=topo1
topo_nodes=node1,node2
src_node=node1
dst_node=node2
do_nodes=node1
ignore_nodes=node2
marker=ape_trans_mysql.topo1

[sinker]
db_type=mysql
sink_type=write
;batch_size批量写入数据条数,1 代表串行
batch_size=20
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
;插入数据时,如果已存在于目标库,是否强行替换,适用于 mysql/pg 的全量/增量任务.默认true
replace=false

[router]
;用于库表映射
tb_map=
col_map=
db_map=

[parallelizer]
;并发类型
parallel_type=serial
;并发线程数
parallel_size=8

[pipeline]
;内存中最多缓存数据的条数,数据同步采用多线程 & 批量写入,故须配置此项
buffer_size=100
;任务当前状态(统计数据,同步位点信息等)写入日志的频率,单位:秒
checkpoint_interval_secs=10

[runtime]
log_dir=./logs
log_level=info
log4rs_file=./log4rs.yaml

一直报
2025-08-29 06:57:13.976256 - WARN - [140381366912704] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:13.986363 - WARN - [140381371139776] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:13.996446 - WARN - [140381366912704] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:14.006563 - WARN - [140381373241024] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:14.023719 - ERROR - [140381373241024] - panic: panicked at /project/dt-task/src/task_runner.rs:458:39:
called Result::unwrap() on an Err value: unexpected binlog data: Too many zero-length reads. Expected data length: 113, read so far: 24

Stack backtrace:
0: anyhow::error::<impl core::convert::From for anyhow::Error>::from
1: <dt_connector::extractor::mysql::mysql_cdc_extractor::MysqlCdcExtractor as dt_connector::Extractor>::extract::{{closure}}
2: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}
3: tokio::runtime::task::core::Core<T,S>::poll
4: tokio::runtime::task::harness::Harness<T,S>::poll
5: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
6: tokio::runtime::scheduler::multi_thread::worker::Context::run
7: tokio::runtime::context::runtime::enter_runtime
8: tokio::runtime::scheduler::multi_thread::worker::run
9: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
10: tokio::runtime::task::core::Core<T,S>::poll
11: tokio::runtime::task::harness::Harness<T,S>::poll
12: tokio::runtime::blocking::pool::Inner::run
13: std::sys::backtrace::__rust_begin_short_backtrace
14: core::ops::function::FnOnce::call_once{{vtable.shim}}
15: std::sys::pal::unix::thread::Thread::new::thread_start
16:
17: __clone
backtrace:
0: dt_task::task_runner::TaskRunner::start_task::{{closure}}::{{closure}}
1: std::panicking::rust_panic_with_hook
2: std::panicking::begin_panic_handler::{{closure}}
3: std::sys::backtrace::__rust_end_short_backtrace
4: rust_begin_unwind
5: core::panicking::panic_fmt
6: core::result::unwrap_failed
7: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}
8: tokio::runtime::task::core::Core<T,S>::poll
9: tokio::runtime::task::harness::Harness<T,S>::poll
10: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
11: tokio::runtime::scheduler::multi_thread::worker::Context::run
12: tokio::runtime::context::runtime::enter_runtime
13: tokio::runtime::scheduler::multi_thread::worker::run
14: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
15: tokio::runtime::task::core::Core<T,S>::poll
16: tokio::runtime::task::harness::Harness<T,S>::poll
17: tokio::runtime::blocking::pool::Inner::run
18: std::sys::backtrace::__rust_begin_short_backtrace
19: core::ops::function::FnOnce::call_once{{vtable.shim}}
20: std::sys::pal::unix::thread::Thread::new::thread_start
21:
22: __clone

2025-08-29 06:57:14.024274 - ERROR - [140381398634688] - panic: panicked at dt-main/src/main.rs:21:39:
called Result::unwrap() on an Err value: task 17 panicked with message "called Result::unwrap() on an Err value: unexpected binlog data: Too many zero-length reads. Expected data length: 113, read so far: 24\n\nStack backtrace:\n 0: anyhow::error::<impl core::convert::From for anyhow::Error>::from\n 1: <dt_connector::extractor::mysql::mysql_cdc_extractor::MysqlCdcExtractor as dt_connector::Extractor>::extract::{{closure}}\n 2: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}\n 3: tokio::runtime::task::core::Core<T,S>::poll\n 4: tokio::runtime::task::harness::Harness<T,S>::poll\n 5: tokio::runtime::scheduler::multi_thread::worker::Context::run_task\n 6: tokio::runtime::scheduler::multi_thread::worker::Context::run\n 7: tokio::runtime::context::runtime::enter_runtime\n 8: tokio::runtime::scheduler::multi_thread::worker::run\n 9: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll\n 10: tokio::runtime::task::core::Core<T,S>::poll\n 11: tokio::runtime::task::harness::Harness<T,S>::poll\n 12: tokio::runtime::blocking::pool::Inner::run\n 13: std::sys::backtrace::__rust_begin_short_backtrace\n 14: core::ops::function::FnOnce::call_once{{vtable.shim}}\n 15: std::sys::pal::unix::thread::Thread::new::thread_start\n 16: \n 17: __clone"

Stack backtrace:
0: anyhow::error::<impl core::convert::From for anyhow::Error>::from
1: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}
2: dt_main::main::{{closure}}
3: tokio::runtime::park::CachedParkThread::block_on
4: dt_main::main
5: std::sys::backtrace::__rust_begin_short_backtrace
6: std::rt::lang_start::{{closure}}
7: std::rt::lang_start_internal
8: main
9:
10: __libc_start_main
11: _start
backtrace:
0: dt_task::task_runner::TaskRunner::start_task::{{closure}}::{{closure}}
1: std::panicking::rust_panic_with_hook
2: std::panicking::begin_panic_handler::{{closure}}
3: std::sys::backtrace::__rust_end_short_backtrace
4: rust_begin_unwind
5: core::panicking::panic_fmt
6: core::result::unwrap_failed
7: dt_main::main::{{closure}}
8: tokio::runtime::park::CachedParkThread::block_on
9: dt_main::main
10: std::sys::backtrace::__rust_begin_short_backtrace
11: std::rt::lang_start::{{closure}}
12: std::rt::lang_start_internal
13: main
14:
15: __libc_start_main
16: _start

Metadata

Metadata

Assignees

No one assigned

    Labels

    Help WantedExtra attention is neededMySQLthe issue about MySQL

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions