-
Notifications
You must be signed in to change notification settings - Fork 87
Description
主--> 从 配置
[extractor]
db_type=mysql
;拉取类型(全量:snapshot,增量:cdc)
extract_type=cdc
binlog_position=173104571
binlog_filename=mysql-bin.000009
server_id=1
;url数据库连接信息
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
heartbeat_interval_secs=1
heartbeat_tb=heartbeat_db.ape_dts_heartbeat
[filter]
;需过滤的库,和 ignore_tbs 取并集
ignore_dbs=
;需同步的库,和 do_tbs 取并集
do_dbs=cgdb,data_exchange,iot_access,iot_core,mortisejointenon,operation,park,parksupervision,tongtongopen,tongtongpay,wukong
do_tbs=
ignore_tbs=
;需同步的事件
do_events=insert,update,delete
;需同步的 ddl,适用于 mysql cdc 任务
do_ddls=create_database,drop_database,alter_database,create_table,drop_table,truncate_table,rename_table,alter_table,create_index,drop_index
[data_marker]
topo_name=topo1
topo_nodes=node1,node2
src_node=node1
dst_node=node2
do_nodes=node1
ignore_nodes=node2
marker=ape_trans_mysql.topo1
[sinker]
db_type=mysql
sink_type=write
;batch_size批量写入数据条数,1 代表串行
batch_size=20
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
;插入数据时,如果已存在于目标库,是否强行替换,适用于 mysql/pg 的全量/增量任务.默认true
replace=false
[router]
;用于库表映射
tb_map=
col_map=
db_map=
[parallelizer]
;并发类型
parallel_type=serial
;并发线程数
parallel_size=8
[pipeline]
;内存中最多缓存数据的条数,数据同步采用多线程 & 批量写入,故须配置此项
buffer_size=100
;任务当前状态(统计数据,同步位点信息等)写入日志的频率,单位:秒
checkpoint_interval_secs=10
[runtime]
log_dir=./logs
log_level=info
log4rs_file=./log4rs.yaml
从--> 主 配置
[extractor]
db_type=mysql
;拉取类型(全量:snapshot,增量:cdc)
extract_type=cdc
binlog_position=221962113
binlog_filename=mysql-bin.000008
server_id=1
;url数据库连接信息
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
heartbeat_interval_secs=1
heartbeat_tb=heartbeat_db.ape_dts_heartbeat
[filter]
;需过滤的库,和 ignore_tbs 取并集
ignore_dbs=
;需同步的库,和 do_tbs 取并集
do_dbs=cgdb,data_exchange,iot_access,iot_core,mortisejointenon,operation,park,parksupervision,tongtongopen,tongtongpay,wukong
do_tbs=
ignore_tbs=
;需同步的事件
do_events=insert,update,delete
;需同步的 ddl,适用于 mysql cdc 任务
do_ddls=create_database,drop_database,alter_database,create_table,drop_table,truncate_table,rename_table,alter_table,create_index,drop_index
[data_marker]
topo_name=topo1
topo_nodes=node1,node2
src_node=node1
dst_node=node2
do_nodes=node1
ignore_nodes=node2
marker=ape_trans_mysql.topo1
[sinker]
db_type=mysql
sink_type=write
;batch_size批量写入数据条数,1 代表串行
batch_size=20
url=mysql://root:Uh0SN%40r%[email protected]:3306?ssl-mode=disabled
;插入数据时,如果已存在于目标库,是否强行替换,适用于 mysql/pg 的全量/增量任务.默认true
replace=false
[router]
;用于库表映射
tb_map=
col_map=
db_map=
[parallelizer]
;并发类型
parallel_type=serial
;并发线程数
parallel_size=8
[pipeline]
;内存中最多缓存数据的条数,数据同步采用多线程 & 批量写入,故须配置此项
buffer_size=100
;任务当前状态(统计数据,同步位点信息等)写入日志的频率,单位:秒
checkpoint_interval_secs=10
[runtime]
log_dir=./logs
log_level=info
log4rs_file=./log4rs.yaml
一直报
2025-08-29 06:57:13.976256 - WARN - [140381366912704] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:13.986363 - WARN - [140381371139776] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:13.996446 - WARN - [140381366912704] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:14.006563 - WARN - [140381373241024] - Stream reading binlog returns zero-length data, Expected data length: 113, read so far: 24
2025-08-29 06:57:14.023719 - ERROR - [140381373241024] - panic: panicked at /project/dt-task/src/task_runner.rs:458:39:
called Result::unwrap() on an Err value: unexpected binlog data: Too many zero-length reads. Expected data length: 113, read so far: 24
Stack backtrace:
0: anyhow::error::<impl core::convert::From for anyhow::Error>::from
1: <dt_connector::extractor::mysql::mysql_cdc_extractor::MysqlCdcExtractor as dt_connector::Extractor>::extract::{{closure}}
2: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}
3: tokio::runtime::task::core::Core<T,S>::poll
4: tokio::runtime::task::harness::Harness<T,S>::poll
5: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
6: tokio::runtime::scheduler::multi_thread::worker::Context::run
7: tokio::runtime::context::runtime::enter_runtime
8: tokio::runtime::scheduler::multi_thread::worker::run
9: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
10: tokio::runtime::task::core::Core<T,S>::poll
11: tokio::runtime::task::harness::Harness<T,S>::poll
12: tokio::runtime::blocking::pool::Inner::run
13: std::sys::backtrace::__rust_begin_short_backtrace
14: core::ops::function::FnOnce::call_once{{vtable.shim}}
15: std::sys::pal::unix::thread::Thread::new::thread_start
16:
17: __clone
backtrace:
0: dt_task::task_runner::TaskRunner::start_task::{{closure}}::{{closure}}
1: std::panicking::rust_panic_with_hook
2: std::panicking::begin_panic_handler::{{closure}}
3: std::sys::backtrace::__rust_end_short_backtrace
4: rust_begin_unwind
5: core::panicking::panic_fmt
6: core::result::unwrap_failed
7: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}
8: tokio::runtime::task::core::Core<T,S>::poll
9: tokio::runtime::task::harness::Harness<T,S>::poll
10: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
11: tokio::runtime::scheduler::multi_thread::worker::Context::run
12: tokio::runtime::context::runtime::enter_runtime
13: tokio::runtime::scheduler::multi_thread::worker::run
14: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
15: tokio::runtime::task::core::Core<T,S>::poll
16: tokio::runtime::task::harness::Harness<T,S>::poll
17: tokio::runtime::blocking::pool::Inner::run
18: std::sys::backtrace::__rust_begin_short_backtrace
19: core::ops::function::FnOnce::call_once{{vtable.shim}}
20: std::sys::pal::unix::thread::Thread::new::thread_start
21:
22: __clone
2025-08-29 06:57:14.024274 - ERROR - [140381398634688] - panic: panicked at dt-main/src/main.rs:21:39:
called Result::unwrap() on an Err value: task 17 panicked with message "called Result::unwrap() on an Err value: unexpected binlog data: Too many zero-length reads. Expected data length: 113, read so far: 24\n\nStack backtrace:\n 0: anyhow::error::<impl core::convert::From for anyhow::Error>::from\n 1: <dt_connector::extractor::mysql::mysql_cdc_extractor::MysqlCdcExtractor as dt_connector::Extractor>::extract::{{closure}}\n 2: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}::{{closure}}\n 3: tokio::runtime::task::core::Core<T,S>::poll\n 4: tokio::runtime::task::harness::Harness<T,S>::poll\n 5: tokio::runtime::scheduler::multi_thread::worker::Context::run_task\n 6: tokio::runtime::scheduler::multi_thread::worker::Context::run\n 7: tokio::runtime::context::runtime::enter_runtime\n 8: tokio::runtime::scheduler::multi_thread::worker::run\n 9: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll\n 10: tokio::runtime::task::core::Core<T,S>::poll\n 11: tokio::runtime::task::harness::Harness<T,S>::poll\n 12: tokio::runtime::blocking::pool::Inner::run\n 13: std::sys::backtrace::__rust_begin_short_backtrace\n 14: core::ops::function::FnOnce::call_once{{vtable.shim}}\n 15: std::sys::pal::unix::thread::Thread::new::thread_start\n 16: \n 17: __clone"
Stack backtrace:
0: anyhow::error::<impl core::convert::From for anyhow::Error>::from
1: dt_task::task_runner::TaskRunner::start_single_task::{{closure}}
2: dt_main::main::{{closure}}
3: tokio::runtime::park::CachedParkThread::block_on
4: dt_main::main
5: std::sys::backtrace::__rust_begin_short_backtrace
6: std::rt::lang_start::{{closure}}
7: std::rt::lang_start_internal
8: main
9:
10: __libc_start_main
11: _start
backtrace:
0: dt_task::task_runner::TaskRunner::start_task::{{closure}}::{{closure}}
1: std::panicking::rust_panic_with_hook
2: std::panicking::begin_panic_handler::{{closure}}
3: std::sys::backtrace::__rust_end_short_backtrace
4: rust_begin_unwind
5: core::panicking::panic_fmt
6: core::result::unwrap_failed
7: dt_main::main::{{closure}}
8: tokio::runtime::park::CachedParkThread::block_on
9: dt_main::main
10: std::sys::backtrace::__rust_begin_short_backtrace
11: std::rt::lang_start::{{closure}}
12: std::rt::lang_start_internal
13: main
14:
15: __libc_start_main
16: _start