diff --git a/logservice/schemastore/persist_storage_ddl_handlers.go b/logservice/schemastore/persist_storage_ddl_handlers.go index b06d69cb2f..b0d46c8b4d 100644 --- a/logservice/schemastore/persist_storage_ddl_handlers.go +++ b/logservice/schemastore/persist_storage_ddl_handlers.go @@ -2692,8 +2692,8 @@ func buildDDLEventForRenameTables(rawEvent *PersistedDDLEvent, tableFilter filte continue } allFiltered = false - if isPartitionTable(rawEvent.TableInfo) { - allPhysicalIDs := getAllPartitionIDs(rawEvent.TableInfo) + if isPartitionTable(tableInfo) { + allPhysicalIDs := getAllPartitionIDs(tableInfo) if !ignorePrevTable { if !notSyncPrevTable { // only when the previous table is not filtered and not NotSync, we add the query and table info diff --git a/logservice/schemastore/persist_storage_test.go b/logservice/schemastore/persist_storage_test.go index b85a56cd57..2505ae73d6 100644 --- a/logservice/schemastore/persist_storage_test.go +++ b/logservice/schemastore/persist_storage_test.go @@ -3459,6 +3459,127 @@ func TestBuildPersistedDDLEventEscapesIdentifiers(t *testing.T) { }) } +func TestBuildDDLEventForRenameTablesForPartitionTable(t *testing.T) { + normalInfo := newEligibleTableInfoForTest(200, "normal_new") + partitionInfo := newEligiblePartitionTableInfoForTest(300, "partition_new", []model.PartitionDefinition{ + {ID: 301}, + {ID: 302}, + }) + partitionInfo2 := newEligiblePartitionTableInfoForTest(400, "partition_new_2", []model.PartitionDefinition{ + {ID: 401}, + {ID: 402}, + }) + + t.Run("normal table then partition table", func(t *testing.T) { + rawEvent := &PersistedDDLEvent{ + Type: byte(model.ActionRenameTables), + SchemaID: 110, + SchemaName: "target_normal", + TableName: normalInfo.Name.O, + TableInfo: normalInfo, + Query: "RENAME TABLE `source_normal`.`normal_old` TO `target_normal`.`normal_new`;" + + "RENAME TABLE `source_partition`.`partition_old` TO `target_partition`.`partition_new`;", + FinishedTs: 1010, + SchemaIDs: []int64{110, 111}, + SchemaNames: []string{"target_normal", "target_partition"}, + ExtraSchemaIDs: []int64{100, 101}, + ExtraSchemaNames: []string{"source_normal", "source_partition"}, + ExtraTableNames: []string{"normal_old", "partition_old"}, + MultipleTableInfos: []*model.TableInfo{ + normalInfo, + partitionInfo, + }, + } + + ddlEvent, ok, err := buildDDLEventForRenameTables(rawEvent, nil, 301) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []int64{common.DDLSpanTableID, 200, 301, 302}, ddlEvent.BlockedTables.TableIDs) + require.Equal(t, []commonEvent.SchemaIDChange{ + {TableID: 200, OldSchemaID: 100, NewSchemaID: 110}, + {TableID: 301, OldSchemaID: 101, NewSchemaID: 111}, + {TableID: 302, OldSchemaID: 101, NewSchemaID: 111}, + }, ddlEvent.UpdatedSchemas) + require.NotNil(t, ddlEvent.TableInfo) + require.Equal(t, int64(300), ddlEvent.GetTableID()) + require.Equal(t, "target_partition", ddlEvent.TableInfo.GetSchemaName()) + require.Equal(t, "partition_new", ddlEvent.TableInfo.GetTableName()) + }) + + t.Run("partition table then normal table", func(t *testing.T) { + rawEvent := &PersistedDDLEvent{ + Type: byte(model.ActionRenameTables), + SchemaID: 111, + SchemaName: "target_partition", + TableName: partitionInfo.Name.O, + TableInfo: partitionInfo, + Query: "RENAME TABLE `source_partition`.`partition_old` TO `target_partition`.`partition_new`;" + + "RENAME TABLE `source_normal`.`normal_old` TO `target_normal`.`normal_new`;", + FinishedTs: 1010, + SchemaIDs: []int64{111, 110}, + SchemaNames: []string{"target_partition", "target_normal"}, + ExtraSchemaIDs: []int64{101, 100}, + ExtraSchemaNames: []string{"source_partition", "source_normal"}, + ExtraTableNames: []string{"partition_old", "normal_old"}, + MultipleTableInfos: []*model.TableInfo{ + partitionInfo, + normalInfo, + }, + } + + ddlEvent, ok, err := buildDDLEventForRenameTables(rawEvent, nil, 200) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []int64{common.DDLSpanTableID, 301, 302, 200}, ddlEvent.BlockedTables.TableIDs) + require.Equal(t, []commonEvent.SchemaIDChange{ + {TableID: 301, OldSchemaID: 101, NewSchemaID: 111}, + {TableID: 302, OldSchemaID: 101, NewSchemaID: 111}, + {TableID: 200, OldSchemaID: 100, NewSchemaID: 110}, + }, ddlEvent.UpdatedSchemas) + require.NotNil(t, ddlEvent.TableInfo) + require.Equal(t, int64(200), ddlEvent.GetTableID()) + require.Equal(t, "target_normal", ddlEvent.TableInfo.GetSchemaName()) + require.Equal(t, "normal_new", ddlEvent.TableInfo.GetTableName()) + }) + + t.Run("multiple partition tables", func(t *testing.T) { + rawEvent := &PersistedDDLEvent{ + Type: byte(model.ActionRenameTables), + SchemaID: 111, + SchemaName: "target_partition", + TableName: partitionInfo.Name.O, + TableInfo: partitionInfo, + Query: "RENAME TABLE `source_partition`.`partition_old` TO `target_partition`.`partition_new`;" + + "RENAME TABLE `source_partition_2`.`partition_old_2` TO `target_partition_2`.`partition_new_2`;", + FinishedTs: 1010, + SchemaIDs: []int64{111, 121}, + SchemaNames: []string{"target_partition", "target_partition_2"}, + ExtraSchemaIDs: []int64{101, 120}, + ExtraSchemaNames: []string{"source_partition", "source_partition_2"}, + ExtraTableNames: []string{"partition_old", "partition_old_2"}, + MultipleTableInfos: []*model.TableInfo{ + partitionInfo, + partitionInfo2, + }, + } + + ddlEvent, ok, err := buildDDLEventForRenameTables(rawEvent, nil, 402) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []int64{common.DDLSpanTableID, 301, 302, 401, 402}, ddlEvent.BlockedTables.TableIDs) + require.Equal(t, []commonEvent.SchemaIDChange{ + {TableID: 301, OldSchemaID: 101, NewSchemaID: 111}, + {TableID: 302, OldSchemaID: 101, NewSchemaID: 111}, + {TableID: 401, OldSchemaID: 120, NewSchemaID: 121}, + {TableID: 402, OldSchemaID: 120, NewSchemaID: 121}, + }, ddlEvent.UpdatedSchemas) + require.NotNil(t, ddlEvent.TableInfo) + require.Equal(t, int64(400), ddlEvent.GetTableID()) + require.Equal(t, "target_partition_2", ddlEvent.TableInfo.GetSchemaName()) + require.Equal(t, "partition_new_2", ddlEvent.TableInfo.GetTableName()) + }) +} + func TestParseRenameTablesQueryInfos(t *testing.T) { cases := []struct { name string diff --git a/tests/integration_tests/csv_storage_multi_tables_ddl/conf/diff_config.toml b/tests/integration_tests/csv_storage_multi_tables_ddl/conf/diff_config.toml index fc562776fc..e88bc52cb1 100644 --- a/tests/integration_tests/csv_storage_multi_tables_ddl/conf/diff_config.toml +++ b/tests/integration_tests/csv_storage_multi_tables_ddl/conf/diff_config.toml @@ -13,7 +13,14 @@ check-struct-only = false target-instance = "mysql1" - target-check-tables = ["multi_tables_ddl_test.t1_7", "multi_tables_ddl_test.t2_7"] + target-check-tables = [ + "multi_tables_ddl_test.t1_7", + "multi_tables_ddl_test.t2_7", + "multi_tables_ddl_test.rename_mix_normal_1_done", + "multi_tables_ddl_test.rename_mix_part_1_done", + "multi_tables_ddl_test.rename_mix_normal_2_done", + "multi_tables_ddl_test.rename_mix_part_2_done", + ] [data-sources] [data-sources.tidb0] diff --git a/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml b/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml index fa4e188af6..a85957996a 100644 --- a/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml +++ b/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml @@ -1,5 +1,21 @@ [filter] -rules = ["multi_tables_ddl_test.t1", "multi_tables_ddl_test.t2", "multi_tables_ddl_test.t3", "multi_tables_ddl_test.t4" ,"multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7", "multi_tables_ddl_test.finish_mark"] +rules = [ + "multi_tables_ddl_test.t1", + "multi_tables_ddl_test.t2", + "multi_tables_ddl_test.t3", + "multi_tables_ddl_test.t4", + "multi_tables_ddl_test.t1_7", + "multi_tables_ddl_test.t2_7", + "multi_tables_ddl_test.rename_mix_normal_1", + "multi_tables_ddl_test.rename_mix_part_1", + "multi_tables_ddl_test.rename_mix_normal_1_done", + "multi_tables_ddl_test.rename_mix_part_1_done", + "multi_tables_ddl_test.rename_mix_normal_2", + "multi_tables_ddl_test.rename_mix_part_2", + "multi_tables_ddl_test.rename_mix_normal_2_done", + "multi_tables_ddl_test.rename_mix_part_2_done", + "multi_tables_ddl_test.finish_mark", +] [sink.csv] -include-commit-ts = true \ No newline at end of file +include-commit-ts = true diff --git a/tests/integration_tests/csv_storage_multi_tables_ddl/data/test.sql b/tests/integration_tests/csv_storage_multi_tables_ddl/data/test.sql index e1a91f809a..5a6047266d 100644 --- a/tests/integration_tests/csv_storage_multi_tables_ddl/data/test.sql +++ b/tests/integration_tests/csv_storage_multi_tables_ddl/data/test.sql @@ -110,6 +110,46 @@ insert into t1_7 values(685477580, 6); insert into t2_7 values(1715679991826145, 7); insert into t2_7 values(2036854775807, 8); +create table rename_mix_normal_1 ( + id int not null, + value32 int not null, + primary key(id) +); + +create table rename_mix_part_1 ( + id int not null, + value32 int not null, + primary key(id) +) partition by hash(id) partitions 2; + +insert into rename_mix_normal_1 values(1, 10), (2, 20); +insert into rename_mix_part_1 values(11, 110), (12, 120); + +rename table rename_mix_normal_1 to rename_mix_normal_1_done, rename_mix_part_1 to rename_mix_part_1_done; + +insert into rename_mix_normal_1_done values(3, 30); +insert into rename_mix_part_1_done values(13, 130); + +create table rename_mix_part_2 ( + id int not null, + value32 int not null, + primary key(id) +) partition by hash(id) partitions 2; + +create table rename_mix_normal_2 ( + id int not null, + value32 int not null, + primary key(id) +); + +insert into rename_mix_part_2 values(21, 210), (22, 220); +insert into rename_mix_normal_2 values(31, 310), (32, 320); + +rename table rename_mix_part_2 to rename_mix_part_2_done, rename_mix_normal_2 to rename_mix_normal_2_done; + +insert into rename_mix_part_2_done values(23, 230); +insert into rename_mix_normal_2_done values(33, 330); + insert into t3 select * from t1_7; insert into t4 select * from t2_7; drop table t3, t4; diff --git a/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh b/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh index 1469055d40..c04c56c037 100755 --- a/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh +++ b/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh @@ -48,6 +48,10 @@ function run() { check_table_exists multi_tables_ddl_test.t66 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} check_table_exists multi_tables_ddl_test.t7 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} check_table_exists multi_tables_ddl_test.t88 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_normal_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_part_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_normal_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_part_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists multi_tables_ddl_test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} echo "check table exists success" diff --git a/tests/integration_tests/multi_tables_ddl/conf/diff_config.toml b/tests/integration_tests/multi_tables_ddl/conf/diff_config.toml index a2dccc4ede..fdc9237e70 100644 --- a/tests/integration_tests/multi_tables_ddl/conf/diff_config.toml +++ b/tests/integration_tests/multi_tables_ddl/conf/diff_config.toml @@ -13,7 +13,14 @@ check-struct-only = false target-instance = "mysql1" - target-check-tables = ["multi_tables_ddl_test.t1_7", "multi_tables_ddl_test.t2_7"] + target-check-tables = [ + "multi_tables_ddl_test.t1_7", + "multi_tables_ddl_test.t2_7", + "multi_tables_ddl_test.rename_mix_normal_1_done", + "multi_tables_ddl_test.rename_mix_part_1_done", + "multi_tables_ddl_test.rename_mix_normal_2_done", + "multi_tables_ddl_test.rename_mix_part_2_done", + ] [data-sources] [data-sources.tidb0] diff --git a/tests/integration_tests/multi_tables_ddl/conf/normal.toml b/tests/integration_tests/multi_tables_ddl/conf/normal.toml index fa4e188af6..a85957996a 100644 --- a/tests/integration_tests/multi_tables_ddl/conf/normal.toml +++ b/tests/integration_tests/multi_tables_ddl/conf/normal.toml @@ -1,5 +1,21 @@ [filter] -rules = ["multi_tables_ddl_test.t1", "multi_tables_ddl_test.t2", "multi_tables_ddl_test.t3", "multi_tables_ddl_test.t4" ,"multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7", "multi_tables_ddl_test.finish_mark"] +rules = [ + "multi_tables_ddl_test.t1", + "multi_tables_ddl_test.t2", + "multi_tables_ddl_test.t3", + "multi_tables_ddl_test.t4", + "multi_tables_ddl_test.t1_7", + "multi_tables_ddl_test.t2_7", + "multi_tables_ddl_test.rename_mix_normal_1", + "multi_tables_ddl_test.rename_mix_part_1", + "multi_tables_ddl_test.rename_mix_normal_1_done", + "multi_tables_ddl_test.rename_mix_part_1_done", + "multi_tables_ddl_test.rename_mix_normal_2", + "multi_tables_ddl_test.rename_mix_part_2", + "multi_tables_ddl_test.rename_mix_normal_2_done", + "multi_tables_ddl_test.rename_mix_part_2_done", + "multi_tables_ddl_test.finish_mark", +] [sink.csv] -include-commit-ts = true \ No newline at end of file +include-commit-ts = true diff --git a/tests/integration_tests/multi_tables_ddl/data/test.sql b/tests/integration_tests/multi_tables_ddl/data/test.sql index e1a91f809a..5a6047266d 100644 --- a/tests/integration_tests/multi_tables_ddl/data/test.sql +++ b/tests/integration_tests/multi_tables_ddl/data/test.sql @@ -110,6 +110,46 @@ insert into t1_7 values(685477580, 6); insert into t2_7 values(1715679991826145, 7); insert into t2_7 values(2036854775807, 8); +create table rename_mix_normal_1 ( + id int not null, + value32 int not null, + primary key(id) +); + +create table rename_mix_part_1 ( + id int not null, + value32 int not null, + primary key(id) +) partition by hash(id) partitions 2; + +insert into rename_mix_normal_1 values(1, 10), (2, 20); +insert into rename_mix_part_1 values(11, 110), (12, 120); + +rename table rename_mix_normal_1 to rename_mix_normal_1_done, rename_mix_part_1 to rename_mix_part_1_done; + +insert into rename_mix_normal_1_done values(3, 30); +insert into rename_mix_part_1_done values(13, 130); + +create table rename_mix_part_2 ( + id int not null, + value32 int not null, + primary key(id) +) partition by hash(id) partitions 2; + +create table rename_mix_normal_2 ( + id int not null, + value32 int not null, + primary key(id) +); + +insert into rename_mix_part_2 values(21, 210), (22, 220); +insert into rename_mix_normal_2 values(31, 310), (32, 320); + +rename table rename_mix_part_2 to rename_mix_part_2_done, rename_mix_normal_2 to rename_mix_normal_2_done; + +insert into rename_mix_part_2_done values(23, 230); +insert into rename_mix_normal_2_done values(33, 330); + insert into t3 select * from t1_7; insert into t4 select * from t2_7; drop table t3, t4; diff --git a/tests/integration_tests/multi_tables_ddl/run.sh b/tests/integration_tests/multi_tables_ddl/run.sh index 17037d380e..2cf634afbc 100755 --- a/tests/integration_tests/multi_tables_ddl/run.sh +++ b/tests/integration_tests/multi_tables_ddl/run.sh @@ -102,6 +102,10 @@ function run() { check_table_exists multi_tables_ddl_test.t66 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} check_table_exists multi_tables_ddl_test.t7 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} check_table_exists multi_tables_ddl_test.t88 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_normal_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_part_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_normal_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} + check_table_exists multi_tables_ddl_test.rename_mix_part_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} # sync_diff can't check non-exist table, so we check expected tables are created in downstream first check_table_exists multi_tables_ddl_test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} echo "check table exists success"