Skip to content

Commit c371797

Browse files
lidezhutenfyzhong
lidezhu
authored and
tenfyzhong
committed
schemastore: fix rename partition table (#4446)
close #4447
1 parent 4476666 commit c371797

File tree

10 files changed

+263
-8
lines changed

10 files changed

+263
-8
lines changed

‎logservice/schemastore/persist_storage_ddl_handlers.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2676,8 +2676,8 @@ func buildDDLEventForRenameTables(rawEvent *PersistedDDLEvent, tableFilter filte
26762676
continue
26772677
}
26782678
allFiltered = false
2679-
if isPartitionTable(rawEvent.TableInfo) {
2680-
allPhysicalIDs := getAllPartitionIDs(rawEvent.TableInfo)
2679+
if isPartitionTable(tableInfo) {
2680+
allPhysicalIDs := getAllPartitionIDs(tableInfo)
26812681
if !ignorePrevTable {
26822682
if !notSyncPrevTable {
26832683
// only when the previous table is not filtered and not NotSync, we add the query and table info

‎logservice/schemastore/persist_storage_test.go‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3459,6 +3459,127 @@ func TestBuildPersistedDDLEventEscapesIdentifiers(t *testing.T) {
34593459
})
34603460
}
34613461

3462+
func TestBuildDDLEventForRenameTablesForPartitionTable(t *testing.T) {
3463+
normalInfo := newEligibleTableInfoForTest(200, "normal_new")
3464+
partitionInfo := newEligiblePartitionTableInfoForTest(300, "partition_new", []model.PartitionDefinition{
3465+
{ID: 301},
3466+
{ID: 302},
3467+
})
3468+
partitionInfo2 := newEligiblePartitionTableInfoForTest(400, "partition_new_2", []model.PartitionDefinition{
3469+
{ID: 401},
3470+
{ID: 402},
3471+
})
3472+
3473+
t.Run("normal table then partition table", func(t *testing.T) {
3474+
rawEvent := &PersistedDDLEvent{
3475+
Type: byte(model.ActionRenameTables),
3476+
SchemaID: 110,
3477+
SchemaName: "target_normal",
3478+
TableName: normalInfo.Name.O,
3479+
TableInfo: normalInfo,
3480+
Query: "RENAME TABLE `source_normal`.`normal_old` TO `target_normal`.`normal_new`;" +
3481+
"RENAME TABLE `source_partition`.`partition_old` TO `target_partition`.`partition_new`;",
3482+
FinishedTs: 1010,
3483+
SchemaIDs: []int64{110, 111},
3484+
SchemaNames: []string{"target_normal", "target_partition"},
3485+
ExtraSchemaIDs: []int64{100, 101},
3486+
ExtraSchemaNames: []string{"source_normal", "source_partition"},
3487+
ExtraTableNames: []string{"normal_old", "partition_old"},
3488+
MultipleTableInfos: []*model.TableInfo{
3489+
normalInfo,
3490+
partitionInfo,
3491+
},
3492+
}
3493+
3494+
ddlEvent, ok, err := buildDDLEventForRenameTables(rawEvent, nil, 301)
3495+
require.NoError(t, err)
3496+
require.True(t, ok)
3497+
require.Equal(t, []int64{common.DDLSpanTableID, 200, 301, 302}, ddlEvent.BlockedTables.TableIDs)
3498+
require.Equal(t, []commonEvent.SchemaIDChange{
3499+
{TableID: 200, OldSchemaID: 100, NewSchemaID: 110},
3500+
{TableID: 301, OldSchemaID: 101, NewSchemaID: 111},
3501+
{TableID: 302, OldSchemaID: 101, NewSchemaID: 111},
3502+
}, ddlEvent.UpdatedSchemas)
3503+
require.NotNil(t, ddlEvent.TableInfo)
3504+
require.Equal(t, int64(300), ddlEvent.GetTableID())
3505+
require.Equal(t, "target_partition", ddlEvent.TableInfo.GetSchemaName())
3506+
require.Equal(t, "partition_new", ddlEvent.TableInfo.GetTableName())
3507+
})
3508+
3509+
t.Run("partition table then normal table", func(t *testing.T) {
3510+
rawEvent := &PersistedDDLEvent{
3511+
Type: byte(model.ActionRenameTables),
3512+
SchemaID: 111,
3513+
SchemaName: "target_partition",
3514+
TableName: partitionInfo.Name.O,
3515+
TableInfo: partitionInfo,
3516+
Query: "RENAME TABLE `source_partition`.`partition_old` TO `target_partition`.`partition_new`;" +
3517+
"RENAME TABLE `source_normal`.`normal_old` TO `target_normal`.`normal_new`;",
3518+
FinishedTs: 1010,
3519+
SchemaIDs: []int64{111, 110},
3520+
SchemaNames: []string{"target_partition", "target_normal"},
3521+
ExtraSchemaIDs: []int64{101, 100},
3522+
ExtraSchemaNames: []string{"source_partition", "source_normal"},
3523+
ExtraTableNames: []string{"partition_old", "normal_old"},
3524+
MultipleTableInfos: []*model.TableInfo{
3525+
partitionInfo,
3526+
normalInfo,
3527+
},
3528+
}
3529+
3530+
ddlEvent, ok, err := buildDDLEventForRenameTables(rawEvent, nil, 200)
3531+
require.NoError(t, err)
3532+
require.True(t, ok)
3533+
require.Equal(t, []int64{common.DDLSpanTableID, 301, 302, 200}, ddlEvent.BlockedTables.TableIDs)
3534+
require.Equal(t, []commonEvent.SchemaIDChange{
3535+
{TableID: 301, OldSchemaID: 101, NewSchemaID: 111},
3536+
{TableID: 302, OldSchemaID: 101, NewSchemaID: 111},
3537+
{TableID: 200, OldSchemaID: 100, NewSchemaID: 110},
3538+
}, ddlEvent.UpdatedSchemas)
3539+
require.NotNil(t, ddlEvent.TableInfo)
3540+
require.Equal(t, int64(200), ddlEvent.GetTableID())
3541+
require.Equal(t, "target_normal", ddlEvent.TableInfo.GetSchemaName())
3542+
require.Equal(t, "normal_new", ddlEvent.TableInfo.GetTableName())
3543+
})
3544+
3545+
t.Run("multiple partition tables", func(t *testing.T) {
3546+
rawEvent := &PersistedDDLEvent{
3547+
Type: byte(model.ActionRenameTables),
3548+
SchemaID: 111,
3549+
SchemaName: "target_partition",
3550+
TableName: partitionInfo.Name.O,
3551+
TableInfo: partitionInfo,
3552+
Query: "RENAME TABLE `source_partition`.`partition_old` TO `target_partition`.`partition_new`;" +
3553+
"RENAME TABLE `source_partition_2`.`partition_old_2` TO `target_partition_2`.`partition_new_2`;",
3554+
FinishedTs: 1010,
3555+
SchemaIDs: []int64{111, 121},
3556+
SchemaNames: []string{"target_partition", "target_partition_2"},
3557+
ExtraSchemaIDs: []int64{101, 120},
3558+
ExtraSchemaNames: []string{"source_partition", "source_partition_2"},
3559+
ExtraTableNames: []string{"partition_old", "partition_old_2"},
3560+
MultipleTableInfos: []*model.TableInfo{
3561+
partitionInfo,
3562+
partitionInfo2,
3563+
},
3564+
}
3565+
3566+
ddlEvent, ok, err := buildDDLEventForRenameTables(rawEvent, nil, 402)
3567+
require.NoError(t, err)
3568+
require.True(t, ok)
3569+
require.Equal(t, []int64{common.DDLSpanTableID, 301, 302, 401, 402}, ddlEvent.BlockedTables.TableIDs)
3570+
require.Equal(t, []commonEvent.SchemaIDChange{
3571+
{TableID: 301, OldSchemaID: 101, NewSchemaID: 111},
3572+
{TableID: 302, OldSchemaID: 101, NewSchemaID: 111},
3573+
{TableID: 401, OldSchemaID: 120, NewSchemaID: 121},
3574+
{TableID: 402, OldSchemaID: 120, NewSchemaID: 121},
3575+
}, ddlEvent.UpdatedSchemas)
3576+
require.NotNil(t, ddlEvent.TableInfo)
3577+
require.Equal(t, int64(400), ddlEvent.GetTableID())
3578+
require.Equal(t, "target_partition_2", ddlEvent.TableInfo.GetSchemaName())
3579+
require.Equal(t, "partition_new_2", ddlEvent.TableInfo.GetTableName())
3580+
})
3581+
}
3582+
34623583
func TestParseRenameTablesQueryInfos(t *testing.T) {
34633584
cases := []struct {
34643585
name string

‎tests/integration_tests/csv_storage_multi_tables_ddl/conf/diff_config.toml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,14 @@ check-struct-only = false
1313

1414
target-instance = "mysql1"
1515

16-
target-check-tables = ["multi_tables_ddl_test.t1_7", "multi_tables_ddl_test.t2_7"]
16+
target-check-tables = [
17+
"multi_tables_ddl_test.t1_7",
18+
"multi_tables_ddl_test.t2_7",
19+
"multi_tables_ddl_test.rename_mix_normal_1_done",
20+
"multi_tables_ddl_test.rename_mix_part_1_done",
21+
"multi_tables_ddl_test.rename_mix_normal_2_done",
22+
"multi_tables_ddl_test.rename_mix_part_2_done",
23+
]
1724

1825
[data-sources]
1926
[data-sources.tidb0]

‎tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
[filter]
2-
rules = ["multi_tables_ddl_test.t1", "multi_tables_ddl_test.t2", "multi_tables_ddl_test.t3", "multi_tables_ddl_test.t4" ,"multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7", "multi_tables_ddl_test.finish_mark"]
2+
rules = [
3+
"multi_tables_ddl_test.t1",
4+
"multi_tables_ddl_test.t2",
5+
"multi_tables_ddl_test.t3",
6+
"multi_tables_ddl_test.t4",
7+
"multi_tables_ddl_test.t1_7",
8+
"multi_tables_ddl_test.t2_7",
9+
"multi_tables_ddl_test.rename_mix_normal_1",
10+
"multi_tables_ddl_test.rename_mix_part_1",
11+
"multi_tables_ddl_test.rename_mix_normal_1_done",
12+
"multi_tables_ddl_test.rename_mix_part_1_done",
13+
"multi_tables_ddl_test.rename_mix_normal_2",
14+
"multi_tables_ddl_test.rename_mix_part_2",
15+
"multi_tables_ddl_test.rename_mix_normal_2_done",
16+
"multi_tables_ddl_test.rename_mix_part_2_done",
17+
"multi_tables_ddl_test.finish_mark",
18+
]
319

420
[sink.csv]
5-
include-commit-ts = true
21+
include-commit-ts = true

‎tests/integration_tests/csv_storage_multi_tables_ddl/data/test.sql‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,46 @@ insert into t1_7 values(685477580, 6);
110110
insert into t2_7 values(1715679991826145, 7);
111111
insert into t2_7 values(2036854775807, 8);
112112

113+
create table rename_mix_normal_1 (
114+
id int not null,
115+
value32 int not null,
116+
primary key(id)
117+
);
118+
119+
create table rename_mix_part_1 (
120+
id int not null,
121+
value32 int not null,
122+
primary key(id)
123+
) partition by hash(id) partitions 2;
124+
125+
insert into rename_mix_normal_1 values(1, 10), (2, 20);
126+
insert into rename_mix_part_1 values(11, 110), (12, 120);
127+
128+
rename table rename_mix_normal_1 to rename_mix_normal_1_done, rename_mix_part_1 to rename_mix_part_1_done;
129+
130+
insert into rename_mix_normal_1_done values(3, 30);
131+
insert into rename_mix_part_1_done values(13, 130);
132+
133+
create table rename_mix_part_2 (
134+
id int not null,
135+
value32 int not null,
136+
primary key(id)
137+
) partition by hash(id) partitions 2;
138+
139+
create table rename_mix_normal_2 (
140+
id int not null,
141+
value32 int not null,
142+
primary key(id)
143+
);
144+
145+
insert into rename_mix_part_2 values(21, 210), (22, 220);
146+
insert into rename_mix_normal_2 values(31, 310), (32, 320);
147+
148+
rename table rename_mix_part_2 to rename_mix_part_2_done, rename_mix_normal_2 to rename_mix_normal_2_done;
149+
150+
insert into rename_mix_part_2_done values(23, 230);
151+
insert into rename_mix_normal_2_done values(33, 330);
152+
113153
insert into t3 select * from t1_7;
114154
insert into t4 select * from t2_7;
115155
drop table t3, t4;

‎tests/integration_tests/csv_storage_multi_tables_ddl/run.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ function run() {
4848
check_table_exists multi_tables_ddl_test.t66 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
4949
check_table_exists multi_tables_ddl_test.t7 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
5050
check_table_exists multi_tables_ddl_test.t88 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
51+
check_table_exists multi_tables_ddl_test.rename_mix_normal_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
52+
check_table_exists multi_tables_ddl_test.rename_mix_part_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
53+
check_table_exists multi_tables_ddl_test.rename_mix_normal_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
54+
check_table_exists multi_tables_ddl_test.rename_mix_part_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
5155
# sync_diff can't check non-exist table, so we check expected tables are created in downstream first
5256
check_table_exists multi_tables_ddl_test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
5357
echo "check table exists success"

‎tests/integration_tests/multi_tables_ddl/conf/diff_config.toml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,14 @@ check-struct-only = false
1313

1414
target-instance = "mysql1"
1515

16-
target-check-tables = ["multi_tables_ddl_test.t1_7", "multi_tables_ddl_test.t2_7"]
16+
target-check-tables = [
17+
"multi_tables_ddl_test.t1_7",
18+
"multi_tables_ddl_test.t2_7",
19+
"multi_tables_ddl_test.rename_mix_normal_1_done",
20+
"multi_tables_ddl_test.rename_mix_part_1_done",
21+
"multi_tables_ddl_test.rename_mix_normal_2_done",
22+
"multi_tables_ddl_test.rename_mix_part_2_done",
23+
]
1724

1825
[data-sources]
1926
[data-sources.tidb0]

‎tests/integration_tests/multi_tables_ddl/conf/normal.toml‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,21 @@
11
[filter]
2-
rules = ["multi_tables_ddl_test.t1", "multi_tables_ddl_test.t2", "multi_tables_ddl_test.t3", "multi_tables_ddl_test.t4" ,"multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7", "multi_tables_ddl_test.finish_mark"]
2+
rules = [
3+
"multi_tables_ddl_test.t1",
4+
"multi_tables_ddl_test.t2",
5+
"multi_tables_ddl_test.t3",
6+
"multi_tables_ddl_test.t4",
7+
"multi_tables_ddl_test.t1_7",
8+
"multi_tables_ddl_test.t2_7",
9+
"multi_tables_ddl_test.rename_mix_normal_1",
10+
"multi_tables_ddl_test.rename_mix_part_1",
11+
"multi_tables_ddl_test.rename_mix_normal_1_done",
12+
"multi_tables_ddl_test.rename_mix_part_1_done",
13+
"multi_tables_ddl_test.rename_mix_normal_2",
14+
"multi_tables_ddl_test.rename_mix_part_2",
15+
"multi_tables_ddl_test.rename_mix_normal_2_done",
16+
"multi_tables_ddl_test.rename_mix_part_2_done",
17+
"multi_tables_ddl_test.finish_mark",
18+
]
319

420
[sink.csv]
5-
include-commit-ts = true
21+
include-commit-ts = true

‎tests/integration_tests/multi_tables_ddl/data/test.sql‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,46 @@ insert into t1_7 values(685477580, 6);
110110
insert into t2_7 values(1715679991826145, 7);
111111
insert into t2_7 values(2036854775807, 8);
112112

113+
create table rename_mix_normal_1 (
114+
id int not null,
115+
value32 int not null,
116+
primary key(id)
117+
);
118+
119+
create table rename_mix_part_1 (
120+
id int not null,
121+
value32 int not null,
122+
primary key(id)
123+
) partition by hash(id) partitions 2;
124+
125+
insert into rename_mix_normal_1 values(1, 10), (2, 20);
126+
insert into rename_mix_part_1 values(11, 110), (12, 120);
127+
128+
rename table rename_mix_normal_1 to rename_mix_normal_1_done, rename_mix_part_1 to rename_mix_part_1_done;
129+
130+
insert into rename_mix_normal_1_done values(3, 30);
131+
insert into rename_mix_part_1_done values(13, 130);
132+
133+
create table rename_mix_part_2 (
134+
id int not null,
135+
value32 int not null,
136+
primary key(id)
137+
) partition by hash(id) partitions 2;
138+
139+
create table rename_mix_normal_2 (
140+
id int not null,
141+
value32 int not null,
142+
primary key(id)
143+
);
144+
145+
insert into rename_mix_part_2 values(21, 210), (22, 220);
146+
insert into rename_mix_normal_2 values(31, 310), (32, 320);
147+
148+
rename table rename_mix_part_2 to rename_mix_part_2_done, rename_mix_normal_2 to rename_mix_normal_2_done;
149+
150+
insert into rename_mix_part_2_done values(23, 230);
151+
insert into rename_mix_normal_2_done values(33, 330);
152+
113153
insert into t3 select * from t1_7;
114154
insert into t4 select * from t2_7;
115155
drop table t3, t4;

‎tests/integration_tests/multi_tables_ddl/run.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ function run() {
102102
check_table_exists multi_tables_ddl_test.t66 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
103103
check_table_exists multi_tables_ddl_test.t7 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
104104
check_table_exists multi_tables_ddl_test.t88 ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
105+
check_table_exists multi_tables_ddl_test.rename_mix_normal_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
106+
check_table_exists multi_tables_ddl_test.rename_mix_part_1_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
107+
check_table_exists multi_tables_ddl_test.rename_mix_normal_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
108+
check_table_exists multi_tables_ddl_test.rename_mix_part_2_done ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
105109
# sync_diff can't check non-exist table, so we check expected tables are created in downstream first
106110
check_table_exists multi_tables_ddl_test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT}
107111
echo "check table exists success"

0 commit comments

Comments
 (0)