diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel index 283ec4a525c9a..5ed89476c3946 100644 --- a/br/pkg/restore/BUILD.bazel +++ b/br/pkg/restore/BUILD.bazel @@ -122,6 +122,7 @@ go_test( "search_test.go", "split_test.go", "stream_metas_test.go", + "systable_restore_test.go", "util_test.go", ], embed = [":restore"], diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go index 8ac9eeb0c3ae3..b51ae57a96386 100644 --- a/br/pkg/restore/client.go +++ b/br/pkg/restore/client.go @@ -206,6 +206,8 @@ type Client struct { // checkpoint information for log restore useCheckpoint bool + + privilegeTableRowsCollateCompatibility bool } // NewRestoreClient returns a new RestoreClient. @@ -1253,7 +1255,13 @@ func (rc *Client) CheckSysTableCompatibility(dom *domain.Domain, tables []*metau table.Info.Name.O, col.Name, col.FieldType.String()) } - if !utils.IsTypeCompatible(backupCol.FieldType, col.FieldType) { + typeEq, collateEq := utils.IsTypeCompatible(backupCol.FieldType, col.FieldType) + collateCompatible := collateEq + if typeEq && !collateEq { + rc.privilegeTableRowsCollateCompatibility = true + collateCompatible = checkSysTableColumnCollateCompatibility(mysql.SystemDB, table.Info.Name.L, col.Name.L, backupCol.GetCollate(), col.GetCollate()) + } + if !(typeEq && collateCompatible) { log.Error("incompatible column", zap.Stringer("table", table.Info.Name), zap.String("col in cluster", fmt.Sprintf("%s %s", col.Name, col.FieldType.String())), diff --git a/br/pkg/restore/client_test.go b/br/pkg/restore/client_test.go index a20b4a2a8b905..babaa1d38d922 100644 --- a/br/pkg/restore/client_test.go +++ b/br/pkg/restore/client_test.go @@ -278,7 +278,44 @@ func TestCheckSysTableCompatibility(t *testing.T) { // other system tables in cluster have more columns(failed) mockedDBTI := dbTI.Clone() - dbTI.Columns = append(dbTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")}) + mockedDBTI.Columns = append(dbTI.Columns, &model.ColumnInfo{Name: model.NewCIStr("new-name")}) + err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{ + DB: tmpSysDB, + Info: mockedDBTI, + }}) + require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err)) + + // skip check collate + mockedDBTI = dbTI.Clone() + mockedDBTI.Columns[1].SetCollate("utf8mb4_bin") + err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{ + DB: tmpSysDB, + Info: mockedDBTI, + }}) + require.NoError(t, err) + + // skip check collate but type mismatch + mockedDBTI = dbTI.Clone() + mockedDBTI.Columns[1].SetCollate("utf8mb4_bin") + mockedDBTI.Columns[1].FieldType.SetFlen(2000) // Columns[1] is `DB` char(64) + err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{ + DB: tmpSysDB, + Info: mockedDBTI, + }}) + require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err)) + + // another column collate mismatch + mockedDBTI = dbTI.Clone() + mockedDBTI.Columns[0].SetCollate("utf8mb4_general_ci") + err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{ + DB: tmpSysDB, + Info: mockedDBTI, + }}) + require.True(t, berrors.ErrRestoreIncompatibleSys.Equal(err)) + + // another column collate mismatch + mockedDBTI = dbTI.Clone() + mockedDBTI.Columns[1].SetCollate("utf8mb4_unicode_ci") err = client.CheckSysTableCompatibility(cluster.Domain, []*metautil.Table{{ DB: tmpSysDB, Info: mockedDBTI, diff --git a/br/pkg/restore/systable_restore.go b/br/pkg/restore/systable_restore.go index 93b503f51b5b0..8aaf77812f18b 100644 --- a/br/pkg/restore/systable_restore.go +++ b/br/pkg/restore/systable_restore.go @@ -13,6 +13,7 @@ import ( "github.com/pingcap/tidb/br/pkg/logutil" "github.com/pingcap/tidb/br/pkg/utils" "github.com/pingcap/tidb/pkg/bindinfo" + "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/parser/mysql" filter "github.com/pingcap/tidb/pkg/util/table-filter" @@ -60,6 +61,32 @@ var unRecoverableTable = map[string]map[string]struct{}{ }, } +type checkPrivilegeTableRowsCollateCompatibilitySQLPair struct { + upstreamCollateSQL string + downstreamCollateSQL string + columns map[string]struct{} +} + +var collateCompatibilityTables = map[string]map[string]checkPrivilegeTableRowsCollateCompatibilitySQLPair{ + "mysql": { + "db": { + upstreamCollateSQL: "SELECT COUNT(1) FROM __TiDB_BR_Temporary_mysql.db", + downstreamCollateSQL: "SELECT COUNT(1) FROM (SELECT Host, DB COLLATE utf8mb4_general_ci, User FROM __TiDB_BR_Temporary_mysql.db GROUP BY Host, DB COLLATE utf8mb4_general_ci, User) as a", + columns: map[string]struct{}{"db": {}}, + }, + "tables_priv": { + upstreamCollateSQL: "SELECT COUNT(1) FROM __TiDB_BR_Temporary_mysql.tables_priv", + downstreamCollateSQL: "SELECT COUNT(1) FROM (SELECT Host, DB COLLATE utf8mb4_general_ci, User, Table_name COLLATE utf8mb4_general_ci FROM __TiDB_BR_Temporary_mysql.tables_priv GROUP BY Host, DB COLLATE utf8mb4_general_ci, User, Table_name COLLATE utf8mb4_general_ci) as a", + columns: map[string]struct{}{"db": {}, "table_name": {}}, + }, + "columns_priv": { + upstreamCollateSQL: "SELECT COUNT(1) FROM __TiDB_BR_Temporary_mysql.columns_priv", + downstreamCollateSQL: "SELECT COUNT(1) FROM (SELECT Host, DB COLLATE utf8mb4_general_ci, User, Table_name COLLATE utf8mb4_general_ci, Column_name COLLATE utf8mb4_general_ci FROM __TiDB_BR_Temporary_mysql.columns_priv GROUP BY Host, DB COLLATE utf8mb4_general_ci, User, Table_name COLLATE utf8mb4_general_ci, Column_name COLLATE utf8mb4_general_ci) as a", + columns: map[string]struct{}{"db": {}, "table_name": {}, "column_name": {}}, + }, + }, +} + func isUnrecoverableTable(schemaName string, tableName string) bool { tableMap, ok := unRecoverableTable[schemaName] if !ok { @@ -252,6 +279,11 @@ func (rc *Client) replaceTemporaryTableToSystable(ctx context.Context, ti *model log.Info("replace into existing table", zap.String("table", tableName), zap.Stringer("schema", db.Name)) + if rc.privilegeTableRowsCollateCompatibility { + if err := rc.checkPrivilegeTableRowsCollateCompatibility(ctx, dbName, tableName, ti, db.ExistingTables[tableName]); err != nil { + return err + } + } // target column order may different with source cluster columnNames := make([]string, 0, len(ti.Columns)) for _, col := range ti.Columns { @@ -283,3 +315,97 @@ func (rc *Client) cleanTemporaryDatabase(ctx context.Context, originDB string) { ) } } + +func checkSysTableColumnCollateCompatibility(dbNameL, tableNameL, columnNameL, upstreamCollate, downstreamCollate string) bool { + if upstreamCollate != "utf8mb4_bin" || downstreamCollate != "utf8mb4_general_ci" { + return false + } + collateCompatibilityTableMap, exists := collateCompatibilityTables[dbNameL] + if !exists { + return false + } + collateCompatibilityColumnMap, exists := collateCompatibilityTableMap[tableNameL] + if !exists { + return false + } + _, exists = collateCompatibilityColumnMap.columns[columnNameL] + return exists +} + +func (rc *Client) checkPrivilegeTableRowsCollateCompatibility( + ctx context.Context, + dbNameL, tableNameL string, + upstreamTable, downstreamTable *model.TableInfo, +) error { + collateCompatibilityTableMap, exists := collateCompatibilityTables[dbNameL] + if !exists { + return nil + } + collateCompatibilityColumnMap, exists := collateCompatibilityTableMap[tableNameL] + if !exists { + return nil + } + colCount := 0 + for _, col := range upstreamTable.Columns { + if _, exists := collateCompatibilityColumnMap.columns[col.Name.L]; exists { + if col.GetCollate() != "utf8mb4_bin" && col.GetCollate() != "utf8mb4_general_ci" { + return errors.Annotatef(berrors.ErrRestoreIncompatibleSys, + "incompatible column collate, upstream table %s.%s column %s collate is %s but should be utf8mb4_bin or utf8mb4_general_ci", + dbNameL, tableNameL, col.Name.L, col.GetCollate()) + } + colCount += 1 + } + } + if colCount != len(collateCompatibilityColumnMap.columns) { + return errors.Annotatef(berrors.ErrRestoreIncompatibleSys, + "incompatible column collate, upstream table %s.%s has only %d compatible columns", + dbNameL, tableNameL, colCount) + } + colCount = 0 + for _, col := range downstreamTable.Columns { + if _, exists := collateCompatibilityColumnMap.columns[col.Name.L]; exists { + if col.GetCollate() != "utf8mb4_general_ci" { + return errors.Annotatef(berrors.ErrRestoreIncompatibleSys, + "incompatible column collate, downstream table %s.%s column %s collate is %s but should be utf8mb4_general_ci", + dbNameL, tableNameL, col.Name.L, col.GetCollate()) + } + colCount += 1 + } + } + if colCount != len(collateCompatibilityColumnMap.columns) { + return errors.Annotatef(berrors.ErrRestoreIncompatibleSys, + "incompatible column collate, downstream table %s.%s has only %d compatible columns", + dbNameL, tableNameL, colCount) + } + ectx := rc.db.se.GetSessionCtx().GetRestrictedSQLExecutor() + rows, _, err := ectx.ExecRestrictedSQL( + kv.WithInternalSourceType(ctx, kv.InternalTxnBR), + nil, + collateCompatibilityColumnMap.upstreamCollateSQL, + ) + if err != nil { + return errors.Annotatef(err, "failed to get the count of privilege rows") + } + if len(rows) == 0 { + return errors.Errorf("failed to get the count of privilege rows") + } + upstreamCount := rows[0].GetInt64(0) + rows, _, err = ectx.ExecRestrictedSQL( + kv.WithInternalSourceType(ctx, kv.InternalTxnBR), + nil, + collateCompatibilityColumnMap.downstreamCollateSQL, + ) + if err != nil { + return errors.Annotatef(err, "failed to get the count of privilege rows") + } + if len(rows) == 0 { + return errors.Errorf("failed to get the count of privilege rows") + } + downstreamCount := rows[0].GetInt64(0) + if upstreamCount != downstreamCount { + return errors.Annotatef(berrors.ErrRestoreIncompatibleSys, + "there are duplicated privilege rows with collate utf8mb4_general_ci [upstream count %d != downstream count %d]", + upstreamCount, downstreamCount) + } + return nil +} diff --git a/br/pkg/restore/systable_restore_test.go b/br/pkg/restore/systable_restore_test.go new file mode 100644 index 0000000000000..8257d61686b61 --- /dev/null +++ b/br/pkg/restore/systable_restore_test.go @@ -0,0 +1,243 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package restore_test + +import ( + "context" + "fmt" + "testing" + + "github.com/pingcap/tidb/br/pkg/glue" + "github.com/pingcap/tidb/br/pkg/gluetidb" + "github.com/pingcap/tidb/br/pkg/restore" + "github.com/pingcap/tidb/pkg/parser/model" + "github.com/stretchr/testify/require" +) + +type mustExecuteSession struct { + ctx context.Context + se glue.Session + t *testing.T +} + +func (se *mustExecuteSession) MustExecute(sql string) { + err := se.se.ExecuteInternal(se.ctx, sql) + require.NoError(se.t, err) +} + +const ( + CreateDBSQL = `CREATE TABLE __TiDB_BR_Temporary_mysql.db ( + Host char(255) NOT NULL, + DB char(64) NOT NULL, + User char(32) NOT NULL, + Select_priv enum('N','Y') NOT NULL DEFAULT 'N', + Insert_priv enum('N','Y') NOT NULL DEFAULT 'N', + Update_priv enum('N','Y') NOT NULL DEFAULT 'N', + Delete_priv enum('N','Y') NOT NULL DEFAULT 'N', + Create_priv enum('N','Y') NOT NULL DEFAULT 'N', + Drop_priv enum('N','Y') NOT NULL DEFAULT 'N', + Grant_priv enum('N','Y') NOT NULL DEFAULT 'N', + References_priv enum('N','Y') NOT NULL DEFAULT 'N', + Index_priv enum('N','Y') NOT NULL DEFAULT 'N', + Alter_priv enum('N','Y') NOT NULL DEFAULT 'N', + Create_tmp_table_priv enum('N','Y') NOT NULL DEFAULT 'N', + Lock_tables_priv enum('N','Y') NOT NULL DEFAULT 'N', + Create_view_priv enum('N','Y') NOT NULL DEFAULT 'N', + Show_view_priv enum('N','Y') NOT NULL DEFAULT 'N', + Create_routine_priv enum('N','Y') NOT NULL DEFAULT 'N', + Alter_routine_priv enum('N','Y') NOT NULL DEFAULT 'N', + Execute_priv enum('N','Y') NOT NULL DEFAULT 'N', + Event_priv enum('N','Y') NOT NULL DEFAULT 'N', + Trigger_priv enum('N','Y') NOT NULL DEFAULT 'N', + PRIMARY KEY (Host,DB,User) /*T![clustered_index] NONCLUSTERED */ +)` + + CreateTableSQL = `CREATE TABLE __TiDB_BR_Temporary_mysql.tables_priv ( + Host char(255) NOT NULL, + DB char(64) NOT NULL, + User char(32) NOT NULL, + Table_name char(64) NOT NULL, + Grantor char(77) DEFAULT NULL, + Timestamp timestamp DEFAULT CURRENT_TIMESTAMP, + Table_priv set('Select','Insert','Update','Delete','Create','Drop','Grant','Index','Alter','Create View','Show View','Trigger','References') DEFAULT NULL, + Column_priv set('Select','Insert','Update','References') DEFAULT NULL, + PRIMARY KEY (Host,DB,User,Table_name) /*T![clustered_index] NONCLUSTERED */ +)` + + CreateColumnSQL = `CREATE TABLE __TiDB_BR_Temporary_mysql.columns_priv ( + Host char(255) NOT NULL, + DB char(64) NOT NULL, + User char(32) NOT NULL, + Table_name char(64) NOT NULL, + Column_name char(64) NOT NULL, + Timestamp timestamp DEFAULT CURRENT_TIMESTAMP, + Column_priv set('Select','Insert','Update','References') DEFAULT NULL, + PRIMARY KEY (Host,DB,User,Table_name,Column_name) /*T![clustered_index] NONCLUSTERED */ +)` +) + +func TestCheckPrivilegeTableRowsCollateCompatibility(t *testing.T) { + cluster := mc + ctx := context.Background() + g := gluetidb.New() + rc := restore.Client{} + defer rc.Close() + err := rc.Init(g, cluster.Storage) + require.NoError(t, err) + + se, err := g.CreateSession(cluster.Storage) + require.NoError(t, err) + defer se.Close() + mse := &mustExecuteSession{ctx, se, t} + mse.MustExecute("CREATE DATABASE __TiDB_BR_Temporary_mysql") + defer mse.MustExecute("DROP DATABASE __TiDB_BR_Temporary_mysql") + + downstreamDBTable, err := rc.GetTableSchema(cluster.Domain, model.NewCIStr("mysql"), model.NewCIStr("db")) + require.NoError(t, err) + downstreamTablesTable, err := rc.GetTableSchema(cluster.Domain, model.NewCIStr("mysql"), model.NewCIStr("tables_priv")) + require.NoError(t, err) + downstreamColumnsTable, err := rc.GetTableSchema(cluster.Domain, model.NewCIStr("mysql"), model.NewCIStr("columns_priv")) + require.NoError(t, err) + // case 1: privilege db + mse.MustExecute(CreateDBSQL) + backupTable, err := rc.GetTableSchema(cluster.Domain, model.NewCIStr("__TiDB_BR_Temporary_mysql"), model.NewCIStr("db")) + require.NoError(t, err) + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.db (Host,DB,User) VALUES ('%','test','newroot')") + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.db (Host,DB,User) VALUES ('%','test','oldroot')") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "db", backupTable, downstreamDBTable) + require.NoError(t, err) + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.db (Host,DB,User) VALUES ('%','Test','newroot')") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "db", backupTable, downstreamDBTable) + require.Error(t, err) + mse.MustExecute("DELETE FROM __TiDB_BR_Temporary_mysql.db WHERE DB = 'Test'") + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.db (Host,DB,User) VALUES ('%','cafe','newroot')") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "db", backupTable, downstreamDBTable) + require.NoError(t, err) + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.db (Host,DB,User) VALUES ('%','café','newroot')") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "db", backupTable, downstreamDBTable) + require.Error(t, err) + mse.MustExecute("DELETE FROM __TiDB_BR_Temporary_mysql.db WHERE DB = 'cafe'") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "db", backupTable, downstreamDBTable) + require.NoError(t, err) + mse.MustExecute("DROP TABLE __TiDB_BR_Temporary_mysql.db") + + // case 2: privilege table + type privCase struct { + insertValues []string + deleteCond []string + } + mse.MustExecute(CreateTableSQL) + backupTable, err = rc.GetTableSchema(cluster.Domain, model.NewCIStr("__TiDB_BR_Temporary_mysql"), model.NewCIStr("tables_priv")) + require.NoError(t, err) + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.tables_priv (Host,DB,User,Table_name) VALUES ('%','test','newroot','ta1')") + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.tables_priv (Host,DB,User,Table_name) VALUES ('%','test','oldroot','ta1')") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "tables_priv", backupTable, downstreamTablesTable) + require.NoError(t, err) + cases := []privCase{ + { + insertValues: []string{"('%','test','newroot','Ta1')"}, + deleteCond: []string{"Table_name = 'Ta1'"}, + }, + { + insertValues: []string{"('%','tEst','newroot','ta1')"}, + deleteCond: []string{"DB = 'tEst'"}, + }, + { + insertValues: []string{"('%','tEst','newroot','Ta1')"}, + deleteCond: []string{"DB = 'tEst'"}, + }, + { + insertValues: []string{"('%','test','newroot','tá1')"}, + deleteCond: []string{"Table_name = 'tá1'"}, + }, + { + insertValues: []string{"('%','tést','newroot','ta1')"}, + deleteCond: []string{"DB = 'tést'"}, + }, + { + insertValues: []string{"('%','tést','newroot','tá1')"}, + deleteCond: []string{"DB = 'tést'"}, + }, + { + insertValues: []string{"('%','tést','newroot','tá1')", "('%','tEst','newroot','Ta1')"}, + deleteCond: []string{"DB = 'tést'", "DB = 'tEst'"}, + }, + } + for _, cs := range cases { + for _, v := range cs.insertValues { + mse.MustExecute(fmt.Sprintf("INSERT INTO __TiDB_BR_Temporary_mysql.tables_priv (Host,DB,User,Table_name) VALUES %s", v)) + } + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "tables_priv", backupTable, downstreamTablesTable) + require.Error(t, err) + for _, v := range cs.deleteCond { + mse.MustExecute(fmt.Sprintf("DELETE FROM __TiDB_BR_Temporary_mysql.tables_priv WHERE %s", v)) + } + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "tables_priv", backupTable, downstreamTablesTable) + require.NoError(t, err) + } + mse.MustExecute("DROP TABLE __TiDB_BR_Temporary_mysql.tables_priv") + + // case 3: privilege column + mse.MustExecute(CreateColumnSQL) + backupTable, err = rc.GetTableSchema(cluster.Domain, model.NewCIStr("__TiDB_BR_Temporary_mysql"), model.NewCIStr("columns_priv")) + require.NoError(t, err) + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.columns_priv (Host,DB,User,Table_name,Column_name) VALUES ('%','test','newroot','ta1','ca1')") + mse.MustExecute("INSERT INTO __TiDB_BR_Temporary_mysql.columns_priv (Host,DB,User,Table_name,Column_name) VALUES ('%','test','oldroot','ta1','ca1')") + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "columns_priv", backupTable, downstreamColumnsTable) + require.NoError(t, err) + cases = []privCase{ + { + insertValues: []string{"('%','test','newroot','ta1','Ca1')"}, + deleteCond: []string{"Column_name = 'Ca1'"}, + }, + { + insertValues: []string{"('%','test','newroot','Ta1','ca1')"}, + deleteCond: []string{"Table_name = 'Ta1'"}, + }, + { + insertValues: []string{"('%','Test','newroot','ta1','ca1')"}, + deleteCond: []string{"DB = 'Test'"}, + }, + { + insertValues: []string{"('%','test','newroot','ta1','cá1')"}, + deleteCond: []string{"Column_name = 'cá1'"}, + }, + { + insertValues: []string{"('%','test','newroot','tá1','ca1')"}, + deleteCond: []string{"Table_name = 'tá1'"}, + }, + { + insertValues: []string{"('%','tést','newroot','ta1','ca1')"}, + deleteCond: []string{"DB = 'tést'"}, + }, + { + insertValues: []string{"('%','tést','newroot','ta1','ca1')", "('%','Test','newroot','ta1','ca1')"}, + deleteCond: []string{"DB = 'tést'", "DB = 'Test'"}, + }, + } + for _, cs := range cases { + for _, v := range cs.insertValues { + mse.MustExecute(fmt.Sprintf("INSERT INTO __TiDB_BR_Temporary_mysql.columns_priv (Host,DB,User,Table_name,Column_name) VALUES %s", v)) + } + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "columns_priv", backupTable, downstreamColumnsTable) + require.Error(t, err) + for _, v := range cs.deleteCond { + mse.MustExecute(fmt.Sprintf("DELETE FROM __TiDB_BR_Temporary_mysql.columns_priv WHERE %s", v)) + } + err = rc.CheckPrivilegeTableRowsCollateCompatibility(ctx, "mysql", "columns_priv", backupTable, downstreamColumnsTable) + require.NoError(t, err) + } + mse.MustExecute("DROP TABLE __TiDB_BR_Temporary_mysql.columns_priv") +} diff --git a/br/pkg/restore/util_test.go b/br/pkg/restore/util_test.go index adcaed80a1514..64dff4964889a 100644 --- a/br/pkg/restore/util_test.go +++ b/br/pkg/restore/util_test.go @@ -3,6 +3,7 @@ package restore import ( + "context" "fmt" "math/rand" "testing" @@ -11,11 +12,20 @@ import ( "github.com/pingcap/kvproto/pkg/import_sstpb" "github.com/pingcap/kvproto/pkg/metapb" recover_data "github.com/pingcap/kvproto/pkg/recoverdatapb" + "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/tablecodec" "github.com/pingcap/tidb/pkg/util/codec" "github.com/stretchr/testify/require" ) +func (rc *Client) CheckPrivilegeTableRowsCollateCompatibility( + ctx context.Context, + dbNameL, tableNameL string, + upstreamTable, downstreamTable *model.TableInfo, +) error { + return rc.checkPrivilegeTableRowsCollateCompatibility(ctx, dbNameL, tableNameL, upstreamTable, downstreamTable) +} + func TestGetKeyRangeByMode(t *testing.T) { file := &backuppb.File{ Name: "file_write.sst", diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index d64e057b40bfe..5ebea4393f160 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -854,7 +854,7 @@ func runRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConf } } - if client.IsFullClusterRestore() && client.HasBackedUpSysDB() { + if cfg.WithSysTable && client.HasBackedUpSysDB() { if err = client.CheckSysTableCompatibility(mgr.GetDomain(), tables); err != nil { return errors.Trace(err) } diff --git a/br/pkg/utils/misc.go b/br/pkg/utils/misc.go index c351f62011a76..693b6a80d8b5e 100644 --- a/br/pkg/utils/misc.go +++ b/br/pkg/utils/misc.go @@ -57,16 +57,17 @@ const ( // - target's flen and decimal should be bigger or equals to src's // - elements in target is superset of elements in src if they're enum or set type // - same charset and collate if they're string types -func IsTypeCompatible(src types.FieldType, target types.FieldType) bool { +func IsTypeCompatible(src types.FieldType, target types.FieldType) (typeEq, collateEq bool) { + collateEq = src.GetCollate() == target.GetCollate() if mysql.HasNotNullFlag(src.GetFlag()) != mysql.HasNotNullFlag(target.GetFlag()) { - return false + return false, collateEq } if mysql.HasUnsignedFlag(src.GetFlag()) != mysql.HasUnsignedFlag(target.GetFlag()) { - return false + return false, collateEq } srcEType, dstEType := src.EvalType(), target.EvalType() if srcEType != dstEType { - return false + return false, collateEq } getFLenAndDecimal := func(tp types.FieldType) (int, int) { @@ -84,7 +85,7 @@ func IsTypeCompatible(src types.FieldType, target types.FieldType) bool { srcFLen, srcDecimal := getFLenAndDecimal(src) targetFLen, targetDecimal := getFLenAndDecimal(target) if srcFLen > targetFLen || srcDecimal > targetDecimal { - return false + return false, collateEq } // if they're not enum or set type, elems will be empty @@ -93,7 +94,7 @@ func IsTypeCompatible(src types.FieldType, target types.FieldType) bool { srcElems := src.GetElems() targetElems := target.GetElems() if len(srcElems) > len(targetElems) { - return false + return false, collateEq } targetElemSet := make(map[string]struct{}) for _, item := range targetElems { @@ -101,11 +102,10 @@ func IsTypeCompatible(src types.FieldType, target types.FieldType) bool { } for _, item := range srcElems { if _, ok := targetElemSet[item]; !ok { - return false + return false, collateEq } } - return src.GetCharset() == target.GetCharset() && - src.GetCollate() == target.GetCollate() + return src.GetCharset() == target.GetCharset(), collateEq } func GRPCConn(ctx context.Context, storeAddr string, tlsConf *tls.Config, opts ...grpc.DialOption) (*grpc.ClientConn, error) { diff --git a/br/pkg/utils/misc_test.go b/br/pkg/utils/misc_test.go index a7678d7785d57..b7b0035e7106b 100644 --- a/br/pkg/utils/misc_test.go +++ b/br/pkg/utils/misc_test.go @@ -31,34 +31,46 @@ func TestIsTypeCompatible(t *testing.T) { src := types.NewFieldType(mysql.TypeInt24) src.AddFlag(mysql.UnsignedFlag) target := types.NewFieldType(mysql.TypeInt24) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) src.DelFlag(mysql.UnsignedFlag) target.AddFlag(mysql.UnsignedFlag) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq = IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // different not null flag src := types.NewFieldType(mysql.TypeInt24) src.AddFlag(mysql.NotNullFlag) target := types.NewFieldType(mysql.TypeInt24) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) src.DelFlag(mysql.NotNullFlag) target.AddFlag(mysql.NotNullFlag) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq = IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // different evaluation type src := types.NewFieldType(mysql.TypeInt24) target := types.NewFieldType(mysql.TypeFloat) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // src flen > target src := types.NewFieldType(mysql.TypeInt24) target := types.NewFieldType(mysql.TypeTiny) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // src flen > target @@ -66,7 +78,9 @@ func TestIsTypeCompatible(t *testing.T) { src.SetFlen(100) target := types.NewFieldType(mysql.TypeVarchar) target.SetFlag(99) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // src decimal > target @@ -74,7 +88,9 @@ func TestIsTypeCompatible(t *testing.T) { src.SetDecimal(5) target := types.NewFieldType(mysql.TypeNewDecimal) target.SetDecimal(4) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // src has more elements @@ -82,7 +98,9 @@ func TestIsTypeCompatible(t *testing.T) { src.SetElems([]string{"a", "b"}) target := types.NewFieldType(mysql.TypeEnum) target.SetElems([]string{"a"}) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // incompatible enum @@ -90,7 +108,9 @@ func TestIsTypeCompatible(t *testing.T) { src.SetElems([]string{"a", "b"}) target := types.NewFieldType(mysql.TypeEnum) target.SetElems([]string{"a", "c", "d"}) - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // incompatible charset @@ -98,7 +118,9 @@ func TestIsTypeCompatible(t *testing.T) { src.SetCharset("gbk") target := types.NewFieldType(mysql.TypeVarchar) target.SetCharset("utf8") - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.False(t, typeEq) + require.True(t, collateEq) } { // incompatible collation @@ -108,7 +130,9 @@ func TestIsTypeCompatible(t *testing.T) { target := types.NewFieldType(mysql.TypeVarchar) target.SetCharset("utf8") target.SetCollate("utf8_general_ci") - require.False(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.True(t, typeEq) + require.False(t, collateEq) } { src := types.NewFieldType(mysql.TypeVarchar) @@ -119,25 +143,33 @@ func TestIsTypeCompatible(t *testing.T) { target.SetFlen(11) target.SetCharset("utf8") target.SetCollate("utf8_bin") - require.True(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.True(t, typeEq) + require.True(t, collateEq) } { src := types.NewFieldType(mysql.TypeBlob) target := types.NewFieldType(mysql.TypeLongBlob) - require.True(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.True(t, typeEq) + require.True(t, collateEq) } { src := types.NewFieldType(mysql.TypeEnum) src.SetElems([]string{"a", "b"}) target := types.NewFieldType(mysql.TypeEnum) target.SetElems([]string{"a", "b", "c"}) - require.True(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.True(t, typeEq) + require.True(t, collateEq) } { src := types.NewFieldType(mysql.TypeTimestamp) target := types.NewFieldType(mysql.TypeTimestamp) target.SetDecimal(3) - require.True(t, IsTypeCompatible(*src, *target)) + typeEq, collateEq := IsTypeCompatible(*src, *target) + require.True(t, typeEq) + require.True(t, collateEq) } }