deps: upgrade tidb release-8.5 dependency (#12589)

ref #11020

Author: OliverS929

cdc/entry/schema_test.go

@@ -130,6 +130,7 @@ func TestAllTables(t *testing.T) {
 	require.Equal(t, job.BinlogInfo.TableInfo.Name.O, tableInfos[0].TableName.Table)
 	// add normal table
 	job = helper.DDL2Job("create table test.t1(id int primary key)")
+	tableIDT1 := job.BinlogInfo.TableInfo.ID
 	require.Nil(t, schema.HandleDDLJob(job))
 	tableInfos, err = schema.AllTables(context.Background(), job.BinlogInfo.FinishedTS)
 	require.Nil(t, err)
@@ -138,7 +139,7 @@ func TestAllTables(t *testing.T) {
 	require.Equal(t, model.TableName{
 		Schema:  "test",
 		Table:   "t1",
-		TableID: 112,
+		TableID: tableIDT1,
 	}, tableName)
 	// add ineligible table
 	job = helper.DDL2Job("create table test.t2(id int)")
@@ -150,7 +151,7 @@ func TestAllTables(t *testing.T) {
 	require.Equal(t, model.TableName{
 		Schema:  "test",
 		Table:   "t1",
-		TableID: 112,
+		TableID: tableIDT1,
 	}, tableName)
 }
 

dm/syncer/ddl.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/go-mysql-org/go-mysql/replication"
+	"github.com/pingcap/errors"
 	"github.com/pingcap/failpoint"
 	tidbddl "github.com/pingcap/tidb/pkg/ddl"
 	"github.com/pingcap/tidb/pkg/meta/metabuild"
@@ -29,6 +30,7 @@ import (
 	"github.com/pingcap/tidb/pkg/table"
 	"github.com/pingcap/tidb/pkg/table/tables"
 	"github.com/pingcap/tidb/pkg/types"
+	"github.com/pingcap/tidb/pkg/util/dbterror"
 	"github.com/pingcap/tidb/pkg/util/filter"
 	tidbmock "github.com/pingcap/tidb/pkg/util/mock"
 	regexprrouter "github.com/pingcap/tidb/pkg/util/regexpr-router"
@@ -1076,27 +1078,35 @@ func (ddl *DDLWorker) handleModifyColumn(qec *queryEventContext, info *ddlInfo,
 	if oldCol == nil {
 		return bf.AlterTable, nil
 	}
-	newCol := table.ToColumn(&model.ColumnInfo{
-		ID:                    oldCol.ID,
-		Offset:                oldCol.Offset,
-		State:                 oldCol.State,
-		OriginDefaultValue:    oldCol.OriginDefaultValue,
-		OriginDefaultValueBit: oldCol.OriginDefaultValueBit,
-		FieldType:             *spec.NewColumns[0].Tp,
-		Name:                  spec.NewColumns[0].Name.Name,
-		Version:               oldCol.Version,
-	})
 
-	// handle charset and collation
-	if err := tidbddl.ProcessColumnCharsetAndCollation(metabuild.NewContext(), oldCol, newCol, ti, spec.NewColumns[0], di); err != nil {
-		ddl.logger.Warn("process column charset and collation failed", zap.Error(err))
+	// Let TiDB build the post-DDL column definition so we stay compatible with its internal
+	// modify-column handling even when lower-level helpers stop being exported.
+	mockCtx := tidbmock.NewContext()
+	mockCtx.GetSessionVars().AllowRemoveAutoInc = true
+	jobW, err := tidbddl.GetModifiableColumnJob(
+		qec.tctx.Ctx,
+		mockCtx,
+		nil,
+		ast.Ident{Schema: di.Name, Name: ti.Name},
+		oldColumnName.Name,
+		di,
+		tbl,
+		spec,
+	)
+	if err != nil {
+		if et, ok := ddl.classifyUnsupportedModifyColumnEvent(info, oldCol, ti, di, spec.NewColumns[0], err); ok {
+			return et, nil
+		}
+		ddl.logger.Warn("build modifiable column job failed", zap.Error(err))
 		return bf.AlterTable, err
 	}
-	// handle column options
-	if err := tidbddl.ProcessModifyColumnOptions(tidbmock.NewContext(), newCol, spec.NewColumns[0].Options); err != nil {
-		ddl.logger.Warn("process column options failed", zap.Error(err))
+	args, ok := jobW.JobArgs.(*model.ModifyColumnArgs)
+	if !ok || args.Column == nil {
+		err := fmt.Errorf("unexpected modify column job args: %T", jobW.JobArgs)
+		ddl.logger.Warn("get modifiable column failed", zap.Error(err))
 		return bf.AlterTable, err
 	}
+	newCol := ddl.normalizeModifyColumnEvent(info, oldCol, args.Column, spec.NewColumns[0])
 
 	if et := ddl.needChangeColumnData(oldCol, newCol); et != bf.AlterTable {
 		return et, nil
@@ -1123,6 +1133,95 @@ func (ddl *DDLWorker) handleModifyColumn(qec *queryEventContext, info *ddlInfo,
 	}
 }
 
+func (ddl *DDLWorker) classifyUnsupportedModifyColumnEvent(
+	info *ddlInfo,
+	oldCol *table.Column,
+	ti *model.TableInfo,
+	di *model.DBInfo,
+	specNewColumn *ast.ColumnDef,
+	jobErr error,
+) (bf.EventType, bool) {
+	cause := errors.Cause(jobErr)
+	if !dbterror.ErrUnsupportedModifyCharset.Equal(cause) && !dbterror.ErrUnsupportedModifyCollation.Equal(cause) {
+		return bf.AlterTable, false
+	}
+
+	// Newer TiDB rejects some MODIFY/CHANGE COLUMN charset transitions while DM still
+	// needs the historical event type so binlog-filter can block them before tracker/execution.
+	newCol := table.ToColumn(&model.ColumnInfo{
+		ID:                    oldCol.ID,
+		Offset:                oldCol.Offset,
+		State:                 oldCol.State,
+		OriginDefaultValue:    oldCol.OriginDefaultValue,
+		OriginDefaultValueBit: oldCol.OriginDefaultValueBit,
+		FieldType:             *specNewColumn.Tp,
+		Name:                  specNewColumn.Name.Name,
+		Version:               oldCol.Version,
+	})
+	if err := tidbddl.ProcessColumnCharsetAndCollation(metabuild.NewContext(), oldCol, newCol, ti, specNewColumn, di); err != nil {
+		ddl.logger.Warn("fallback classify modify column charset/collation failed",
+			zap.String("origin_sql", info.originDDL),
+			zap.Error(jobErr),
+			zap.Error(err),
+		)
+		return bf.AlterTable, false
+	}
+
+	switch {
+	case oldCol.GetCharset() != newCol.GetCharset():
+		return bf.ModifyCharset, true
+	case oldCol.GetCollate() != newCol.GetCollate():
+		return bf.ModifyCollation, true
+	default:
+		return bf.AlterTable, false
+	}
+}
+
+func (ddl *DDLWorker) normalizeModifyColumnEvent(
+	info *ddlInfo,
+	oldCol *table.Column,
+	jobCol *model.ColumnInfo,
+	specNewColumn *ast.ColumnDef,
+) *table.Column {
+	eventCol := table.ToColumn(jobCol.Clone())
+
+	// TiDB's modify-column job builder preserves existing key flags so the DDL can
+	// execute correctly. DM's incompatible-DDL classification is historical and
+	// spec-based: omitted key attributes in CHANGE/MODIFY COLUMN should still be
+	// treated as removals for event typing.
+	const indexFlagMask = mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.MultipleKeyFlag
+	eventCol.DelFlag(indexFlagMask)
+	eventCol.AddFlag(specNewColumn.Tp.GetFlag() & indexFlagMask)
+
+	hasExplicitDefault := false
+	for _, opt := range specNewColumn.Options {
+		if opt.Tp == ast.ColumnOptionDefaultValue {
+			hasExplicitDefault = true
+			break
+		}
+	}
+	if !hasExplicitDefault {
+		// Preserve the tracked schema's default when the statement does not specify
+		// one. This matches DM's previous event classification semantics and avoids
+		// misclassifying a key drop as a default-value change after earlier skipped DDLs.
+		eventCol.DefaultValue = oldCol.DefaultValue
+		eventCol.DefaultValueBit = oldCol.DefaultValueBit
+		eventCol.DefaultIsExpr = oldCol.DefaultIsExpr
+	}
+
+	ddl.logger.Debug("normalized modify column event",
+		zap.String("origin_sql", info.originDDL),
+		zap.Uint64("old_flags", uint64(oldCol.GetFlag())),
+		zap.Uint64("job_flags", uint64(jobCol.GetFlag())),
+		zap.Uint64("event_flags", uint64(eventCol.GetFlag())),
+		zap.Any("old_default", oldCol.GetDefaultValue()),
+		zap.Any("job_default", jobCol.GetDefaultValue()),
+		zap.Any("event_default", eventCol.GetDefaultValue()),
+	)
+
+	return eventCol
+}
+
 // AstToDDLEvent returns filter.DDLEvent.
 func (ddl *DDLWorker) AstToDDLEvent(qec *queryEventContext, info *ddlInfo) (et bf.EventType) {
 	defer func() {

dm/syncer/ddl_test.go

@@ -22,8 +22,13 @@ import (
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/go-mysql-org/go-mysql/mysql"
 	"github.com/pingcap/check"
+	"github.com/pingcap/tidb/pkg/meta/model"
 	"github.com/pingcap/tidb/pkg/parser"
 	"github.com/pingcap/tidb/pkg/parser/ast"
+	pmodel "github.com/pingcap/tidb/pkg/parser/model"
+	tmysql "github.com/pingcap/tidb/pkg/parser/mysql"
+	fieldtypes "github.com/pingcap/tidb/pkg/parser/types"
+	"github.com/pingcap/tidb/pkg/table"
 	"github.com/pingcap/tidb/pkg/util/filter"
 	regexprrouter "github.com/pingcap/tidb/pkg/util/regexpr-router"
 	router "github.com/pingcap/tidb/pkg/util/table-router"
@@ -36,6 +41,7 @@ import (
 	"github.com/pingcap/tiflow/dm/pkg/terror"
 	"github.com/pingcap/tiflow/dm/syncer/metrics"
 	onlineddl "github.com/pingcap/tiflow/dm/syncer/online-ddl-tools"
+	bf "github.com/pingcap/tiflow/pkg/binlog-filter"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 )
@@ -841,6 +847,110 @@ func TestAdjustCollation(t *testing.T) {
 	}
 }
 
+func TestNormalizeModifyColumnEventPrefersDropUniqueOverDefault(t *testing.T) {
+	tctx := tcontext.Background().WithLogger(log.With(zap.String("test", "TestNormalizeModifyColumnEventPrefersDropUniqueOverDefault")))
+	ddlWorker := &DDLWorker{logger: tctx.Logger}
+
+	p := parser.New()
+	stmt, err := p.ParseOneStmt("alter table test.t change c_int c_int int", "", "")
+	require.NoError(t, err)
+	alterStmt, ok := stmt.(*ast.AlterTableStmt)
+	require.True(t, ok)
+	require.Len(t, alterStmt.Specs, 1)
+	spec := alterStmt.Specs[0]
+
+	oldCol := table.ToColumn(&model.ColumnInfo{
+		ID:                 1,
+		Name:               pmodel.NewCIStr("c_int"),
+		Offset:             0,
+		DefaultValue:       int64(1),
+		OriginDefaultValue: int64(1),
+		FieldType:          *fieldtypes.NewFieldType(tmysql.TypeLong),
+	})
+	oldCol.AddFlag(tmysql.UniqueKeyFlag)
+
+	jobCol := oldCol.ColumnInfo.Clone()
+	jobCol.DefaultValue = nil
+
+	eventCol := ddlWorker.normalizeModifyColumnEvent(&ddlInfo{originDDL: "alter table test.t change c_int c_int int"}, oldCol, jobCol, spec.NewColumns[0])
+	require.Equal(t, oldCol.GetDefaultValue(), eventCol.GetDefaultValue())
+	require.False(t, tmysql.HasUniKeyFlag(eventCol.GetFlag()))
+
+	eventType := ddlWorker.needChangeColumnData(oldCol, eventCol)
+	if eventType == bf.AlterTable {
+		switch {
+		case tmysql.HasAutoIncrementFlag(oldCol.GetFlag()) && !tmysql.HasAutoIncrementFlag(eventCol.GetFlag()):
+			eventType = bf.RemoveAutoIncrement
+		case tmysql.HasPriKeyFlag(oldCol.GetFlag()) && !tmysql.HasPriKeyFlag(eventCol.GetFlag()):
+			eventType = bf.DropPrimaryKey
+		case tmysql.HasUniKeyFlag(oldCol.GetFlag()) && !tmysql.HasUniKeyFlag(eventCol.GetFlag()):
+			eventType = bf.DropUniqueKey
+		case oldCol.GetDefaultValue() != eventCol.GetDefaultValue():
+			eventType = bf.ModifyDefaultValue
+		}
+	}
+	require.Equal(t, bf.DropUniqueKey, eventType)
+}
+
+func TestClassifyUnsupportedModifyColumnEventAsModifyCharset(t *testing.T) {
+	tctx := tcontext.Background().WithLogger(log.With(zap.String("test", "TestClassifyUnsupportedModifyColumnEventAsModifyCharset")))
+
+	tableInfo := &model.TableInfo{
+		ID:      1,
+		Name:    pmodel.NewCIStr("t"),
+		Charset: "utf8mb4",
+		Collate: "utf8mb4_bin",
+		State:   model.StatePublic,
+		Columns: []*model.ColumnInfo{
+			{
+				ID:        1,
+				Name:      pmodel.NewCIStr("c_char"),
+				Offset:    0,
+				State:     model.StatePublic,
+				FieldType: *fieldtypes.NewFieldType(tmysql.TypeString),
+			},
+		},
+	}
+	tableInfo.Columns[0].SetCharset("utf8mb4")
+	tableInfo.Columns[0].SetCollate("utf8mb4_bin")
+
+	dbInfo := &model.DBInfo{
+		Name:    pmodel.NewCIStr("test"),
+		Charset: "utf8mb4",
+		Collate: "utf8mb4_bin",
+	}
+
+	ddlWorker := &DDLWorker{
+		logger: tctx.Logger,
+		getTableInfo: func(_ *tcontext.Context, _, _ *filter.Table) (*model.TableInfo, error) {
+			return tableInfo.Clone(), nil
+		},
+		getDBInfoFromDownstream: func(_ *tcontext.Context, _, _ *filter.Table) (*model.DBInfo, error) {
+			return dbInfo, nil
+		},
+	}
+
+	p := parser.New()
+	stmt, err := p.ParseOneStmt("alter table test.t modify c_char char(4) character set latin1", "", "")
+	require.NoError(t, err)
+	alterStmt, ok := stmt.(*ast.AlterTableStmt)
+	require.True(t, ok)
+	require.Len(t, alterStmt.Specs, 1)
+
+	qec := &queryEventContext{
+		eventContext: &eventContext{tctx: tctx},
+	}
+	info := &ddlInfo{
+		originDDL:    "alter table test.t modify c_char char(4) character set latin1",
+		sourceTables: []*filter.Table{{Schema: "test", Name: "t"}},
+		targetTables: []*filter.Table{{Schema: "test", Name: "t"}},
+	}
+
+	eventType, err := ddlWorker.handleModifyColumn(qec, info, alterStmt.Specs[0])
+	require.NoError(t, err)
+	require.Equal(t, bf.ModifyCharset, eventType)
+}
+
 type mockOnlinePlugin struct {
 	toFinish map[string]struct{}
 }

engine/pkg/promutil/registry_test.go

@@ -100,8 +100,7 @@ func TestMustRegister(t *testing.T) {
 	require.Len(t, reg.collectorByWorker["worker4"], 2)
 }
 
-// prometheus metric name and label name only support [a-z][A-Z][0-9][_][:]
-// label name can't have prefix '__'
+// metric name must not be empty, and label name can't have prefix '__'
 func TestMustRegisterNotValidName(t *testing.T) {
 	t.Parallel()
 
@@ -116,7 +115,7 @@ func TestMustRegisterNotValidName(t *testing.T) {
 
 	// not a valid metric name
 	reg.MustRegister("worker", prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "counter-5",
+		Name: "",
 	}))
 }