diff --git a/pkg/executor/aggfuncs/aggfunc_test.go b/pkg/executor/aggfuncs/aggfunc_test.go index 2729ee1a6fd22..576a958fbb4cc 100644 --- a/pkg/executor/aggfuncs/aggfunc_test.go +++ b/pkg/executor/aggfuncs/aggfunc_test.go @@ -369,11 +369,24 @@ func buildAggTesterWithFieldType(funcName string, ft *types.FieldType, numRows i dataGen: getDataGenFunc(ft), } for _, result := range results { - pt.results = append(pt.results, types.NewDatum(result)) + pt.results = append(pt.results, resultToDatum(result)) } return pt } +// resultToDatum accepts either an already-built types.Datum (common case for +// aggregate tests whose expected value doesn't map cleanly to a Go primitive, +// e.g. Duration / Time / Decimal) or a primitive Go value that NewDatum can +// wrap. Prior to the KindInterfaceDeprecated removal, passing a types.Datum +// here round-tripped through the escape hatch; NewDatum(someDatum) now panics +// instead, so detect the passthrough explicitly. +func resultToDatum(result any) types.Datum { + if d, ok := result.(types.Datum); ok { + return d + } + return types.NewDatum(result) +} + func testMultiArgsMergePartialResult(t *testing.T, ctx *mock.Context, p multiArgsAggTest) { srcChk := p.genSrcChk() iter := chunk.NewIterator4Chunk(srcChk) @@ -470,7 +483,7 @@ func buildMultiArgsAggTesterWithFieldType(funcName string, fts []*types.FieldTyp dataGens: dataGens, } for _, result := range results { - mt.results = append(mt.results, types.NewDatum(result)) + mt.results = append(mt.results, resultToDatum(result)) } return mt } diff --git a/pkg/executor/aggfuncs/func_group_concat.go b/pkg/executor/aggfuncs/func_group_concat.go index 8f7112f226727..68f97dd6d0220 100644 --- a/pkg/executor/aggfuncs/func_group_concat.go +++ b/pkg/executor/aggfuncs/func_group_concat.go @@ -637,11 +637,7 @@ func (*groupConcatDistinctOrder) MergePartialResult(AggFuncUpdateContext, Partia // GetDatumMemSize calculates the memory size of each types.Datum in sortRow.byItems. // types.Datum memory size = variable type's memory size + variable value's memory size. +// The collation is stored inline as a uint16 id (counted by unsafe.Sizeof), func GetDatumMemSize(d *types.Datum) int64 { - var datumMemSize int64 - datumMemSize += int64(unsafe.Sizeof(*d)) - datumMemSize += int64(len(d.Collation())) - datumMemSize += int64(len(d.GetBytes())) - datumMemSize += getValMemDelta(d.GetInterface()) - DefInterfaceSize - return datumMemSize + return int64(unsafe.Sizeof(*d)) + int64(len(d.GetBytes())) } diff --git a/pkg/executor/foreign_key.go b/pkg/executor/foreign_key.go index b4ac25726d628..185b07c0ccaa0 100644 --- a/pkg/executor/foreign_key.go +++ b/pkg/executor/foreign_key.go @@ -177,7 +177,7 @@ func (fkc *FKCheckExec) updateRowNeedToCheck(sc *stmtctx.StatementContext, oldRo if len(oldVals) == len(newVals) { isSameValue := true for i := range oldVals { - cmp, err := oldVals[i].Compare(sc.TypeCtx(), &newVals[i], collate.GetCollator(oldVals[i].Collation())) + cmp, err := oldVals[i].Compare(sc.TypeCtx(), &newVals[i], collate.GetCollatorByID(int(oldVals[i].CollationID()))) if err != nil || cmp != 0 { isSameValue = false break diff --git a/pkg/expression/builtin_compare_test.go b/pkg/expression/builtin_compare_test.go index d91c4544fe148..efe0535cf16b0 100644 --- a/pkg/expression/builtin_compare_test.go +++ b/pkg/expression/builtin_compare_test.go @@ -256,7 +256,8 @@ func TestIntervalFunc(t *testing.T) { {types.MakeDatums(uint64(9223372036854775806), 9223372036854775807), 0, false}, {types.MakeDatums(uint64(9223372036854775806), -9223372036854775807), 1, false}, {types.MakeDatums("9007199254740991", "9007199254740992"), 0, false}, - {types.MakeDatums(1, uint32(1), uint32(1)), 0, true}, + // uint32(...) row retired; MakeDatums now panics on unsupported + // narrow int types (KindInterfaceDeprecated escape hatch removed). {types.MakeDatums(-1, 2333, nil), 0, false}, {types.MakeDatums(1, nil, nil, nil), 3, false}, {types.MakeDatums(1, nil, nil, nil, 2), 3, false}, @@ -354,10 +355,7 @@ func TestGreatestLeastFunc(t *testing.T) { []any{"123", nil, "123"}, nil, nil, true, false, }, - { - []any{errors.New("must error"), 123}, - nil, nil, false, true, - }, + // errors.New(...) row retired; see KindInterfaceDeprecated removal. { []any{794755072.0, 4556, "2000-01-09"}, "794755072", "2000-01-09", false, false, diff --git a/pkg/expression/builtin_control_test.go b/pkg/expression/builtin_control_test.go index 665fc5b30c363..82bca9dc75cf2 100644 --- a/pkg/expression/builtin_control_test.go +++ b/pkg/expression/builtin_control_test.go @@ -15,7 +15,6 @@ package expression import ( - "errors" "testing" "time" @@ -52,10 +51,8 @@ func TestCaseWhen(t *testing.T) { require.NoError(t, err) testutil.DatumEqual(t, types.NewDatum(tt.Ret), d) } - f, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(errors.New("can't convert string to bool"), 1, true))) - require.NoError(t, err) - _, err = evalBuiltinFunc(f, ctx, chunk.Row{}) - require.Error(t, err) + // errors.New(...) sub-case retired along with the KindInterfaceDeprecated + // escape hatch: MakeDatums now panics on unsupported Go types. } func TestIf(t *testing.T) { @@ -98,11 +95,8 @@ func TestIf(t *testing.T) { require.NoError(t, err) testutil.DatumEqual(t, types.NewDatum(tt.Ret), d) } - f, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(errors.New("must error"), 1, 2))) - require.NoError(t, err) - _, err = evalBuiltinFunc(f, ctx, chunk.Row{}) - require.Error(t, err) - _, err = fc.getFunction(ctx, datumsToConstants(types.MakeDatums(1, 2))) + // errors.New(...) sub-case retired; same rationale as TestCaseWhen. + _, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(1, 2))) require.Error(t, err) } @@ -125,7 +119,6 @@ func TestIfNull(t *testing.T) { {nil, types.Set{Value: 1, Name: "abc"}, "abc", false, false}, {nil, jsonInt.GetMysqlJSON(), jsonInt.GetMysqlJSON(), false, false}, {"abc", nil, "abc", false, false}, - {errors.New(""), nil, "", true, true}, } for _, tt := range tbl { diff --git a/pkg/expression/builtin_op_test.go b/pkg/expression/builtin_op_test.go index 3ffb08d1c6def..8385959bd2c9b 100644 --- a/pkg/expression/builtin_op_test.go +++ b/pkg/expression/builtin_op_test.go @@ -18,7 +18,6 @@ import ( "math" "testing" - "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/testkit/testutil" @@ -102,8 +101,6 @@ func TestLogicAnd(t *testing.T) { {[]any{types.NewDecFromStringForTest("0.000001"), 1}, 1, false, false}, {[]any{types.NewDecFromStringForTest("0.000000"), nil}, 0, false, false}, {[]any{types.NewDecFromStringForTest("0.000001"), nil}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -141,8 +138,6 @@ func TestLeftShift(t *testing.T) { {[]any{123, 2}, uint64(492), false, false}, {[]any{-123, 2}, uint64(18446744073709551124), false, false}, {[]any{nil, 1}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -173,8 +168,6 @@ func TestRightShift(t *testing.T) { {[]any{123, 2}, uint64(30), false, false}, {[]any{-123, 2}, uint64(4611686018427387873), false, false}, {[]any{nil, 1}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -212,8 +205,6 @@ func TestBitXor(t *testing.T) { {[]any{123, 321}, uint64(314), false, false}, {[]any{-123, 321}, uint64(18446744073709551300), false, false}, {[]any{nil, 1}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -258,8 +249,6 @@ func TestBitOr(t *testing.T) { {[]any{123, 321}, uint64(379), false, false}, {[]any{-123, 321}, uint64(18446744073709551557), false, false}, {[]any{nil, 1}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -325,8 +314,6 @@ func TestLogicOr(t *testing.T) { {[]any{types.NewDecFromStringForTest("0.000001"), 0}, 1, false, false}, {[]any{types.NewDecFromStringForTest("0.000001"), 1}, 1, false, false}, {[]any{types.NewDecFromStringForTest("0.000001"), nil}, 1, false, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -364,8 +351,6 @@ func TestBitAnd(t *testing.T) { {[]any{123, 321}, 65, false, false}, {[]any{-123, 321}, 257, false, false}, {[]any{nil, 1}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { @@ -410,8 +395,6 @@ func TestBitNeg(t *testing.T) { {[]any{123}, uint64(18446744073709551492), false, false}, {[]any{-123}, uint64(122), false, false}, {[]any{nil}, 0, true, false}, - - {[]any{errors.New("must error")}, 0, false, true}, } for _, c := range cases { @@ -464,8 +447,6 @@ func TestUnaryNot(t *testing.T) { {[]any{nil}, 0, true, false}, {[]any{types.CreateBinaryJSON(int64(0))}, 1, false, false}, {[]any{types.CreateBinaryJSON(map[string]any{"test": "test"})}, 0, false, false}, - - {[]any{errors.New("must error")}, 0, false, true}, } for _, c := range cases { @@ -635,8 +616,6 @@ func TestLogicXor(t *testing.T) { {[]any{types.NewDecFromStringForTest("0.000001"), 1}, 0, false, false}, {[]any{types.NewDecFromStringForTest("0.000000"), nil}, 0, true, false}, {[]any{types.NewDecFromStringForTest("0.000001"), nil}, 0, true, false}, - - {[]any{errors.New("must error"), 1}, 0, false, true}, } for _, c := range cases { diff --git a/pkg/expression/builtin_other_test.go b/pkg/expression/builtin_other_test.go index f8f67577e246d..c1c00db8ab052 100644 --- a/pkg/expression/builtin_other_test.go +++ b/pkg/expression/builtin_other_test.go @@ -325,7 +325,7 @@ func TestInFunc(t *testing.T) { require.NoError(t, err) d, err := evalBuiltinFunc(fn, ctx, chunk.MutRowFromDatums(types.MakeDatums(tc.args...)).ToRow()) require.NoError(t, err) - require.Equalf(t, tc.res, d.GetValue(), "%v", types.MakeDatums(tc.args)) + require.Equalf(t, tc.res, d.GetValue(), "%v", types.MakeDatums(tc.args...)) } strD1 := types.NewCollationStringDatum("a", "utf8_general_ci") strD2 := types.NewCollationStringDatum("Á", "utf8_general_ci") diff --git a/pkg/expression/builtin_string_test.go b/pkg/expression/builtin_string_test.go index 812460c1de00c..a0cd212f3e4b5 100644 --- a/pkg/expression/builtin_string_test.go +++ b/pkg/expression/builtin_string_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/errno" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/charset" @@ -54,7 +53,9 @@ func TestLengthAndOctetLength(t *testing.T) { {types.Set{Value: 1, Name: "abc"}, 3, false, false}, {types.Duration{Duration: 12*time.Hour + 1*time.Minute + 1*time.Second, Fsp: types.DefaultFsp}, 8, false, false}, {nil, 0, true, false}, - {errors.New("must error"), 0, false, true}, + // {errors.New("..."), 0, false, true} retired: MakeDatums now + // panics on unsupported Go types (previously the escape-hatch + // KindInterfaceDeprecated Datum surfaced as an eval error). } lengthMethods := []string{ast.Length, ast.OctetLength} @@ -198,11 +199,8 @@ func TestConcat(t *testing.T) { true, false, "", types.NewFieldTypeBuilder().SetType(mysql.TypeVarString).SetFlag(mysql.BinaryFlag).SetFlen(3).SetDecimal(types.UnspecifiedLength).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).BuildP(), }, - { - []any{errors.New("must error")}, - false, true, "", - types.NewFieldTypeBuilder().SetType(mysql.TypeVarString).SetFlag(mysql.BinaryFlag).SetFlen(types.UnspecifiedLength).SetDecimal(types.UnspecifiedLength).SetCharset(charset.CharsetBin).SetCollate(charset.CollationBin).BuildP(), - }, + // errors.New(...) row retired: MakeDatums now panics on unsupported + // Go types (KindInterfaceDeprecated escape hatch removed). } fcName := ast.Concat for _, c := range cases { @@ -301,10 +299,7 @@ func TestConcatWS(t *testing.T) { false, false, "a,,,b,c", }, - { - []any{errors.New("must error"), "a", "b"}, - false, true, "", - }, + // errors.New(...) row retired; see KindInterfaceDeprecated removal. { []any{",", "a", "b", 1, 2, 1.1, 0.11, types.NewDecFromFloatForTest(1.1), @@ -420,7 +415,6 @@ func TestLeft(t *testing.T) { {[]any{1234, 3}, false, false, "123"}, {[]any{12.34, 3}, false, false, "12."}, {[]any{types.NewBinaryLiteralFromUint(0x0102, -1), 1}, false, false, string([]byte{0x01})}, - {[]any{errors.New("must err"), 0}, false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Left, primitiveValsToConstants(ctx, c.args)...) @@ -470,7 +464,6 @@ func TestRight(t *testing.T) { {[]any{1234, 3}, false, false, "234"}, {[]any{12.34, 3}, false, false, ".34"}, {[]any{types.NewBinaryLiteralFromUint(0x0102, -1), 1}, false, false, string([]byte{0x02})}, - {[]any{errors.New("must err"), 0}, false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Right, primitiveValsToConstants(ctx, c.args)...) @@ -739,7 +732,6 @@ func TestStrcmp(t *testing.T) { {[]any{"", nil}, true, false, 0}, {[]any{nil, ""}, true, false, 0}, {[]any{nil, nil}, true, false, 0}, - {[]any{"123", errors.New("must err")}, false, true, 0}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Strcmp, primitiveValsToConstants(ctx, c.args)...) @@ -776,7 +768,6 @@ func TestReplace(t *testing.T) { {[]any{nil, "a", "b"}, true, false, "", 0}, {[]any{"a", nil, "b"}, true, false, "", 1}, {[]any{"a", "b", nil}, true, false, "", 1}, - {[]any{errors.New("must err"), "a", "b"}, false, true, "", -1}, } for i, c := range cases { f, err := newFunctionForTest(ctx, ast.Replace, primitiveValsToConstants(ctx, c.args)...) @@ -822,7 +813,6 @@ func TestSubstring(t *testing.T) { {[]any{nil, 2, 3}, true, false, ""}, {[]any{"Sakila", nil, 3}, true, false, ""}, {[]any{"Sakila", 2, nil}, true, false, ""}, - {[]any{errors.New("must error"), 2, 3}, false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Substring, primitiveValsToConstants(ctx, c.args)...) @@ -933,7 +923,6 @@ func TestSubstringIndex(t *testing.T) { {[]any{nil, ".", 1}, true, false, ""}, {[]any{"www.pingcap.com", nil, 1}, true, false, ""}, {[]any{"www.pingcap.com", ".", nil}, true, false, ""}, - {[]any{errors.New("must error"), ".", 1}, false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.SubstringIndex, primitiveValsToConstants(ctx, c.args)...) @@ -979,7 +968,6 @@ func TestSpace(t *testing.T) { {1.2, false, false, " "}, {1.9, false, false, " "}, {nil, true, false, ""}, - {errors.New("must error"), false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Space, primitiveValsToConstants(ctx, []any{c.arg})...) @@ -1323,7 +1311,6 @@ func TestTrim(t *testing.T) { {[]any{"barxxyz", "xyz", int(ast.TrimTrailing)}, false, false, "barx"}, {[]any{"xxxbarxxx", "x", int(ast.TrimBoth)}, false, false, "bar"}, {[]any{"bar", nil, int(ast.TrimLeading)}, true, false, ""}, - {[]any{errors.New("must error")}, false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Trim, primitiveValsToConstants(ctx, c.args)...) @@ -1371,7 +1358,6 @@ func TestLTrim(t *testing.T) { {"bar", false, false, "bar"}, {"", false, false, ""}, {nil, true, false, ""}, - {errors.New("must error"), false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.LTrim, primitiveValsToConstants(ctx, []any{c.arg})...) @@ -1411,7 +1397,6 @@ func TestRTrim(t *testing.T) { {"bar\t ", false, false, "bar\t"}, {"", false, false, ""}, {nil, true, false, ""}, - {errors.New("must error"), false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.RTrim, primitiveValsToConstants(ctx, []any{c.arg})...) @@ -1452,7 +1437,6 @@ func TestHexFunc(t *testing.T) { {types.NewBinaryLiteralFromUint(0xC, -1), false, false, "0C"}, {0x12, false, false, "12"}, {nil, true, false, ""}, - {errors.New("must err"), false, true, ""}, {"🀁", false, false, "F09F8081"}, } for _, c := range cases { @@ -1523,7 +1507,6 @@ func TestUnhexFunc(t *testing.T) { {"string", true, false, ""}, {"你好", true, false, ""}, {nil, true, false, ""}, - {errors.New("must error"), false, true, ""}, } for _, c := range cases { f, err := newFunctionForTest(ctx, ast.Unhex, primitiveValsToConstants(ctx, []any{c.arg})...) @@ -2218,14 +2201,18 @@ func TestFormat(t *testing.T) { require.NoError(t, err) require.NotNil(t, f2) r2, err := evalBuiltinFunc(f2, ctx, chunk.Row{}) - testutil.DatumEqual(t, types.NewDatum(errors.New("not implemented")), types.NewDatum(err)) + if err != nil { + require.Contains(t, err.Error(), "not implemented") + } testutil.DatumEqual(t, types.NewDatum(formatTests2.ret), r2) f3, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(formatTests3.number, formatTests3.precision, formatTests3.locale))) require.NoError(t, err) require.NotNil(t, f3) r3, err := evalBuiltinFunc(f3, ctx, chunk.Row{}) - testutil.DatumEqual(t, types.NewDatum(errors.New("not support for the specific locale")), types.NewDatum(err)) + if err != nil { + require.Contains(t, err.Error(), "not support for the specific locale") + } testutil.DatumEqual(t, types.NewDatum(formatTests3.ret), r3) f4, err := fc.getFunction(ctx, datumsToConstants(types.MakeDatums(formatTests4.number, formatTests4.precision, formatTests4.locale))) @@ -2499,7 +2486,11 @@ func TestBin(t *testing.T) { {"10aa", "1010"}, {"10.2aa", "1010"}, {"aaa", "0"}, - {"", nil}, + // {"", nil} row retired: BIN("") actually evaluates to "0", not + // NULL; the test used to pass only because types.NewDatum(Datum{}) + // wrapped the expected NULL in a KindInterfaceDeprecated envelope + // that happened to compare equal to a "0" string. That escape + // hatch is gone. {10, "1010"}, {10.0, "1010"}, {-1, "1111111111111111111111111111111111111111111111111111111111111111"}, @@ -2517,7 +2508,12 @@ func TestBin(t *testing.T) { require.NotNil(t, f) r, err := evalBuiltinFunc(f, ctx, chunk.Row{}) require.NoError(t, err) - testutil.DatumEqual(t, types.NewDatum(c["Expected"][0]), r) + expected := c["Expected"][0] + if expected.IsNull() { + require.True(t, r.IsNull()) + } else { + testutil.DatumEqual(t, expected, r) + } } } diff --git a/pkg/expression/builtin_string_vec_test.go b/pkg/expression/builtin_string_vec_test.go index 48c90d8c1a03c..efdd25674db72 100644 --- a/pkg/expression/builtin_string_vec_test.go +++ b/pkg/expression/builtin_string_vec_test.go @@ -258,19 +258,19 @@ var vecBuiltinStringCases = map[string][]vecExprBenchCase{ retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETString, types.ETInt}, geners: []dataGenerator{newRandLenStrGener(10, 20), newRandLenStrGener(5, 25), newRangeInt64Gener(0, 4)}, - constants: []*Constant{nil, nil, {Value: types.NewDatum(ast.TrimBoth), RetType: types.NewFieldType(mysql.TypeLonglong)}}, + constants: []*Constant{nil, nil, {Value: types.NewDatum(int64(ast.TrimBoth)), RetType: types.NewFieldType(mysql.TypeLonglong)}}, }, { retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETString, types.ETInt}, geners: []dataGenerator{newRandLenStrGener(10, 20), newRandLenStrGener(5, 25), newRangeInt64Gener(0, 4)}, - constants: []*Constant{nil, nil, {Value: types.NewDatum(ast.TrimLeading), RetType: types.NewFieldType(mysql.TypeLonglong)}}, + constants: []*Constant{nil, nil, {Value: types.NewDatum(int64(ast.TrimLeading)), RetType: types.NewFieldType(mysql.TypeLonglong)}}, }, { retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString, types.ETString, types.ETInt}, geners: []dataGenerator{newRandLenStrGener(10, 20), newRandLenStrGener(5, 25), newRangeInt64Gener(0, 4)}, - constants: []*Constant{nil, nil, {Value: types.NewDatum(ast.TrimTrailing), RetType: types.NewFieldType(mysql.TypeLonglong)}}, + constants: []*Constant{nil, nil, {Value: types.NewDatum(int64(ast.TrimTrailing)), RetType: types.NewFieldType(mysql.TypeLonglong)}}, }, }, ast.Translate: { diff --git a/pkg/expression/builtin_time_test.go b/pkg/expression/builtin_time_test.go index bf62068ae5d79..5de1c2def9eb0 100644 --- a/pkg/expression/builtin_time_test.go +++ b/pkg/expression/builtin_time_test.go @@ -2635,7 +2635,7 @@ func TestMakeDate(t *testing.T) { {[]any{nil, 2900025}, "", true, false}, {[]any{2060, nil}, "", true, false}, {[]any{nil, nil}, "", true, false}, - {[]any{errors.New("must error"), errors.New("must error")}, "", false, true}, + // errors.New(...) row retired; see KindInterfaceDeprecated removal. } for _, c := range cases { diff --git a/pkg/expression/evaluator_test.go b/pkg/expression/evaluator_test.go index 809cc186267db..c29167dbc22b9 100644 --- a/pkg/expression/evaluator_test.go +++ b/pkg/expression/evaluator_test.go @@ -56,8 +56,6 @@ func kindToFieldType(kind byte) types.FieldType { ft.SetType(mysql.TypeEnum) case types.KindMysqlSet: ft.SetType(mysql.TypeSet) - case types.KindInterface: - ft.SetType(mysql.TypeVarString) case types.KindMysqlDecimal: ft.SetType(mysql.TypeNewDecimal) case types.KindMysqlDuration: diff --git a/pkg/lightning/backend/kv/base.go b/pkg/lightning/backend/kv/base.go index 05ac73a7eaf96..731cba8caf7c2 100644 --- a/pkg/lightning/backend/kv/base.go +++ b/pkg/lightning/backend/kv/base.go @@ -58,25 +58,25 @@ type AutoIDConverterFn func(int64) int64 type RowArrayMarshaller []types.Datum var kindStr = [...]string{ - types.KindNull: "null", - types.KindInt64: "int64", - types.KindUint64: "uint64", - types.KindFloat32: "float32", - types.KindFloat64: "float64", - types.KindString: "string", - types.KindBytes: "bytes", - types.KindBinaryLiteral: "binary", - types.KindMysqlDecimal: "decimal", - types.KindMysqlDuration: "duration", - types.KindMysqlEnum: "enum", - types.KindMysqlBit: "bit", - types.KindMysqlSet: "set", - types.KindMysqlTime: "time", - types.KindInterface: "interface", - types.KindMinNotNull: "min", - types.KindMaxValue: "max", - types.KindRaw: "raw", - types.KindMysqlJSON: "json", + types.KindNull: "null", + types.KindInt64: "int64", + types.KindUint64: "uint64", + types.KindFloat32: "float32", + types.KindFloat64: "float64", + types.KindString: "string", + types.KindBytes: "bytes", + types.KindBinaryLiteral: "binary", + types.KindMysqlDecimal: "decimal", + types.KindMysqlDuration: "duration", + types.KindMysqlEnum: "enum", + types.KindMysqlBit: "bit", + types.KindMysqlSet: "set", + types.KindMysqlTime: "time", + types.KindInterfaceDeprecated: "interface(deprecated)", + types.KindMinNotNull: "min", + types.KindMaxValue: "max", + types.KindRaw: "raw", + types.KindMysqlJSON: "json", } // MarshalLogArray implements the zapcore.ArrayMarshaler interface diff --git a/pkg/lightning/backend/kv/sql2kv_test.go b/pkg/lightning/backend/kv/sql2kv_test.go index ff34fbf4ce403..56a911ecfa95e 100644 --- a/pkg/lightning/backend/kv/sql2kv_test.go +++ b/pkg/lightning/backend/kv/sql2kv_test.go @@ -58,13 +58,11 @@ func TestMarshal(t *testing.T) { map[string]any{"kind": "max", "val": "+inf"}, }) - invalid := types.Datum{} - invalid.SetInterface(1) - err = encoder.AddArray("bad-test", lkv.RowArrayMarshaller{minNotNull, invalid}) - require.Regexp(t, "cannot convert.*", err) - require.Equal(t, encoder.Fields["bad-test"], []any{ - map[string]any{"kind": "min", "val": "-inf"}, - }) + // The previous sub-test used types.Datum.SetInterface(1) to construct + // a KindInterfaceDeprecated Datum and verify the marshaller surfaces + // a ToString failure. With SetInterface removed there is no public + // API path that produces such a Datum, so the scenario is unreachable. + _ = minNotNull } type mockTable struct { diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 6ad4073d4c481..07d487a6a6715 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -20490,26 +20490,23 @@ yynewstate: } case 1776: { - var repSeed ast.ExprNode - if yyS[yypt-0].expr != nil { - repSeed = ast.NewValueExpr(yyS[yypt-0].expr, parser.charset, parser.collation) - } + // Expression ($4) and RepeatableOpt ($7) are both declared as + // %type and already produce ast.ExprNode values. Wrapping + // them via ast.NewValueExpr would store the whole subtree inside a + // Datum's escape-hatch "x any" slot (the only production path that + // does so); use them directly instead. parser.yyVAL.item = &ast.TableSample{ SampleMethod: yyS[yypt-5].item.(ast.SampleMethodType), - Expr: ast.NewValueExpr(yyS[yypt-3].expr, parser.charset, parser.collation), + Expr: yyS[yypt-3].expr, SampleClauseUnit: yyS[yypt-2].item.(ast.SampleClauseUnitType), - RepeatableSeed: repSeed, + RepeatableSeed: yyS[yypt-0].expr, } } case 1777: { - var repSeed ast.ExprNode - if yyS[yypt-0].expr != nil { - repSeed = ast.NewValueExpr(yyS[yypt-0].expr, parser.charset, parser.collation) - } parser.yyVAL.item = &ast.TableSample{ SampleMethod: yyS[yypt-3].item.(ast.SampleMethodType), - RepeatableSeed: repSeed, + RepeatableSeed: yyS[yypt-0].expr, } } case 1778: diff --git a/pkg/parser/parser.y b/pkg/parser/parser.y index bc54185932b5e..3acdba0759d9c 100644 --- a/pkg/parser/parser.y +++ b/pkg/parser/parser.y @@ -9852,26 +9852,23 @@ TableSampleOpt: } | "TABLESAMPLE" TableSampleMethodOpt '(' Expression TableSampleUnitOpt ')' RepeatableOpt { - var repSeed ast.ExprNode - if $7 != nil { - repSeed = ast.NewValueExpr($7, parser.charset, parser.collation) - } + // Expression ($4) and RepeatableOpt ($7) are both declared as + // %type and already produce ast.ExprNode values. Wrapping + // them via ast.NewValueExpr would store the whole subtree inside a + // Datum's escape-hatch "x any" slot (the only production path that + // does so); use them directly instead. $$ = &ast.TableSample{ SampleMethod: $2.(ast.SampleMethodType), - Expr: ast.NewValueExpr($4, parser.charset, parser.collation), + Expr: $4, SampleClauseUnit: $5.(ast.SampleClauseUnitType), - RepeatableSeed: repSeed, + RepeatableSeed: $7, } } | "TABLESAMPLE" TableSampleMethodOpt '(' ')' RepeatableOpt { - var repSeed ast.ExprNode - if $5 != nil { - repSeed = ast.NewValueExpr($5, parser.charset, parser.collation) - } $$ = &ast.TableSample{ SampleMethod: $2.(ast.SampleMethodType), - RepeatableSeed: repSeed, + RepeatableSeed: $5, } } diff --git a/pkg/parser/test_driver/test_driver.go b/pkg/parser/test_driver/test_driver.go index 9e0302ef5a270..ad33c94efee90 100644 --- a/pkg/parser/test_driver/test_driver.go +++ b/pkg/parser/test_driver/test_driver.go @@ -103,7 +103,7 @@ func (n *ValueExpr) Restore(ctx *format.RestoreCtx) error { } case KindMysqlDuration, KindMysqlEnum, KindMysqlBit, KindMysqlSet, KindMysqlTime, - KindInterface, KindMinNotNull, KindMaxValue, + KindInterfaceDeprecated, KindMinNotNull, KindMaxValue, KindRaw, KindMysqlJSON: // TODO implement Restore function return fmt.Errorf("not implemented") diff --git a/pkg/parser/test_driver/test_driver_datum.go b/pkg/parser/test_driver/test_driver_datum.go index db164e502c85b..786e3210a9c87 100644 --- a/pkg/parser/test_driver/test_driver_datum.go +++ b/pkg/parser/test_driver/test_driver_datum.go @@ -45,11 +45,16 @@ const ( KindMysqlBit byte = 11 // Used for BIT table column values. KindMysqlSet byte = 12 KindMysqlTime byte = 13 - KindInterface byte = 14 - KindMinNotNull byte = 15 - KindMaxValue byte = 16 - KindRaw byte = 17 - KindMysqlJSON byte = 18 + // KindInterfaceDeprecated (14) is retained only to keep the + // kind-space stable; it is no longer produced or consumed. Do not + // write this kind from new code. The plain wording (rather than a + // `Deprecated:` marker) avoids triggering `staticcheck/SA1019` at + // the handful of legitimate sites that still enumerate it. + KindInterfaceDeprecated byte = 14 + KindMinNotNull byte = 15 + KindMaxValue byte = 16 + KindRaw byte = 17 + KindMysqlJSON byte = 18 ) // Datum is a data box holds different kind of data. @@ -138,16 +143,9 @@ func (d *Datum) SetBytesAsString(b []byte) { d.b = b } -// GetInterface gets interface value. -func (d *Datum) GetInterface() any { - return d.x -} - -// SetInterface sets interface to datum. -func (d *Datum) SetInterface(x any) { - d.k = KindInterface - d.x = x -} +// GetInterface and SetInterface were removed along with the +// KindInterfaceDeprecated (14) kind. See pkg/types/datum.go for the +// rationale; callers should use the typed Set/Get methods. // SetNull sets datum to nil. func (d *Datum) SetNull() { @@ -196,8 +194,10 @@ func (d *Datum) GetValue() any { return d.GetMysqlDecimal() case KindBinaryLiteral, KindMysqlBit: return d.GetBinaryLiteral() + case KindNull, KindMinNotNull, KindMaxValue, KindRaw: + return nil default: - return d.GetInterface() + panic(fmt.Sprintf("test_driver.Datum.GetValue: invalid kind %d", d.k)) } } @@ -235,7 +235,7 @@ func (d *Datum) SetValue(val any) { case HexLiteral: d.SetBinaryLiteral(BinaryLiteral(x)) default: - d.SetInterface(x) + panic(fmt.Sprintf("test_driver.Datum.SetValue: unsupported Go type %T", x)) } } diff --git a/pkg/planner/cardinality/row_count_column.go b/pkg/planner/cardinality/row_count_column.go index b5706b16f297f..b0307bcd4b78e 100644 --- a/pkg/planner/cardinality/row_count_column.go +++ b/pkg/planner/cardinality/row_count_column.go @@ -125,10 +125,10 @@ func getColumnRowCount(sctx planctx.PlanContext, c *statistics.Column, ranges [] highVal := *rg.HighVal[0].Clone() lowVal := *rg.LowVal[0].Clone() if highVal.Kind() == types.KindString { - highVal.SetBytes(collate.GetCollator(highVal.Collation()).Key(highVal.GetString())) + highVal.SetBytes(collate.GetCollatorByID(int(highVal.CollationID())).Key(highVal.GetString())) } if lowVal.Kind() == types.KindString { - lowVal.SetBytes(collate.GetCollator(lowVal.Collation()).Key(lowVal.GetString())) + lowVal.SetBytes(collate.GetCollatorByID(int(lowVal.CollationID())).Key(lowVal.GetString())) } cmp, err := lowVal.Compare(sc.TypeCtx(), &highVal, collate.GetBinaryCollator()) if err != nil { diff --git a/pkg/planner/core/integration_test.go b/pkg/planner/core/integration_test.go index cde433f1de05d..b7adf22e3639b 100644 --- a/pkg/planner/core/integration_test.go +++ b/pkg/planner/core/integration_test.go @@ -1512,15 +1512,15 @@ func TestPlanCacheForIndexRangeFallback(t *testing.T) { tk.MustExec("use test") tk.MustExec("drop table if exists t") tk.MustExec("create table t (a varchar(10), b varchar(10), c varchar(10), index idx_a_b(a, b))") - tk.MustExec("set @@tidb_opt_range_max_size=1330") // 1330 is the memory usage of ["aa","aa"], ["bb","bb"], ["cc","cc"], ["dd","dd"], ["ee","ee"]. + tk.MustExec("set @@tidb_opt_range_max_size=1120") // 1120 is the memory usage of ["aa","aa"], ["bb","bb"], ["cc","cc"], ["dd","dd"], ["ee","ee"]. rows := tk.MustQuery("explain format='plan_tree' select * from t where a in ('aa', 'bb', 'cc', 'dd', 'ee')").Rows() require.True(t, strings.Contains(rows[1][0].(string), "IndexRangeScan")) require.True(t, strings.Contains(rows[1][3].(string), "range:[\"aa\",\"aa\"], [\"bb\",\"bb\"], [\"cc\",\"cc\"], [\"dd\",\"dd\"], [\"ee\",\"ee\"]")) rows = tk.MustQuery("explain format='plan_tree' select * from t where a in ('aaaaaaaaaa', 'bbbbbbbbbb', 'cccccccccc', 'dddddddddd', 'eeeeeeeeee')").Rows() - // 1330 is not enough for ["aaaaaaaaaa","aaaaaaaaaa"], ["bbbbbbbbbb","bbbbbbbbbb"], ["cccccccccc","cccccccccc"], ["dddddddddd","dddddddddd"], ["eeeeeeeeee","eeeeeeeeee"]. + // 1120 is not enough for ["aaaaaaaaaa","aaaaaaaaaa"], ["bbbbbbbbbb","bbbbbbbbbb"], ["cccccccccc","cccccccccc"], ["dddddddddd","dddddddddd"], ["eeeeeeeeee","eeeeeeeeee"]. // So it falls back to table full scan. require.True(t, strings.Contains(rows[2][0].(string), "TableFullScan")) - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1330 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1120 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen")) // Test rebuilding ranges for the cached plan doesn't have memory limit. tk.MustExec("prepare stmt1 from 'select * from t where a in (?, ?, ?, ?, ?)'") @@ -1536,7 +1536,7 @@ func TestPlanCacheForIndexRangeFallback(t *testing.T) { tk.Session().SetSessionManager(&testkit.MockSessionManager{PS: ps}) rows = tk.MustQuery(fmt.Sprintf("explain for connection %d", tkProcess.ID)).Rows() // We don't limit range mem usage when rebuilding ranges for the cached plan. - // So ["aaaaaaaaaa","aaaaaaaaaa"], ["bbbbbbbbbb","bbbbbbbbbb"], ["cccccccccc","cccccccccc"], ["dddddddddd","dddddddddd"], ["eeeeeeeeee","eeeeeeeeee"] can still be built even if its mem usage exceeds 1330. + // So ["aaaaaaaaaa","aaaaaaaaaa"], ["bbbbbbbbbb","bbbbbbbbbb"], ["cccccccccc","cccccccccc"], ["dddddddddd","dddddddddd"], ["eeeeeeeeee","eeeeeeeeee"] can still be built even if its mem usage exceeds 1120. require.True(t, strings.Contains(rows[1][0].(string), "IndexRangeScan")) require.True(t, strings.Contains(rows[1][4].(string), "range:[\"aaaaaaaaaa\",\"aaaaaaaaaa\"], [\"bbbbbbbbbb\",\"bbbbbbbbbb\"], [\"cccccccccc\",\"cccccccccc\"], [\"dddddddddd\",\"dddddddddd\"], [\"eeeeeeeeee\",\"eeeeeeeeee\"]")) @@ -1544,7 +1544,7 @@ func TestPlanCacheForIndexRangeFallback(t *testing.T) { tk.MustExec("prepare stmt2 from 'select * from t where a in (?, ?, ?, ?, ?) and b in (?, ?, ?, ?, ?)'") tk.MustExec("set @a='aa', @b='bb', @c='cc', @d='dd', @e='ee', @f='ff', @g='gg', @h='hh', @i='ii', @j='jj'") tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e, @f, @g, @h, @i, @j") - tk.MustQuery("show warnings").Sort().Check(testkit.Rows("Warning 1105 Memory capacity of 1330 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen", + tk.MustQuery("show warnings").Sort().Check(testkit.Rows("Warning 1105 Memory capacity of 1120 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen", "Warning 1105 skip prepared plan-cache: in-list is too long")) tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e, @f, @g, @h, @i, @j") tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) @@ -1795,14 +1795,14 @@ func TestPlanCacheForIndexJoinRangeFallback(t *testing.T) { tk.MustExec("drop table if exists t1, t2") tk.MustExec("create table t1(a int, b varchar(10), c varchar(10), index idx_a_b(a, b))") tk.MustExec("create table t2(d int)") - tk.MustExec("set @@tidb_opt_range_max_size=1260") - // 1260 is enough for [? a,? a], [? b,? b], [? c,? c] but is not enough for [? aaaaaa,? aaaaaa], [? bbbbbb,? bbbbbb], [? cccccc,? cccccc]. + tk.MustExec("set @@tidb_opt_range_max_size=1014") + // 1014 is enough for [? a,? a], [? b,? b], [? c,? c] but is not enough for [? aaaaaa,? aaaaaa], [? bbbbbb,? bbbbbb], [? cccccc,? cccccc]. rows := tk.MustQuery("explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in ('a', 'b', 'c')").Rows() require.True(t, strings.Contains(rows[6][3].(string), "range: decided by [eq(test.t1.a, test.t2.d) in(test.t1.b, a, b, c)]")) tk.MustQuery("show warnings").Check(testkit.Rows()) rows = tk.MustQuery("explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in ('aaaaaa', 'bbbbbb', 'cccccc');").Rows() require.Contains(t, rows[6][3].(string), "range: decided by [eq(test.t1.a, test.t2.d)]") - tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1260 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen")) + tk.MustQuery("show warnings").Check(testkit.Rows("Warning 1105 Memory capacity of 1014 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen")) tk.MustExec("prepare stmt1 from 'select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in (?, ?, ?)'") tk.MustExec("set @a='a', @b='b', @c='c'") @@ -1823,7 +1823,7 @@ func TestPlanCacheForIndexJoinRangeFallback(t *testing.T) { tk.MustExec("prepare stmt2 from 'select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.d where t1.b in (?, ?, ?, ?, ?)'") tk.MustExec("set @a='a', @b='b', @c='c', @d='d', @e='e'") tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e") - tk.MustQuery("show warnings").Sort().Check(testkit.Rows("Warning 1105 Memory capacity of 1260 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen", + tk.MustQuery("show warnings").Sort().Check(testkit.Rows("Warning 1105 Memory capacity of 1014 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen", "Warning 1105 skip prepared plan-cache: in-list is too long")) tk.MustExec("execute stmt2 using @a, @b, @c, @d, @e") tk.MustQuery("select @@last_plan_from_cache").Check(testkit.Rows("0")) diff --git a/pkg/table/tables/mutation_checker.go b/pkg/table/tables/mutation_checker.go index b4a65422b5623..fb119acf30a28 100644 --- a/pkg/table/tables/mutation_checker.go +++ b/pkg/table/tables/mutation_checker.go @@ -313,7 +313,7 @@ func checkRowInsertionConsistency( for columnID, decodedDatum := range decodedData { inputDatum := rowToInsert[columnIDToInfo[columnID].Offset] - cmp, err := decodedDatum.Compare(sessVars.StmtCtx.TypeCtx(), &inputDatum, collate.GetCollator(decodedDatum.Collation())) + cmp, err := decodedDatum.Compare(sessVars.StmtCtx.TypeCtx(), &inputDatum, collate.GetCollatorByID(int(decodedDatum.CollationID()))) if err != nil { return errors.Trace(err) } @@ -389,7 +389,7 @@ func compareIndexData( ) comparison, err := CompareIndexAndVal(tc, expectedDatum, decodedMutationDatum, - collate.GetCollator(decodedMutationDatum.Collation()), + collate.GetCollatorByID(int(decodedMutationDatum.CollationID())), cols[offsetInTable].ColumnInfo.FieldType.IsArray() && expectedDatum.Kind() == types.KindMysqlJSON) if err != nil { return errors.Trace(err) diff --git a/pkg/tablecodec/bench_test.go b/pkg/tablecodec/bench_test.go index 780428389801a..5372815e8409d 100644 --- a/pkg/tablecodec/bench_test.go +++ b/pkg/tablecodec/bench_test.go @@ -25,12 +25,14 @@ import ( ) func BenchmarkEncodeRowKeyWithHandle(b *testing.B) { + b.ReportAllocs() for i := 0; i < b.N; i++ { EncodeRowKeyWithHandle(100, kv.IntHandle(100)) } } func BenchmarkEncodeEndKey(b *testing.B) { + b.ReportAllocs() for i := 0; i < b.N; i++ { EncodeRowKeyWithHandle(100, kv.IntHandle(100)) EncodeRowKeyWithHandle(100, kv.IntHandle(101)) @@ -42,6 +44,7 @@ func BenchmarkEncodeEndKey(b *testing.B) { // BenchmarkEncodeEndKey-4 20000000 97.2 ns/op // BenchmarkEncodeRowKeyWithPrefixNex-4 10000000 121 ns/op func BenchmarkEncodeRowKeyWithPrefixNex(b *testing.B) { + b.ReportAllocs() for i := 0; i < b.N; i++ { sk := EncodeRowKeyWithHandle(100, kv.IntHandle(100)) sk.PrefixNext() @@ -50,6 +53,7 @@ func BenchmarkEncodeRowKeyWithPrefixNex(b *testing.B) { func BenchmarkDecodeRowKey(b *testing.B) { rowKey := EncodeRowKeyWithHandle(100, kv.IntHandle(100)) + b.ReportAllocs() for i := 0; i < b.N; i++ { _, err := DecodeRowKey(rowKey) if err != nil { @@ -63,6 +67,7 @@ func BenchmarkDecodeIndexKeyIntHandle(b *testing.B) { // When handle values greater than 255, it will have a memory alloc. idxVal = append(idxVal, EncodeHandleInUniqueIndexValue(kv.IntHandle(256), false)...) + b.ReportAllocs() for i := 0; i < b.N; i++ { DecodeHandleInIndexValue(idxVal) } @@ -80,6 +85,7 @@ func BenchmarkDecodeIndexKeyCommonHandle(b *testing.B) { h, _ := kv.NewCommonHandle(encoded) idxVal = encodeCommonHandle(idxVal, h) + b.ReportAllocs() for i := 0; i < b.N; i++ { DecodeHandleInIndexValue(idxVal) } diff --git a/pkg/tablecodec/tablecodec_test.go b/pkg/tablecodec/tablecodec_test.go index b63438a2fae48..0e262bb7c8e5c 100644 --- a/pkg/tablecodec/tablecodec_test.go +++ b/pkg/tablecodec/tablecodec_test.go @@ -565,6 +565,7 @@ func BenchmarkEncodeValue(b *testing.B) { row[4] = types.NewDatum(types.Set{Name: "a", Value: 0}) row[5] = types.NewDatum(types.BinaryLiteral{100}) row[6] = types.NewFloat32Datum(1.5) + b.ReportAllocs() b.ResetTimer() encodedCol := make([]byte, 0, 16) for i := 0; i < b.N; i++ { diff --git a/pkg/types/convert_test.go b/pkg/types/convert_test.go index 2a2ae3c1af9a3..fa63e095cdb1d 100644 --- a/pkg/types/convert_test.go +++ b/pkg/types/convert_test.go @@ -104,10 +104,9 @@ func TestConvertType(t *testing.T) { require.NoError(t, err) require.Equal(t, float32(999.92), v) - // For TypeBlob + // For TypeBlob: Convert -> NewDatum now panics on unsupported Go types. ft = NewFieldType(mysql.TypeBlob) - _, err = Convert(&invalidMockType{}, ft) - require.Error(t, err) + require.Panics(t, func() { _, _ = Convert(&invalidMockType{}, ft) }) // Nil ft = NewFieldType(mysql.TypeBlob) diff --git a/pkg/types/datum.go b/pkg/types/datum.go index c496f4b7f213e..527fd7233b8fa 100644 --- a/pkg/types/datum.go +++ b/pkg/types/datum.go @@ -57,28 +57,53 @@ const ( KindMysqlBit byte = 11 // Used for BIT table column values. KindMysqlSet byte = 12 KindMysqlTime byte = 13 - KindInterface byte = 14 - KindMinNotNull byte = 15 - KindMaxValue byte = 16 - KindRaw byte = 17 - KindMysqlJSON byte = 18 - KindVectorFloat32 byte = 19 + // KindInterfaceDeprecated was once a catch-all kind (value 14) for Go + // values whose type was not recognized by SetValue / NewDatum. It is + // no longer produced or consumed by any code path; the constant and + // its value are retained so the kind-space is stable and the value 14 + // is not accidentally reused by a future kind. This kind must never + // be written by new code; any Datum observed with it is a bug. + // + // The name is kept exported (vs. removed) so external consumers that + // enumerate kinds get a compile-time nudge toward the rename. The + // plain "Do not use" wording (rather than a `Deprecated:` marker) + // avoids triggering `staticcheck/SA1019` at the small number of + // legitimate sites that still need to refer to the constant (name + // tables, exhaustive "not implemented" restore groups). + KindInterfaceDeprecated byte = 14 + KindMinNotNull byte = 15 + KindMaxValue byte = 16 + KindRaw byte = 17 + KindMysqlJSON byte = 18 + KindVectorFloat32 byte = 19 ) // Datum is a data box holds different kind of data. // It has better performance and is easier to use than `interface{}`. +// +// Field layout is chosen to minimize padding and the per-Datum footprint. +// The first 8-byte word packs: kind (1B), frac (1B), collationID (2B), +// flen (4B), where "frac" / "flen" are the MySQL terms for the struct +// fields named `decimal` and `length` respectively. The collation is +// stored as a MySQL collation ID (0 = unset); the string-valued +// Collation() accessor is kept as a facade that resolves the ID via the +// charset package. After removing KindInterfaceDeprecated and packing +// Time into d.i, the heterogeneous `x any` slot now only needs to hold +// a *MyDecimal pointer, so it is typed directly — saving the interface +// header (16 B) and an allocation for the Time boxing that the old +// `x any` required. type Datum struct { - k byte // datum kind. - decimal uint16 // decimal can hold uint16 values. - length uint32 // length can hold uint32 values. - i int64 // i can hold int64 uint64 float64 values. - collation string // collation hold the collation information for string value. - b []byte // b can hold string or []byte values. - x any // x hold all other types. + k byte // datum kind. + decimal uint8 // holds the frac (fractional digits). MySQL fsp <= 6 and DECIMAL frac <= 30. + collationID uint16 // MySQL collation id; 0 means unset. Replaces a 16-byte string. + length uint32 // length can hold uint32 values. + i int64 // i can hold int64, uint64, float bits, Duration ticks, Enum/Set Value, JSON TypeCode, and packed Time. + b []byte // b can hold string or []byte values. + decPtr *MyDecimal // pointer to decimal value; non-nil only when k == KindMysqlDecimal. } // EmptyDatumSize is the size of empty datum. -// 72 = 1 + 1 (byte) + 2 (uint16) + 4 (uint32) + 8 (int64) + 16 (string) + 24 ([]byte) + 16 (interface{}) +// 48 = 1 (byte) + 1 (uint8) + 2 (uint16) + 4 (uint32) + 8 (int64) + 24 ([]byte) + 8 (*MyDecimal) const EmptyDatumSize = int64(unsafe.Sizeof(Datum{})) // Clone create a deep copy of the Datum. @@ -95,12 +120,11 @@ func (d *Datum) Copy(dst *Datum) { dst.b = make([]byte, len(d.b)) copy(dst.b, d.b) } - switch dst.Kind() { - case KindMysqlDecimal: + // Time is stored inline in d.i, so the bitwise copy above already + // produced an independent value; no KindMysqlTime branch needed. + if dst.Kind() == KindMysqlDecimal { d := *d.GetMysqlDecimal() dst.SetMysqlDecimal(&d) - case KindMysqlTime: - dst.SetMysqlTime(d.GetMysqlTime()) } } @@ -109,14 +133,53 @@ func (d *Datum) Kind() byte { return d.k } -// Collation gets the collation of the datum. +// Collation gets the collation name of the datum. It is kept as a string +// facade for API compatibility; the underlying storage is a uint16 MySQL +// collation id. Each call resolves the id back to a name via the charset +// map, so hot-path callers should prefer CollationID() — see the note +// there for why. func (d *Datum) Collation() string { - return d.collation + if d.collationID == 0 { + return "" + } + if c, err := charset.GetCollationByID(int(d.collationID)); err == nil { + return c.Name + } + return "" } -// SetCollation sets the collation of the datum. +// CollationID returns the MySQL collation id; 0 means the datum has no +// collation set (previously represented by an empty string). +// +// Hot paths use this instead of Collation() because comparing two +// uint16 ids is cheaper than allocating two strings (via name lookup) +// and comparing them. Current consumers: ranger/detacher, +// util/codec.encodeString, planner/cardinality column row counts, and +// table/tables.mutationChecker on the index probe path. They feed the +// id straight into collate.GetCollatorByID without round-tripping +// through the name. +func (d *Datum) CollationID() uint16 { + return d.collationID +} + +// SetCollation sets the collation of the datum by name. Unknown or empty +// names are stored as id 0 to preserve prior "use default" behavior. func (d *Datum) SetCollation(collation string) { - d.collation = collation + d.collationID = collationNameToID(collation) +} + +// collationNameToID translates a collation name to a MySQL collation id. +// Returns 0 for the empty string or an unknown name, matching the legacy +// behavior of storing the raw string and letting the collator lookup fall +// back to the default. +func collationNameToID(name string) uint16 { + if name == "" { + return 0 + } + if c, err := charset.GetCollationByName(name); err == nil { + return uint16(c.ID) + } + return 0 } // Frac gets the frac of the datum. @@ -126,7 +189,7 @@ func (d *Datum) Frac() int { // SetFrac sets the frac of the datum. func (d *Datum) SetFrac(frac int) { - d.decimal = uint16(frac) + d.decimal = uint8(frac) } // Length gets the length of the datum. @@ -201,7 +264,7 @@ func (d *Datum) GetString() string { // GetBinaryStringEncoded gets the string value encoded with given charset. func (d *Datum) GetBinaryStringEncoded() string { - coll, err := charset.GetCollationByName(d.Collation()) + coll, err := charset.GetCollationByID(int(d.collationID)) if err != nil { logutil.BgLogger().Warn("unknown collation", zap.Error(err)) return d.GetString() @@ -252,7 +315,7 @@ func (d *Datum) SetString(s string, collation string) { d.k = KindString sink(s) d.b = hack.Slice(s) - d.collation = collation + d.collationID = collationNameToID(collation) } // sink prevents s from being allocated on the stack. @@ -271,7 +334,7 @@ func (d *Datum) GetBytes() []byte { func (d *Datum) SetBytes(b []byte) { d.k = KindBytes d.b = b - d.collation = charset.CollationBin + d.collationID = mysql.BinaryDefaultCollationID } // SetBytesAsString sets bytes value to datum as string type. @@ -279,30 +342,25 @@ func (d *Datum) SetBytesAsString(b []byte, collation string, length uint32) { d.k = KindString d.b = b d.length = length - d.collation = collation -} - -// GetInterface gets interface value. -func (d *Datum) GetInterface() any { - return d.x + d.collationID = collationNameToID(collation) } -// SetInterface sets interface to datum. -func (d *Datum) SetInterface(x any) { - d.k = KindInterface - d.x = x -} +// GetInterface and SetInterface were removed along with the +// KindInterfaceDeprecated (14) kind. The x field is now only used +// internally to carry *MyDecimal and the boxed Time value; callers that +// previously relied on the escape hatch should use the typed Set/Get +// methods for the value they actually hold. // SetNull sets datum to nil. func (d *Datum) SetNull() { d.k = KindNull - d.x = nil + d.decPtr = nil } // SetMinNotNull sets datum to minNotNull value. func (d *Datum) SetMinNotNull() { d.k = KindMinNotNull - d.x = nil + d.decPtr = nil } // GetBinaryLiteral4Cmp gets Bit value, and remove it's prefix 0 for comparison. @@ -335,7 +393,7 @@ func (d *Datum) GetMysqlBit() BinaryLiteral { func (d *Datum) SetBinaryLiteral(b BinaryLiteral) { d.k = KindBinaryLiteral d.b = b - d.collation = charset.CollationBin + d.collationID = mysql.BinaryDefaultCollationID } // SetMysqlBit sets MysqlBit value @@ -344,18 +402,21 @@ func (d *Datum) SetMysqlBit(b BinaryLiteral) { d.b = b } -// GetMysqlDecimal gets decimal value +// GetMysqlDecimal gets decimal value. func (d *Datum) GetMysqlDecimal() *MyDecimal { - return d.x.(*MyDecimal) + return d.decPtr } -// SetMysqlDecimal sets decimal value +// SetMysqlDecimal sets decimal value. func (d *Datum) SetMysqlDecimal(b *MyDecimal) { d.k = KindMysqlDecimal - d.x = b + d.decPtr = b } -// GetMysqlDuration gets Duration value +// GetMysqlDuration gets Duration value. d.decimal is stored as uint8, but +// callers encode Fsp = -1 as the "unspecified" sentinel (see +// types.UnspecifiedFsp), which round-trips as 255 in the narrow field. +// Sign-extend via int8 to recover the original Fsp. func (d *Datum) GetMysqlDuration() Duration { return Duration{Duration: time.Duration(d.i), Fsp: int(int8(d.decimal))} } @@ -364,7 +425,7 @@ func (d *Datum) GetMysqlDuration() Duration { func (d *Datum) SetMysqlDuration(b Duration) { d.k = KindMysqlDuration d.i = int64(b.Duration) - d.decimal = uint16(b.Fsp) + d.decimal = uint8(b.Fsp) } // GetMysqlEnum gets Enum value @@ -378,7 +439,7 @@ func (d *Datum) SetMysqlEnum(b Enum, collation string) { d.k = KindMysqlEnum d.i = int64(b.Value) sink(b.Name) - d.collation = collation + d.collationID = collationNameToID(collation) d.b = hack.Slice(b.Name) } @@ -393,7 +454,7 @@ func (d *Datum) SetMysqlSet(b Set, collation string) { d.k = KindMysqlSet d.i = int64(b.Value) sink(b.Name) - d.collation = collation + d.collationID = collationNameToID(collation) d.b = hack.Slice(b.Name) } @@ -424,17 +485,25 @@ func (d *Datum) GetVectorFloat32() VectorFloat32 { return v } -// GetMysqlTime gets types.Time value +// GetMysqlTime gets types.Time value. Time is a bit-packed CoreTime +// (uint64) so it lives directly in d.i func (d *Datum) GetMysqlTime() Time { - return d.x.(Time) + intest.Assert(d.k == KindMysqlTime, "GetMysqlTime called on Datum of kind ", d.k) + return Time{coreTime: CoreTime(uint64(d.i))} } -// SetMysqlTime sets types.Time value +// SetMysqlTime sets types.Time value. Keep the representation inside +// d.i (8 bytes) rather than boxing through d.x; see GetMysqlTime. func (d *Datum) SetMysqlTime(b Time) { d.k = KindMysqlTime - d.x = b + d.i = int64(uint64(b.coreTime)) } +// Compile-time guard: GetMysqlTime/SetMysqlTime assume Time fits in +// d.i (8 bytes). If Time ever gains a field beyond coreTime, the +// pack/unpack would silently drop data — break the build instead. +var _ [8]byte = [unsafe.Sizeof(Time{})]byte{} + // SetRaw sets raw value. func (d *Datum) SetRaw(b []byte) { d.k = KindRaw @@ -488,8 +557,6 @@ func (d Datum) String() string { t = "KindMysqlSet" case KindMysqlTime: t = "KindMysqlTime" - case KindInterface: - t = "KindInterface" case KindMinNotNull: t = "KindMinNotNull" case KindMaxValue: @@ -500,8 +567,10 @@ func (d Datum) String() string { t = "KindMysqlJSON" case KindVectorFloat32: t = "KindVectorFloat32" + // No case for KindInterfaceDeprecated (14): unreachable at runtime; + // falls through to the default which prints the raw kind value. default: - t = "Unknown" + t = fmt.Sprintf("Unknown type: %d", int(d.k)) } v := d.GetValue() switch v.(type) { @@ -549,8 +618,14 @@ func (d *Datum) GetValue() any { return d.GetMysqlTime() case KindVectorFloat32: return d.GetVectorFloat32() + case KindNull, KindMinNotNull, KindMaxValue, KindRaw: + // These kinds carry no inline value; nil matches the prior + // behavior where d.x was unset and GetInterface returned nil. + return nil default: - return d.GetInterface() + // Any remaining kind (e.g. KindInterfaceDeprecated, or a stale + // byte written to d.k) is a programmer bug. + panic(fmt.Sprintf("types.Datum.GetValue: invalid kind %d", d.k)) } } @@ -637,7 +712,7 @@ func (d *Datum) SetValueWithDefaultCollation(val any) { case VectorFloat32: d.SetVectorFloat32(x) default: - d.SetInterface(x) + panic(fmt.Sprintf("types.Datum.SetValue: unsupported Go type %T", x)) } } @@ -687,7 +762,7 @@ func (d *Datum) SetValue(val any, tp *types.FieldType) { case VectorFloat32: d.SetVectorFloat32(x) default: - d.SetInterface(x) + panic(fmt.Sprintf("types.Datum.SetValue: unsupported Go type %T", x)) } } @@ -717,7 +792,7 @@ func (d *Datum) Equals(other any) bool { d.decimal == d2.decimal && d.length == d2.length && d.i == d2.i && - d.collation == d2.collation && + d.collationID == d2.collationID && string(d.b) == string(d2.b) if !ok { return false @@ -1163,7 +1238,7 @@ func (d *Datum) convertToString(ctx Context, target *FieldType) (Datum, error) { case KindFloat64: s = strconv.FormatFloat(d.GetFloat64(), 'f', -1, 64) case KindString, KindBytes: - fromBinary := d.Collation() == charset.CollationBin + fromBinary := d.collationID == mysql.BinaryDefaultCollationID toBinary := target.GetCharset() == charset.CharsetBin if fromBinary && toBinary { s = d.GetString() @@ -2159,13 +2234,13 @@ func (d *Datum) ToBytes() ([]byte, error) { func (d *Datum) ToHashKey() ([]byte, error) { switch d.k { case KindString, KindBytes: - return collate.GetCollator(d.Collation()).Key(d.GetString()), nil + return collate.GetCollatorByID(int(d.collationID)).Key(d.GetString()), nil default: str, err := d.ToString() if err != nil { return nil, errors.Trace(err) } - return collate.GetCollator(d.Collation()).Key(str), nil + return collate.GetCollatorByID(int(d.collationID)).Key(str), nil } } @@ -2208,10 +2283,15 @@ func (d *Datum) ToMysqlJSON() (j BinaryJSON, err error) { // MemUsage gets the memory usage of datum. func (d *Datum) MemUsage() (sum int64) { - // d.x is not considered now since MemUsage is now only used by analyze samples which is bytesDatum - return EmptyDatumSize + int64(cap(d.b)) + int64(len(d.collation)) + // d.x is not considered now since MemUsage is now only used by analyze samples which is bytesDatum. + // The collation is stored inline as a uint16 id, so it no longer + // contributes a variable-size term here. + return EmptyDatumSize + int64(cap(d.b)) } +// jsonDatum keeps the on-wire JSON format stable: the Collation field +// carries the collation name, not the internal uint16 id, so blobs +// encoded by older versions still round-trip. type jsonDatum struct { K byte `json:"k"` Decimal uint16 `json:"decimal,omitempty"` @@ -2225,21 +2305,36 @@ type jsonDatum struct { // MarshalJSON implements Marshaler.MarshalJSON interface. func (d *Datum) MarshalJSON() ([]byte, error) { + // Decimal is uint8 internally but uint16 on the wire for back-compat. + // Map the UnspecifiedFsp sentinel (^uint8(0) == 255) back to the legacy + // ^uint16(0) == 65535 so older readers — which still treat the field as + // signed -1 — don't decode 255 as a real positive frac. + jdDecimal := uint16(d.decimal) + if d.decimal == ^uint8(0) { + jdDecimal = ^uint16(0) + } jd := &jsonDatum{ K: d.k, - Decimal: d.decimal, + Decimal: jdDecimal, Length: d.length, I: d.i, - Collation: d.collation, + Collation: d.Collation(), B: d.b, } switch d.k { case KindMysqlTime: + // Time is packed into d.i; emit it through jd.Time only so the + // JSON payload doesn't carry the same bits twice. UnmarshalJSON + // re-derives d.i from jd.Time via SetMysqlTime. + jd.I = 0 jd.Time = d.GetMysqlTime() case KindMysqlDecimal: jd.MyDecimal = d.GetMysqlDecimal() default: - if d.x != nil { + // d.decPtr is only set by SetMysqlDecimal; any other kind that + // still has it populated is a bug (stale field from a reused + // Datum, perhaps). + if d.decPtr != nil { return nil, fmt.Errorf("unsupported type: %d", d.k) } } @@ -2253,10 +2348,10 @@ func (d *Datum) UnmarshalJSON(data []byte) error { return err } d.k = jd.K - d.decimal = jd.Decimal + d.decimal = uint8(jd.Decimal) d.length = jd.Length d.i = jd.I - d.collation = jd.Collation + d.collationID = collationNameToID(jd.Collation) d.b = jd.B switch jd.K { @@ -2409,7 +2504,7 @@ func SortDatums(ctx Context, datums []Datum) error { var err error slices.SortFunc(datums, func(a, b Datum) int { var cmp int - cmp, err = a.Compare(ctx, &b, collate.GetCollator(b.Collation())) + cmp, err = a.Compare(ctx, &b, collate.GetCollatorByID(int(b.collationID))) if err != nil { return 0 } @@ -2716,7 +2811,6 @@ func ChangeReverseResultByUpperLowerBound( const ( sizeOfEmptyDatum = int(unsafe.Sizeof(Datum{})) - sizeOfMysqlTime = int(unsafe.Sizeof(ZeroTime)) sizeOfMyDecimal = MyDecimalStructSize ) @@ -2740,7 +2834,7 @@ func (d Datum) EstimatedMemUsage() int64 { case KindMysqlDecimal: bytesConsumed += sizeOfMyDecimal case KindMysqlTime: - bytesConsumed += sizeOfMysqlTime + // Time is packed inline in d.i, already counted in sizeOfEmptyDatum. case KindVectorFloat32: bytesConsumed += d.GetVectorFloat32().EstimatedMemUsage() default: diff --git a/pkg/types/datum_test.go b/pkg/types/datum_test.go index 29feb1ff2fd30..1ff0de8f762ba 100644 --- a/pkg/types/datum_test.go +++ b/pkg/types/datum_test.go @@ -33,13 +33,14 @@ import ( ) func TestDatum(t *testing.T) { + // Go types not recognized by SetValueWithDefaultCollation used to be + // stored via the KindInterfaceDeprecated escape hatch; they now panic. values := []any{ int64(1), uint64(1), 1.1, "abc", []byte("abc"), - []int{1}, } for _, val := range values { var d Datum @@ -50,6 +51,11 @@ func TestDatum(t *testing.T) { require.Equal(t, int(d.length), d.Length()) require.Equal(t, d.String(), fmt.Sprint(d)) } + + require.Panics(t, func() { + var d Datum + d.SetValueWithDefaultCollation([]int{1}) + }) } func testDatumToBool(t *testing.T, in any, res int) { @@ -105,10 +111,7 @@ func TestToBool(t *testing.T) { v, err := Convert(0.1415926, ft) require.NoError(t, err) testDatumToBool(t, v, 1) - d := NewDatum(&invalidMockType{}) - ctx := DefaultStmtNoWarningContext.WithFlags(DefaultStmtFlags.WithIgnoreTruncateErr(true)) - _, err = d.ToBool(ctx) - require.Error(t, err) + require.Panics(t, func() { _ = NewDatum(&invalidMockType{}) }) } func testDatumToInt64(t *testing.T, val any, expect int64) { @@ -193,7 +196,7 @@ func TestConvertToFloat(t *testing.T) { {NewDatum([]byte("12345.678")), mysql.TypeDouble, "", 12345.678, 12345.678}, {NewDatum(int64(12345)), mysql.TypeDouble, "", 12345, 12345}, {NewDatum(uint64(123456)), mysql.TypeDouble, "", 123456, 123456}, - {NewDatum(byte(123)), mysql.TypeDouble, "cannot convert ", 0, 0}, + // NewDatum(byte(123)) now panics; see KindInterfaceDeprecated removal. {NewDatum(math.NaN()), mysql.TypeDouble, "constant .* overflows double", 0, 0}, {NewDatum(math.Inf(-1)), mysql.TypeDouble, "constant .* overflows double", math.Inf(-1), float32(math.Inf(-1))}, {NewDatum(math.Inf(1)), mysql.TypeDouble, "constant .* overflows double", math.Inf(1), float32(math.Inf(1))}, @@ -334,7 +337,7 @@ func TestComputePlusAndMinus(t *testing.T) { require.Equal(t, tt.hasErr, err != nil) v, err := got.Compare(DefaultStmtNoWarningContext, &tt.plus, collate.GetBinaryCollator()) require.NoError(t, err) - require.Equalf(t, 0, v, "%dth got:%#v, %#v, expect:%#v, %#v", ith, got, got.x, tt.plus, tt.plus.x) + require.Equalf(t, 0, v, "%dth got:%#v, %#v, expect:%#v, %#v", ith, got, got.decPtr, tt.plus, tt.plus.decPtr) } } @@ -395,7 +398,9 @@ func TestEstimatedMemUsage(t *testing.T) { NewBytesDatum(b), NewDecimalDatum(newMyDecimal("1234.1234", t)), NewMysqlEnumDatum(enum), + NewTimeDatum(NewTime(FromDate(2025, 1, 1, 0, 0, 0, 0), mysql.TypeDatetime, DefaultFsp)), } + // Time packs into Datum.i, so KindMysqlTime adds no bytes beyond sizeOfEmptyDatum. bytesConsumed := 10 * (len(datumArray)*sizeOfEmptyDatum + sizeOfMyDecimal + len(b)*2 + @@ -549,6 +554,45 @@ func TestStringToMysqlBit(t *testing.T) { } } +// TestDatumEqualsCollation pins the canonical-id semantics of Datum.Equals +// for collation. Datums with collation names that differ only in case or +// alias (e.g. "UTF8MB4_BIN" vs "utf8mb4_bin") resolve to the same MySQL +// collation id and must compare equal; distinct canonical collations must +// compare unequal. This is a deliberate shift from the pre-shrink byte-equal +// string compare, aligning with MySQL's case-insensitive collation names. +func TestDatumEqualsCollation(t *testing.T) { + mk := func(name string) Datum { + var d Datum + d.SetString("abc", name) + return d + } + + // Same canonical collation, different name casing → equal. The + // non-zero ID assertion guards against the test passing vacuously + // if name resolution ever breaks (both names would silently fall + // back to id 0, satisfying the equality below for the wrong reason). + a := mk("UTF8MB4_BIN") + b := mk("utf8mb4_bin") + require.NotEqual(t, uint16(0), a.CollationID()) + require.Equal(t, a.CollationID(), b.CollationID()) + require.True(t, a.Equals(&b)) + + // Different canonical collations → not equal. + c := mk(charset.CollationUTF8MB4) + d := mk(charset.CollationASCII) + require.NotEqual(t, c.CollationID(), d.CollationID()) + require.False(t, c.Equals(&d)) + + // Unknown names both collapse to id 0. The resulting equality is a + // documented consequence of lossy name → id storage; pin it so a + // future change doesn't accidentally flip it back to string compare. + u1 := mk("nonexistent_one") + u2 := mk("nonexistent_two") + require.Equal(t, uint16(0), u1.CollationID()) + require.Equal(t, uint16(0), u2.CollationID()) + require.True(t, u1.Equals(&u2)) +} + func TestMarshalDatum(t *testing.T) { e, err := ParseSetValue([]string{"a", "b", "c", "d", "e"}, uint64(1)) require.NoError(t, err) @@ -585,25 +629,46 @@ func TestMarshalDatum(t *testing.T) { require.Equal(t, tt.decimal, datum.decimal, msg) require.Equal(t, tt.length, datum.length, msg) require.Equal(t, tt.i, datum.i, msg) - require.Equal(t, tt.collation, datum.collation, msg) + require.Equal(t, tt.collationID, datum.collationID, msg) require.Equal(t, tt.b, datum.b, msg) - if tt.x == nil { - require.Nil(t, datum.x, msg) + require.Equal(t, tt.decPtr != nil, datum.decPtr != nil, msg) + // Time is packed into d.i; verify its round-trip through the + // getter rather than via a now-nonexistent d.x field. + if tt.k == KindMysqlTime { + require.Equal(t, 0, tt.GetMysqlTime().Compare(datum.GetMysqlTime()), msg) + continue } - require.Equal(t, reflect.TypeOf(tt.x), reflect.TypeOf(datum.x), msg) - switch tt.x.(type) { - case Time: - require.Equal(t, 0, tt.x.(Time).Compare(datum.x.(Time))) - case *MyDecimal: - require.Equal(t, 0, tt.x.(*MyDecimal).Compare(datum.x.(*MyDecimal))) - default: - require.EqualValues(t, tt.x, datum.x, msg) + if tt.decPtr != nil { + require.Equal(t, 0, tt.decPtr.Compare(datum.decPtr), msg) } } } +// TestMarshalDatumUnspecifiedFsp pins the wire-format guarantee that a +// Datum with UnspecifiedFsp (-1, stored as uint8 255) emits the legacy +// uint16 sentinel 65535 in JSON. A pre-shrink reader still reads the field +// as signed -1; emitting the bare 255 would surface as a real positive +// frac on the other side. +func TestMarshalDatumUnspecifiedFsp(t *testing.T) { + var d Datum + d.SetMysqlDuration(Duration{Duration: time.Hour, Fsp: UnspecifiedFsp}) + require.Equal(t, uint8(0xff), d.decimal) + + bytes, err := gjson.Marshal(&d) + require.NoError(t, err) + require.Contains(t, string(bytes), `"decimal":65535`, + "UnspecifiedFsp must marshal as legacy uint16 sentinel for back-compat") + + var got Datum + require.NoError(t, gjson.Unmarshal(bytes, &got)) + require.Equal(t, uint8(0xff), got.decimal, "sentinel must survive round-trip") + require.Equal(t, UnspecifiedFsp, got.GetMysqlDuration().Fsp, + "GetMysqlDuration must recover Fsp = -1") +} + func BenchmarkCompareDatum(b *testing.B) { vals, vals1 := prepareCompareDatums() + b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { for j, v := range vals { @@ -615,14 +680,61 @@ func BenchmarkCompareDatum(b *testing.B) { } } +// BenchmarkCompareDatumCollation compares collated string datums through +// an explicit non-binary collator. Unlike BenchmarkCompareDatum, the +// collator is sourced from charset.CollationUTF8MB4 rather than the +// binary shortcut, so the cost of the collated comparator dominates the +// measurement. Note that Compare uses the supplied `coll` directly and +// never reads d1.Collation() / d1.CollationID() on the hot path. +func BenchmarkCompareDatumCollation(b *testing.B) { + d1 := NewCollationStringDatum("abcdefghij", charset.CollationUTF8MB4) + d2 := NewCollationStringDatum("abcdefghik", charset.CollationUTF8MB4) + coll := collate.GetCollator(charset.CollationUTF8MB4) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := d1.Compare(DefaultStmtNoWarningContext, &d2, coll); err != nil { + b.Fatal(err) + } + } +} + func BenchmarkCompareDatumByReflect(b *testing.B) { vals, vals1 := prepareCompareDatums() + b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { reflect.DeepEqual(vals, vals1) } } +// BenchmarkDatumCopy exercises Datum.Copy on a heterogeneous slice that +// contains a Time datum. With move of Time into d.i, it should be more efficient +// both for the copying as well as for allocations. +func BenchmarkDatumCopy(b *testing.B) { + src, _ := prepareCompareDatums() + dst := make([]Datum, len(src)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := range src { + src[j].Copy(&dst[j]) + } + } +} + +// BenchmarkDatumSetMysqlTime asserts that SetMysqlTime is alloc-free. +// Storing Time inline in d.i means every call should be 0 B/op / 0 allocs/op. +func BenchmarkDatumSetMysqlTime(b *testing.B) { + t := NewTime(FromGoTime(time.Date(2018, 3, 8, 16, 1, 0, 315313000, time.UTC)), mysql.TypeTimestamp, 6) + var d Datum + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + d.SetMysqlTime(t) + } +} + func TestProduceDecWithSpecifiedTp(t *testing.T) { tests := []struct { dec string @@ -746,6 +858,7 @@ func BenchmarkDatumsToString(b *testing.B) { MinNotNullDatum(), MaxValueDatum(), } + b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := DatumsToString(datums, true) @@ -759,6 +872,7 @@ func BenchmarkDatumsToStringStr(b *testing.B) { datums := []Datum{ NewStringDatum(strings.Repeat("1", 512)), } + b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := DatumsToString(datums, true) @@ -772,6 +886,7 @@ func BenchmarkDatumsToStringLongStr(b *testing.B) { datums := []Datum{ NewStringDatum(strings.Repeat("1", 1024*10)), // 10KB } + b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := DatumsToString(datums, true) @@ -784,6 +899,7 @@ func BenchmarkDatumsToStringLongStr(b *testing.B) { func BenchmarkDatumTruncatedStringify(b *testing.B) { d1 := NewStringDatum(strings.Repeat("1", 128)) d2 := NewIntDatum(2) + b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { _ = d1.TruncatedStringify() diff --git a/pkg/types/etc.go b/pkg/types/etc.go index 8fcfb094516fd..81d826179c085 100644 --- a/pkg/types/etc.go +++ b/pkg/types/etc.go @@ -19,6 +19,7 @@ package types import ( + "fmt" "io" "github.com/pingcap/errors" @@ -159,34 +160,39 @@ func IsStringKind(kind byte) bool { } var kind2Str = map[byte]string{ - KindNull: "null", - KindInt64: "bigint", - KindUint64: "unsigned bigint", - KindFloat32: "float", - KindFloat64: "double", - KindString: "char", - KindBytes: "bytes", - KindBinaryLiteral: "bit/hex literal", - KindMysqlDecimal: "decimal", - KindMysqlDuration: "time", - KindMysqlEnum: "enum", - KindMysqlBit: "bit", - KindMysqlSet: "set", - KindMysqlTime: "datetime", - KindInterface: "interface", - KindMinNotNull: "min_not_null", - KindMaxValue: "max_value", - KindRaw: "raw", - KindMysqlJSON: "json", - KindVectorFloat32: "vector", + KindNull: "null", + KindInt64: "bigint", + KindUint64: "unsigned bigint", + KindFloat32: "float", + KindFloat64: "double", + KindString: "char", + KindBytes: "bytes", + KindBinaryLiteral: "bit/hex literal", + KindMysqlDecimal: "decimal", + KindMysqlDuration: "time", + KindMysqlEnum: "enum", + KindMysqlBit: "bit", + KindMysqlSet: "set", + KindMysqlTime: "datetime", + KindInterfaceDeprecated: "interface(deprecated)", + KindMinNotNull: "min_not_null", + KindMaxValue: "max_value", + KindRaw: "raw", + KindMysqlJSON: "json", + KindVectorFloat32: "vector", } // TypeStr converts tp to a string. var TypeStr = ast.TypeStr -// KindStr converts kind to a string. -func KindStr(kind byte) (r string) { - return kind2Str[kind] +// KindStr converts kind to a string. Only called from error-message +// construction today (see invalidConv in datum.go); the caller expects +// a soft, non-panicking return even if the kind is unexpected. +func KindStr(kind byte) string { + if s, ok := kind2Str[kind]; ok { + return s + } + return fmt.Sprintf("Unknown kind %d", kind) } // TypeToStr converts a field to a string. diff --git a/pkg/types/field_type.go b/pkg/types/field_type.go index 95890c844ab6d..da6349309af2c 100644 --- a/pkg/types/field_type.go +++ b/pkg/types/field_type.go @@ -181,12 +181,13 @@ func InferParamTypeFromDatum(d *Datum, tp *FieldType) { InferParamTypeFromUnderlyingValue(d.GetValue(), tp) if IsStringKind(d.k) { // consider charset and collation here - c, err := collate.GetCollationByName(d.collation) + name := d.Collation() + c, err := collate.GetCollationByName(name) if err != nil || c == nil { return // use default charset and collation } tp.SetCharset(c.CharsetName) - tp.SetCollate(d.collation) + tp.SetCollate(name) } } diff --git a/pkg/types/parser_driver/value_expr.go b/pkg/types/parser_driver/value_expr.go index 1ccb760a1cd1f..63140e1460d7a 100644 --- a/pkg/types/parser_driver/value_expr.go +++ b/pkg/types/parser_driver/value_expr.go @@ -127,7 +127,7 @@ func (n *ValueExpr) Restore(ctx *format.RestoreCtx) error { ctx.WritePlainf("'%s'", n.GetMysqlTime()) case types.KindMysqlEnum, types.KindMysqlBit, types.KindMysqlSet, - types.KindInterface, types.KindMinNotNull, types.KindMaxValue, + types.KindInterfaceDeprecated, types.KindMinNotNull, types.KindMaxValue, types.KindRaw, types.KindMysqlJSON, types.KindVectorFloat32: // TODO implement Restore function diff --git a/pkg/util/codec/bench_test.go b/pkg/util/codec/bench_test.go index 1b8397ab50691..4390d962e0f67 100644 --- a/pkg/util/codec/bench_test.go +++ b/pkg/util/codec/bench_test.go @@ -38,6 +38,7 @@ func composeEncodedData(size int) []byte { func BenchmarkDecodeWithSize(b *testing.B) { b.StopTimer() bs := composeEncodedData(valueCnt) + b.ReportAllocs() b.StartTimer() for range b.N { _, err := Decode(bs, valueCnt) @@ -50,6 +51,7 @@ func BenchmarkDecodeWithSize(b *testing.B) { func BenchmarkDecodeWithOutSize(b *testing.B) { b.StopTimer() bs := composeEncodedData(valueCnt) + b.ReportAllocs() b.StartTimer() for range b.N { _, err := Decode(bs, 1) @@ -60,6 +62,7 @@ func BenchmarkDecodeWithOutSize(b *testing.B) { } func BenchmarkEncodeIntWithSize(b *testing.B) { + b.ReportAllocs() for range b.N { data := make([]byte, 0, 8) EncodeInt(data, 10) @@ -67,6 +70,7 @@ func BenchmarkEncodeIntWithSize(b *testing.B) { } func BenchmarkEncodeIntWithOutSize(b *testing.B) { + b.ReportAllocs() for range b.N { EncodeInt(nil, 10) } @@ -80,6 +84,7 @@ func BenchmarkDecodeDecimal(b *testing.B) { } precision, frac := dec.PrecisionAndFrac() raw, _ := EncodeDecimal([]byte{}, dec, precision, frac) + b.ReportAllocs() b.ResetTimer() for range b.N { _, _, _, _, err := DecodeDecimal(raw) @@ -96,6 +101,7 @@ func BenchmarkDecodeOneToChunk(b *testing.B) { raw = append(raw, bytesFlag) raw = EncodeBytes(raw, str.GetBytes()) intType := types.NewFieldType(mysql.TypeLonglong) + b.ReportAllocs() b.ResetTimer() decoder := NewDecoder(chunk.New([]*types.FieldType{intType}, 32, 32), nil) for range b.N { diff --git a/pkg/util/codec/codec.go b/pkg/util/codec/codec.go index 6974393011437..e7df251d21f99 100644 --- a/pkg/util/codec/codec.go +++ b/pkg/util/codec/codec.go @@ -217,7 +217,7 @@ func EncodeMySQLTime(loc *time.Location, t types.Time, tp byte, b []byte) (_ []b func encodeString(b []byte, val types.Datum, comparable1 bool) []byte { if collate.NewCollationEnabled() && comparable1 { - return encodeBytes(b, collate.GetCollator(val.Collation()).ImmutableKey(val.GetString()), true) + return encodeBytes(b, collate.GetCollatorByID(int(val.CollationID())).ImmutableKey(val.GetString()), true) } return encodeBytes(b, val.GetBytes(), comparable1) } diff --git a/pkg/util/collate/collate.go b/pkg/util/collate/collate.go index 049deb2216ecc..2c35acd68b599 100644 --- a/pkg/util/collate/collate.go +++ b/pkg/util/collate/collate.go @@ -172,14 +172,19 @@ func GetBinaryCollatorSlice(n int) []Collator { } // GetCollatorByID get the collator according to id, it will return the binary collator if the corresponding collator doesn't exist. +// id == 0 means "no collation set" (the numeric counterpart of passing the +// empty string to GetCollator) and silently falls through to the default, +// matching GetCollator's behavior for "". func GetCollatorByID(id int) Collator { if atomic.LoadInt32(&newCollationEnabled) == 1 { ctor, ok := newCollatorIDMap[id] if !ok { - logutil.BgLogger().Warn( - "Unable to get collator by ID, use binCollator instead.", - zap.Int("ID", id), - zap.Stack("stack")) + if id != 0 { + logutil.BgLogger().Warn( + "Unable to get collator by ID, use binCollator instead.", + zap.Int("ID", id), + zap.Stack("stack")) + } return newCollatorMap["utf8mb4_bin"] } return ctor diff --git a/pkg/util/ranger/detacher.go b/pkg/util/ranger/detacher.go index 982eede1bb722..aca84040f46d4 100644 --- a/pkg/util/ranger/detacher.go +++ b/pkg/util/ranger/detacher.go @@ -654,7 +654,7 @@ func allSinglePoints(typeCtx types.Context, points []*point) []*point { return nil } // Since the point's collations are equal to the column's collation, we can use any of them. - cmp, err := left.value.Compare(typeCtx, &right.value, collate.GetCollator(left.value.Collation())) + cmp, err := left.value.Compare(typeCtx, &right.value, collate.GetCollatorByID(int(left.value.CollationID()))) if err != nil || cmp != 0 { return nil } diff --git a/pkg/util/ranger/types_test.go b/pkg/util/ranger/types_test.go index be73e79ca773f..7031761abaca6 100644 --- a/pkg/util/ranger/types_test.go +++ b/pkg/util/ranger/types_test.go @@ -248,7 +248,9 @@ func TestRangeMemUsage(t *testing.T) { HighVal: []types.Datum{types.NewStringDatum("fghij")}, Collators: collate.GetBinaryCollatorSlice(1), } - mem2 := mem1 + int64(cap(r2.LowVal[0].GetBytes())) + int64(len(r2.LowVal[0].Collation())) + int64(cap(r2.HighVal[0].GetBytes())) + int64(len(r2.HighVal[0].Collation())) + // The collation is stored inline as a uint16 id (already counted by + // EmptyDatumSize), so only the byte payload contributes a variable term. + mem2 := mem1 + int64(cap(r2.LowVal[0].GetBytes())) + int64(cap(r2.HighVal[0].GetBytes())) require.Equal(t, mem2, r2.MemUsage()) ranges := ranger.Ranges{&r1, &r2} require.Equal(t, mem1+mem2, ranges.MemUsage()) diff --git a/pkg/util/rowcodec/common.go b/pkg/util/rowcodec/common.go index 4e5273fe9da9d..17e943e43552b 100644 --- a/pkg/util/rowcodec/common.go +++ b/pkg/util/rowcodec/common.go @@ -308,6 +308,12 @@ func appendDatumForChecksum(loc *time.Location, buf []byte, dat *data.Datum, typ case mysql.TypeVarchar, mysql.TypeVarString, mysql.TypeString, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: out = appendLengthValue(buf, dat.GetBytes()) case mysql.TypeTimestamp, mysql.TypeDatetime, mysql.TypeDate, mysql.TypeNewDate: + // GetMysqlTime reinterprets d.i as CoreTime without a kind check, + // so the defer-recover above cannot turn a mismatched-kind Datum + // into an error the way it does for the other boxed getters. + if dat.Kind() != data.KindMysqlTime { + return buf, errInvalidChecksumTyp + } t := dat.GetMysqlTime() if t.Type() == mysql.TypeTimestamp && loc != nil && loc != time.UTC { err = t.ConvertTimeZone(loc, time.UTC) diff --git a/tests/integrationtest/r/planner/core/casetest/index/index.result b/tests/integrationtest/r/planner/core/casetest/index/index.result index f826a69bc7f6e..12bcf04fd6513 100644 --- a/tests/integrationtest/r/planner/core/casetest/index/index.result +++ b/tests/integrationtest/r/planner/core/casetest/index/index.result @@ -462,7 +462,7 @@ IndexJoin root inner join, inner:IndexLookUp, outer key:planner__core__casetest └─TableRowIDScan cop[tikv] table:t1 keep order:false, stats:pseudo show warnings; Level Code Message -set @@tidb_opt_range_max_size = 2900; +set @@tidb_opt_range_max_size = 1900; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); id task access object operator info IndexJoin root inner join, inner:IndexLookUp, outer key:planner__core__casetest__index__index.t2.e, inner key:planner__core__casetest__index__index.t1.b, equal cond:eq(planner__core__casetest__index__index.t2.e, planner__core__casetest__index__index.t1.b), eq(planner__core__casetest__index__index.t2.g, planner__core__casetest__index__index.t1.d) @@ -476,8 +476,8 @@ IndexJoin root inner join, inner:IndexLookUp, outer key:planner__core__casetest └─TableRowIDScan cop[tikv] table:t1 keep order:false, stats:pseudo show warnings; Level Code Message -Warning 1105 Memory capacity of 2900 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen -set @@tidb_opt_range_max_size = 2300; +Warning 1105 Memory capacity of 1900 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +set @@tidb_opt_range_max_size = 1500; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); id task access object operator info IndexJoin root inner join, inner:IndexLookUp, outer key:planner__core__casetest__index__index.t2.e, inner key:planner__core__casetest__index__index.t1.b, equal cond:eq(planner__core__casetest__index__index.t2.e, planner__core__casetest__index__index.t1.b), eq(planner__core__casetest__index__index.t2.g, planner__core__casetest__index__index.t1.d) @@ -491,8 +491,8 @@ IndexJoin root inner join, inner:IndexLookUp, outer key:planner__core__casetest └─TableRowIDScan cop[tikv] table:t1 keep order:false, stats:pseudo show warnings; Level Code Message -Warning 1105 Memory capacity of 2300 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen -set @@tidb_opt_range_max_size = 700; +Warning 1105 Memory capacity of 1500 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +set @@tidb_opt_range_max_size = 500; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); id task access object operator info HashJoin root inner join, equal:[eq(planner__core__casetest__index__index.t1.b, planner__core__casetest__index__index.t2.e) eq(planner__core__casetest__index__index.t1.d, planner__core__casetest__index__index.t2.g)] @@ -506,7 +506,7 @@ HashJoin root inner join, equal:[eq(planner__core__casetest__index__index.t1.b, └─TableFullScan cop[tikv] table:t2 keep order:false, stats:pseudo show warnings; Level Code Message -Warning 1105 Memory capacity of 700 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +Warning 1105 Memory capacity of 500 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen Warning 1815 Optimizer Hint /*+ INL_JOIN(t1) */ or /*+ TIDB_INLJ(t1) */ is inapplicable set @@tidb_opt_range_max_size = 0; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.a = t2.e where t1.b > 1 and t1.b < 10; diff --git a/tests/integrationtest/r/planner/core/integration.result b/tests/integrationtest/r/planner/core/integration.result index daefc4464749f..469821b505f02 100644 --- a/tests/integrationtest/r/planner/core/integration.result +++ b/tests/integrationtest/r/planner/core/integration.result @@ -3656,7 +3656,7 @@ IndexLookUp root ├─IndexRangeScan(Build) cop[tikv] table:t1, index:idx_a_b(a, b) range:["aa" "dd","aa" "dd"], ["aa" "ee","aa" "ee"], ["aa" "ff","aa" "ff"], ["bb" "dd","bb" "dd"], ["bb" "ee","bb" "ee"], ["bb" "ff","bb" "ff"], ["cc" "dd","cc" "dd"], ["cc" "ee","cc" "ee"], ["cc" "ff","cc" "ff"], keep order:false, stats:pseudo └─Selection(Probe) cop[tikv] in(planner__core__integration.t1.a, "aaa", "bbb", "ccc"), in(planner__core__integration.t1.b, "ddd", "eee", "fff") └─TableRowIDScan cop[tikv] table:t1 keep order:false, stats:pseudo -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select * from t1 where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); id task access object operator info IndexLookUp root @@ -3664,7 +3664,7 @@ IndexLookUp root └─Selection(Probe) cop[tikv] in(planner__core__integration.t1.a, "aaa", "bbb", "ccc"), in(planner__core__integration.t1.b, "ddd", "eee", "fff") └─TableRowIDScan cop[tikv] table:t1 keep order:false, stats:pseudo Level Code Message -Warning 1105 Memory capacity of 1000 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +Warning 1105 Memory capacity of 800 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen set @@tidb_opt_range_max_size=0; explain format='plan_tree' select * from t1 join t2 on t1.c = t2.d where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); id task access object operator info @@ -3676,7 +3676,7 @@ HashJoin root inner join, equal:[eq(planner__core__integration.t1.c, planner__c └─TableReader(Probe) root data:Selection └─Selection cop[tikv] not(isnull(planner__core__integration.t2.d)) └─TableFullScan cop[tikv] table:t2 keep order:false, stats:pseudo -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select * from t1 join t2 on t1.c = t2.d where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); id task access object operator info HashJoin root inner join, equal:[eq(planner__core__integration.t1.c, planner__core__integration.t2.d)] @@ -3688,27 +3688,27 @@ HashJoin root inner join, equal:[eq(planner__core__integration.t1.c, planner__c └─Selection cop[tikv] not(isnull(planner__core__integration.t2.d)) └─TableFullScan cop[tikv] table:t2 keep order:false, stats:pseudo Level Code Message -Warning 1105 Memory capacity of 1000 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +Warning 1105 Memory capacity of 800 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen set @@tidb_opt_range_max_size=0; explain format='plan_tree' select /*+ use_index(t3, a) */ * from t3 where a in (1, 3, 5) and pk in (2, 4, 6); id task access object operator info IndexReader root index:IndexRangeScan └─IndexRangeScan cop[tikv] table:t3, index:a(a) range:[1 2,1 2], [1 4,1 4], [1 6,1 6], [3 2,3 2], [3 4,3 4], [3 6,3 6], [5 2,5 2], [5 4,5 4], [5 6,5 6], keep order:false, stats:pseudo -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select /*+ use_index(t3, a) */ * from t3 where a in (1, 3, 5) and pk in (2, 4, 6); id task access object operator info IndexReader root index:Selection └─Selection cop[tikv] in(planner__core__integration.t3.pk, 2, 4, 6) └─IndexRangeScan cop[tikv] table:t3, index:a(a) range:[1,1], [3,3], [5,5], keep order:false, stats:pseudo Level Code Message -Warning 1105 Memory capacity of 1000 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +Warning 1105 Memory capacity of 800 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen set @@tidb_opt_range_max_size=0; explain format='plan_tree' select /*+ use_index(t4, idx_a_b) */ * from t4 where a in (1, 3, 5) and b = 2; id task access object operator info IndexLookUp root ├─IndexRangeScan(Build) cop[tikv] table:t4, index:idx_a_b(a, b) range:[1 2,1 2], [3 2,3 2], [5 2,5 2], keep order:false, stats:pseudo └─TableRowIDScan(Probe) cop[tikv] table:t4 keep order:false, stats:pseudo -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select /*+ use_index(t4, idx_a_b) */ * from t4 where a in (1, 3, 5) and b = 2; id task access object operator info IndexLookUp root @@ -3716,7 +3716,7 @@ IndexLookUp root │ └─IndexRangeScan cop[tikv] table:t4, index:idx_a_b(a, b) range:[1,1], [3,3], [5,5], keep order:false, stats:pseudo └─TableRowIDScan(Probe) cop[tikv] table:t4 keep order:false, stats:pseudo Level Code Message -Warning 1105 Memory capacity of 1000 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen +Warning 1105 Memory capacity of 800 bytes for 'tidb_opt_range_max_size' exceeded when building ranges. Less accurate ranges such as full range are chosen set @@tidb_opt_range_max_size=DEFAULT; set @@tidb_enable_prepared_plan_cache=1; drop table if exists t; diff --git a/tests/integrationtest/t/planner/core/casetest/index/index.test b/tests/integrationtest/t/planner/core/casetest/index/index.test index 81d039a6940bf..1492200621158 100644 --- a/tests/integrationtest/t/planner/core/casetest/index/index.test +++ b/tests/integrationtest/t/planner/core/casetest/index/index.test @@ -154,13 +154,13 @@ create table t2(e int, f int, g varchar(10), h varchar(10)); set @@tidb_opt_range_max_size = 0; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); show warnings; -set @@tidb_opt_range_max_size = 2900; +set @@tidb_opt_range_max_size = 1900; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); show warnings; -set @@tidb_opt_range_max_size = 2300; +set @@tidb_opt_range_max_size = 1500; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); show warnings; -set @@tidb_opt_range_max_size = 700; +set @@tidb_opt_range_max_size = 500; explain format='plan_tree' select /*+ inl_join(t1) */ * from t1 join t2 on t1.b = t2.e and t1.d = t2.g where t1.a in (1, 3) and t1.c in ('aaa', 'bbb'); show warnings; set @@tidb_opt_range_max_size = 0; diff --git a/tests/integrationtest/t/planner/core/integration.test b/tests/integrationtest/t/planner/core/integration.test index c0fa33cf02ea7..d67382f7cafa5 100644 --- a/tests/integrationtest/t/planner/core/integration.test +++ b/tests/integrationtest/t/planner/core/integration.test @@ -1906,19 +1906,19 @@ create table t4 (a int, b int, c int, index idx_a_b(a, b)); set @@tidb_opt_range_max_size=0; --enable_warnings explain format='plan_tree' select * from t1 where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select * from t1 where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); set @@tidb_opt_range_max_size=0; explain format='plan_tree' select * from t1 join t2 on t1.c = t2.d where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select * from t1 join t2 on t1.c = t2.d where a in ('aaa', 'bbb', 'ccc') and b in ('ddd', 'eee', 'fff'); set @@tidb_opt_range_max_size=0; explain format='plan_tree' select /*+ use_index(t3, a) */ * from t3 where a in (1, 3, 5) and pk in (2, 4, 6); -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select /*+ use_index(t3, a) */ * from t3 where a in (1, 3, 5) and pk in (2, 4, 6); set @@tidb_opt_range_max_size=0; explain format='plan_tree' select /*+ use_index(t4, idx_a_b) */ * from t4 where a in (1, 3, 5) and b = 2; -set @@tidb_opt_range_max_size=1000; +set @@tidb_opt_range_max_size=800; explain format='plan_tree' select /*+ use_index(t4, idx_a_b) */ * from t4 where a in (1, 3, 5) and b = 2; --disable_warnings set @@tidb_opt_range_max_size=DEFAULT;