expression: fix signed bigint subtraction overflow | tidb-test=pr/2742

pkg/expression/builtin_arithmetic.go

@@ -519,7 +519,8 @@ func (s *builtinArithmeticMinusIntSig) overflowCheck(isLHSUnsigned, isRHSUnsigne
 				return true
 			}
 		} else {
-			if a > 0 && b < 0 {
+			// 0 - math.MinInt64 exceeds MaxInt64, so zero belongs to the non-negative path.
+			if a >= 0 && b < 0 {
 				resUnsigned = true
 			} else if a < 0 && b > 0 && res >= 0 {
 				return true

pkg/expression/builtin_arithmetic_test.go

@@ -206,6 +206,47 @@ func TestArithmeticMinus(t *testing.T) {
 	require.False(t, isNull)
 	require.Equal(t, int64(11), intResult)
 
+	for _, tc := range []struct {
+		name   string
+		args   []any
+		expect any
+		errStr string
+	}{
+		{
+			name:   "zero minus signed min overflows",
+			args:   []any{int64(0), int64(math.MinInt64)},
+			errStr: "BIGINT value is out of range",
+		},
+		{
+			name:   "positive minus signed min overflows",
+			args:   []any{int64(1), int64(math.MinInt64)},
+			errStr: "BIGINT value is out of range",
+		},
+		{
+			name:   "negative one minus signed min stays in range",
+			args:   []any{int64(-1), int64(math.MinInt64)},
+			expect: int64(math.MaxInt64),
+		},
+		{
+			name:   "zero minus negative one stays in range",
+			args:   []any{int64(0), int64(-1)},
+			expect: int64(1),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			sig, err := funcs[ast.Minus].getFunction(ctx, datumsToConstants(types.MakeDatums(tc.args...)))
+			require.NoError(t, err)
+			require.NotNil(t, sig)
+			val, err := evalBuiltinFunc(sig, ctx, chunk.Row{})
+			if tc.errStr == "" {
+				require.NoError(t, err)
+				testutil.DatumEqual(t, types.NewDatum(tc.expect), val)
+			} else {
+				require.ErrorContains(t, err, tc.errStr)
+			}
+		})
+	}
+
 	// case 2
 	args = []any{1.01001, -0.01}
 

pkg/expression/builtin_arithmetic_vec_test.go

@@ -198,6 +198,20 @@ func TestVectorizedBuiltinArithmeticFunc(t *testing.T) {
 
 func TestVectorizedDecimalErrOverflow(t *testing.T) {
 	ctx := mock.NewContext()
+	t.Run("int_minus", func(t *testing.T) {
+		fts := []*types.FieldType{types.NewFieldType(mysql.TypeLonglong), types.NewFieldType(mysql.TypeLonglong)}
+		input := chunk.New(fts, 1, 1)
+		input.AppendInt64(0, 0)
+		input.AppendInt64(1, math.MinInt64)
+		cols := []Expression{&Column{Index: 0, RetType: fts[0]}, &Column{Index: 1, RetType: fts[1]}}
+		baseFunc, err := funcs[ast.Minus].getFunction(ctx, cols)
+		require.NoError(t, err)
+		require.True(t, baseFunc.vectorized() && baseFunc.isChildrenVectorized())
+		result := chunk.NewColumn(eType2FieldType(types.ETInt), 1)
+		err = vecEvalType(ctx, baseFunc, types.ETInt, input, result)
+		require.EqualError(t, err, "[types:1690]BIGINT value is out of range in '(Column#0 - Column#0)'")
+	})
+
 	testCases := []struct {
 		args     []float64
 		funcName string