Skip to content

expression: simple expression IsTruthWithNull/IsTruthWithoutNull with CAST #61076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

expression: simple expression IsTruthWithNull/IsTruthWithoutNull with CAST #61076

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a48ce5a
planner: fix incorrect query result may caused by join
hawkingrei May 12, 2025
d048d40
executor: simple expression IsTruthWithNull/IsTruthWithNull with CAST
hawkingrei May 13, 2025
5539919
executor: simple expression IsTruthWithNull/IsTruthWithNull with CAST
hawkingrei May 13, 2025
c6d2944
update
hawkingrei May 13, 2025
8ec7e0c
update
hawkingrei May 13, 2025
3dc6f01
update
hawkingrei May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 16 additions & 5 deletions pkg/expression/expression.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1171,26 +1171,37 @@ func wrapWithIsTrue(ctx BuildContext, keepNull bool, arg Expression, wrapForInt
}
}
var fc *isTrueOrFalseFunctionClass
var fn ast.CIStr
if keepNull {
fc = &isTrueOrFalseFunctionClass{baseFunctionClass{ast.IsTruthWithNull, 1, 1}, opcode.IsTruth, keepNull}
fn = ast.NewCIStr(ast.IsTruthWithNull)
} else {
fc = &isTrueOrFalseFunctionClass{baseFunctionClass{ast.IsTruthWithoutNull, 1, 1}, opcode.IsTruth, keepNull}
fn = ast.NewCIStr(ast.IsTruthWithoutNull)
}
f, err := fc.getFunction(ctx, []Expression{arg})
f, err := fc.getFunction(ctx, []Expression{simpleExpression(arg)})
if err != nil {
return nil, err
}
sf := &ScalarFunction{
FuncName: ast.NewCIStr(ast.IsTruthWithoutNull),
FuncName: fn,
Function: f,
RetType: f.getRetTp(),
}
if keepNull {
sf.FuncName = ast.NewCIStr(ast.IsTruthWithNull)
}
return FoldConstant(ctx, sf), nil
}

func simpleExpression(arg Expression) Expression {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The convert is too aggressive, istrue(cast(xxx as xxx)) is not always equal to istrue(xxx)
For example:


mysql> select istrue_with_null('0.1');
+-------------------------+
| istrue_with_null('0.1') |
+-------------------------+
|                       1 |
+-------------------------+
1 row in set (0.01 sec)

mysql> select istrue_with_null(cast('0.1' as signed));
+-----------------------------------------+
| istrue_with_null(cast('0.1' as signed)) |
+-----------------------------------------+
|                                       0 |
+-----------------------------------------+
1 row in set, 1 warning (0.00 sec)

switch fn := arg.(type) {
case *ScalarFunction:
if fn.FuncName.L == ast.Cast {
return simpleExpression(fn.GetArgs()[0])
}
return arg
}
return arg
}

// PropagateType propagates the type information to the `expr`.
// Note: For now, we only propagate type for the function CastDecimalAsDouble.
//
Expand Down
20 changes: 20 additions & 0 deletions pkg/expression/test/simplify/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
load("@io_bazel_rules_go//go:def.bzl", "go_test")

go_test(
name = "simplify_test",
timeout = "short",
srcs = [
"main_test.go",
"simplify_test.go",
],
flaky = True,
deps = [
"//pkg/config",
"//pkg/testkit",
"//pkg/testkit/testmain",
"//pkg/testkit/testsetup",
"//pkg/util/timeutil",
"@com_github_tikv_client_go_v2//tikv",
"@org_uber_go_goleak//:goleak",
],
)
57 changes: 57 additions & 0 deletions pkg/expression/test/simplify/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright 2025 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package simplify

import (
"testing"

"github.com/pingcap/tidb/pkg/config"
"github.com/pingcap/tidb/pkg/testkit/testmain"
"github.com/pingcap/tidb/pkg/testkit/testsetup"
"github.com/pingcap/tidb/pkg/util/timeutil"
"github.com/tikv/client-go/v2/tikv"
"go.uber.org/goleak"
)

func TestMain(m *testing.M) {
testsetup.SetupForCommonTest()
testmain.ShortCircuitForBench(m)

config.UpdateGlobal(func(conf *config.Config) {
conf.TiKVClient.AsyncCommit.SafeWindow = 0
conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
conf.Experimental.AllowsExpressionIndex = true
})
tikv.EnableFailpoints()

// Some test depends on the values of timeutil.SystemLocation()
// If we don't SetSystemTZ() here, the value would change unpredictable.
// Affected by the order whether a testsuite runs before or after integration test.
// Note, SetSystemTZ() is a sync.Once operation.
timeutil.SetSystemTZ("system")

opts := []goleak.Option{
goleak.IgnoreTopFunction("github.com/golang/glog.(*fileSink).flushDaemon"),
goleak.IgnoreTopFunction("github.com/bazelbuild/rules_go/go/tools/bzltestutil.RegisterTimeoutHandler.func1"),
goleak.IgnoreTopFunction("github.com/lestrrat-go/httprc.runFetchWorker"),
goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"),
goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"),
goleak.IgnoreTopFunction("github.com/pingcap/tidb/pkg/ttl/ttlworker.(*ttlScanWorker).loop"),
goleak.IgnoreTopFunction("github.com/pingcap/tidb/pkg/ttl/client.(*mockClient).WatchCommand.func1"),
goleak.IgnoreTopFunction("github.com/pingcap/tidb/pkg/ttl/ttlworker.(*JobManager).jobLoop"),
}

goleak.VerifyTestMain(m, opts...)
}
92 changes: 92 additions & 0 deletions pkg/expression/test/simplify/simplify_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Copyright 2025 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package simplify

import (
"testing"

"github.com/pingcap/tidb/pkg/testkit"
)

func TestIsTruthXXXWithCast(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
// https://github.com/pingcap/tidb/issues/61062
tk.MustExec(`CREATE TABLE t0(c0 FLOAT UNSIGNED);`)
tk.MustExec(`CREATE TABLE t1 LIKE t0;`)
tk.MustExec(`INSERT IGNORE INTO t0 VALUES (0.5);`)
tk.MustExec(`INSERT IGNORE INTO t1 VALUES (NULL);`)
tk.MustQuery(`SELECT t0.c0, t1.c0 FROM t0 INNER JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).
Check(testkit.Rows())
tk.MustQuery(`explain format=brief SELECT t0.c0, t1.c0 FROM t0 INNER JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).
Check(testkit.Rows("HashJoin 100000.00 root CARTESIAN inner join",
"├─TableReader(Build) 10.00 root data:Selection",
"│ └─Selection 10.00 cop[tikv] not(istrue_with_null(test.t0.c0))",
"│ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
"└─TableReader(Probe) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery(`SELECT t0.c0, t1.c0 FROM t0 LEFT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)))
INTERSECT
SELECT t0.c0, t1.c0 FROM t0 RIGHT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).Check(testkit.Rows())
tk.MustQuery(`explain format=brief SELECT t0.c0, t1.c0 FROM t0 LEFT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)))
INTERSECT
SELECT t0.c0, t1.c0 FROM t0 RIGHT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).Check(testkit.Rows(
"HashJoin 6400.00 root semi join, left side:HashAgg, equal:[nulleq(test.t0.c0, test.t0.c0) nulleq(test.t1.c0, test.t1.c0)]",
"├─HashJoin(Build) 100000.00 root CARTESIAN inner join",
"│ ├─TableReader(Build) 10.00 root data:Selection",
"│ │ └─Selection 10.00 cop[tikv] not(istrue_with_null(test.t0.c0))",
"│ │ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
"│ └─TableReader(Probe) 10000.00 root data:TableFullScan",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─HashAgg(Probe) 8000.00 root group by:test.t0.c0, test.t1.c0, funcs:firstrow(test.t0.c0)->test.t0.c0, funcs:firstrow(test.t1.c0)->test.t1.c0",
" └─HashJoin 100000.00 root CARTESIAN left outer join, left side:TableReader",
" ├─TableReader(Build) 10.00 root data:Selection",
" │ └─Selection 10.00 cop[tikv] not(istrue_with_null(test.t0.c0))",
" │ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
" └─TableReader(Probe) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"))
// https://github.com/pingcap/tidb/issues/51359
tk.MustExec("DROP TABLE t0;")
tk.MustExec("CREATE TABLE t0(c0 BOOL);")
tk.MustExec("REPLACE INTO t0(c0) VALUES (false), (true);")
tk.MustExec("CREATE VIEW v0(c0) AS SELECT (REGEXP_LIKE(t0.c0, t0.c0)) FROM t0 WHERE t0.c0 GROUP BY t0.c0 HAVING 1;")
tk.MustQuery(`SELECT t0.c0 FROM v0, t0 WHERE (SUBTIME('2001-11-28 06', '252 10') OR ('' IS NOT NULL));`).Check(testkit.Rows("0", "1"))
tk.MustQuery(`explain format='brief' SELECT t0.c0 FROM v0, t0 WHERE (SUBTIME('2001-11-28 06', '252 10') OR ('' IS NOT NULL));`).Check(testkit.Rows(
"HashJoin 27265706.67 root CARTESIAN inner join",
"├─Selection(Build) 3408.21 root or(istrue_with_null(cast(subtime(\"2001-11-28 06\", \"252 10\"), double BINARY)), 1)",
"│ └─HashAgg 4260.27 root group by:test.t0.c0, funcs:firstrow(1)->Column#7",
"│ └─Selection 5325.33 root or(istrue_with_null(cast(subtime(\"2001-11-28 06\", \"252 10\"), double BINARY)), 1)",
"│ └─TableReader 6656.67 root data:Selection",
"│ └─Selection 6656.67 cop[tikv] test.t0.c0",
"│ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
"└─Selection(Probe) 8000.00 root or(istrue_with_null(cast(subtime(\"2001-11-28 06\", \"252 10\"), double BINARY)), 1)",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo"))
tk.MustQuery(`SELECT t0.c0 FROM v0, t0 WHERE (SUBTIME('2001-11-28 06', '252 10') OR ('' IS NOT NULL)) AND v0.c0;`).
Check(testkit.Rows("0", "1"))
tk.MustQuery(`explain format='brief' SELECT t0.c0 FROM v0, t0 WHERE (SUBTIME('2001-11-28 06', '252 10') OR ('' IS NOT NULL)) AND v0.c0;`).
Check(testkit.Rows(
"HashJoin 27265706.67 root CARTESIAN inner join",
"├─Selection(Build) 3408.21 root or(istrue_with_null(cast(subtime(\"2001-11-28 06\", \"252 10\"), double BINARY)), 1)",
"│ └─HashAgg 4260.27 root group by:test.t0.c0, funcs:firstrow(Column#9)->Column#8",
"│ └─TableReader 4260.27 root data:HashAgg",
"│ └─HashAgg 4260.27 cop[tikv] group by:test.t0.c0, funcs:firstrow(1)->Column#9",
"│ └─Selection 5325.33 cop[tikv] regexp_like(cast(test.t0.c0, var_string(20)), cast(test.t0.c0, var_string(20))), test.t0.c0",
"│ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
"└─Selection(Probe) 8000.00 root or(istrue_with_null(cast(subtime(\"2001-11-28 06\", \"252 10\"), double BINARY)), 1)",
" └─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo"))
}
2 changes: 1 addition & 1 deletion pkg/planner/core/issuetest/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ go_test(
data = glob(["testdata/**"]),
flaky = True,
race = "on",
shard_count = 8,
shard_count = 9,
deps = [
"//pkg/parser",
"//pkg/planner",
Expand Down
39 changes: 39 additions & 0 deletions pkg/planner/core/issuetest/planner_issue_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,3 +259,42 @@ func TestIssue59902(t *testing.T) {
" └─Selection 1.00 cop[tikv] not(isnull(test.t2.a))",
" └─IndexRangeScan 1.00 cop[tikv] table:t2, index:idx(a) range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo"))
}

func TestIssue61062(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test;")
tk.MustExec(`CREATE TABLE t0(c0 FLOAT UNSIGNED);`)
tk.MustExec(`CREATE TABLE t1 LIKE t0;`)
tk.MustExec(`INSERT IGNORE INTO t0 VALUES (0.5);`)
tk.MustExec(`INSERT IGNORE INTO t1 VALUES (NULL);`)
tk.MustQuery(`SELECT t0.c0, t1.c0 FROM t0 INNER JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).
Check(testkit.Rows())
tk.MustQuery(`explain format=brief SELECT t0.c0, t1.c0 FROM t0 INNER JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).
Check(testkit.Rows("HashJoin 100000.00 root CARTESIAN inner join",
"├─TableReader(Build) 10.00 root data:Selection",
"│ └─Selection 10.00 cop[tikv] not(istrue_with_null(test.t0.c0))",
"│ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
"└─TableReader(Probe) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"))
tk.MustQuery(`SELECT t0.c0, t1.c0 FROM t0 LEFT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)))
INTERSECT
SELECT t0.c0, t1.c0 FROM t0 RIGHT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).Check(testkit.Rows())
tk.MustQuery(`explain format=brief SELECT t0.c0, t1.c0 FROM t0 LEFT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)))
INTERSECT
SELECT t0.c0, t1.c0 FROM t0 RIGHT JOIN t1 ON true WHERE (NOT (CAST(t0.c0 AS DATETIME)));`).Check(testkit.Rows(
"HashJoin 6400.00 root semi join, left side:HashAgg, equal:[nulleq(test.t0.c0, test.t0.c0) nulleq(test.t1.c0, test.t1.c0)]",
"├─HashJoin(Build) 100000.00 root CARTESIAN inner join",
"│ ├─TableReader(Build) 10.00 root data:Selection",
"│ │ └─Selection 10.00 cop[tikv] not(istrue_with_null(test.t0.c0))",
"│ │ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
"│ └─TableReader(Probe) 10000.00 root data:TableFullScan",
"│ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
"└─HashAgg(Probe) 8000.00 root group by:test.t0.c0, test.t1.c0, funcs:firstrow(test.t0.c0)->test.t0.c0, funcs:firstrow(test.t1.c0)->test.t1.c0",
" └─HashJoin 100000.00 root CARTESIAN left outer join, left side:TableReader",
" ├─TableReader(Build) 10.00 root data:Selection",
" │ └─Selection 10.00 cop[tikv] not(istrue_with_null(test.t0.c0))",
" │ └─TableFullScan 10000.00 cop[tikv] table:t0 keep order:false, stats:pseudo",
" └─TableReader(Probe) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"))
}