move pruning of NO_NULL_INDEX to deriveStats

YangKeao · YangKeao · commit 68aae641fb93 · 2025-05-15T13:09:10.000+08:00
Signed-off-by: Yang Keao &lt;yangkeao@chunibyo.icu&gt;
diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go
@@ -1163,37 +1163,7 @@ func skylinePruning(ds *logicalop.DataSource, prop *property.PhysicalProperty) [
 			candidates = append(candidates, getIndexMergeCandidate(ds, path, prop))
 			continue
 		}
-		// If the index contains NO_NULL_INDEX column, it may not contain all rows
-		if path.Index != nil && len(path.Index.NoNullIdxColOffsets) > 0 {
-			includeNullRange := false
-
-			for _, col := range path.Index.NoNullIdxColOffsets {
-				for _, ran := range path.Ranges {
-					if len(ran.LowVal) <= col || len(ran.HighVal) <= col ||
-						ran.LowVal[col].IsNull() || ran.HighVal[col].IsNull() {
-						includeNullRange = true
-						break
-					}
-				}
-
-				if includeNullRange {
-					// If the condition can make sure the column is not null, we can also use this index.
-					// TODO: maybe only considering the conditions except accessConds is enough, because the ranges
-					// don't contain NULL.
-					if ranger.CheckColumnIsNotNullWithCNFConditions(ds.SCtx(), path.FullIdxCols[col], ds.AllConds) {
-						includeNullRange = false
-						continue
-					}
 
-					break
-				}
-			}
-
-			// conditions that the column is not null
-			if includeNullRange {
-				continue
-			}
-		}
 		// if we already know the range of the scan is empty, just return a TableDual
 		if len(path.Ranges) == 0 {
 			return []*candidatePath{{path: path}}
diff --git a/pkg/planner/core/stats.go b/pkg/planner/core/stats.go
@@ -123,6 +123,7 @@ func deriveStats4DataSource(lp base.LogicalPlan, colGroups [][]*expression.Colum
 		ds.SetStats(ds.TableStats.Scale(selectivity))
 		return ds.StatsInfo(), nil
 	}
+
 	if ds.SCtx().GetSessionVars().StmtCtx.EnableOptimizerDebugTrace {
 		debugtrace.EnterContextCommon(ds.SCtx())
 		defer debugtrace.LeaveContextCommon(ds.SCtx())
@@ -156,6 +157,8 @@ func deriveStats4DataSource(lp base.LogicalPlan, colGroups [][]*expression.Colum
 		return nil, err
 	}
 
+	removeInvalidPathsForDataSource(ds)
+
 	if ds.SCtx().GetSessionVars().StmtCtx.EnableOptimizerDebugTrace {
 		debugTraceAccessPaths(ds.SCtx(), ds.PossibleAccessPaths)
 	}
@@ -168,6 +171,73 @@ func deriveStats4DataSource(lp base.LogicalPlan, colGroups [][]*expression.Colum
 	return ds.StatsInfo(), nil
 }
 
+// checkNoNullIndexForPath checks whether a path is valid considering `NO_NULL_INDEX`
+func checkNoNullIndexForPath(ds *logicalop.DataSource, path *util.AccessPath, conditions []expression.Expression) (valid bool) {
+	includeNullRange := false
+
+	for _, col := range path.Index.NoNullIdxColOffsets {
+		for _, ran := range path.Ranges {
+			if len(ran.LowVal) <= col || len(ran.HighVal) <= col ||
+				ran.LowVal[col].IsNull() || ran.HighVal[col].IsNull() {
+				includeNullRange = true
+				break
+			}
+		}
+
+		if includeNullRange {
+			// If the condition can make sure the column is not null, we can also use this index.
+			// TODO: maybe only considering the conditions except accessConds is enough, because the ranges
+			// do contain NULL.
+			if ranger.CheckColumnIsNotNullWithCNFConditions(ds.SCtx(), path.FullIdxCols[col], conditions) {
+				includeNullRange = false
+				continue
+			}
+
+			break
+		}
+	}
+
+	return !includeNullRange
+}
+
+// removeInvalidPaths will remove invalid paths from PossibleAccessPaths.
+// Some paths are not available because they may use index which contains NO_NULL_INDEX column,
+// but cannot make sure the column is not null.
+func removeInvalidPathsForDataSource(ds *logicalop.DataSource) {
+	for i := len(ds.PossibleAccessPaths) - 1; i >= 0; i-- {
+		path := ds.PossibleAccessPaths[i]
+
+		if path.IsTablePath() {
+			continue
+		}
+
+		// If the index contains NO_NULL_INDEX column, it may not contain all rows
+		if path.Index != nil && len(path.Index.NoNullIdxColOffsets) > 0 {
+			if !checkNoNullIndexForPath(ds, path, ds.AllConds) {
+				ds.PossibleAccessPaths = slices.Delete(ds.PossibleAccessPaths, i, i+1)
+				continue
+			}
+		}
+
+		if path.PartialIndexPaths != nil {
+			// consider whether this path is valid for the `NO_NULL_INDEX` column.
+			isValid := true
+			for _, partialPath := range path.PartialIndexPaths {
+				if partialPath.Index != nil && len(partialPath.Index.NoNullIdxColOffsets) > 0 {
+					if !checkNoNullIndexForPath(ds, partialPath, append(partialPath.IndexFilters, path.TableFilters...)) {
+						isValid = false
+						break
+					}
+				}
+			}
+			if !isValid {
+				ds.PossibleAccessPaths = slices.Delete(ds.PossibleAccessPaths, i, i+1)
+				continue
+			}
+		}
+	}
+}
+
 func fillIndexPath(ds *logicalop.DataSource, path *util.AccessPath, conds []expression.Expression) error {
 	if ds.SCtx().GetSessionVars().StmtCtx.EnableOptimizerDebugTrace {
 		debugtrace.EnterContextCommon(ds.SCtx())
diff --git a/tests/integrationtest/r/planner/core/integration.result b/tests/integrationtest/r/planner/core/integration.result
@@ -4391,3 +4391,42 @@ IndexLookUp	9980.01	root
 select * from t use index (idx3) where col1 is not null and col3 is not null;
 id	col1	col2	col3
 3	1	NULL	1
+drop table if exists t;
+create table t (id int primary key, col1 int no_null_index, col2 int, col3 int);
+alter table t add index idx2(col1, col2);
+alter table t add index idx3(col1, col3);
+insert into t values (2, null, 1, 1);
+select * FROM t WHERE col1 is null and col2 < 100 and col3 < 100;
+id	col1	col2	col3
+2	NULL	1	1
+select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and col2 < 100 and col3 < 100;
+Error 1815 (HY000): Internal : Can't find a proper physical plan for this query
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and col2 < 100 and col3 < 100;
+Error 1815 (HY000): Internal : Can't find a proper physical plan for this query
+drop table if exists t;
+create table t (id int primary key, col1 int no_null_index, col2 int no_null_index);
+alter table t add index idx1(col1);
+alter table t add index idx2(col2);
+insert into t values (1, 1, NULL);
+insert into t values (2, NULL, 1);
+select /*+ USE_INDEX_MERGE(t, idx1, idx2) */ * FROM t WHERE col1 > 5 or col2 > 5;
+id	col1	col2
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx1, idx2) */ * FROM t WHERE col1 > 5 or col2 > 5;
+id	estRows	task	access object	operator info
+IndexMerge	5555.56	root		type: union
+├─IndexRangeScan(Build)	3333.33	cop[tikv]	table:t, index:idx1(col1)	range:(5,+inf], keep order:false, stats:pseudo
+├─IndexRangeScan(Build)	3333.33	cop[tikv]	table:t, index:idx2(col2)	range:(5,+inf], keep order:false, stats:pseudo
+└─TableRowIDScan(Probe)	5555.56	cop[tikv]	table:t	keep order:false, stats:pseudo
+drop table if exists t;
+create table t (id int primary key, col1 int no_null_index, col2 json, col3 json);
+alter table t add index idx2(col1, (CAST(col2->'$.path' AS SIGNED ARRAY)) );
+alter table t add index idx3(col1, (CAST(col3->'$.path' AS SIGNED ARRAY)) );
+insert into t values (2, null, '{"path":[1]}', '{"path":[1]}');
+select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
+id	col1	col2	col3
+2	NULL	{"path": [1]}	{"path": [1]}
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
+id	estRows	task	access object	operator info
+TableReader	8.00	root		data:Selection
+└─Selection	8.00	cop[tikv]		isnull(planner__core__integration.t.col1), json_contains(json_extract(planner__core__integration.t.col2, "$.path"), cast("1", json BINARY)), json_contains(json_extract(planner__core__integration.t.col3, "$.path"), cast("1", json BINARY))
+  └─TableFullScan	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
diff --git a/tests/integrationtest/t/planner/core/integration.test b/tests/integrationtest/t/planner/core/integration.test
@@ -2407,3 +2407,32 @@ explain format='brief' select * from t use index (idx1) where col1 > 0 and col3
 select * from t use index (idx2) where col1 is not null and col3 is not null;
 explain format='brief' select * from t use index (idx2) where col1 is not null and col3 is not null;
 select * from t use index (idx3) where col1 is not null and col3 is not null;
+
+# TestNoNullIndexWithIndexMerge
+drop table if exists t;
+create table t (id int primary key, col1 int no_null_index, col2 int, col3 int);
+alter table t add index idx2(col1, col2);
+alter table t add index idx3(col1, col3);
+insert into t values (2, null, 1, 1);
+select * FROM t WHERE col1 is null and col2 < 100 and col3 < 100;
+-- error 1815
+select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and col2 < 100 and col3 < 100;
+-- error 1815
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and col2 < 100 and col3 < 100;
+
+drop table if exists t;
+create table t (id int primary key, col1 int no_null_index, col2 int no_null_index);
+alter table t add index idx1(col1);
+alter table t add index idx2(col2);
+insert into t values (1, 1, NULL);
+insert into t values (2, NULL, 1);
+select /*+ USE_INDEX_MERGE(t, idx1, idx2) */ * FROM t WHERE col1 > 5 or col2 > 5;
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx1, idx2) */ * FROM t WHERE col1 > 5 or col2 > 5;
+
+drop table if exists t;
+create table t (id int primary key, col1 int no_null_index, col2 json, col3 json);
+alter table t add index idx2(col1, (CAST(col2->'$.path' AS SIGNED ARRAY)) );
+alter table t add index idx3(col1, (CAST(col3->'$.path' AS SIGNED ARRAY)) );
+insert into t values (2, null, '{"path":[1]}', '{"path":[1]}');
+select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
