fix the issue that IndexMerge OR Path may generate invalid plan

Signed-off-by: Yang Keao <[email protected]>

Author: YangKeao

pkg/planner/core/stats.go

@@ -200,7 +200,7 @@ func checkNoNullIndexForPath(ds *logicalop.DataSource, path *util.AccessPath, co
 	return !includeNullRange
 }
 
-// removeInvalidPaths will remove invalid paths from PossibleAccessPaths.
+// removeInvalidPathsForDataSource will remove invalid paths from PossibleAccessPaths.
 // Some paths are not available because they may use index which contains NO_NULL_INDEX column,
 // but cannot make sure the column is not null.
 func removeInvalidPathsForDataSource(ds *logicalop.DataSource) {
@@ -223,19 +223,56 @@ func removeInvalidPathsForDataSource(ds *logicalop.DataSource) {
 			// consider whether this path is valid for the `NO_NULL_INDEX` column.
 			isValid := true
 			for _, partialPath := range path.PartialIndexPaths {
-				if partialPath.Index != nil && len(partialPath.Index.NoNullIdxColOffsets) > 0 {
-					if !checkNoNullIndexForPath(ds, partialPath, append(partialPath.IndexFilters, path.TableFilters...)) {
-						isValid = false
-						break
+				if !checkPartialPathValid(ds, partialPath, path.TableFilters) {
+					isValid = false
+					break
+				}
+			}
+			if !isValid {
+				ds.PossibleAccessPaths = slices.Delete(ds.PossibleAccessPaths, i, i+1)
+				continue
+			}
+		}
+
+		if len(path.PartialAlternativeIndexPaths) > 0 {
+			isValid := true
+			// For each branch, remove the invalid path.
+			for i := 0; i < len(path.PartialAlternativeIndexPaths); i++ {
+				partialPaths := path.PartialAlternativeIndexPaths[i]
+				for j := len(partialPaths) - 1; j >= 0; j-- {
+					partialPath := partialPaths[j]
+					if !checkPartialPathValid(ds, partialPath, path.TableFilters) {
+						partialPaths = slices.Delete(partialPaths, j, j+1)
 					}
 				}
+
+				// If all paths are invalid, remove the whole possibleAccessPath.
+				if len(partialPaths) == 0 {
+					isValid = false
+					break
+				}
+
+				// If there are still valid paths, we need to update the PartialAlternativeIndexPaths.
+				path.PartialAlternativeIndexPaths[i] = partialPaths
 			}
 			if !isValid {
 				ds.PossibleAccessPaths = slices.Delete(ds.PossibleAccessPaths, i, i+1)
 				continue
 			}
 		}
+
+	}
+}
+
+// checkPartialPathValid checks whether a partial path of index merge is valid
+func checkPartialPathValid(ds *logicalop.DataSource, partialPath *util.AccessPath, tableFilters []expression.Expression) bool {
+	if partialPath.Index != nil && len(partialPath.Index.NoNullIdxColOffsets) > 0 {
+		if !checkNoNullIndexForPath(ds, partialPath, append(partialPath.IndexFilters, tableFilters...)) {
+			return false
+		}
 	}
+
+	return true
 }
 
 func fillIndexPath(ds *logicalop.DataSource, path *util.AccessPath, conds []expression.Expression) error {

tests/integrationtest/r/planner/core/integration.result

@@ -4430,3 +4430,34 @@ id	estRows	task	access object	operator info
 TableReader	8.00	root		data:Selection
 └─Selection	8.00	cop[tikv]		isnull(planner__core__integration.t.col1), json_contains(json_extract(planner__core__integration.t.col2, "$.path"), cast("1", json BINARY)), json_contains(json_extract(planner__core__integration.t.col3, "$.path"), cast("1", json BINARY))
   └─TableFullScan	10000.00	cop[tikv]	table:t	keep order:false, stats:pseudo
+drop table if exists t;
+create table t (a int, b int, c int no_null_index, key a(a), key b(b), key ac(a, c), key bc(b, c));
+insert into t values (NULL, 1, 1);
+insert into t values (1, NULL, 1);
+insert into t values (1, 1, NULL);
+select * from t where a is null or b is null order by c;
+a	b	c
+NULL	1	1
+1	NULL	1
+explain format='verbose' select * from t where a is null or b is null order by c;
+id	estRows	estCost	task	access object	operator info
+Sort_5	19.99	5431.39	root		planner__core__integration.t.c
+└─IndexMerge_14	19.99	1019.04	root		type: union
+  ├─IndexRangeScan_11(Build)	10.00	2035.00	cop[tikv]	table:t, index:a(a)	range:[NULL,NULL], keep order:false, stats:pseudo
+  ├─IndexRangeScan_12(Build)	10.00	2035.00	cop[tikv]	table:t, index:b(b)	range:[NULL,NULL], keep order:false, stats:pseudo
+  └─TableRowIDScan_13(Probe)	19.99	4881.56	cop[tikv]	table:t	keep order:false, stats:pseudo
+drop table if exists t;
+create table t (a int no_null_index, b int, c int, key a(a), key b(b), key ac(a, c), key bc(b, c));
+insert into t values (NULL, 1, 1);
+insert into t values (1, NULL, 1);
+insert into t values (1, 1, NULL);
+select * from t where a is null or b is null order by c;
+a	b	c
+NULL	1	1
+1	NULL	1
+explain format='verbose' select * from t where a is null or b is null order by c;
+id	estRows	estCost	task	access object	operator info
+Sort_5	19.99	200732.32	root		planner__core__integration.t.c
+└─TableReader_10	19.99	196319.98	root		data:Selection_9
+  └─Selection_9	19.99	2941000.00	cop[tikv]		or(isnull(planner__core__integration.t.a), isnull(planner__core__integration.t.b))
+    └─TableFullScan_8	10000.00	2442000.00	cop[tikv]	table:t	keep order:false, stats:pseudo

tests/integrationtest/t/planner/core/integration.test

@@ -2435,4 +2435,20 @@ alter table t add index idx2(col1, (CAST(col2->'$.path' AS SIGNED ARRAY)) );
 alter table t add index idx3(col1, (CAST(col3->'$.path' AS SIGNED ARRAY)) );
 insert into t values (2, null, '{"path":[1]}', '{"path":[1]}');
 select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
-explain format='brief' select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
+explain format='brief' select /*+ USE_INDEX_MERGE(t, idx2, idx3) */ * FROM t WHERE col1 is null and json_contains(col2->'$.path', '1') and json_contains(col3->'$.path', '1');
+
+drop table if exists t;
+create table t (a int, b int, c int no_null_index, key a(a), key b(b), key ac(a, c), key bc(b, c));
+insert into t values (NULL, 1, 1);
+insert into t values (1, NULL, 1);
+insert into t values (1, 1, NULL);
+select * from t where a is null or b is null order by c;
+explain format='verbose' select * from t where a is null or b is null order by c;
+
+drop table if exists t;
+create table t (a int no_null_index, b int, c int, key a(a), key b(b), key ac(a, c), key bc(b, c));
+insert into t values (NULL, 1, 1);
+insert into t values (1, NULL, 1);
+insert into t values (1, 1, NULL);
+select * from t where a is null or b is null order by c;
+explain format='verbose' select * from t where a is null or b is null order by c;