cardinality: tighten full-range fast path for index estimation

Two fixes raised in the release-8.5 cherry-pick review (PR #66695) that
also apply on master:

1. Move canSkipIndexEstimation ahead of IndexStatsIsInvalid so the
   full-range fast path short-circuits before IndexStatsIsInvalid
   queues an unnecessary async histogram load when the index stats are
   not fully loaded. Guarded by idx != nil so the pseudo fallback is
   preserved.

2. Reject ranges with LowExclude or HighExclude in
   isFullRangeIncludingNulls. A range like (NULL, +inf) drops the
   NULL endpoint and shrinks the result, so it must not trigger the
   fast path.

Tighten TestCanSkipIndexEstimation to actually exercise these cases:
model 10 NULL rows in the index histogram (NullCount=10) so the
not-null estimate is strictly below RealtimeCount, switch the
assertion to require.Less, and add a case covering (NULL, +inf) with
an exclusive lower bound.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: terry1purcell

pkg/planner/cardinality/row_count_index.go

@@ -43,6 +43,14 @@ func GetRowCountByIndexRanges(sctx planctx.PlanContext, coll *statistics.HistCol
 	sc := sctx.GetSessionVars().StmtCtx
 	idx := coll.GetIdx(idxID)
 	recordUsedItemStatsStatus(sctx, idx, coll.PhysicalID, idxID)
+	// Fast-path: a full-range scan over a non-MV, non-partial index returns exactly
+	// RealtimeCount regardless of histogram availability, so we can short-circuit
+	// before IndexStatsIsInvalid — which would otherwise queue an unnecessary async
+	// histogram load whenever the index stats are not fully loaded.
+	if idx != nil && canSkipIndexEstimation(idx, indexRanges) {
+		realtimeCnt, _ := coll.GetScaledRealtimeAndModifyCnt(idx)
+		return statistics.DefaultRowEst(float64(realtimeCnt)), nil
+	}
 	if statistics.IndexStatsIsInvalid(sctx, idx, coll, idxID) {
 		if hasColumnStats(sctx, coll, idxCols) && !ranger.HasFullRange(indexRanges, false) {
 			count, maxCount, err = getPseudoRowCountWithPartialStats(sctx, coll, indexRanges, float64(coll.RealtimeCount), idxCols)
@@ -58,9 +66,6 @@ func GetRowCountByIndexRanges(sctx planctx.PlanContext, coll *statistics.HistCol
 		return result, err
 	}
 	realtimeCnt, modifyCount := coll.GetScaledRealtimeAndModifyCnt(idx)
-	if canSkipIndexEstimation(idx, indexRanges) {
-		return statistics.DefaultRowEst(float64(realtimeCnt)), nil
-	}
 	if idx.CMSketch != nil && idx.StatsVer == statistics.Version1 {
 		count, err = getIndexRowCountForStatsV1(sctx, coll, idxID, indexRanges)
 		result = statistics.DefaultRowEst(count)
@@ -608,12 +613,18 @@ func canSkipIndexEstimation(idx *statistics.Index, indexRanges []*ranger.Range)
 }
 
 // isFullRangeIncludingNulls checks if a single range covers all values including NULLs.
-// Unlike ranger.IsFullRange, this requires the low bound to be NULL (KindNull),
-// not KindMinNotNull, ensuring NULL rows are included in the count.
+// Unlike ranger.IsFullRange, this requires the low bound to be NULL (KindNull) inclusive,
+// not KindMinNotNull and not an exclusive lower bound, so NULL rows are guaranteed to be
+// included in the count.
 func isFullRangeIncludingNulls(ran *ranger.Range) bool {
 	if len(ran.LowVal) != len(ran.HighVal) || len(ran.LowVal) == 0 {
 		return false
 	}
+	// An exclusive bound on NULL (low) or +inf (high) would drop those endpoints
+	// and shrink the range, so the fast path must not apply.
+	if ran.LowExclude || ran.HighExclude {
+		return false
+	}
 	for i := range ran.LowVal {
 		if ran.LowVal[i].Kind() != types.KindNull {
 			return false

pkg/planner/cardinality/selectivity_test.go

@@ -549,29 +549,39 @@ func TestCanSkipIndexEstimation(t *testing.T) {
 	tblInfo := tb.Meta()
 
 	// Use mock stats so Idx2ColUniqueIDs is populated (required by getIndexRowCountForStatsV2).
-	realtimeCount := int64(51) // 50 rows + 1 NULL
+	// 50 distinct non-NULL values + 10 NULL rows, so the not-null range produces a
+	// strictly smaller estimate than RealtimeCount and catches a buggy fast-path that
+	// returns RealtimeCount for [MinNotNull,+inf).
+	const nonNullCount = 50
+	const nullCount = 10
+	realtimeCount := int64(nonNullCount + nullCount)
 	statsTbl := mockStatsTable(tblInfo, realtimeCount)
-	colValues, err := generateIntDatum(1, 51)
+	colValues, err := generateIntDatum(1, nonNullCount)
 	require.NoError(t, err)
 	for i := 1; i <= 2; i++ {
+		colHist := mockStatsHistogram(int64(i), colValues, 1, types.NewFieldType(mysql.TypeLonglong))
+		colHist.NullCount = nullCount
 		statsTbl.SetCol(int64(i), &statistics.Column{
-			Histogram:         *mockStatsHistogram(int64(i), colValues, 1, types.NewFieldType(mysql.TypeLonglong)),
+			Histogram:         *colHist,
 			Info:              tblInfo.Columns[i-1],
 			StatsLoadedStatus: statistics.NewStatsFullLoadStatus(),
 			StatsVer:          2,
 		})
 	}
 	// Index histogram must store encoded key bytes (same as getIndexRowCountForStatsV2 uses for l/r).
-	idxValues := make([]types.Datum, 51)
-	for i := 0; i < 51; i++ {
+	idxValues := make([]types.Datum, nonNullCount)
+	for i := range idxValues {
 		enc, err := codec.EncodeKey(time.UTC, nil, types.NewIntDatum(int64(i)))
 		require.NoError(t, err)
 		idxValues[i].SetBytes(enc)
 	}
+	idxHist := mockStatsHistogram(tblInfo.Indices[0].ID, idxValues, 1, types.NewFieldType(mysql.TypeBlob))
+	idxHist.NullCount = nullCount
 	statsTbl.SetIdx(tblInfo.Indices[0].ID, &statistics.Index{
-		Histogram: *mockStatsHistogram(tblInfo.Indices[0].ID, idxValues, 1, types.NewFieldType(mysql.TypeBlob)),
-		Info:      tblInfo.Indices[0],
-		StatsVer:  2,
+		Histogram:         *idxHist,
+		Info:              tblInfo.Indices[0],
+		StatsLoadedStatus: statistics.NewStatsFullLoadStatus(),
+		StatsVer:          2,
 	})
 	generateMapsForMockStatsTbl(statsTbl)
 
@@ -586,20 +596,36 @@ func TestCanSkipIndexEstimation(t *testing.T) {
 	require.Equal(t, float64(realtimeCount), countResult.Est,
 		"full range [NULL,+inf) should use fast path and return RealtimeCount")
 
-	// Full range excluding NULLs [MinNotNull, +inf) should NOT use fast path.
-	// It goes through histogram estimation.
+	// Full range excluding NULLs [MinNotNull, +inf) must NOT use the fast path.
+	// With nullCount > 0, the histogram estimate must be strictly below RealtimeCount;
+	// equality would mean the fast path was wrongly taken.
 	fullNotNullRanges := ranger.FullNotNullRange()
 	countResult2, err := cardinality.GetRowCountByIndexRanges(sctx, &statsTbl.HistColl, idxID, fullNotNullRanges, nil)
 	require.NoError(t, err)
-	require.LessOrEqual(t, countResult2.Est, float64(realtimeCount),
-		"full not-null range excludes NULLs, estimate should be <= RealtimeCount")
+	require.Less(t, countResult2.Est, float64(realtimeCount),
+		"full not-null range excludes %d NULL row(s), estimate must be < RealtimeCount", nullCount)
 
 	// Bounded range should NOT use fast path.
 	boundedRanges := getRange(1, 10)
 	countResult3, err := cardinality.GetRowCountByIndexRanges(sctx, &statsTbl.HistColl, idxID, boundedRanges, nil)
 	require.NoError(t, err)
 	require.Less(t, countResult3.Est, float64(realtimeCount),
 		"bounded range should use histogram estimation, not fast path")
+
+	// (NULL, +inf) with an exclusive lower bound drops the NULL endpoint, so the
+	// fast path must not apply — otherwise the NULL row would be silently counted.
+	var nullDatum types.Datum
+	nullDatum.SetNull()
+	exclusiveNullRanges := []*ranger.Range{{
+		LowVal:     []types.Datum{nullDatum},
+		HighVal:    []types.Datum{types.MaxValueDatum()},
+		LowExclude: true,
+		Collators:  collate.GetBinaryCollatorSlice(1),
+	}}
+	countResult4, err := cardinality.GetRowCountByIndexRanges(sctx, &statsTbl.HistColl, idxID, exclusiveNullRanges, nil)
+	require.NoError(t, err)
+	require.Less(t, countResult4.Est, float64(realtimeCount),
+		"exclusive lower bound on NULL must drop the NULL row, estimate must be < RealtimeCount")
 }
 
 func TestEstimationForUnknownValuesAfterModify(t *testing.T) {