fix stats for block filters (#15532)

fix stats for block filters

Approved by: @ouyuanning, @sukki37

Author: badboynt1

pkg/sql/plan/stats.go

@@ -36,6 +36,7 @@ import (
 const DefaultBlockMaxRows = 8192
 const BlockNumForceOneCN = 200
 const blockSelectivityThreshHold = 0.95
+const blockNDVThreshHold = 100
 const highNDVcolumnThreshHold = 0.95
 
 // stats cache is small, no need to use LRU for now
@@ -544,10 +545,13 @@ func estimateFilterWeight(expr *plan.Expr, w float64) float64 {
 }
 
 // harsh estimate of block selectivity, will improve it in the future
-func estimateFilterBlockSelectivity(ctx context.Context, expr *plan.Expr, tableDef *plan.TableDef, tableCnt float64) float64 {
+func estimateFilterBlockSelectivity(ctx context.Context, expr *plan.Expr, tableDef *plan.TableDef, builder *QueryBuilder) float64 {
 	if !ExprIsZonemappable(ctx, expr) {
 		return 1
 	}
+	if expr.Selectivity < 0.01 {
+		return expr.Selectivity * 90
+	}
 	col := extractColRefInFilter(expr)
 	if col != nil {
 		switch GetSortOrder(tableDef, col.Name) {
@@ -559,11 +563,11 @@ func estimateFilterBlockSelectivity(ctx context.Context, expr *plan.Expr, tableD
 			return math.Min(expr.Selectivity*10, 0.9)
 		}
 	}
-	if tableCnt > 10000 {
-		return 0.9
-	} else {
+	if getExprNdv(expr, builder) < blockNDVThreshHold {
 		return 1
 	}
+	// do not know selectivity for this expr, default 0.5
+	return 0.5
 }
 
 func rewriteFilterListByStats(ctx context.Context, nodeID int32, builder *QueryBuilder) {
@@ -993,7 +997,7 @@ func calcScanStats(node *plan.Node, builder *QueryBuilder) *plan.Stats {
 	var blockExprList []*plan.Expr
 	for i := range node.FilterList {
 		node.FilterList[i].Selectivity = estimateExprSelectivity(node.FilterList[i], builder)
-		currentBlockSel := estimateFilterBlockSelectivity(builder.GetContext(), node.FilterList[i], node.TableDef, stats.TableCnt)
+		currentBlockSel := estimateFilterBlockSelectivity(builder.GetContext(), node.FilterList[i], node.TableDef, builder)
 		if currentBlockSel < blockSelectivityThreshHold {
 			copyOfExpr := DeepCopyExpr(node.FilterList[i])
 			copyOfExpr.Selectivity = currentBlockSel