executor: use analyze NDV rate option

Author: 0xPoe

pkg/executor/analyze_utils_test.go

@@ -17,8 +17,10 @@ package executor
 import (
 	"context"
 	"fmt"
+	"math"
 	"testing"
 
+	"github.com/pingcap/tidb/pkg/parser/ast"
 	"github.com/pingcap/tidb/pkg/planner/core"
 	"github.com/pingcap/tidb/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/types"
@@ -72,6 +74,20 @@ func BuildExecutorForTest(ctx context.Context, stmt *ExecStmt) error {
 }
 
 func TestEstimateSamplingNDV(t *testing.T) {
+	t.Run("configured NDV sample rate", func(t *testing.T) {
+		require.Equal(t, float64(statistics.DefaultNDVSampleRate), configuredAnalyzeNDVSampleRate(nil))
+		require.Equal(t, 0.25, configuredAnalyzeNDVSampleRate(map[ast.AnalyzeOptionType]uint64{
+			ast.AnalyzeOptNDVRate: math.Float64bits(0.25),
+		}))
+	})
+
+	t.Run("effective NDV sample rate clamp", func(t *testing.T) {
+		// configured >= rowSampleRate: pass through.
+		require.Equal(t, 0.25, effectiveAnalyzeNDVSampleRate(0.25, 0.1))
+		// configured < rowSampleRate: clamp up.
+		require.Equal(t, 0.5, effectiveAnalyzeNDVSampleRate(0.25, 0.5))
+	})
+
 	rootCollector := statistics.NewReservoirRowSampleCollector(1, 2)
 	rootCollector.Count = 100
 	rootCollector.FMSketches = append(rootCollector.FMSketches,

pkg/executor/builder.go

@@ -3169,7 +3169,6 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown(
 		modifyCount = int64(val.(int))
 	})
 	sampleRate := new(float64)
-	ndvRate := float64(statistics.DefaultNDVSampleRate)
 	var sampleRateReason string
 	if opts[ast.AnalyzeOptNumSamples] == 0 {
 		*sampleRate = math.Float64frombits(opts[ast.AnalyzeOptSampleRate])
@@ -3195,10 +3194,30 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown(
 			}
 		}
 	}
-	// NDV estimation needs at least as many rows as the row sample to remain statistically
-	// useful; align ndvRate up to sampleRate when a smaller value was configured.
-	if ndvRate < *sampleRate {
-		ndvRate = *sampleRate
+	configuredNDVRate := configuredAnalyzeNDVSampleRate(opts)
+	ndvRate := effectiveAnalyzeNDVSampleRate(configuredNDVRate, *sampleRate)
+	if ndvRate > configuredNDVRate {
+		// handleAnalyzeOptions already rejects the case where both samplerate and
+		// ndvrate are explicit and ndvrate is smaller, so this only fires when
+		// samplerate was auto-adjusted above the user-configured ndvrate.
+		if task.PartitionName != "" {
+			sc.AppendNote(errors.NewNoStackErrorf(
+				`Analyze raised NDV sample rate from %f to %f to match the auto-adjusted row sample rate for table %s.%s's partition %s`,
+				configuredNDVRate,
+				ndvRate,
+				task.DBName,
+				task.TableName,
+				task.PartitionName,
+			))
+		} else {
+			sc.AppendNote(errors.NewNoStackErrorf(
+				`Analyze raised NDV sample rate from %f to %f to match the auto-adjusted row sample rate for table %s.%s`,
+				configuredNDVRate,
+				ndvRate,
+				task.DBName,
+				task.TableName,
+			))
+		}
 	}
 	job := &statistics.AnalyzeJob{
 		DBName:           task.DBName,
@@ -3250,6 +3269,28 @@ func (b *executorBuilder) buildAnalyzeSamplingPushdown(
 	return &analyzeTask{taskType: colTask, colExec: e, job: job}
 }
 
+// configuredAnalyzeNDVSampleRate returns the NDV sample rate explicitly set in
+// the analyze options, or DefaultNDVSampleRate when the option is absent.
+func configuredAnalyzeNDVSampleRate(opts map[ast.AnalyzeOptionType]uint64) float64 {
+	if val, ok := opts[ast.AnalyzeOptNDVRate]; ok {
+		return math.Float64frombits(val)
+	}
+	return float64(statistics.DefaultNDVSampleRate)
+}
+
+// effectiveAnalyzeNDVSampleRate raises the configured NDV sample rate up to
+// rowSampleRate. TiKV builds singleton sketches at ndvrate first and then
+// draws the row sample from that stream at samplerate, so ndvrate < samplerate
+// is incoherent — the row sample cannot exceed the population it is drawn
+// from. The usual trigger is a small table whose samplerate gets auto-adjusted
+// up to 1.0 (full scan) above a smaller configured ndvrate.
+func effectiveAnalyzeNDVSampleRate(configured, rowSampleRate float64) float64 {
+	if configured < rowSampleRate {
+		return rowSampleRate
+	}
+	return configured
+}
+
 // getAdjustedSampleRate calculate the sample rate by the table size. If we cannot get the table size. We use the 0.001 as the default sample rate.
 // From the paper "Random sampling for histogram construction: how much is enough?"'s Corollary 1 to Theorem 5,
 // for a table size n, histogram size k, maximum relative error in bin size f, and error probability gamma,

pkg/executor/test/analyzetest/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
         "main_test.go",
     ],
     flaky = True,
-    shard_count = 38,
+    shard_count = 40,
     deps = [
         "//pkg/config",
         "//pkg/config/kerneltype",

pkg/executor/test/analyzetest/analyze_test.go

@@ -455,6 +455,83 @@ func TestSnapshotAnalyzeAndMaxTSAnalyze(t *testing.T) {
 	}
 }
 
+func TestAnalyzeUsesNDVRateOption(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+
+	tk.MustExec("use test")
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("create table t_ndv_rate(a int, b int, index idx_b(b))")
+	// 200 rows: col a is fully unique; col b has 10 distinct values, each repeated 20 times.
+	values := make([]string, 0, 200)
+	for i := 1; i <= 200; i++ {
+		values = append(values, fmt.Sprintf("(%d,%d)", i, i%10))
+	}
+	tk.MustExec("insert into t_ndv_rate values " + strings.Join(values, ","))
+
+	tbl, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t_ndv_rate"))
+	require.NoError(t, err)
+	tblInfo := tbl.Meta()
+	readNDV := func(histID int64, isIndex int) int64 {
+		rows := tk.MustQuery(fmt.Sprintf(
+			"select distinct_count from mysql.stats_histograms where table_id = %d and hist_id = %d and is_index = %d",
+			tblInfo.ID,
+			histID,
+			isIndex,
+		)).Rows()
+		require.Len(t, rows, 1)
+		n, err := strconv.ParseInt(rows[0][0].(string), 10, 64)
+		require.NoError(t, err)
+		return n
+	}
+
+	t.Run("exact path with ndvrate=1", func(t *testing.T) {
+		tk.MustExec("analyze table t_ndv_rate with 1 samplerate, 1 ndvrate")
+		require.Equal(t, int64(200), readNDV(tblInfo.Columns[0].ID, 0))
+		require.Equal(t, int64(10), readNDV(tblInfo.Columns[1].ID, 0))
+		require.Equal(t, int64(10), readNDV(tblInfo.Indices[0].ID, 1))
+	})
+
+	t.Run("singleton-sketch path with ndvrate<1", func(t *testing.T) {
+		// Pin the unistore analyze RNG seed so the Bernoulli sketch sampling and the
+		// resulting GEE estimate are deterministic.
+		require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/store/mockstore/unistore/cophandler/mockAnalyzeSamplingSeed", "return(1)"))
+		defer func() {
+			require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/store/mockstore/unistore/cophandler/mockAnalyzeSamplingSeed"))
+		}()
+		tk.MustExec("analyze table t_ndv_rate with 0.5 samplerate, 0.5 ndvrate")
+		// True NDVs are 200/10/10. Column `a` lands at 141 (instead of 200) because
+		// GEE under-estimates NDV on fully-unique columns from a half-sample; this
+		// confirms the singleton-sketch path is exercised rather than the exact one.
+		require.Equal(t, int64(141), readNDV(tblInfo.Columns[0].ID, 0))
+		require.Equal(t, int64(10), readNDV(tblInfo.Columns[1].ID, 0))
+		require.Equal(t, int64(10), readNDV(tblInfo.Indices[0].ID, 1))
+	})
+}
+
+func TestAnalyzeRaisesNDVRateNote(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+	tk := testkit.NewTestKit(t, store)
+
+	tk.MustExec("use test")
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("create table t_ndv_note(a int)")
+	tk.MustExec("insert into t_ndv_note values (1),(2),(3)")
+
+	// Force stats_meta to a tiny count so getAdjustedSampleRate returns 1, which
+	// will exceed the user's configured ndvrate of 0.5 and trigger the clamp.
+	statsHandle := dom.StatsHandle()
+	tbl, err := dom.InfoSchema().TableByName(context.Background(), ast.NewCIStr("test"), ast.NewCIStr("t_ndv_note"))
+	require.NoError(t, err)
+	tk.MustExec(fmt.Sprintf("update mysql.stats_meta set count = 3 where table_id = %d", tbl.Meta().ID))
+	require.NoError(t, statsHandle.Update(context.Background(), dom.InfoSchema()))
+
+	tk.MustExec("analyze table t_ndv_note with 0.5 ndvrate")
+	tk.MustQuery("show warnings").CheckContain(
+		"Analyze raised NDV sample rate from 0.500000 to 1.000000 to match the auto-adjusted row sample rate for table test.t_ndv_note",
+	)
+}
+
 func TestAdjustSampleRateNote(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 	tk := testkit.NewTestKit(t, store)

pkg/planner/core/physical_plan_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/pingcap/tidb/pkg/planner/util"
 	"github.com/pingcap/tidb/pkg/planner/util/coretestsdk"
 	"github.com/pingcap/tidb/pkg/session"
+	"github.com/pingcap/tidb/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/store/mockstore"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/external"
@@ -61,10 +62,18 @@ func TestAnalyzeBuildSucc(t *testing.T) {
 			sql:  "analyze table t with 0.1 samplerate",
 			succ: true,
 		},
+		{
+			sql:  "analyze table t with 0.5 ndvrate",
+			succ: true,
+		},
 		{
 			sql:  "analyze table t with 10 samplerate",
 			succ: false,
 		},
+		{
+			sql:  "analyze table t with 2 ndvrate",
+			succ: false,
+		},
 		{
 			sql:  "analyze table t with 0.1 samplerate, 100000 samples",
 			succ: false,
@@ -101,20 +110,29 @@ func TestAnalyzeSetRate(t *testing.T) {
 	tk.MustExec("use test")
 	tk.MustExec("create table t(a int)")
 	tests := []struct {
-		sql  string
-		rate float64
+		sql     string
+		rate    float64
+		ndvRate float64
 	}{
 		{
-			sql:  "analyze table t",
-			rate: -1,
+			sql:     "analyze table t",
+			rate:    -1,
+			ndvRate: statistics.DefaultNDVSampleRate,
 		},
 		{
-			sql:  "analyze table t with 0.1 samplerate",
-			rate: 0.1,
+			sql:     "analyze table t with 0.1 samplerate",
+			rate:    0.1,
+			ndvRate: statistics.DefaultNDVSampleRate,
+		},
+		{
+			sql:     "analyze table t with 0.25 ndvrate",
+			rate:    -1,
+			ndvRate: 0.25,
 		},
 		{
-			sql:  "analyze table t with 10000 samples",
-			rate: -1,
+			sql:     "analyze table t with 10000 samples",
+			rate:    -1,
+			ndvRate: statistics.DefaultNDVSampleRate,
 		},
 	}
 
@@ -132,6 +150,7 @@ func TestAnalyzeSetRate(t *testing.T) {
 		require.NoError(t, err, comment)
 		ana := p.(*core.Analyze)
 		require.Equal(t, tt.rate, math.Float64frombits(ana.Opts[ast.AnalyzeOptSampleRate]))
+		require.Equal(t, tt.ndvRate, math.Float64frombits(ana.Opts[ast.AnalyzeOptNDVRate]))
 	}
 }
 

pkg/planner/core/planbuilder.go

@@ -3111,6 +3111,7 @@ var analyzeOptionLimit = map[ast.AnalyzeOptionType]uint64{
 	ast.AnalyzeOptCMSketchDepth: CMSketchSizeLimit,
 	ast.AnalyzeOptNumSamples:    5000000,
 	ast.AnalyzeOptSampleRate:    math.Float64bits(1),
+	ast.AnalyzeOptNDVRate:       math.Float64bits(statistics.DefaultNDVSampleRate),
 }
 
 // TopN reduced from 500 to 100 due to concerns over large number of TopN values collected for customers with many tables.
@@ -3122,6 +3123,7 @@ var analyzeOptionDefaultV2 = map[ast.AnalyzeOptionType]uint64{
 	ast.AnalyzeOptCMSketchDepth: 5,
 	ast.AnalyzeOptNumSamples:    0,
 	ast.AnalyzeOptSampleRate:    math.Float64bits(-1),
+	ast.AnalyzeOptNDVRate:       math.Float64bits(statistics.DefaultNDVSampleRate),
 }
 
 // GetAnalyzeOptionDefaultV2ForTest returns the default analyze options for test.
@@ -3133,7 +3135,7 @@ func GetAnalyzeOptionDefaultV2ForTest() map[ast.AnalyzeOptionType]uint64 {
 // explicitly specified in the statement.
 func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint64, error) {
 	optMap := make(map[ast.AnalyzeOptionType]uint64, len(analyzeOptionDefaultV2))
-	sampleNum, sampleRate := uint64(0), 0.0
+	sampleNum, sampleRate, ndvRate := uint64(0), 0.0, 0.0
 	for _, opt := range opts {
 		datumValue := opt.Value.(*driver.ValueExpr).Datum
 		switch opt.Type {
@@ -3143,7 +3145,7 @@ func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint
 				return nil, errors.Errorf("Value of analyze option %s should not be larger than %d", ast.AnalyzeOptionString[opt.Type], analyzeOptionLimit[opt.Type])
 			}
 			optMap[opt.Type] = v
-		case ast.AnalyzeOptSampleRate:
+		case ast.AnalyzeOptSampleRate, ast.AnalyzeOptNDVRate:
 			// Only Int/Float/decimal is accepted, so pass nil here is safe.
 			fVal, err := datumValue.ToFloat64(types.DefaultStmtNoWarningContext)
 			if err != nil {
@@ -3153,7 +3155,12 @@ func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint
 			if fVal <= 0 || fVal > limit {
 				return nil, errors.Errorf("Value of analyze option %s should not larger than %f, and should be greater than 0", ast.AnalyzeOptionString[opt.Type], limit)
 			}
-			sampleRate = fVal
+			switch opt.Type {
+			case ast.AnalyzeOptSampleRate:
+				sampleRate = fVal
+			case ast.AnalyzeOptNDVRate:
+				ndvRate = fVal
+			}
 			optMap[opt.Type] = math.Float64bits(fVal)
 		default:
 			v := datumValue.GetUint64()
@@ -3169,6 +3176,15 @@ func handleAnalyzeOptions(opts []ast.AnalyzeOpt) (map[ast.AnalyzeOptionType]uint
 	if sampleNum > 0 && sampleRate > 0 {
 		return nil, errors.Errorf("You can only either set the value of the sample num or set the value of the sample rate. Don't set both of them")
 	}
+	// TiKV builds singleton sketches at ndvrate first and then draws the row
+	// sample from that stream at samplerate, so ndvrate < samplerate is
+	// incoherent by construction — the row sample cannot exceed the population
+	// it is drawn from. Reject the explicit conflict here so the user sees the
+	// mistake instead of letting execution-time clamping silently raise the
+	// value.
+	if ndvRate > 0 && sampleRate > 0 && ndvRate < sampleRate {
+		return nil, errors.Errorf("Value of analyze option NDVRATE (%f) must not be smaller than SAMPLERATE (%f)", ndvRate, sampleRate)
+	}
 
 	return optMap, nil
 }

pkg/planner/core/planbuilder_test.go

@@ -670,6 +670,16 @@ func TestHandleAnalyzeOptions(t *testing.T) {
 			},
 			ExpectedErr: "Value of analyze option SAMPLERATE should not larger than 1.000000, and should be greater than 0",
 		},
+		{
+			name: "Too big NDVRate option",
+			opts: []ast.AnalyzeOpt{
+				{
+					Type:  ast.AnalyzeOptNDVRate,
+					Value: ast.NewValueExpr(2, "", ""),
+				},
+			},
+			ExpectedErr: "Value of analyze option NDVRATE should not larger than 1.000000, and should be greater than 0",
+		},
 		{
 			name: "Too big NumBuckets option",
 			opts: []ast.AnalyzeOpt{
@@ -694,6 +704,20 @@ func TestHandleAnalyzeOptions(t *testing.T) {
 			},
 			ExpectedErr: "ou can only either set the value of the sample num or set the value of the sample rate. Don't set both of them",
 		},
+		{
+			name: "NDVRate below SampleRate",
+			opts: []ast.AnalyzeOpt{
+				{
+					Type:  ast.AnalyzeOptSampleRate,
+					Value: ast.NewValueExpr(0.5, "", ""),
+				},
+				{
+					Type:  ast.AnalyzeOptNDVRate,
+					Value: ast.NewValueExpr(0.1, "", ""),
+				},
+			},
+			ExpectedErr: "Value of analyze option NDVRATE (0.100000) must not be smaller than SAMPLERATE (0.500000)",
+		},
 	}
 
 	for _, tt := range tests {