[DNM]*: a demo for no_null_index and use_index(tiflash)

pkg/ddl/ddl_api.go

@@ -1198,6 +1198,8 @@ func columnDefToCol(ctx sessionctx.Context, offset int, colDef *ast.ColumnDef, o
 				col.DelFlag(mysql.NotNullFlag)
 				removeOnUpdateNowFlag(col)
 				hasNullFlag = true
+			case ast.ColumnOptionNoNullIndex:
+				col.NoNullIndex = true
 			case ast.ColumnOptionAutoIncrement:
 				col.AddFlag(mysql.AutoIncrementFlag | mysql.NotNullFlag)
 			case ast.ColumnOptionPrimaryKey:
@@ -5688,6 +5690,8 @@ func ProcessModifyColumnOptions(ctx sessionctx.Context, col *table.Column, optio
 			return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify with check"))
 		// Ignore ColumnOptionAutoRandom. It will be handled later.
 		case ast.ColumnOptionAutoRandom:
+		case ast.ColumnOptionNoNullIndex:
+			col.NoNullIndex = true
 		default:
 			return errors.Trace(dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs(fmt.Sprintf("unknown column option type: %d", opt.Tp)))
 		}
@@ -5862,6 +5866,10 @@ func GetModifiableColumnJob(
 		return nil, errors.Trace(err)
 	}
 
+	if err = checkModifyColumnWithNoNullIndex(t.Meta(), col.ColumnInfo, specNewColumn.Options); err != nil {
+		return nil, errors.Trace(err)
+	}
+
 	// Copy index related options to the new spec.
 	indexFlags := col.FieldType.GetFlag() & (mysql.PriKeyFlag | mysql.UniqueKeyFlag | mysql.MultipleKeyFlag)
 	newCol.FieldType.AddFlag(indexFlags)
@@ -9516,3 +9524,30 @@ func NewDDLReorgMeta(ctx sessionctx.Context) *model.DDLReorgMeta {
 		Version:           model.CurrentReorgMetaVersion,
 	}
 }
+
+func checkModifyColumnWithNoNullIndex(tbInfo *model.TableInfo, col *model.ColumnInfo, options []*ast.ColumnOption) error {
+	if col.NoNullIndex {
+		return nil
+	}
+
+	addNoNullIndex := false
+	for _, opt := range options {
+		if opt.Tp == ast.ColumnOptionNoNullIndex {
+			addNoNullIndex = true
+			break
+		}
+	}
+	if !addNoNullIndex {
+		return nil
+	}
+
+	// cannot add no_null_index if the column is part of an existing index
+	for _, indexInfo := range tbInfo.Indices {
+		for _, idxCol := range indexInfo.Columns {
+			if col.Name.L == idxCol.Name.L && addNoNullIndex {
+				return dbterror.ErrUnsupportedModifyColumn.GenWithStackByArgs("can't modify column to no_null_index with existing index")
+			}
+		}
+	}
+	return nil
+}

pkg/ddl/index.go

@@ -323,6 +323,9 @@ func BuildIndexInfo(
 		MVIndex: mvIndex,
 	}
 
+	// Primary index is not null, so it's fine to use no_null_index even if it's a primary index (as it'll have no effect).
+	idxInfo.NoNullIdxColOffsets = buildNoNullIdxColOffsets(idxInfo.Columns, allTableColumns)
+
 	if indexOption != nil {
 		idxInfo.Comment = indexOption.Comment
 		if indexOption.Visibility == ast.IndexVisibilityInvisible {
@@ -342,6 +345,18 @@ func BuildIndexInfo(
 	return idxInfo, nil
 }
 
+func buildNoNullIdxColOffsets(indexColumns []*model.IndexColumn, allTableColumns []*model.ColumnInfo) []int {
+	var noNullIdxColOffsets []int
+
+	for i, col := range indexColumns {
+		if allTableColumns[col.Offset].NoNullIndex {
+			noNullIdxColOffsets = append(noNullIdxColOffsets, i)
+		}
+	}
+
+	return noNullIdxColOffsets
+}
+
 // AddIndexColumnFlag aligns the column flags of columns in TableInfo to IndexInfo.
 func AddIndexColumnFlag(tblInfo *model.TableInfo, indexInfo *model.IndexInfo) {
 	if indexInfo.Primary {

pkg/ddl/index_cop.go

@@ -29,7 +29,9 @@ import (
 	"github.com/pingcap/tidb/pkg/expression"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/metrics"
+	"github.com/pingcap/tidb/pkg/parser/ast"
 	"github.com/pingcap/tidb/pkg/parser/model"
+	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/parser/terror"
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
@@ -340,6 +342,9 @@ func getRestoreData(tblInfo *model.TableInfo, targetIdx, pkIdx *model.IndexInfo,
 	return dtToRestored
 }
 
+// buildDAGPB builds a DAGRequest for the DDL.
+// Usually, it's a table scan executor. If the `colInfo` contains `NoNullIndex` attribute, it'll attach a selection executor.
+// to only fetch the not null rows.
 func buildDAGPB(sCtx sessionctx.Context, tblInfo *model.TableInfo, colInfos []*model.ColumnInfo) (*tipb.DAGRequest, error) {
 	dagReq := &tipb.DAGRequest{}
 	dagReq.TimeZoneName, dagReq.TimeZoneOffset = timeutil.Zone(sCtx.GetSessionVars().Location())
@@ -348,11 +353,62 @@ func buildDAGPB(sCtx sessionctx.Context, tblInfo *model.TableInfo, colInfos []*m
 	for i := range colInfos {
 		dagReq.OutputOffsets = append(dagReq.OutputOffsets, uint32(i))
 	}
-	execPB, err := constructTableScanPB(sCtx, tblInfo, colInfos)
+	tblScanPB, err := constructTableScanPB(sCtx, tblInfo, colInfos)
 	if err != nil {
 		return nil, err
 	}
-	dagReq.Executors = append(dagReq.Executors, execPB)
+
+	noNullSelection := make([]expression.Expression, 0)
+	for i, colInfo := range colInfos {
+		if colInfo.NoNullIndex {
+			isNullExpr, err := expression.NewFunction(sCtx.GetExprCtx(), ast.IsNull, types.NewFieldType(mysql.TypeTiny),
+				&expression.Column{Index: i, RetType: &colInfo.FieldType},
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			notNullExpr, err := expression.NewFunction(sCtx.GetExprCtx(), ast.UnaryNot, types.NewFieldType(mysql.TypeTiny), isNullExpr)
+			if err != nil {
+				return nil, err
+			}
+
+			noNullSelection = append(noNullSelection, notNullExpr)
+		}
+	}
+	if len(noNullSelection) > 0 {
+		// build an OR for these expressions, because "multi-schema change" will pack all needed column into this selection.
+		// FIXME: this behavior is not correct, because the `colInfos` also contains the dependency of generated columns which are
+		// not used by the index. It's kept here for demo.
+		var expr expression.Expression
+		for _, e := range noNullSelection {
+			if expr == nil {
+				expr = e
+			} else {
+				expr, err = expression.NewFunction(sCtx.GetExprCtx(), ast.LogicOr, types.NewFieldType(mysql.TypeTiny), expr, e)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+
+		conditions, err := expression.ExpressionsToPBList(sCtx.GetExprCtx().GetEvalCtx(), []expression.Expression{expr}, sCtx.GetClient())
+		if err != nil {
+			return nil, err
+		}
+
+		selectionPB := &tipb.Executor{
+			Tp: tipb.ExecType_TypeSelection,
+			Selection: &tipb.Selection{
+				Conditions: conditions,
+				Child:      tblScanPB,
+			},
+		}
+		dagReq.Executors = append(dagReq.Executors, tblScanPB, selectionPB)
+	} else {
+		dagReq.Executors = append(dagReq.Executors, tblScanPB)
+	}
+
 	distsql.SetEncodeType(sCtx.GetDistSQLCtx(), dagReq)
 	return dagReq, nil
 }

pkg/executor/builder.go

@@ -481,23 +481,27 @@ func buildIndexLookUpChecker(b *executorBuilder, p *plannercore.PhysicalIndexLoo
 }
 
 func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) exec.Executor {
-	noMVIndexOrPrefixIndex := true
+	canUseFastCheck := true
 	for _, idx := range v.IndexInfos {
 		if idx.MVIndex {
-			noMVIndexOrPrefixIndex = false
+			canUseFastCheck = false
+			break
+		}
+		if len(idx.NoNullIdxColOffsets) != 0 {
+			canUseFastCheck = false
 			break
 		}
 		for _, col := range idx.Columns {
 			if col.Length != types.UnspecifiedLength {
-				noMVIndexOrPrefixIndex = false
+				canUseFastCheck = false
 				break
 			}
 		}
-		if !noMVIndexOrPrefixIndex {
+		if !canUseFastCheck {
 			break
 		}
 	}
-	if b.ctx.GetSessionVars().FastCheckTable && noMVIndexOrPrefixIndex {
+	if b.ctx.GetSessionVars().FastCheckTable && canUseFastCheck {
 		e := &FastCheckTableExec{
 			BaseExecutor: exec.NewBaseExecutor(b.ctx, v.Schema(), v.ID()),
 			dbName:       v.DBName,

pkg/executor/executor.go

@@ -940,7 +940,7 @@ func (e *CheckTableExec) Next(ctx context.Context, _ *chunk.Chunk) error {
 
 	idxNames := make([]string, 0, len(e.indexInfos))
 	for _, idx := range e.indexInfos {
-		if idx.MVIndex {
+		if idx.MVIndex || len(idx.NoNullIdxColOffsets) != 0 {
 			continue
 		}
 		idxNames = append(idxNames, idx.Name.O)
@@ -963,7 +963,7 @@ func (e *CheckTableExec) Next(ctx context.Context, _ *chunk.Chunk) error {
 	// TODO: Make the value of concurrency adjustable. And we can consider the number of records.
 	if len(e.srcs) == 1 {
 		err = e.checkIndexHandle(ctx, e.srcs[0])
-		if err == nil && e.srcs[0].index.MVIndex {
+		if err == nil && shouldCheckTableRecordForIndex(e.srcs[0].index) {
 			err = e.checkTableRecord(ctx, 0)
 		}
 		if err != nil {
@@ -987,7 +987,7 @@ func (e *CheckTableExec) Next(ctx context.Context, _ *chunk.Chunk) error {
 					select {
 					case src := <-taskCh:
 						err1 := e.checkIndexHandle(ctx, src)
-						if err1 == nil && src.index.MVIndex {
+						if err1 == nil && shouldCheckTableRecordForIndex(src.index) {
 							for offset, idx := range e.indexInfos {
 								if idx.ID == src.index.ID {
 									err1 = e.checkTableRecord(ctx, offset)
@@ -1018,6 +1018,10 @@ func (e *CheckTableExec) Next(ctx context.Context, _ *chunk.Chunk) error {
 	}
 }
 
+func shouldCheckTableRecordForIndex(idx *model.IndexInfo) bool {
+	return idx.MVIndex || len(idx.NoNullIdxColOffsets) != 0
+}
+
 func (e *CheckTableExec) checkTableRecord(ctx context.Context, idxOffset int) error {
 	idxInfo := e.indexInfos[idxOffset]
 	txn, err := e.Ctx().Txn(true)

pkg/executor/show.go

@@ -1091,6 +1091,9 @@ func constructResultOfShowCreateTable(ctx sessionctx.Context, dbName *model.CISt
 		if tableInfo.PKIsHandle && mysql.HasPriKeyFlag(col.GetFlag()) {
 			pkCol = col
 		}
+		if col.NoNullIndex {
+			buf.WriteString(" NO_NULL_INDEX")
+		}
 	}
 
 	if pkCol != nil {

pkg/executor/test/tiflashtest/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
     ],
     flaky = True,
     race = "on",
-    shard_count = 43,
+    shard_count = 44,
     deps = [
         "//pkg/config",
         "//pkg/domain",

pkg/executor/test/tiflashtest/tiflash_test.go

@@ -548,6 +548,23 @@ func TestPartitionTable(t *testing.T) {
 	failpoint.Disable("github.com/pingcap/tidb/pkg/executor/checkUseMPP")
 }
 
+func TestUseIndexTiFlash(t *testing.T) {
+	store := testkit.CreateMockStore(t, withMockTiFlash(2))
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec(`create table t (id int primary key, a int, b int, c int, key(a), key(b), key(c))`)
+	tk.MustExec(`alter table t set tiflash replica 1`)
+	tb := external.GetTableByName(t, tk, "test", "t")
+	err := domain.GetDomain(tk.Session()).DDL().UpdateTableReplicaInfo(tk.Session(), tb.Meta().ID, true)
+	require.NoError(t, err)
+
+	tk.MustUseIndex(`select /*+ use_index(t, a, b, c, tiflash) */ 1 from t where a=1`, "a")
+	tk.MustUseIndex(`select /*+ use_index(t, a, b, c, tiflash) */ 1 from t where b=1`, "b")
+	tk.MustUseIndex(`select /*+ use_index(t, a, b, c, tiflash) */ 1 from t where b=1`, "b")
+	plan := tk.MustQuery(`explain select /*+ use_index(t, a, b, c, tiflash) */ 1 from t`).Rows()
+	require.Equal(t, plan[len(plan)-1][2], "mpp[tiflash]")
+}
+
 func TestMppEnum(t *testing.T) {
 	store := testkit.CreateMockStore(t, withMockTiFlash(2))
 	tk := testkit.NewTestKit(t, store)

pkg/parser/ast/ddl.go

@@ -526,6 +526,7 @@ const (
 	ColumnOptionColumnFormat
 	ColumnOptionStorage
 	ColumnOptionAutoRandom
+	ColumnOptionNoNullIndex
 )
 
 var (
@@ -598,6 +599,8 @@ func (n *ColumnOption) Restore(ctx *format.RestoreCtx) error {
 		ctx.WriteKeyWord("UNIQUE KEY")
 	case ColumnOptionNull:
 		ctx.WriteKeyWord("NULL")
+	case ColumnOptionNoNullIndex:
+		ctx.WriteKeyWord("NO_NULL_INDEX")
 	case ColumnOptionOnUpdate:
 		ctx.WriteKeyWord("ON UPDATE ")
 		if err := n.Expr.Restore(ctx); err != nil {

pkg/parser/misc.go

@@ -552,6 +552,7 @@ var tokenMap = map[string]int{
 	"NEXT":                     next,
 	"NEXTVAL":                  nextval,
 	"NO_WRITE_TO_BINLOG":       noWriteToBinLog,
+	"NO_NULL_INDEX":            noNullIndex,
 	"NO":                       no,
 	"NOCACHE":                  nocache,
 	"NOCYCLE":                  nocycle,

pkg/parser/model/model.go

@@ -117,6 +117,7 @@ type ColumnInfo struct {
 	FieldType           types.FieldType     `json:"type"`
 	State               SchemaState         `json:"state"`
 	Comment             string              `json:"comment"`
+	NoNullIndex         bool                `json:"no_null_index"`
 	// A hidden column is used internally(expression index) and are not accessible by users.
 	Hidden           bool `json:"hidden"`
 	*ChangeStateInfo `json:"change_state_info"`
@@ -1473,19 +1474,20 @@ const (
 // It corresponds to the statement `CREATE INDEX Name ON Table (Column);`
 // See https://dev.mysql.com/doc/refman/5.7/en/create-index.html
 type IndexInfo struct {
-	ID            int64          `json:"id"`
-	Name          CIStr          `json:"idx_name"` // Index name.
-	Table         CIStr          `json:"tbl_name"` // Table name.
-	Columns       []*IndexColumn `json:"idx_cols"` // Index columns.
-	State         SchemaState    `json:"state"`
-	BackfillState BackfillState  `json:"backfill_state"`
-	Comment       string         `json:"comment"`      // Comment
-	Tp            IndexType      `json:"index_type"`   // Index type: Btree, Hash or Rtree
-	Unique        bool           `json:"is_unique"`    // Whether the index is unique.
-	Primary       bool           `json:"is_primary"`   // Whether the index is primary key.
-	Invisible     bool           `json:"is_invisible"` // Whether the index is invisible.
-	Global        bool           `json:"is_global"`    // Whether the index is global.
-	MVIndex       bool           `json:"mv_index"`     // Whether the index is multivalued index.
+	ID                  int64          `json:"id"`
+	Name                CIStr          `json:"idx_name"` // Index name.
+	Table               CIStr          `json:"tbl_name"` // Table name.
+	Columns             []*IndexColumn `json:"idx_cols"` // Index columns.
+	NoNullIdxColOffsets []int          `json:"no_null_idx_cols"`
+	State               SchemaState    `json:"state"`
+	BackfillState       BackfillState  `json:"backfill_state"`
+	Comment             string         `json:"comment"`      // Comment
+	Tp                  IndexType      `json:"index_type"`   // Index type: Btree, Hash or Rtree
+	Unique              bool           `json:"is_unique"`    // Whether the index is unique.
+	Primary             bool           `json:"is_primary"`   // Whether the index is primary key.
+	Invisible           bool           `json:"is_invisible"` // Whether the index is invisible.
+	Global              bool           `json:"is_global"`    // Whether the index is global.
+	MVIndex             bool           `json:"mv_index"`     // Whether the index is multivalued index.
 }
 
 // Clone clones IndexInfo.