IGNOREME Feature: ivfflat cronjob reindex

pkg/bootstrap/versions/v4_0_0/cluster_upgrade_list.go

@@ -25,6 +25,7 @@ import (
 var clusterUpgEntries = []versions.UpgradeEntry{
 	upg_mo_iscp_log_new,
 	upg_mo_iscp_task,
+	upg_mo_index_update_new,
 	upg_create_mo_branch_metadata,
 	upg_rename_system_stmt_info_4000,
 	upg_create_system_stmt_info_4000,
@@ -53,6 +54,16 @@ var upg_mo_iscp_task = versions.UpgradeEntry{
 	},
 }
 
+var upg_mo_index_update_new = versions.UpgradeEntry{
+	Schema:    catalog.MO_CATALOG,
+	TableName: catalog.MO_INDEX_UPDATE,
+	UpgType:   versions.CREATE_NEW_TABLE,
+	UpgSql:    frontend.MoCatalogMoIndexUpdateDDL,
+	CheckFunc: func(txn executor.TxnExecutor, accountId uint32) (bool, error) {
+		return versions.CheckTableDefinition(txn, accountId, catalog.MO_CATALOG, catalog.MO_INDEX_UPDATE)
+	},
+}
+
 var upg_create_mo_branch_metadata = versions.UpgradeEntry{
 	Schema:    catalog.MO_CATALOG,
 	TableName: catalog.MO_BRANCH_METADATA,

pkg/catalog/types.go

@@ -174,6 +174,8 @@ const (
 	MO_ISCP_LOG         = "mo_iscp_log"
 	MO_STORED_PROCEDURE = "mo_stored_procedure"
 
+	MO_INDEX_UPDATE = "mo_index_update"
+
 	MO_BRANCH_METADATA = "mo_branch_metadata"
 )
 

pkg/cnservice/server_task.go

@@ -38,6 +38,7 @@ import (
 	db_holder "github.com/matrixorigin/matrixone/pkg/util/export/etl/db"
 	ie "github.com/matrixorigin/matrixone/pkg/util/internalExecutor"
 	"github.com/matrixorigin/matrixone/pkg/util/metric/mometric"
+	"github.com/matrixorigin/matrixone/pkg/vectorindex/idxcron"
 	"github.com/matrixorigin/matrixone/pkg/vm/engine/disttae"
 	"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
 	"go.uber.org/zap"
@@ -338,4 +339,13 @@ func (s *service) registerExecutorsLocked() {
 			common.ISCPAllocator,
 		),
 	)
+
+	s.task.runner.RegisterExecutor(task.TaskCode_IndexUpdateTaskExecutor,
+		idxcron.IndexUpdateTaskExecutorFactory(
+			s.cfg.UUID,
+			s.storeEngine,
+			s._txnClient,
+			common.ISCPAllocator,
+		),
+	)
 }

pkg/container/bytejson/bytejson.go

@@ -126,7 +126,14 @@ func (bj ByteJson) GetUint64() uint64 {
 }
 
 func (bj ByteJson) GetFloat64() float64 {
-	return math.Float64frombits(bj.GetUint64())
+	switch bj.Type {
+	case TpCodeInt64:
+		return float64(bj.GetInt64())
+	case TpCodeUint64:
+		return float64(bj.GetUint64())
+	default:
+		return math.Float64frombits(bj.GetUint64())
+	}
 }
 
 func (bj ByteJson) GetString() []byte {

pkg/frontend/authenticate.go

@@ -933,6 +933,7 @@ var (
 		catalog.MO_TABLE_STATS:        0,
 		catalog.MO_MERGE_SETTINGS:     0,
 		catalog.MO_ISCP_LOG:           0,
+		catalog.MO_INDEX_UPDATE:       0,
 		catalog.MO_BRANCH_METADATA:    0,
 	}
 	sysAccountTables = map[string]struct{}{
@@ -979,6 +980,7 @@ var (
 		catalog.MO_ACCOUNT_LOCK:       0,
 		catalog.MO_MERGE_SETTINGS:     0,
 		catalog.MO_ISCP_LOG:           0,
+		catalog.MO_INDEX_UPDATE:       0,
 		catalog.MO_BRANCH_METADATA:    0,
 	}
 	createDbInformationSchemaSql = "create database information_schema;"
@@ -1021,6 +1023,7 @@ var (
 		MoCatalogMergeSettingsDDL,
 		MoCatalogMergeSettingsInitData,
 		MoCatalogMoISCPLogDDL,
+		MoCatalogMoIndexUpdateDDL,
 		MoCatalogBranchMetadataDDL,
 	}
 
@@ -7829,7 +7832,9 @@ func createTablesInMoCatalogOfGeneralTenant2(bh BackgroundExec, ca *createAccoun
 		if strings.HasPrefix(sql, fmt.Sprintf("CREATE TABLE mo_catalog.%s", catalog.MO_ISCP_LOG)) {
 			return true
 		}
-
+		if strings.HasPrefix(sql, fmt.Sprintf("CREATE TABLE mo_catalog.%s", catalog.MO_INDEX_UPDATE)) {
+			return true
+		}
 		if strings.HasPrefix(sql, fmt.Sprintf("create table mo_catalog.%s", catalog.MO_BRANCH_METADATA)) {
 			return true
 		}

pkg/frontend/predefined.go

@@ -282,6 +282,20 @@ var (
 				primary key(account_id, table_id, job_name, job_id)
 			)`
 
+	MoCatalogMoIndexUpdateDDL = `CREATE TABLE mo_catalog.mo_index_update (
+                                account_id INT UNSIGNED NOT NULL,
+                                table_id BIGINT UNSIGNED NOT NULL,
+				db_name VARCHAR NOT NULL,
+				table_name VARCHAR NOT NULL,
+				index_name VARCHAR NOT NULL,
+				action VARCHAR NOT NULL,
+				metadata JSON NOT NULL,
+				status JSON NOT NULL,
+                                create_at TIMESTAMP NOT NULL,
+                                last_update_at TIMESTAMP NULL, 
+                                primary key(account_id, table_id, index_name, action)
+                        )`
+
 	MoCatalogMoSessionsDDL       = `CREATE VIEW mo_catalog.mo_sessions AS SELECT node_id, conn_id, session_id, account, user, host, db, session_start, command, info, txn_id, statement_id, statement_type, query_type, sql_source_type, query_start, client_host, role, proxy_host FROM mo_sessions() AS mo_sessions_tmp`
 	MoCatalogMoConfigurationsDDL = `CREATE VIEW mo_catalog.mo_configurations AS SELECT node_type, node_id, name, current_value, default_value, internal FROM mo_configurations() AS mo_configurations_tmp`
 	MoCatalogMoLocksDDL          = `CREATE VIEW mo_catalog.mo_locks AS SELECT cn_id, txn_id, table_id, lock_key, lock_content, lock_mode, lock_status, lock_wait FROM mo_locks() AS mo_locks_tmp`

pkg/frontend/variables.go

@@ -3552,7 +3552,7 @@ var gSysVarsDefs = map[string]SystemVariable{
 		Dynamic:           true,
 		SetVarHintApplies: false,
 		Type:              InitSystemVariableBoolType("experimental_ivf_index"),
-		Default:           int64(0),
+		Default:           int8(0),
 	},
 	"ivf_threads_build": {
 		Name:              "ivf_threads_build",
@@ -3583,8 +3583,8 @@ var gSysVarsDefs = map[string]SystemVariable{
 		Scope:             ScopeBoth,
 		Dynamic:           true,
 		SetVarHintApplies: false,
-		Type:              InitSystemVariableIntType("kmeans_train_percent", 1, 100, false),
-		Default:           int64(10),
+		Type:              InitSystemVariableDoubleType("kmeans_train_percent", 0, 100),
+		Default:           float64(10),
 	},
 	"kmeans_max_iteration": {
 		Name:              "kmeans_max_iteration",
@@ -3608,7 +3608,7 @@ var gSysVarsDefs = map[string]SystemVariable{
 		Dynamic:           true,
 		SetVarHintApplies: false,
 		Type:              InitSystemVariableBoolType("experimental_fulltext_index"),
-		Default:           int64(0),
+		Default:           int8(0),
 	},
 	"ft_relevancy_algorithm": {
 		Name:              fulltext.FulltextRelevancyAlgo,
@@ -3624,7 +3624,7 @@ var gSysVarsDefs = map[string]SystemVariable{
 		Dynamic:           true,
 		SetVarHintApplies: false,
 		Type:              InitSystemVariableBoolType("experimental_hnsw_index"),
-		Default:           int64(0),
+		Default:           int8(0),
 	},
 	"hnsw_threads_build": {
 		Name:              "hnsw_threads_build",