*: support the vector index

Makefile

@@ -363,15 +363,16 @@ endif
 #	make bench-daily TO=/path/to/file.json
 .PHONY: bench-daily
 bench-daily:
-	go test github.com/pingcap/tidb/pkg/session -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/pkg/distsql -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/pkg/executor -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/pkg/executor/test/splittest -run TestBenchDaily -bench Ignore --outfile bench_daily.json
-	go test github.com/pingcap/tidb/pkg/tablecodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/pkg/expression -run TestBenchDaily -bench Ignore --outfile bench_daily.json
-	go test github.com/pingcap/tidb/pkg/util/rowcodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
-	go test github.com/pingcap/tidb/pkg/util/codec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
-	go test github.com/pingcap/tidb/pkg/distsql -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/pkg/planner/core/tests/partition -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/pkg/session -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/pkg/statistics -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/pkg/tablecodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/pkg/util/codec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/pkg/util/rowcodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/pkg/util/benchdaily -run TestBenchDaily -bench Ignore \
 		-date `git log -n1 --date=unix --pretty=format:%cd` \
 		-commit `git log -n1 --pretty=format:%h` \
@@ -565,7 +566,15 @@ dumpling_bins:
 
 .PHONY: generate_grafana_scripts
 generate_grafana_scripts:
-	@cd metrics/grafana && mv tidb_summary.json tidb_summary.json.committed && ./generate_json.sh && diff -u tidb_summary.json.committed tidb_summary.json && rm tidb_summary.json.committed
+	@cd pkg/metrics/grafana && \
+	  mv tidb_summary.json tidb_summary.json.committed && \
+	  mv tidb_resource_control.json tidb_resource_control.json.committed && \
+	  ./generate_json.sh && \
+	  diff -u tidb_summary.json.committed tidb_summary.json && \
+	  diff -u tidb_resource_control.json.committed tidb_resource_control.json && \
+	  rm tidb_summary.json.committed && \
+	  rm tidb_resource_control.json.committed
+
 
 .PHONY: bazel_ci_prepare
 bazel_ci_prepare:
@@ -623,6 +632,14 @@ bazel_coverage_test: failpoint-enable bazel_ci_simple_prepare
 		-- //... -//cmd/... -//tests/graceshutdown/... \
 		-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...
 
+.PHONY: bazel_coverage_test_ddlargsv1
+bazel_coverage_test_ddlargsv1: failpoint-enable bazel_ci_simple_prepare
+	bazel $(BAZEL_GLOBAL_CONFIG) --nohome_rc coverage $(BAZEL_CMD_CONFIG) $(BAZEL_INSTRUMENTATION_FILTER) --jobs=35 --build_tests_only --test_keep_going=false \
+		--@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock,intest \
+		--define gc_linkopts=-X,github.com/pingcap/tidb/pkg/ddl.ForceDDLJobVersionToV1InTest=true \
+		-- //... -//cmd/... -//tests/graceshutdown/... \
+		-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...
+
 .PHONY: bazel_build
 bazel_build:
 	mkdir -p bin

OWNERS

@@ -98,6 +98,7 @@ approvers:
   - winoros
   - WizardXiao
   - wjhuang2016
+  - wk989898
   - wshwsh12
   - xhebox
   - xiongjiwei
@@ -127,6 +128,7 @@ reviewers:
   - dhysum
   - fengou1
   - fzzf678
+  - ghazalfamilyusa
   - iamxy
   - JmPotato
   - js00070
@@ -145,6 +147,7 @@ reviewers:
   - shihongzhi
   - spongedu
   - tangwz
+  - terry1purcell
   - Tjianke
   - TonsnakeLin
   - tsthght

br/cmd/br/restore.go

@@ -74,14 +74,14 @@ func runRestoreCommand(command *cobra.Command, cmdName string) error {
 
 	if err := task.RunRestore(GetDefaultContext(), tidbGlue, cmdName, &cfg); err != nil {
 		log.Error("failed to restore", zap.Error(err))
-		printWorkaroundOnFullRestoreError(command, err)
+		printWorkaroundOnFullRestoreError(err)
 		return errors.Trace(err)
 	}
 	return nil
 }
 
 // print workaround when we met not fresh or incompatible cluster error on full cluster restore
-func printWorkaroundOnFullRestoreError(command *cobra.Command, err error) {
+func printWorkaroundOnFullRestoreError(err error) {
 	if !errors.ErrorEqual(err, berrors.ErrRestoreNotFreshCluster) &&
 		!errors.ErrorEqual(err, berrors.ErrRestoreIncompatibleSys) {
 		return

br/pkg/checkpoint/BUILD.bazel

@@ -5,23 +5,30 @@ go_library(
     srcs = [
         "backup.go",
         "checkpoint.go",
+        "external_storage.go",
         "log_restore.go",
         "restore.go",
+        "storage.go",
         "ticker.go",
     ],
     importpath = "github.com/pingcap/tidb/br/pkg/checkpoint",
     visibility = ["//visibility:public"],
     deps = [
+        "//br/pkg/glue",
         "//br/pkg/logutil",
         "//br/pkg/metautil",
         "//br/pkg/pdutil",
         "//br/pkg/rtree",
         "//br/pkg/storage",
         "//br/pkg/summary",
         "//br/pkg/utils",
+        "//pkg/domain",
+        "//pkg/kv",
         "//pkg/meta/model",
         "//pkg/parser/model",
         "//pkg/util",
+        "//pkg/util/sqlexec",
+        "@com_github_google_uuid//:uuid",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_pingcap_kvproto//pkg/brpb",
@@ -38,11 +45,13 @@ go_test(
     srcs = ["checkpoint_test.go"],
     flaky = True,
     race = "on",
-    shard_count = 5,
+    shard_count = 6,
     deps = [
         ":checkpoint",
+        "//br/pkg/gluetidb",
         "//br/pkg/pdutil",
         "//br/pkg/storage",
+        "//br/pkg/utiltest",
         "//pkg/meta/model",
         "//pkg/parser/model",
         "@com_github_pingcap_kvproto//pkg/brpb",

br/pkg/checkpoint/backup.go

@@ -57,13 +57,13 @@ func StartCheckpointBackupRunnerForTest(
 	tick time.Duration,
 	timer GlobalTimer,
 ) (*CheckpointRunner[BackupKeyType, BackupValueType], error) {
-	runner := newCheckpointRunner[BackupKeyType, BackupValueType](
-		ctx, storage, cipher, timer, flushPositionForBackup(), valueMarshalerForBackup)
-
-	err := runner.initialLock(ctx)
+	checkpointStorage, err := newExternalCheckpointStorage(ctx, storage, timer)
 	if err != nil {
-		return nil, errors.Annotate(err, "Failed to initialize checkpoint lock.")
+		return nil, errors.Trace(err)
 	}
+	runner := newCheckpointRunner[BackupKeyType, BackupValueType](
+		checkpointStorage, cipher, valueMarshalerForBackup)
+
 	runner.startCheckpointMainLoop(ctx, tick, tick, tick)
 	return runner, nil
 }
@@ -74,13 +74,13 @@ func StartCheckpointRunnerForBackup(
 	cipher *backuppb.CipherInfo,
 	timer GlobalTimer,
 ) (*CheckpointRunner[BackupKeyType, BackupValueType], error) {
-	runner := newCheckpointRunner[BackupKeyType, BackupValueType](
-		ctx, storage, cipher, timer, flushPositionForBackup(), valueMarshalerForBackup)
-
-	err := runner.initialLock(ctx)
+	checkpointStorage, err := newExternalCheckpointStorage(ctx, storage, timer)
 	if err != nil {
 		return nil, errors.Trace(err)
 	}
+	runner := newCheckpointRunner[BackupKeyType, BackupValueType](
+		checkpointStorage, cipher, valueMarshalerForBackup)
+
 	runner.startCheckpointMainLoop(
 		ctx,
 		defaultTickDurationForFlush,