Merge remote-tracking branch 'origin/master' into feature/dual-password

Author: takaidohigasi

.agents/skills/tidb-test-guidelines/references/lightning-case-map.md

@@ -11,24 +11,24 @@
 ### Tests
 - `pkg/lightning/backend/backend_test.go` - lightning/backend: Tests open/close import cleanup engine.
 
-## pkg/lightning/backend/external
-
-### Tests
-- `pkg/lightning/backend/external/bench_test.go` - lightning/backend/external: Tests compare writer.
-- `pkg/lightning/backend/external/byte_reader_test.go` - lightning/backend/external: Tests byte reader.
-- `pkg/lightning/backend/external/codec_test.go` - lightning/backend/external: Tests range property codec.
-- `pkg/lightning/backend/external/concurrent_reader_test.go` - lightning/backend/external: Tests concurrent read.
-- `pkg/lightning/backend/external/engine_test.go` - lightning/backend/external: Tests memory ingest data.
-- `pkg/lightning/backend/external/file_test.go` - lightning/backend/external: Tests add key-value maintain range property.
-- `pkg/lightning/backend/external/iter_test.go` - lightning/backend/external: Tests merge KV iter.
-- `pkg/lightning/backend/external/merge_test.go` - lightning/backend/external: Tests split data files.
-- `pkg/lightning/backend/external/misc_bench_test.go` - lightning/backend/external: Tests misc bench.
-- `pkg/lightning/backend/external/onefile_writer_test.go` - lightning/backend/external: Tests onefile writer basic.
-- `pkg/lightning/backend/external/reader_test.go` - lightning/backend/external: Tests read all data basic.
-- `pkg/lightning/backend/external/sort_test.go` - lightning/backend/external: Tests global sort local basic.
-- `pkg/lightning/backend/external/split_test.go` - lightning/backend/external: Tests general properties.
-- `pkg/lightning/backend/external/util_test.go` - lightning/backend/external: Tests seek props offsets.
-- `pkg/lightning/backend/external/writer_test.go` - lightning/backend/external: Tests writer.
+## pkg/ingestor/globalsort
+
+### Tests
+- `pkg/ingestor/globalsort/bench_test.go` - ingestor/globalsort: Tests compare writer.
+- `pkg/ingestor/globalsort/byte_reader_test.go` - ingestor/globalsort: Tests byte reader.
+- `pkg/ingestor/globalsort/codec_test.go` - ingestor/globalsort: Tests range property codec.
+- `pkg/ingestor/globalsort/concurrent_reader_test.go` - ingestor/globalsort: Tests concurrent read.
+- `pkg/ingestor/globalsort/engine_test.go` - ingestor/globalsort: Tests memory ingest data.
+- `pkg/ingestor/globalsort/file_test.go` - ingestor/globalsort: Tests add key-value maintain range property.
+- `pkg/ingestor/globalsort/iter_test.go` - ingestor/globalsort: Tests merge KV iter.
+- `pkg/ingestor/globalsort/merge_test.go` - ingestor/globalsort: Tests split data files.
+- `pkg/ingestor/globalsort/misc_bench_test.go` - ingestor/globalsort: Tests misc bench.
+- `pkg/ingestor/globalsort/onefile_writer_test.go` - ingestor/globalsort: Tests onefile writer basic.
+- `pkg/ingestor/globalsort/reader_test.go` - ingestor/globalsort: Tests read all data basic.
+- `pkg/ingestor/globalsort/sort_test.go` - ingestor/globalsort: Tests global sort local basic.
+- `pkg/ingestor/globalsort/split_test.go` - ingestor/globalsort: Tests general properties.
+- `pkg/ingestor/globalsort/util_test.go` - ingestor/globalsort: Tests seek props offsets.
+- `pkg/ingestor/globalsort/writer_test.go` - ingestor/globalsort: Tests writer.
 
 ## pkg/lightning/backend/kv
 

.github/licenserc.yml

@@ -57,10 +57,6 @@ header:
     - "pkg/objstore/mockobjstore/*_mock.go"
     - "pkg/extension/enterprise/"
     - "lightning/pkg/**/mock/*_mock.go"
-    - "lightning/pkg/web/res_vfsdata.go"
-    - "lightning/web/docs/api.yaml"
-    - "lightning/web/public/index.html"
-    - "lightning/web/webpack.config.js"
     - "lightning/pkg/checkpoints/checkpointspb/file_checkpoints.pb.go"
     - "pkg/lightning/manual/manual.go"
     - "pkg/lightning/manual/manual_nocgo.go"

.github/workflows/check-bazel-prepare.yml

@@ -0,0 +1,42 @@
+name: Check Bazel Prepare
+on:
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+concurrency:
+  group: check-bazel-prepare-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    name: Check Bazel Prepare
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: 1.25
+          cache: false
+      - name: Set up Bazelisk
+        uses: bazel-contrib/[email protected]
+        with:
+          bazelisk-cache: false
+          repository-cache: false
+          external-cache: false
+      - name: Check Bazel Prepare
+        shell: bash
+        run: |
+          set -euo pipefail
+          # Avoid the TiDB CI path that assumes Jenkins-owned temp directories.
+          unset CI
+          make check-bazel-prepare

.github/workflows/generate-bazel-files.yml

@@ -42,7 +42,16 @@ jobs:
           sed -i '/ats.apps.svc/d' DEPS.bzl
           sed -i '/bazel-cache/d' WORKSPACE
           sed -i '/ats.apps.svc/d' WORKSPACE
-          make bazel_prepare
+          bazel run //:gazelle
+          bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro DEPS.bzl%go_deps -build_file_proto_mode=disable -prune
+          bazel run \
+            --run_under="cd ${PWD} && " \
+            //tools/tazel:tazel
+
+          tmp_out="$(mktemp -d -t tidbbzl.XXXXXX)"
+          trap 'rm -rf "${tmp_out}"' EXIT
+          bazel run //cmd/mirror -- --mirror > "${tmp_out}/DEPS.bzl"
+          cp "${tmp_out}/DEPS.bzl" DEPS.bzl
       - name: Restore non-generated files
         shell: bash
         run: |
@@ -62,10 +71,16 @@ jobs:
             fi
 
             case "${path}" in
+              build/BUILD.bazel|build/*/BUILD.bazel)
+                ;;
+              WORKSPACE|WORKSPACE.bazel|build|build/*)
+                echo "Unsafe generated file path: ${path}" >&2
+                exit 1
+                ;;
               DEPS.bzl|*.bazel|*.bzl)
                 ;;
               *)
-                echo "Unexpected file changed by bazel_prepare: ${path}" >&2
+                echo "Unexpected generated file path: ${path}" >&2
                 exit 1
                 ;;
             esac

.github/workflows/update-bazel-files.yml

@@ -59,11 +59,10 @@ jobs:
 
           if [[ "${head_repo}" != "${REPOSITORY}" ]]; then
             echo "same_repo=false" >> "${GITHUB_OUTPUT}"
-            exit 0
+          else
+            echo "same_repo=true" >> "${GITHUB_OUTPUT}"
           fi
 
-          echo "same_repo=true" >> "${GITHUB_OUTPUT}"
-
           if [[ "${pr_head_sha}" != "${RUN_HEAD_SHA}" ]]; then
             echo "stale=true" >> "${GITHUB_OUTPUT}"
             exit 0
@@ -75,6 +74,42 @@ jobs:
         shell: bash
         run: |
           echo "Skip stale bazel update for PR #${{ steps.pr.outputs.pr_number }}."
+      - name: Download Fork Bazel Files Artifact
+        if: steps.pr.outputs.same_repo != 'true' && steps.pr.outputs.stale != 'true'
+        uses: actions/download-artifact@v4
+        with:
+          name: bazel-files
+          path: bazel-artifact
+          repository: ${{ github.repository }}
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+      - name: Report fork PR Bazel changes
+        if: steps.pr.outputs.same_repo != 'true' && steps.pr.outputs.stale != 'true'
+        shell: bash
+        env:
+          PR_NUMBER: ${{ steps.pr.outputs.pr_number }}
+          REPOSITORY: ${{ github.repository }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          set -euo pipefail
+
+          patch_path="${PWD}/bazel-artifact/bazel.patch"
+          if [[ ! -f "${patch_path}" ]]; then
+            echo "Missing bazel.patch artifact for fork PR #${PR_NUMBER}." >&2
+            exit 1
+          fi
+
+          if [[ ! -s "${patch_path}" ]]; then
+            echo "No generated Bazel changes for fork PR #${PR_NUMBER}."
+            exit 0
+          fi
+
+          {
+            echo "Generated Bazel changes cannot be committed automatically for fork PR #${PR_NUMBER}."
+            echo "Run: gh run download ${RUN_ID} --repo ${REPOSITORY} --name bazel-files --dir bazel-artifact && git apply bazel-artifact/bazel.patch"
+          } | tee -a "${GITHUB_STEP_SUMMARY}"
+
+          exit 1
       - name: Checkout PR branch
         if: steps.pr.outputs.same_repo == 'true' && steps.pr.outputs.stale != 'true'
         uses: actions/checkout@v6
@@ -101,7 +136,7 @@ jobs:
           set -euo pipefail
 
           patch_path="${PWD}/bazel-artifact/bazel.patch"
-          summary="$(git apply --summary "${patch_path}")"
+          summary="$(git -C pr apply --summary "${patch_path}")"
 
           while IFS= read -r line; do
             case "${line}" in
@@ -126,6 +161,12 @@ jobs:
                 echo "Unexpected patch path: ${path}" >&2
                 exit 1
                 ;;
+              build/BUILD.bazel|build/*/BUILD.bazel)
+                ;;
+              WORKSPACE|WORKSPACE.bazel|build|build/*)
+                echo "Unsafe patch path: ${path}" >&2
+                exit 1
+                ;;
               DEPS.bzl|*.bazel|*.bzl)
                 ;;
               *)
@@ -139,7 +180,7 @@ jobs:
               exit 1
             fi
             seen_paths["${path}"]=1
-          done < <(git apply --numstat -z "${patch_path}")
+          done < <(git -C pr apply --numstat -z "${patch_path}")
 
           git -C pr apply --check --index "${patch_path}"
           git -C pr apply --index "${patch_path}"

.gitignore

@@ -80,3 +80,7 @@ var
 # Personal config files
 /*config.toml
 .cache
+
+# Claude Code runtime state (per-user, not part of repo)
+.claude/scheduled_tasks.lock
+.claude/settings.local.json

DEPS.bzl

@@ -6478,13 +6478,13 @@ def go_deps():
         name = "com_github_pingcap_errors",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/errors",
-        sha256 = "e90414b6322256cb88c6c39569070e23c074a95a803df4d2cedb10136e1d121c",
-        strip_prefix = "github.com/pingcap/[email protected].20260310054046-9c8b3586e4b2",
+        sha256 = "8843ed815adc22e020095250e3d0ac3e2ffd974260dbb58c050491bb3ba2f926",
+        strip_prefix = "github.com/pingcap/[email protected].20260508054701-306e305bcf41",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260310054046-9c8b3586e4b2.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260310054046-9c8b3586e4b2.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260310054046-9c8b3586e4b2.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260310054046-9c8b3586e4b2.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260508054701-306e305bcf41.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260508054701-306e305bcf41.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260508054701-306e305bcf41.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/errors/com_github_pingcap_errors-v0.11.5-0.20260508054701-306e305bcf41.zip",
         ],
     )
     go_repository(
@@ -6582,13 +6582,13 @@ def go_deps():
         name = "com_github_pingcap_tipb",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/tipb",
-        sha256 = "2d6aaef873e175599c39f6fe3cf85cabce88a69c157ff7ec4c22fbcbc97648dd",
-        strip_prefix = "github.com/pingcap/[email protected]20260210113932-1447c9d7e9fe",
+        sha256 = "68768a27ed6c35716fcb01a0b4a15ff13e5c1a5dc11acc7a3d44ba02a2742077",
+        strip_prefix = "github.com/pingcap/[email protected]20260414032333-da912b84de6f",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
-            "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260210113932-1447c9d7e9fe.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260414032333-da912b84de6f.zip",
+            "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260414032333-da912b84de6f.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260414032333-da912b84de6f.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260414032333-da912b84de6f.zip",
         ],
     )
     go_repository(
@@ -7319,19 +7319,6 @@ def go_deps():
             "https://storage.googleapis.com/pingcapmirror/gomod/github.com/shurcooL/httpfs/com_github_shurcool_httpfs-v0.0.0-20230704072500-f1e31cf0ba5c.zip",
         ],
     )
-    go_repository(
-        name = "com_github_shurcool_httpgzip",
-        build_file_proto_mode = "disable_global",
-        importpath = "github.com/shurcooL/httpgzip",
-        sha256 = "70ef73fce2f89d622f828cb439fd6c7b48a7fe63600410a8c0a936042c0e4631",
-        strip_prefix = "github.com/shurcooL/[email protected]",
-        urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/shurcooL/httpgzip/com_github_shurcool_httpgzip-v0.0.0-20190720172056-320755c1c1b0.zip",
-            "http://ats.apps.svc/gomod/github.com/shurcooL/httpgzip/com_github_shurcool_httpgzip-v0.0.0-20190720172056-320755c1c1b0.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/shurcooL/httpgzip/com_github_shurcool_httpgzip-v0.0.0-20190720172056-320755c1c1b0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/shurcooL/httpgzip/com_github_shurcool_httpgzip-v0.0.0-20190720172056-320755c1c1b0.zip",
-        ],
-    )
     go_repository(
         name = "com_github_shurcool_vfsgen",
         build_file_proto_mode = "disable_global",
@@ -7805,13 +7792,13 @@ def go_deps():
         build_tags = ["nextgen", "intest"],
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "a5e70899561f437abf34016a1fd907ab6394ab4771f75ba509b61dd5bb806351",
-        strip_prefix = "github.com/tikv/client-go/[email protected].20260414033830-1adc54c38a51",
+        sha256 = "239c5c831ab7ca4f27c0e2b3e73b2df5d828c38fb30ce2700c9d7f52b0ae4008",
+        strip_prefix = "github.com/tikv/client-go/[email protected].20260429090231-839228f8c022",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260414033830-1adc54c38a51.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260414033830-1adc54c38a51.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260414033830-1adc54c38a51.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260414033830-1adc54c38a51.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260429090231-839228f8c022.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260429090231-839228f8c022.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260429090231-839228f8c022.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20260429090231-839228f8c022.zip",
         ],
     )
     go_repository(

Makefile

@@ -256,6 +256,11 @@ else
 	CGO_ENABLED=1 $(GOBUILD) -gcflags="all=-N -l" $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o '$(TARGET)' ./cmd/tidb-server
 endif
 
+.PHONY: server_failpoint
+server_failpoint: failpoint-enable ## Build TiDB server binary with failpoints enabled
+	$(SERVER_BUILD_CMD) || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(FAILPOINT_DISABLE)
+
 .PHONY: init-submodule
 init-submodule:
 	git submodule init && git submodule update --force
@@ -384,10 +389,6 @@ tools/bin/golangci-lint:
 	$(eval GOLANGCI_LINT_VERSION := $(shell grep 'github.com/golangci/golangci-lint/v2' go.mod | awk '{print $$2}'))
 	GOBIN=$(shell pwd)/tools/bin $(GO) install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 
-.PHONY: tools/bin/vfsgendev
-tools/bin/vfsgendev:
-	GOBIN=$(shell pwd)/tools/bin $(GO) install github.com/shurcooL/vfsgen/cmd/vfsgendev@0d455de
-
 .PHONY: tools/bin/gotestsum
 tools/bin/gotestsum:
 	GOBIN=$(shell pwd)/tools/bin $(GO) install gotest.tools/[email protected]
@@ -447,10 +448,6 @@ bench-daily:
 .PHONY: build_tools
 build_tools: build_br build_lightning build_lightning-ctl ## Build all BR and Lightning tools
 
-.PHONY: lightning_web
-lightning_web: ## Build Lightning web UI
-	@cd lightning/web && npm install && npm run build
-
 .PHONY: build_br
 build_br: ## Build BR (backup and restore) tool 
 ifeq ($(shell echo $(GOOS) | tr A-Z a-z),darwin)
@@ -461,10 +458,6 @@ else
 	CGO_ENABLED=0 $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(BR_BIN) ./br/cmd/br
 endif
 
-.PHONY: build_lightning_for_web
-build_lightning_for_web:
-	CGO_ENABLED=1 $(GOBUILD_NO_TAGS) -tags dev $(RACE_FLAG) -ldflags '$(LDFLAGS) $(CHECK_FLAG)' -o $(LIGHTNING_BIN) lightning/cmd/tidb-lightning/main.go
-
 .PHONY: build_lightning
 build_lightning: ## Build TiDB Lightning data import tool
 ifeq ("$(GOOS)", "freebsd")
@@ -616,9 +609,8 @@ br_bins:
 	@rm tmp_parser.go
 
 .PHONY: data_parsers
-data_parsers: tools/bin/vfsgendev pkg/lightning/mydump/parser_generated.go lightning_web
+data_parsers: pkg/lightning/mydump/parser_generated.go
 	PATH="$(GOPATH)/bin":"$(PATH)":"$(TOOLS)" protoc -I. -I"$(GOMODCACHE)" lightning/pkg/checkpoints/checkpointspb/file_checkpoints.proto --gogofaster_out=.
-	tools/bin/vfsgendev -source='"github.com/pingcap/tidb/lightning/pkg/web".Res' && mv res_vfsdata.go lightning/pkg/web/
 
 .PHONY: build_dumpling
 build_dumpling: ## Build Dumpling data export tool

OWNERS_ALIASES

@@ -10,12 +10,15 @@ aliases:
   sig-critical-approvers-tidb-server:
     - yudongusa
     - terry1purcell
+    - likidu
   sig-critical-approvers-tidb-lightning:
     - yudongusa
     - terry1purcell
+    - likidu
   sig-critical-approvers-parser:
     - yudongusa
     - terry1purcell
+    - likidu
   sig-critical-approvers-dep:
     - bb7133
     - cfzjywxk

br/.gitignore

@@ -7,8 +7,4 @@
 .DS_Store
 /go.mod
 /go.sum
-
-# for the web interface
-web/node_modules/
-web/dist/
 .vscode/