diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index c21a6fa1d..79958c2f8 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -1,9 +1,17 @@ # -name: Create and publish a develop Docker image +name: Create and publish a KVCM Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: workflow_dispatch: + inputs: + flavor: + description: Image flavor + type: choice + options: + - dev + - integration + default: dev # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -12,7 +20,60 @@ env: # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: + integration: + if: ${{ inputs.flavor == 'integration' }} + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build server package + env: + BUILD_IMAGE: ghcr.io/alibaba/tair-kvcache-kvcm-dev:2026_02_13_12_03_24230b1 + run: | + docker run --rm --privileged \ + -v "${GITHUB_WORKSPACE}:/workspace" \ + -w /workspace \ + "${BUILD_IMAGE}" \ + bash -lc ' + git config --global --add safe.directory /workspace && + bazelisk build //package:kv_cache_manager_server --config=ci_fast && + cp bazel-bin/package/kv_cache_manager_server.tar.gz open_source/docker/ && + chown "$(stat -c %u /workspace):$(stat -c %g /workspace)" \ + open_source/docker/kv_cache_manager_server.tar.gz + ' + + - name: Prepare image tag and context + id: integration_image + run: | + VERSION="integration-$(date -u '+%Y_%m_%d_%H_%M')-${GITHUB_SHA::7}" + IMAGE="${REGISTRY}/${IMAGE_NAME}:${VERSION}" + test -r open_source/docker/kv_cache_manager_server.tar.gz + echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" + echo "KVCM integration image: ${IMAGE}" >> "$GITHUB_STEP_SUMMARY" + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish integration image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: open_source/docker + file: open_source/docker/Dockerfile.prod + platforms: linux/amd64 + push: true + tags: ${{ steps.integration_image.outputs.image }} + build-args: BINARY_PACKAGE_TAR=kv_cache_manager_server.tar.gz + build: + if: ${{ inputs.flavor != 'integration' }} strategy: fail-fast: false matrix: @@ -97,6 +158,7 @@ jobs: retention-days: 1 merge: + if: ${{ inputs.flavor != 'integration' }} runs-on: ubuntu-latest needs: - build diff --git a/docs/README.md b/docs/README.md index d6bb75192..7cf06090d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,7 @@ - [模块架构与关联关系](design/module_architecture.md) - 各模块职责、依赖方向、控制流与数据流，附 Mermaid 图 - [基本概念](design/basic_concepts.md) - Storage、Instance Group、Instance、Block、CacheLocation 等核心概念 - [ReportEvent 增量上报与权威快照设计](design/report_event_snapshot_uri_version.md) - 增量/快照协同、提交屏障、故障恢复、性能取舍与 Subscriber 集成 +- [ReportEvent / GetHostCacheState 小 block 性能记录](design/report_event_performance.md) - local/Redis 指标解释、锁与可见性语义、有界并发、容量基准及后续优化边界 - [高可用与选主机制](design/ha_leader_elector.md) - HA 架构、LeaderElector 状态机、CoordinationBackend、Leader 发现 - [CacheReclaimer 异步删除与过度逐出优化](design/cache_reclaimer_async_delete.md) - 异步删除生命周期、in-flight credit、反压与无进展退避 - [后台扫描 GC](design/cache_garbage_collector.md) - 基于 authoritative cursor 的后台全量巡检；V1 清理长期 orphan WRITING 和普通 SERVING storage-missing，并提供无副作用读取、精确值条件 CAS 与 HA 生命周期 diff --git a/docs/api/report_event.md b/docs/api/report_event.md index e8e88c0e7..ac6aed39d 100644 --- a/docs/api/report_event.md +++ b/docs/api/report_event.md @@ -16,7 +16,8 @@ 5. 客户端不得在 URI 中填写 `s_version`；KVCM 会自动追加。 6. `committed_snapshot_version` 是 32 位十六进制 opaque generation，不能按大小比较；成功完整 snapshot 后它会成为该 reporter 的严格查询栅栏。 -7. `snapshot_required=true` 是“当前 KVCM 进程还没有该 reporter generation”的提示，不会阻止合法增量。 +7. `snapshot_required=true` 表示请求到达时当前进程还没有该 reporter generation；创建 generation + 的首条 ADD/DELETE 自身仍返回 `true`，下一条事件才返回 `false`。该提示不会阻止合法增量。 8. 当前进程尚未收到该 reporter 的任何合法事件，或节点 unavailable 时，查询不会返回该 节点的数据；节点存活状态是硬门槛。 9. snapshot 失败、进程重启或从未成功 snapshot 的 soft 模式可能返回 stale cache candidate； @@ -55,8 +56,16 @@ KVCM 更新的最小逻辑身份是： block_key + medium + spec.name ``` +HTTP/protobuf 中 `block_key` 使用十进制字符串。调用方既可以发送 signed int64，也可以发送 +unsigned uint64；KVCM 按相同的 64-bit bit pattern 归一化，例如 +`"18446744073709551615"` 与 `"-1"` 指向同一个内部 block key。这与 vLLM 外部 block hash +使用 uint64 的约定兼容；超出 uint64 或 int64 表示范围、带正号或空白的字符串会被拒绝。 + - `medium` 是 block 级属性，例如 `gpu`、`hbm`、`memory`、`disk`； -- `spec.name` 区分一个 block 内的多个物理组成部分，例如不同 TP、full attention 或 mamba state； +- `spec.name` 区分一个 block 内的多个物理组成部分，例如 `F0`、`L1` 等 cache group； +- ADD、DELETE 和 SNAPSHOT 中使用的每个 spec name 都必须已在 + `RegisterInstance.location_spec_infos` 中注册；未注册的 spec 对应 item 返回 + `INVALID_ARGUMENT`，且不会修改 metadata。同一请求中的其他合法 item 仍可生效； - 同一 `block_key` 可以同时存在于多个 medium； - 同一 `block_key + medium` 可以包含多个不同的 spec name； - 同一 `block_key + medium + spec.name` 是集合语义，不是引用计数。 @@ -140,7 +149,8 @@ Instance/InstanceGroup 和 cache metadata 会持久化；reporter 节点表、li 2. 第一条 HEARTBEAT、ADD、DELETE 或 SNAPSHOT 会自动重建 reporter 节点状态，不要求再次 REGISTER； 3. 如果第一条只是 HEARTBEAT，响应为 `snapshot_required=true`、 `committed_snapshot_version=""`，格式合法的历史 metadata 可以重新成为 cache candidate； -4. 如果第一条是 ADD/DELETE，它正常成功并建立新 generation； +4. 如果第一条是 ADD/DELETE，它正常成功并建立新 generation；该次响应仍为 + `snapshot_required=true`，后续事件复用 generation 时变为 `false`； 5. 该增量只更新明确涉及的 spec，不删除其他历史 block/spec； 6. snapshot-capable 调用方可以稍后补一次完整 snapshot； 7. realtime-only 调用方可以继续只发增量。 @@ -196,9 +206,11 @@ InstanceGroup 必须把对应 EventReport storage 配置在 行为： - 建立或刷新该 reporter 的节点状态； -- 重复 REGISTER 可安全重试，medium 列表会合并；但每次成功 REGISTER 都是新的 reporter - lifecycle 栅栏，会使更早生命周期中尚未落盘的 mutation/cleanup 失效，因此不能把 - REGISTER 当作 HEARTBEAT 高频发送，也不应与普通数据请求无序并发； +- 重复 REGISTER 可安全重试，medium 列表会合并；同一请求中的多个合法 REGISTER 分别校验、 + 合并后只执行一次实际注册，因此至多推进一次 lifecycle，所有合法 REGISTER item 获得相同 + 注册结果，非法 REGISTER 只影响自身 item；跨请求的每次成功 REGISTER 都会建立新的 reporter + lifecycle 栅栏，使更早生命周期中尚未落盘的 mutation/cleanup 失效，因此不能把 REGISTER + 当作 HEARTBEAT 高频发送，也不应与普通数据请求无序并发； - REGISTER 本身不创建 `committed_snapshot_version`； - REGISTER 后可以直接发 ADD/DELETE； - KVCM 重启后不必重新 REGISTER； @@ -240,11 +252,11 @@ InstanceGroup 必须把对应 EventReport storage 配置在 "medium": "gpu", "specs": [ { - "name": "full_attention:group=0:tp=0", + "name": "F0", "uri": "event_report://10.0.0.8:9600/gpu/123?size=4096" }, { - "name": "mamba_state:group=0:tp=0", + "name": "L1", "uri": "event_report://10.0.0.8:9600/gpu/123?size=1024" } ] @@ -257,7 +269,7 @@ InstanceGroup 必须把对应 EventReport storage 配置在 - `block_key` 必须是可解析的十进制整数文本； - `medium` 非空，且不能包含 location id 分隔符 `#`； - `specs` 非空； -- 每个 `spec.name` 非空，且同一事件内不能重复； +- 每个 `spec.name` 非空、已在 `RegisterInstance.location_spec_infos` 中注册，且同一事件内不能重复； - 每个 URI 必须合法； - 客户端 URI 不能带 `s_version`； - 旧字段 `block_add.uri` 已废弃，只有 `specs` 生效。 @@ -279,7 +291,7 @@ InstanceGroup 必须把对应 EventReport storage 配置在 "block_key": "123", "medium": "gpu", "spec_names": [ - "full_attention:group=0:tp=0" + "F0" ] } } @@ -289,7 +301,7 @@ InstanceGroup 必须把对应 EventReport storage 配置在 - `block_key`、`medium` 非空且合法；`medium` 不能包含 location id 分隔符 `#`； - `spec_names` 非空； -- spec name 非空且同一事件内不能重复。 +- spec name 非空、已在 `RegisterInstance.location_spec_infos` 中注册，且同一事件内不能重复。 行为： @@ -312,11 +324,11 @@ InstanceGroup 必须把对应 EventReport storage 配置在 "medium": "gpu", "specs": [ { - "name": "full_attention:group=0:tp=0", + "name": "F0", "uri": "event_report://10.0.0.8:9600/gpu/123?size=4096" }, { - "name": "mamba_state:group=0:tp=0", + "name": "L1", "uri": "event_report://10.0.0.8:9600/gpu/123?size=1024" } ] @@ -326,7 +338,7 @@ InstanceGroup 必须把对应 EventReport storage 配置在 "medium": "memory", "specs": [ { - "name": "full_attention:group=0:tp=0", + "name": "F0", "uri": "event_report://10.0.0.8:9600/memory/456" } ] @@ -344,7 +356,8 @@ Snapshot 的完整性规则： - 同一 `block_key + medium` 只能出现一次； - 相同 block key 可以出现在不同 medium； - `medium` 不能包含 location id 分隔符 `#`； -- 每个 block 的 specs 必须非空，且 spec name 不能重复； +- 每个 block 的 specs 必须非空，且 spec name 必须已在 + `RegisterInstance.location_spec_infos` 中注册并且不能重复； - `blocks=[]` 表示该 reporter 当前没有任何 cache，会异步清理其全部旧 location。 Snapshot 的更新语义： @@ -496,14 +509,17 @@ SNAPSHOT_IN_PROGRESS -> SNAPSHOT 失败时重发完整 snapshot ### 9.2 snapshot_required -`snapshot_required` 等价于“当前进程还没有该 reporter generation”： +`snapshot_required` 表示“本次请求到达时当前进程还没有该 reporter generation”。对于首条合法 +ADD/DELETE，KVCM 会在请求内创建 generation 并返回到 `committed_snapshot_version`，但本次响应 +仍保留 `snapshot_required=true`；下一条事件复用该 generation 时才返回 `false`： | 场景 | 值 | | --- | --- | | fresh REGISTER 后 | `true` | | KVCM 重启后的第一条 HEARTBEAT | `true` | | 只有 REGISTER/HEARTBEAT、还没有合法 mutation | `true` | -| 第一条合法 ADD/DELETE 后 | `false` | +| 创建 generation 的第一条合法 ADD/DELETE | `true` | +| 后续复用已有 generation 的 ADD/DELETE | `false` | | 成功 SNAPSHOT 后 | `false` | | invalid-only 首批增量 | 仍为 `true` | | realtime-only reporter | 可忽略该提示，继续发合法增量 | @@ -607,6 +623,23 @@ HTTP 接口为 `POST /api/getCacheLocation`： `ST_EVENT_REPORT_L1P5`、`ST_EVENT_REPORT_L2` 等 backend，适合验证两种 EventReport storage 的 隔离状态。 +`location_spec_names` 不只是返回结果的投影条件，也是 backend/peer 选择前按 query key 生效的候选条件： + +- 为空时，location 中任意合法 spec 都可使该 location 成为候选； +- 非空时，数组长度必须等于 query key 数量，且每个 name 都不能为空；第 i 个 name 只过滤第 i 个 key； +- 第 i 个 key 的 location 必须包含对应的 spec name 才能成为候选，selector 也从该 spec URI 提取 peer； +- 同一 EventReport location 由 `storage_type + medium + host_ip_port` 标识，其中各 spec 必须属于 + 同一个 reporter endpoint； +- 多个 peer 的 prefix/coverage 相同时按 endpoint 字典序选择，避免容器遍历顺序引起选择抖动； +- peer 选择完成后，第 i 个 key 的响应仍只保留其对应 name 指定的 spec。 + +因此 spec name 是 reporter 与查询方之间的稳定协议字段，不能用 object size 代替：不同 cache +group 即使 byte size 相同，也必须使用不同且稳定的 spec name。调用方必须让 +`location_spec_names` 与 `block_keys`（或由 token 生成的 query keys）同序对齐。同一个 block key +可以在不同位置重复并请求不同 spec，用于 mixed-attention/Mamba groups。长度不匹配或包含空 +name 会返回 `INVALID_ARGUMENT`。确定性 tie-break 只消除无序遍历造成的抖动；各 key 经过 +spec 过滤后的候选 peer 集合仍可能不同。 + ### 11.4 GetHostCacheState HTTP 接口为 `POST /api/getHostCacheState`： @@ -633,7 +666,9 @@ HTTP 接口为 `POST /api/getHostCacheState`： "hosts": [ { "host_ip_port": "10.0.0.8:8080", - "prefix_match_blocks": "3" + "local": "3", + "p2p_1_fetch": "0", + "p2p_1_total_match": "3" } ] } @@ -649,11 +684,17 @@ HTTP 接口为 `POST /api/getHostCacheState`： - `QT_UNSPECIFIED` 使用 RegisterInstance 时配置的 `default_query_type`； - 支持 `QT_PREFIX_MATCH` 和 `QT_PREFIX_MATCH_WITH_MAMBA`，其他类型返回参数错误； - 同一个 host 在多个 backend 的有效 cache 会按 host 汇总参与匹配； +- `local` 包含同一 host 的 subscriber 与 Vineyard 上报； +- 非混合注意力对 full local-miss 使用 Prefix 选择远端 Vineyard；混合注意力先对 + FullAttention group 使用 Prefix，再对 Mamba local-miss spec 使用 Coverage； +- `p2p_1_fetch` 表示各 P2P 阶段实际选中并拉取的 spec 所属的去重 block key 数； +- `p2p_1_total_match` 表示本地 cache 与实际选中的远端 spec 合并后的最终前缀； +- 远端 P2P 候选只使用 `ST_EVENT_REPORT_L2`，且不会让 `local` 为 0 的 host 出现在响应中； - reporter unavailable 时，该 host 对应的 event-report location 不参与匹配。 成功完整 snapshot 后，`GetHostCacheState` 会立即忽略完全属于旧 generation 的 location。 snapshot 失败、KVCM 重启恢复或 realtime-only reporter 仍使用 soft metadata，因此 -`prefix_match_blocks` 在这些模式下仍可能是 false positive。 +`local` 在这些模式下仍可能是 false positive。 ## 12. 节点生命周期与查询 @@ -728,12 +769,38 @@ cache 发起物理 DELETE。清理以稳定 location 为粒度：如果一次增 - Backend UT：`kv_cache_manager/data_storage/test/event_report_backend_test.cc` - Manager UT：`kv_cache_manager/manager/test/cache_manager_test.cc` - Meta UT：`kv_cache_manager/manager/test/meta_searcher_test.cc` +- PR HTTP 集成：`integration_test/meta_service/http_interface_test.py` - 基础集成：`integration_test/meta_service/test_report_event.py` - Snapshot 集成：`integration_test/meta_service/test_report_event_snapshot.py` - 重启集成：`integration_test/meta_service/test_report_event_restart.py` -后两项对应的 Bazel target 带 `manual` 标签，不属于默认 GitHub CI。覆盖结论必须以显式执行结果为准； -同样，普通 GitHub check 通过不代表已经执行 ASAN。 +PR HTTP 集成由 `//integration_test/meta_service:http_interface_test` 启动真实 KVCM 进程，属于默认 +`//integration_test/...` CI。基础 ReportEvent 脚本当前没有 Bazel target；Snapshot target 带 +`manual` 标签，重启脚本也需显式执行，因此三者不属于默认 GitHub CI。覆盖结论必须以显式执行 +结果为准；同样，普通 GitHub check 通过不代表已经执行 ASAN。 + +Snapshot target 是外部服务测试，单独执行 `bazel test` 不会替它启动 KVCM。先按脚本头部说明 +启动 meta/admin HTTP 服务，再显式传入端口。功能与容量入口分别为： + +```bash +bazel run //integration_test/meta_service:test_report_event_snapshot -- \ + --host localhost --http_port 56020 --admin_http_port 56040 \ + --instance_id event_report_functional --skip-bench + +bazel run //integration_test/meta_service:test_report_event_snapshot -- \ + --host localhost --http_port 56020 --admin_http_port 56040 \ + --instance_id event_report_bench --only-bench + +# 仅运行小 block / 大批量单请求基准；默认使用进程内 local metadata backend +bazel run //integration_test/meta_service:test_report_event_snapshot -- \ + --host localhost --http_port 56020 --admin_http_port 56040 \ + --instance_id event_report_large_delta \ + --bench-test test_20_large_single_request_delta_scaling +``` + +同一 KVCM 进程上重复执行时，fixture 会通过 `listStorage` 校验已有 storage 的 type 和 EventReport +时序配置；配置不一致应立即失败，不能把任意 `addStorage` 错误当作“可能已存在”后继续测试。 +heartbeat/grace 短时序测试使用独立 storage/instance group，不得缩短功能与容量用例的主 storage。 | ID | 用户行为 | 自动化覆盖 | | --- | --- | --- | @@ -759,6 +826,7 @@ cache 发起物理 DELETE。清理以稳定 location 为粒度：如果一次增 | D-12 | KVCM 只追加一个合法 s_version，不改变客户端 URI 其他部分 | 基础集成 `_assert_profile_specs_in_locations`；snapshot 集成 `_assert_reporter_scope` | | D-13 | 第一条 delta 已创建 generation、metadata 写失败时准确报错，重试复用 generation | `TestReportEventFirstDeltaMetadataFailureReportsFailureAndReusesGeneration` | | D-14 | 同一 spec 的折叠事件共享最终 metadata 写入失败结果 | `TestReportEventFoldedDeltaEventsShareFinalWriteFailure` | +| D-15 | 大于 32 KiB 的部分失败批次保持逐项结果与输入索引严格对齐，只重试失败项后复用 generation 并最终收敛 | snapshot 集成 `test_34_large_partial_batch_preserves_item_alignment_and_retry` | | S-01 | snapshot 跨 medium 完整上报、响应返回 generation | `TestReportEventSnapshotReplacesCompleteSpecSetPerBlock`；snapshot 集成 `test_17/22` | | S-02 | 同 block 跨 medium 合法，同 block+medium 重复非法 | `TestReportEventRejectsCanonicalDuplicateSnapshotKeysButAllowsDifferentMedia`；snapshot 集成 `test_27_*` | | S-03 | snapshot block 内重复 spec name 被拒绝 | `TestReportEventRejectsDuplicateSpecNamesWithinSnapshotBlock` | @@ -781,6 +849,10 @@ cache 发起物理 DELETE。清理以稳定 location 为粒度：如果一次增 | Q-07 | GetCacheLocationsByBackend 同样执行 reporter liveness 过滤 | `TestGetCacheLocationsByBackendWithBackendSelectors`；snapshot 集成 `test_16a` | | Q-08 | 三个查询入口在 timeout 隐藏和 HEARTBEAT 恢复时结果一致 | snapshot 集成 `test_16a_heartbeat_timeout_then_recovery` | | Q-09 | snapshot 成功后即使异步 cleanup 尚未运行，遗漏的旧 version block 也立即不可见 | `TestSuccessfulSnapshotImmediatelyFencesOmittedOldVersionBeforeCleanup`、`TestGetCacheLocationEnforcesReporterLifecycleAndBatchOrdering` | +| Q-10 | backend 查询在 peer 选择前按 spec 过滤，并对同覆盖率 peer 确定性择优 | `TestGetCacheLocationsByBackendWithBackendSelectors`；`EventReportPrefixFiltersRequestedSpecBeforePeerSelection`；`EventReportCoverageFiltersRequestedSpecBeforePeerSelection`；`EventReportPrefixTieBreaksByPeerAddress`；`EventReportCoverageTieBreaksByPeerAddress`；HTTP 集成 `test_event_report_requested_spec_filters_before_peer_selection` | +| Q-11 | requested spec 不存在时 Prefix/Coverage 都返回与输入等长的空结果，不回退到其他 spec | `EventReportUnknownRequestedSpecReturnsNoCandidate`；HTTP 集成 `test_event_report_requested_spec_filters_before_peer_selection` | +| Q-12 | requested spec 按 any-of 语义匹配，重复 name 不改变结果；匹配同一 location 的非首个 spec 时仍使用该 reporter endpoint | `EventReportRequestedSpecMatchesAnyNameIncludingNonFirstSpec` | +| Q-13 | requested-spec gap 会终止 Prefix，但 Coverage 可跳过 gap 继续返回后续命中；响应投影后 `spec_size` 始终等于实际 specs 数量 | `EventReportRequestedSpecGapStopsPrefixButNotCoverage`；`TestGetCacheLocationsByBackendWithBackendSelectors` | | L-01 | 自动 heartbeat timeout 隐藏、grace 内恢复原 generation | `MightExistFollowsAutomaticLivenessAndFullReporterLifecycle`；snapshot 集成 `test_16a` | | L-02 | unavailable 期间增量可写但保持不可见，HEARTBEAT 后恢复 | `TestReportEventLazilyRestoresReporterWithoutRegisterOrSnapshot`；snapshot 集成 `test_16a` | | L-03 | 超过 grace 后按 generation 原子 unregister，最终 metadata 删除持有 generation lease，旧 cleanup 不伤重新注册数据 | `HeartbeatRecoveryFencesCleanupAlreadySelectedByLivenessLoop`、`ConditionalUnregisterCannotRemoveNewGeneration`、`CleanupLeaseFencesReregisterThroughFinalDeleteStage`；snapshot 集成 `test_16b` | @@ -797,6 +869,7 @@ cache 发起物理 DELETE。清理以稳定 location 为粒度：如果一次增 | C-04 | 已进入 metadata I/O 的旧 lifecycle 请求不能在 HOST_DOWN、REGISTER、新 snapshot 后恢复写入 | `TestOldDeltaCannotCrossReporterLifecycleAfterReregisterAndSnapshot` | | C-05 | snapshot 等待 active delta 超时后 abort candidate、保留 committed generation 并重新打开 delta 写门 | `SnapshotDrainTimeoutAbortsCandidateAndReopensWriteGate` | | C-06 | delta 等待 in-flight snapshot 超时后返回可重试错误，且不会中止 snapshot；节点校验不会在超时栅栏前无限等待 | `DeltaWaitTimeoutReturnsSnapshotInProgressWithoutAbortingSnapshot`、`TestReportEventDeltaGateTimeoutReturnsSnapshotInProgressAndCanRetry` | +| C-07 | mutation lease 每个 RMW 阶段只获取一次，但阻塞在 metadata read 的旧请求仍可被 HOST_DOWN/新 lifecycle 抢占 | `TestBatchMutationWriteLeaseIsAcquiredOncePerRmwPhase`、`TestBatchMutationWriteLeaseFailurePreventsAllWrites`、`TestHostDownCancelsSnapshotAlreadyWritingMetadata`、`TestHostDownMakesAlreadyAdmittedDeltaInvisibleWithoutDeadlock`、`TestOldDeltaCannotCrossReporterLifecycleAfterReregisterAndSnapshot` | | V-01 | reporter host、ADD/DELETE/SNAPSHOT medium 含 `#` 时拒绝且无写入副作用 | `RegisterNodeWithMediums`、`TestReportEventRejectsInvalidRequestsAndMapsItemErrors`；snapshot 集成 `test_19_*` | | V-02 | instance/host/storage type/instance backend 的公共字段校验 | `TestReportEventRejectsInvalidRequestsAndMapsItemErrors`；snapshot 集成 `test_14_*` | | V-03 | event_type 与 oneof payload 缺失/错配时该 item fail closed；同批其他合法 mutation 可独立懒初始化并生效 | `TestReportEventRejectsMismatchedPayloadsWithoutSideEffects`；snapshot 集成 `test_13/33` | @@ -807,6 +880,7 @@ cache 发起物理 DELETE。清理以稳定 location 为粒度：如果一次增 | CFG-01 | snapshot interval/drain timeout 默认值、正数校验、负数拒绝、JSON/proto round trip，并由 backend 加载 | `TestEventReportStorageSpecSnapshotSettingsDefaultAndValidation`、`TestEventReportStorageSpecJsonRoundTripIncludesSnapshotSettings`、`EventReportStorageSpecProtoRoundTripPreservesSnapshotSettings`、`BasicAccessors` | | PERF-01 | 100 线程共 1 万 ADD、50 线程共 5000 混合批次均无错误 | `EventReportBenchTest.test_17/18` | | PERF-02 | 10 reporter × 每台 5000 blocks 完整 snapshot，并查询每台首/中/末 block | `EventReportBenchTest.test_19_ten_reporters_full_snapshot_capacity` | +| PERF-03 | 单请求 512 个跨重复 medium 的增量正确落盘；手工容量测试记录 100/1000/5000/20000 个 ADD 的总 RT 与单 event 开销 | `TestReportEventLargeDeltaBatchAcrossRepeatedMediums`、`EventReportBenchTest.test_20_large_single_request_delta_scaling` | 上表中的参数解析、故障注入、CAS 竞态等内部边界使用 UT 验证；跨 HTTP 的正常流程、并发流程、 节点生命周期和 Redis/KVCM 重启使用集成测试验证。 diff --git a/docs/configuration.md b/docs/configuration.md index 15c8347f3..c3083d79b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,16 @@ kvcm.schedule_plan_executor_thread_count=8 # 0 < migration_worker_budget < executor_thread_count kvcm.schedule_plan_migration_worker_budget=3 +# GetHostCacheState 大请求使用的独立有界 metadata query executor。 +# worker_count 包含当前 RPC caller；默认 4 表示 caller + 3 个后台线程，设为 1 可禁用并行。 +kvcm.meta_query.worker_count=4 + +# key/投影元素数达到该阈值才进入并行路径；小请求保持串行。 +kvcm.meta_query.parallel_threshold=256 + +# 每个并行任务一次领取的连续元素数，必须不大于 parallel_threshold。 +kvcm.meta_query.chunk_size=128 + # CacheReclaimer 删除 Future 在 delay 结束后可继续抵扣水位的最长时间；到期只关闭 credit， # 不取消底层删除。默认 60000ms。 kvcm.cache_reclaimer.inflight_delete_timeout_ms=60000 @@ -167,6 +177,25 @@ executor_thread_count > 1 新增迁移策略，因此不能根据启动时的策略状态放宽校验。`2/1`、`8/3` 是合法配置，`1/1`、`8/8` 和 `8/0` 会导致 `ServerConfig::Check` 或 `CacheManager::Init` 失败。已有单线程自定义配置需要至少调整为 `2/1`。 +### GetHostCacheState metadata query executor + +`kvcm.meta_query.*` 控制独立于 `SchedulePlanExecutor` 的进程级有界查询池，仅服务大规模 +GetHostCacheState metadata read 和 host 投影/归约。线程不会按 instance 或请求创建：所有 MetaIndexer +共享一个 executor，RPC caller 始终参与工作，因此 `worker_count=4` 只创建 3 个后台线程。队列饱和时 +请求由 caller 继续执行，不会无限堆积任务。 + +只有单一 `local` metadata backend 的大批 key 读取会并发；`redis`、`cached` 和其他 backend 保持原有 +batch 调用。CPU 投影使用同一有界池。参数约束为： + +```TEXT +1 <= worker_count <= 64 +0 < chunk_size <= parallel_threshold +``` + +默认值是 `4/256/128`。`worker_count=1` 是线上回退开关；调大 worker 前应同时对比单请求和并发请求 +p99、CPU 与 ReportEvent RT。设计、指标含义、测试命令见 +[`design/report_event_performance.md`](design/report_event_performance.md)。 + 同一个 `migration_config` 中，`strategies` 的 `(source_storage_name, target_storage_name)` 组合必须唯一。 相同 source 迁移到不同 target、不同 source 迁移到相同 target，以及 `hot -> warm -> cold` 级联均可配置；只有 完全相同的 source/target route 会被拒绝。重复 route 无法明确选择各自的 threshold、method、retention 和 Mark diff --git a/docs/design/cache_reclaimer_async_delete.md b/docs/design/cache_reclaimer_async_delete.md index 45704c4ef..5056f4c13 100644 --- a/docs/design/cache_reclaimer_async_delete.md +++ b/docs/design/cache_reclaimer_async_delete.md @@ -221,6 +221,14 @@ size 的 Location 按 0 bytes 记账，但仍加入 pending 并受数量上限 只有本次请求覆盖某个 block 的全部有效 Location 时，才增加一个 `predicted_deleted_keys`。V1 不跨 多个 `DeleteHandler` 合并推断，允许保守少计。 +EventReport Location 由外部 reporter 拥有，只能由 ReportEvent snapshot、delta 或 host lifecycle 清理，不能进入 +通用物理存储回收请求。它仍然是 metadata key 上的有效 Location：若一个 block 同时包含 EventReport 与普通 +Location，删除全部普通 Location 后 key 仍然存在，因此不得产生 `predicted_deleted_keys` credit。EventReport usage +不参与按 storage type 的水位，但仍计入 group 总 byte 水位；通用 Reclaimer 即使因此触发，也只能选择普通 +Location，EventReport-only 场景会按 no-progress 退避，不能进入物理删除。EventReport Location 也不能作为 +migration cold-tier spec coverage。key-count 水位继续使用 MetaIndexer 的官方总 key 数，无法证明可删除时保持 +fail-closed、允许保守多触发而不能提前抵扣。 + 水位判断改为： ```text @@ -362,6 +370,10 @@ V1 提供以下指标： 有效 Location。 15. Meta 异步请求跳过 `CLS_DELETING`，重复提交不会再次安排物理删除。 16. Admission 已进入队列但尚未执行时停止 Executor，cancel callback 使 Future 以错误终态完成。 +17. EventReport Location 不进入物理删除请求，但与普通 Location 共存时仍阻止错误的 + `predicted_deleted_keys` credit。 +18. EventReport usage 仍可触发 group 总 byte 水位，但不触发 EventReport storage-type 水位。 +19. reporter URI host 即使与 migration target storage 同名，也不能补齐 cold-tier spec coverage。 ### 8.2 集成测试关注点 diff --git a/docs/design/report_event_followup_todo.md b/docs/design/report_event_followup_todo.md index 0cfbf528b..ff71c9ce1 100644 --- a/docs/design/report_event_followup_todo.md +++ b/docs/design/report_event_followup_todo.md @@ -1,140 +1,82 @@ -# ReportEvent follow-up TODO +# ReportEvent follow-up 状态与 TODO -本文记录 PR #233 完成后暂缓处理的 Review 项。它们不属于当前 PR 的已实现能力， -也不应在发布说明或测试结论中描述为已覆盖。 +本文最初记录 PR #233 暂缓处理的 Review 项。后续 hardened/performance 分支已经解决其中多数 +问题；为避免后续 AI 重复修改或把已覆盖能力误报为缺口，本文件现在同时维护“仍未完成事项” +和“已解决证据”。以当前工作分支相对 main 的完整 diff 为准，不能只看最早两个 feature commit。 后续修改仍应以 [`report_event.md`](../api/report_event.md) 和 [`report_event_snapshot_uri_version.md`](report_event_snapshot_uri_version.md) 中的接口与生命周期契约为准，并为并发问题优先补充可控阻塞点的确定性测试。 +小 block / 大批量性能问题的已完成优化、禁止破坏的 HOST_DOWN 抢占窗口和后续目标化 Redis +读取方案记录在 [`report_event_performance.md`](report_event_performance.md)。 -## P1：生命周期与重试语义 +## 仍未完成 -### 1. Snapshot 最终提交需要覆盖完整 reporter lifecycle +### 1. 运维 CLI 的 snapshot delta drain timeout 兼容性 -当前 lifecycle generation lease 覆盖 metadata replace，但 replace 完成后的 `Sync` -和 snapshot version commit 不在同一个 generation 栅栏内。旧 snapshot 可能在 metadata -写入完成后被阻塞，期间新 REGISTER 启动新的 reporter lifecycle，随后旧请求恢复并提交 -旧 snapshot version。 +仓库内 Admin proto、proto/config 转换、序列化、默认值和正数/零/负数校验已经覆盖；若生产使用的 +`kvcm_ops` 来自外部运维仓库，仍需在该仓库确认 add/update storage 暴露 +`snapshot_delta_drain_timeout_ms`，且旧服务端不会因未知字段受影响。 -建议： - -- snapshot candidate 绑定创建时的 lifecycle generation； -- lifecycle 变化时取消旧 candidate； -- 提供原子的 `CommitSnapshotVersionIfGeneration(expected_generation)` 或等价操作， - 使 generation 校验、candidate 校验和最终 commit 不可分割； -- 明确 `Sync` 失败、generation 失效和 candidate 被替换时的可重试错误。 - -验收测试： - -- 旧 snapshot 阻塞在 `Sync`； -- 同一 reporter 执行 REGISTER 并进入新 lifecycle； -- 恢复旧 snapshot； -- 断言旧请求不能 commit，新 lifecycle 的 snapshot version 和 metadata 不受影响。 - -### 2. Delta 部分失败的 `item_results` 需要满足安全重试契约 - -ADD 和 DELETE 分阶段写 metadata 时，同一 block/location 上的事件可能存在依赖。例如： - -```text -event 0: ADD {A, B} -event 1: DELETE {A} -期望最终状态：只有 B -``` - -若 ADD B 失败、DELETE A 成功，接口可能返回 event 0 失败、event 1 成功。调用方按当前文档 -只重试失败的 event 0 后，最终状态会变成 A、B 均存在。 - -建议优先保持现有“只重试失败项”契约，并将失败传播到同一 block/location 的事件依赖闭包。 -如果选择要求调用方重试整个相关 delta batch，需要先评估兼容性并同步修改接口文档。 - -验收测试应覆盖同一请求内的 ADD→DELETE、DELETE→ADD、重复 ADD/DELETE，以及每个 metadata -阶段分别部分失败的组合。 - -### 3. EventReport storage 在线变更需要 backend incarnation 栅栏 - -在线 disable、update 或 remove EventReport storage 时存在以下待完善边界： - -- backend lookup 需要检查 backend 是否仍然 available； -- 已持有旧 `shared_ptr` 的请求需要在状态锁内检查 backend 是否仍 open/available； -- HOST_DOWN/liveness cleanup 不能只携带可能从头计数的数值 generation； -- backend 重建后，旧请求和旧 cleanup 不能命中新 backend 中同值 generation 的 reporter。 - -建议为每次 backend 创建分配不可复用的 incarnation，并让请求 admission、generation lease -和异步 cleanup 同时校验 incarnation + reporter generation。 - -验收测试： - -- 阻塞旧请求或旧 cleanup； -- disable/update/remove storage 并重建 backend； -- 注册 reporter 并写入新 metadata； -- 恢复旧操作； -- 断言旧操作失败或提前退出，且不能修改新 backend 的状态。 - -### 4. 重复 RegisterInstance 需要校验 instance group - -同一 `instance_id` 的重复注册一致性校验还需要包含 `instance_group_name`。否则调用方使用 -另一个 instance group 重复注册时可能先收到成功，随后 ReportEvent 仍按持久化的旧 group -查找 backend，并返回不直观的 `EventReportBackend not found`。 - -建议在重复注册时直接返回明确的冲突错误，并在错误信息中同时给出已有 group 和请求 group。 - -验收测试应覆盖相同 group 幂等注册、不同 group 冲突，以及冲突后原 instance 仍可正常查询 -和上报。 - -## P2：接口、配置与运维一致性 - -### 5. 完成 snapshot delta drain timeout 的端到端配置 - -- 在 `kvcm_ops add_storage` / `update_storage` 暴露 - `snapshot_delta_drain_timeout_ms`； -- Admin Proto 输入负数时返回参数错误，不应静默忽略并回落到默认值； -- 保持配置文件、Admin API、运维 CLI 和运行时 update 使用同一校验范围； -- 为默认值、合法边界、零值和负值补齐端到端测试。 - -### 6. 明确并实现同一请求内 HEARTBEAT 的事件顺序语义 - -当前 HEARTBEAT 延迟到事件解析完成后处理。对于 unavailable reporter 的 -`[HEARTBEAT, ADD]`，ADD 可能先捕获旧 lifecycle，随后 HEARTBEAT 更新 generation， -导致按输入顺序本应可执行的 ADD 失败。 - -需要明确 ReportEvent 是严格按序执行，还是仅对 mutation 定义顺序。如果接口保持按序语义， -应让 HEARTBEAT 的 lifecycle 变更在后续 mutation 捕获 lease 前生效，并补 -`HEARTBEAT→ADD/DELETE/SNAPSHOT` 组合测试。 - -### 7. 多个 REGISTER 的逐 item 结果应与实际处理一致 - -同一请求内多个 REGISTER 当前会预聚合 mediums 并只执行一次。后续非法 REGISTER 可能使 -前面的合法 REGISTER 一并失败，与逐 item 的结果契约不完全一致。 - -后续应选择并文档化一种语义： - -- 按输入顺序逐项验证和执行；或 -- 将 REGISTER 定义为请求级原子操作，并让所有相关 item 返回一致且明确的结果。 - -需要覆盖合法/非法 REGISTER 混合、重复 medium 和 REGISTER 与其他事件混合的测试。 - -### 8. 收敛 ReportEvent 成功路径的逐请求 INFO 日志 - -HTTP 层和 ServiceImpl 层仍可能为同一 ReportEvent 请求各输出一条 INFO 日志。高 QPS 下 -会产生重复日志和额外 I/O。 - -建议只保留一处必要的成功日志，或改为 DEBUG/采样日志；错误日志仍需保留 trace id、 -instance id、reporter、storage type、event type 和错误码，且不能打印完整大 snapshot。 - -## P3：低优先级维护项 - -### 9. 避免在热路径依赖 `std::random_device` +### 2. 避免在热路径依赖 `std::random_device` 评估 snapshot version/token 生成对随机性的真实要求。若不要求密码学安全，可改为进程级 初始化的生成器或无锁/低锁的单调序列与随机前缀组合，避免某些平台上 `std::random_device` 阻塞或产生不稳定延迟。修改后需继续保证 token 格式、进程内唯一性和 并发安全。 -### 10. 保持 PR/发布说明中的验证口径可审计 +### 3. 保持 PR/发布说明中的验证口径可审计 - 带 `manual` 标签的 snapshot/restart HTTP 集成测试不属于默认 GitHub CI； - 没有实际 ASAN workflow 或本地执行记录时，不应写成“ASAN 已覆盖”； - force-push 或新增 commit 后，应区分“此前 head 的全量结果”和“当前 head 的定向结果”； - 发布前记录实际执行的 commit SHA、命令、target、模式（内源/外源、debug/release、 ASAN/UBSAN）和结果。 + +## 当前分支已解决的历史项 + +- Snapshot candidate 绑定 reporter lifecycle generation，并通过 + `CommitSnapshotVersionIfGeneration` 原子校验 generation/candidate 后提交；测试覆盖 REGISTER + 抢占旧 snapshot、cleanup 与后续 attempt 的 fence。 +- Delta 同一 block/location 的 ADD/DELETE 建立安全重试依赖闭包；任一阶段失败会传播到相关 + item，避免仅重试失败项反转 last-operation-wins 结果。 +- 在线 disable/remove/rebuild 会检查 backend open/available；异步 cleanup 同时携带旧 backend + `shared_ptr` 和 generation，旧 incarnation 不能清理新 backend。等待中的 delta/snapshot 在 + `Close()` 或 disable 时会被主动唤醒并二次检查 admission；disable 遗留的 candidate 会被 abort， + re-enable 后写门可继续使用，不能等待完整的 snapshot drain timeout。 +- 重复 `RegisterInstance` 校验 `instance_group_name`，不能把既有 instance 移到另一 group。 +- unavailable reporter 的同请求 HEARTBEAT 恢复会把已准入 ADD/DELETE/SNAPSHOT 统一收敛到恢复后 + generation；正反事件顺序均有测试。 +- 同一请求内多个 REGISTER 逐 item 校验，合法 item 的 mediums 合并并只执行一次实际注册，因此 + 每个请求至多推进一次 lifecycle；非法 sibling 只影响自身结果。 +- HTTP 与 ServiceImpl 的 ReportEvent 成功入口日志已降为 DEBUG；错误日志仍保留诊断字段。 +- ReportEvent delta 热路径已压平为 block 哈希表和 location/spec 小向量，并直接生成最终 metadata task； + `LocationSpec`/`CacheLocation` 的失效 move、重复 URI parse、BatchMerge task 深拷贝和 ordered-map spec + merge 已修正。同机 Release/O2、纯 local 的 20k create/update 相对直接父提交下降约 21%/25%；具体 + A/B、语义约束与剩余并行边界见 `report_event_performance.md` 2.4 和 5.5。 +- `snapshot_delta_drain_timeout_ms` 已进入 Admin proto/config 转换和运行时验证，非正值会被配置 + 校验拒绝；仓库内 round-trip 和非法边界测试已覆盖。 +- 查询有界线程池对部分线程创建失败和 `ParallelFor` 分配/入队异常执行显式清理；`noexcept` 热路径 + 不会因资源异常直接 `std::terminate`，已入队 helper 也不能越过请求返回边界访问调用方引用。 +- cached backend 恢复一旦由 SCAN cursor 得到非空批次，就保留该批精确 key，直到 Get 与 + PutIfAbsent 全部成功后才推进 cursor；失败重试不得重新 SCAN 同一 cursor，否则变化中的 Redis + 集合可能让原失败 key 在发布 Running 前被跳过。 +- `MetaStorageBackendManager` 的 `Init()` 只在所有 backend factory 都成功后一次性发布对象， + `noexcept` 内部的分配/工厂异常会转成 `EC_ERROR`；成功初始化后拒绝重复 Init。`Open()` 同样 + 拒绝重复调用，cache open 或 recovery-thread 创建失败会回滚两侧 backend，避免半初始化状态和 + 给 joinable `std::thread` 再赋值触发 `std::terminate`。 +- Python Manager Client 的 route-refresh 线程启动失败会事务回滚 HTTP/session-discovery 资源； + `close()` 超时后由 refresh worker 延迟关闭仍在使用的 discovery client。HTTP 非 200 和公共响应 + envelope 损坏统一归类为 `requests.RequestException` 子类（并兼容旧 `AssertionError`），包括 + `check_response=False` 路径，供 Vineyard 等上层正确推进熔断器；明确的 Manager 非 OK 业务状态 + 保持原异常语义。 + +关键回归入口包括 `SnapshotCommitRejectsChangedLifecycleGeneration`、 +`TestReportEventDeltaFailureMarksSafeRetryDependencyClosure`、 +`TestHostCleanupCannotCrossEventBackendIncarnations`、 +`CloseUnblocksSnapshotAndDeltaWaiters`、 +`DisableWhileSnapshotDrainsAbortsCandidateAndReopensGate`、 +`TestReportEventHeartbeatRecoveryCarriesSameRequestMutationsIntoNewLifecycle`、 +`TestReportEventValidatesMultipleRegisterItemsIndependently` 和 +`TestReportEventCoalescesMultipleValidRegistersIntoOneLifecycle`。 diff --git a/docs/design/report_event_performance.md b/docs/design/report_event_performance.md new file mode 100644 index 000000000..ea88980f6 --- /dev/null +++ b/docs/design/report_event_performance.md @@ -0,0 +1,1871 @@ +# ReportEvent 小 block / 大批量性能优化记录 + +本文记录 `ReportEvent` 在 block size 较小、单次或并发上报 block 数较多时的性能分析、已经实施的 +低风险优化、必须保持的并发语义，以及后续继续优化的边界。后续 AI 或开发者应先阅读本文和 +[`report_event_snapshot_uri_version.md`](report_event_snapshot_uri_version.md)，不要仅根据某个 RT +指标直接引入并行。 + +## 1. 现象与指标解释 + +线上曾观察到 `ReportEvent` 与查询 RT 接近 100ms，同时 `meta_indexer.get_io_time_us` 可占约 +80ms，查询侧 `manager.prefix_match_time_us` 也偏高。这里最容易误判的是指标名： + +- `meta_indexer.get_io_time_us` 是 metadata backend 调用的墙钟时间，不等于磁盘或 Redis 网络 I/O。 + `storage_type=local` 时完全不经过 Redis，时间来自 sharded LRU lookup、每个 item 的 shared lock、 + location 容器复制/分配、revisit 统计及 CPU/cache miss； +- `storage_type=cached` 才是 local cache 加 persistent backend fallback/recovery；`storage_type=redis` + 才会进入 Redis pipeline、reply 传输和 HSCAN/HMGET。分析监控前必须先确认实例实际 storage type； +- `manager.prefix_match_time_us` 包含 metadata read、location 可见性判断、URI 解析、host 投影和前缀 + 归约。其内层指标不能与它相加； +- ReportEvent 原有的节点表独占锁和 lifecycle lease 放大会增加 CPU/排队与 heap allocation。纯 local + 模式观察到 80ms 时，应先查 O(block 数) 的串行读和容器复制，不能归因于不存在的 Redis。 + +## 2. 已实施的低风险优化 + +### 2.1 请求内 medium 注册去重 + +一个 ReportEvent 请求内，相同 reporter/medium 的每个 ADD/DELETE 原来都会调用 +`EnsureNodeRegistered`。现在只在该 medium 第一次成功时调用；成功 REGISTER 声明的 medium 也会 +写入请求内集合。失败不会缓存，因此后续有序 REGISTER 或下一事件仍可重试，保持 +“delta 可以出现在显式 REGISTER 前”的既有语义。 +节点是否已经确保与 medium 集合分开记录：fresh reporter 的空 snapshot 即使没有 medium，也必须 +调用一次 `EnsureNodeRegistered` 完成懒初始化，不能被“缺少待注册 medium”的快路径跳过。 + +`EventReportBackend::EnsureNodeRegistered` 对已存在且 medium 已知的节点使用 shared lock 快路径； +只有缺少 medium 时才释放 shared lock、获取 unique lock 并二次检查。节点 map 和 mediums 仍始终 +受 `nodes_mutex_` 保护，不能改成无锁读取或用一个原子布尔值替代整个 map 的一致性。 + +### 2.2 lifecycle lease 从每 key 收敛到每次 metadata mutation 一次 + +原实现会在 modifier 的每个 key 上查 reporter fence、分配 `shared_lock`；已有 location 的 ADD +经过 block-create 和 targeted-location 两阶段时还会重复一轮。当前 `BatchMergeLocationSpecs` 已按 +2.5 合并为一次 targeted RMW，在该次 RMW 第一次进入 modifier 时获取一个 lease，后续 key 复用： + +```text +metadata read（可被 lifecycle writer 抢占） + | +non-blocking lifecycle lease（本次 fused RMW 一次） + | +本阶段全部 metadata mutation + | +释放 lease +``` + +lease 不能在 metadata read 前获取，也不能无条件持有整个 ReportEvent。HOST_DOWN/REGISTER 的 +锁序是 `lifecycle -> metadata`；如果旧请求阻塞在 metadata I/O，lifecycle writer 必须能先完成。 +旧请求恢复后获取 generation-pinned lease 失败并放弃写入。fused RMW 从读取 key/目标 location 到 +upsert 始终处于同一个 metadata shard 临界区，lease 从 read 后持有到 upsert 返回。确定性 +HOST_DOWN/重注册竞态测试是这个优化的强制回归项。 + +### 2.3 GetHostCacheState 的 local 大查询路径 + +小 block 会放大一次请求的 block key 数。旧查询对每个 key 串行读取 local LRU，并把完整 +`unordered_map>` 克隆到请求结果；随后每个 location 又重复查 +instance group、data-storage backend 和 reporter node lock。当前实现做了以下收敛： + +1. `MetaLocalBackend::GetLocationValues` 只复制不可变 `CacheLocation` 的 `shared_ptr`，不复制 map node、 + hash bucket 和 location-id 字符串。原有 `GetLocations` 保留给需要按 id 查找的调用方； +2. 仅在“单一 persistent backend 且类型为 `local`”时启用渐进读取：先同步读取 4096-key probe，若候选 + prefix 尚未终止，再以 16384-key chunk 有界并发读取后缀。`cached`、`redis`、dummy 等模式仍执行一次 + 原有 batch 调用，避免放大远端请求或破坏 recovery 语义； +3. 全进程所有 MetaIndexer 共享一个有界 `QueryExecutor`。配置的 worker 数包含 RPC caller，默认 4 + 表示 caller + 3 个后台线程，不为每个请求创建线程。队列满时 caller 自己完成剩余 chunk；若 caller + 已完成全部工作，尚未启动的 helper 会取消，不能为了一个排队中的空任务制造队头阻塞。线程池部分 + 构造失败时会停止并 join 已创建线程；`ParallelFor` 的分配/入队失败会先阻止 queued helper 再进入 + callback、等待 active helper 退出并返回失败，不能因 `noexcept` 直接终止进程，也不能让 helper 在 + 请求返回后继续访问 request-local 引用； +4. 首个有界 metadata range 读取完成后，第一次处理 event-report location 时用 `std::call_once` 为该请求 + 抓取 reporter liveness 与 committed-version 快照。每个 backend 只持有一次 `nodes_mutex_` shared lock， + 后续 range 的 `(block, location)` 只读不可变快照； +5. host/spec 投影和候选 host 前缀归约复用同一个有界 executor，输出仍按 host 字典序构造，普通 prefix、 + Mamba、Eagle pop 和 medium filter 的结果语义不变； +6. 普通 prefix 只为首 key 建立排序后的候选 host，后续 key 直接写入按候选编号组织的 packed bitset， + 不再为每个 key 构造 `map>`，也不保存普通 prefix 根本不需要的 spec name。 + Mamba 把 required spec 和每个 host 的 state presence 都编码为 64-bit words，不再为每个 key 建红黑树， + 也不再保留一字节一个 `(key, host)` 的 state matrix； +7. GetHostCacheState 专用可见性 checker 在校验 EventReport reporter 状态和 URI 时一并返回已经解析的 + medium/host。host 投影复用该结果，不再对同一 location id 做第二次 split。EventReport URI 的查询侧 + 校验直接在不可变字符串上单次扫描，用 `string_view` 比较 generation；不再为每个 spec 拆分 + protocol/host/path、构造 query-param `std::map` 或复制 token。普通 prefix 对一个 EventReport location + 只做一次候选 host 标记； +8. service access log 默认只记录 key count、首末 key、query type、medium count、返回 host 数和最大 + prefix，不再把数千 key 的 protobuf 完整转 JSON 后再 parse 成 DOM。诊断时可临时设置 + `KVCM_GET_HOST_CACHE_STATE_FULL_ACCESS_LOG=true` 恢复完整 request/response，压测时应保持关闭； +9. local metadata read 不再让每个 query worker 对每个 block 直接更新共享 revisit histogram counters。 + 每个 128-key chunk 先在本地累计 bucket/count/sum，再按非零 bucket 提交原子增量。Prometheus 最终值与 + 逐 key `Observe` 完全一致，但避免十万级原子 RMW 争抢同一组 cache line；这段时间属于 + `meta_indexer.get_io_time_us`； +10. prefix 只把首个 `EC_NOENT` 当作正常终止。首个 miss 之后的 speculative read 结果不影响已经确定的 + 前缀；但 miss 之前的 `EC_ERROR`、`EC_MISMATCH` 等硬错误必须原样返回，不能伪装成较短的 cache miss。 + 普通 prefix 与 Mamba 路径遵循相同规则； +11. `p2p_host_count>0` 不能把 local miss 直接当作全局 stop，因为 Vineyard peer 仍可能延续覆盖。该路径改用 + ordered compact visitor：普通 prefix 在线维护候选 peer 的前缀交集；Mamba 最多分 local、peer plan、最终投影 + 三个有界 pass，没有实际 peer plan 时跳过最终 pass。非 local backend 只读取一次 compact batch 并在内存中 + 重放，均不再构造逐 key 的 `map/set` 图。 + +可见性快照在首个有界 metadata range 读取之后、其 projection 开始时采集。采集前已经可见的 HOST_DOWN +会被当前请求过滤；采集后的 HOST_DOWN 允许当前请求继续看到旧状态，但采集完成后本请求不再变化，下一 +请求会重新采集。百万 key 流式查询不能等全部 range 读完再开始 projection，否则必须保留全量 location +或再次扫描 metadata；因此这里明确把快照线性化点放在首 range 与后续并行 range 之间。由于 `available` +是逐 reporter atomic，多个 reporter 与并发 liveness 变化之间不承诺一个全局事务时间点；保证的是 +request-stable 结果，避免同一 reporter 在一次长 projection 中前半段 up、后半段 down。 + +这不是用“原子变量 + 双重检查”替换 node map。`available` 本身可以是 atomic，但 reporter 的存在性、 +lifecycle generation、strict flag 与 committed token 必须在同一个受保护快照中一致；只原子化一个布尔值 +会产生 host 已换代但仍配旧 token 的组合。`EnsureNodeRegistered` 的 shared-lock fast path + unique-lock +二次检查仍用于写路径，请求级只读快照用于大查询路径，两者解决的问题不同。 + +相关启动参数如下，修改后需重启： + +| 参数 | 默认值 | 约束/含义 | +| --- | ---: | --- | +| `kvcm.meta_query.worker_count` | 4 | 1..64；包含 caller，设为 1 可回退为串行 | +| `kvcm.meta_query.parallel_threshold` | 256 | key/投影元素数小于该值时不并发 | +| `kvcm.meta_query.chunk_size` | 128 | `0 < chunk_size <= threshold` | + +默认值是保守起点，不是固定 SLA。调参必须同时看单请求 p50/p99、并发查询 p99、CPU、RPC worker 排队和 +ReportEvent RT；worker 并非越多越好。 + +### 2.4 ReportEvent 折叠、URI 与 Location 拷贝收敛 + +2026-08-05 的进一步分段和同机 A/B 表明，纯 local 模式下两阶段 RMW 的真实 backend I/O 只占少数， +主要成本是请求内聚合以及 modifier 在 metadata shard 锁内做的 Location/spec 深拷贝。当前实现做了以下 +不改变持久化语义的收敛： + +1. `LocationSpec` 和 `CacheLocation` 因显式析构函数而没有隐式 move，原来多处看似 + `std::move` 的代码实际退化为深拷贝；`set_location_specs(vector&&)` 也错误地执行了拷贝赋值。现在显式 + 提供 `noexcept` move，并真正移动 vector。该修复同时覆盖新建 Location、spec merge、迁移等已有调用点； +2. delta 请求从“三层 `map` + 每个 mutation 的 event vector”改为 block 哈希表以及通常很小的 + location/spec 连续 vector。每个稳定 `(block, location, spec name)` 仍按请求顺序原地覆盖，最后按 + block/location/spec 排序后直接生成 ADD/DELETE task；删除了 + `delta_spec_mutations -> block_to_add/del -> merged_entries -> tasks` 的重复聚合和拷贝链； +3. 重试依赖保留为每个稳定 block/location 的有序 event 引用。只有已经 materialize 且参与最终 ADD 或 + DELETE phase 的事件先接收该 phase 的写错误，随后再按原有规则闭包传播。因此 last-operation-wins、 + admission failure、两 phase 不同错误以及逐 item 返回语义均未因扁平化改变； +4. 协议 URI 在入口完整校验时保存已经解析的 `DataStorageUri`，追加 `s_version` 时复用，不再为同一 URI + 重复 parse；BatchMerge 的版本一致性校验也复用本轮已解析对象，同时仍在 API 边界保留 raw 参数计数， + duplicate `s_version` 仍会 fail closed； +5. `BatchMergeLocationSpecs` 直接让 caller-owned task 与目标 location id 对齐，不再复制 location id 和 + 整组 spec/URI。existing Location 仍做一次必要的 copy-on-write，之后在其小 vector 内按 name 原地 + 覆盖、兼容 legacy 重名并恢复字典序；不再构造每 key 的 ordered map 和第二份完整 spec vector； +6. 单 spec/task 的常见路径不创建去重 hash set，多 spec 时 set 保存 `string_view`；storage usage 使用 + 一段 flat vector + offsets，不再为每 key 分配 vector。入口 URI 校验时顺带累计新 spec 的 `size`， + merge 只解析旧 spec，避免写入成功后再次遍历、解析全部新 URI。 + +这些优化不改变 lifecycle lease、shard lock 或 HOST_DOWN 的锁序。收益来自减少进入和持有 metadata +shard 锁期间的 CPU/分配工作，因此既降低单请求 RT，也缩短并发请求的锁占用窗口；不能把它解释成 +“把锁换成原子变量”。 + +### 2.5 BLOCK_ADD fused targeted RMW + +2026-08-06 在独立分支落地了此前刻意延后的 fused 原语。旧的 existing-location ADD 先通过 +`ReadModifyWriteBlock(GetLocationIds)` 枚举 block 的全部 location id，再通过 +`ReadModifyWriteLocation` 读取目标 location 并 merge/upsert；纯 local 下至少产生两次 metadata read、 +一次 upsert 和两轮 RMW 容器。当前路径改为： + +```text +GetLocationsWithKeyStatus(key, requested_location_ids) + | 同时返回 key 是否存在 + 各目标 location 的值/错误 + v +modifier：create 或 copy-on-write merge + | +同一 shard-lock 临界区内一次 Upsert +``` + +必须保持以下不变量： + +1. `key missing` 与 `key exists but target location missing` 严格区分。只有前者进入 + `put_global_indices`，参与 `max_key_count` 检查并在 upsert 成功后增加 `key_count`；已有 key 新增 + location 不增加 key 数，容量已满时仍允许更新已有 key。若一个 internal upsert batch 同时包含已有 + key 更新和超容量的新 key，只给新 key 返回 `EC_NOSPC`，不能把已有更新连带拒绝； +2. local backend 在一次 LRU lookup/item shared-lock 中返回上述两个层次的状态。返回 `EC_OK` 的 location + 必须非空且 id 与请求一致，否则 indexer fail closed；输出 vector 每次重新初始化，miss 不能泄漏 caller + 复用缓冲区中的旧指针； +3. `MetaStorageBackendManager` 在 cached recovery 模式仅对真正的 cache key miss 回源 persistent；cache + 中已有 key 但缺目标 location 时，cache 状态仍是 authoritative。generic backend fallback 仅对 + “所有目标均 NOENT”的歧义行补一次 `Exists`；纯 local 主路径不走该 fallback； +4. generation-pinned lifecycle lease 在 targeted read 后、modifier 第一次 mutation 前获取一次，并持有到 + upsert 返回；不能提前到 metadata read 前，也不能在 read 与 write 之间释放； +5. 逐 location read/type/modifier/write 错误保持原位，部分成功只更新成功 location 的 storage usage。 + legacy duplicate spec name 仍按原 last-value-wins 规则归一化；malformed backend shape、空/错 id、 + `key missing + target EC_OK` 等矛盾状态全部拒绝写入； +6. 当前实现只是把两个逻辑 RMW 合为一个 targeted read-modify-upsert；local backend 的 upsert 仍会再次 + lookup 并获取 item unique-lock。没有把 backend item 指针或锁暴露给 modifier，也没有引入批内线程。 + +## 3. 当前明确不做的事情 + +暂不并行执行 ReportEvent 内的 metadata batch。原因不是并行永远无效，而是 5.10 的同吞吐 A/B 已证明 +把单次 L2 batch 控制在约 2k 能显著降低 ReportEvent、Get 和 Heartbeat 尾延迟；服务端再增加 writer +会与优先级更高的查询争夺 CPU、allocator 和 local LRU/item lock，同时扩大 key-count、请求内顺序与 +lifecycle fencing 的并发面。若后续仍要并行，必须使用独立有界 executor、按互斥 shard 分组并保留串行 +回退，不能创建 request-local thread 或复用查询 executor。 + +暂不把 local targeted read 与 upsert 进一步合成“持有一个 item unique-lock、在 backend 内执行 +modifier”的原语。那会让 manager callback 进入 backend 临界区、扩大锁序与异常安全边界；当前一次 +targeted read + 一次 upsert 已消除整轮 block-id 枚举，同时保持 backend API 分层。 + +GetHostCacheState 也不并发 Redis/cached backend batch，不创建 request-local thread，不复用只有少量 +worker 且承载回收/迁移的 `SchedulePlanExecutor`。查询池独立且有界，避免长 metadata 请求饿死系统任务。 + +## 4. 非 local backend 的扩展边界 + +当前部署和本轮交付门槛是纯 local metadata；真实 Redis 不在本轮验证范围。通用 +`GetLocationsWithKeyStatus` fallback 已保证正确性，但当所有目标 location 均不存在时会额外调用 +`Exists`。若未来启用 Redis，应实现原生 pipeline，在一个 round trip 中同时返回： + +1. key 是否存在（用于 `key_count/max_key_count`）； +2. 请求指定 location id 的值与逐项错误； +3. 不枚举、不传输无关 location value。 + +Redis 实现应把 EXISTS 与目标 HMGET 放进同一 pipeline round trip，并单独跑真实 Redis 的新 key、已有 +key 缺 location、已有 location、properties-only key、部分 I/O 失败与恢复测试。在完成这些验证前， +不能把本轮纯 local 性能数据外推到 Redis，也不能删除 generic fallback。 + +## 5. 验证与观测清单 + +- UT：请求内 512 个跨重复 medium 的 ADD；并发 EnsureNodeRegistered；fused RMW 单 lease 与失败原子性； + HOST_DOWN、REGISTER、新 snapshot 抢占阻塞 metadata read；同请求 ADD/DELETE 顺序；新 key、已有 key + 缺 location、已有 location、容量已满更新、部分 backend 错误、malformed response shape 与 storage + usage 精确性。 +- 手工容量：`EventReportBenchTest.test_20_large_single_request_delta_scaling` 分别记录 100/1000/5000/20000 + 个新 block ADD 与相同 block 再次 ADD 的总 RT、单 event RT，并查询首/中/末 block，不能只看 + HTTP 成功码。 + 可在启动 KVCM 后用 `--bench-test test_20_large_single_request_delta_scaling` 单独执行。纯 cache + 部署保持默认 local metadata backend；只有明确验证 Redis 部署形态时才传 `--meta-storage-uri`。 +- 线上对比：至少拆分 request parse/fold、node ensure、lifecycle lease wait/fail、RMW lock wait、 + `get_io_time_us`、serialize、enqueue/upsert 和完整 ReportEvent RT；同时观察查询 p50/p99。 +- local 路径已经使用目标化 backend read；若 `get_io_time_us` 仍接近总 RT，应先看 LRU/item-lock wait 与 + request key-count 分桶，不能再归因于已删除的全 location-id 枚举。 +- 若 backend I/O 已显著下降但 CPU/锁等待仍主导，再评估按互斥 shard 分组的有界并行（建议先从 + 2~4 并发开始），并对查询 p99、连接池等待和 Redis CPU 设置回退阈值。 + +GetHostCacheState 的新增分段指标： + +- `meta_searcher.indexer_get_time_us`：整个 MetaIndexer 读取； +- `meta_indexer.get_io_time_us`：其内部 backend 调用墙钟时间； +- `meta_searcher.host_projection_time_us`：location 可见性、URI/host/spec 投影； +- `meta_searcher.host_prefix_reduce_time_us`：按 host 计算普通或 Mamba 前缀； +- `manager.prefix_match_time_us`：上述阶段及少量管理层开销的外层总时间。 + +UT 覆盖单线程/4-worker 结果对照、缺失 key 和重复 key、普通/Mamba 大于阈值、medium filter、Eagle pop、 +HOST_DOWN 发生在 metadata read 期间、快照 instance 隔离、并发读写 local item、executor 队列饱和、 +callback 异常、嵌套调用与线程池异常清理。可选 HTTP benchmark： + +```bash +python integration_test/meta_service/test_report_event_snapshot.py \ + --host localhost --http_port 56020 --admin_http_port 56040 \ + --instance_id event_report_cluster_0 \ + --bench-test test_21_get_host_cache_state_local_scaling +``` + +它会对 100/1000/5000/20000 个命中 block（再加一个尾部 miss）记录 20 次串行请求以及 16-way 并发请求的 +p50/p99，并逐次校验 host prefix。对照串行基线时分别用 +`kvcm.meta_query.worker_count=1` 和默认 `4` 重启服务运行；不要在一次进程内动态改配置。 + +### 5.1 2026-08-04 本地 Redis 基准记录 + +在同机 Redis 7.2.5、debug KVCM、真实 meta/admin HTTP 接口下，使用本文新增的 benchmark 得到： + +| events/request | 新建 block ADD | 已有 block/location 再次 ADD | +| ---: | ---: | ---: | +| 100 | 7.61ms（0.0761ms/event） | 10.40ms（0.1040ms/event） | +| 1000 | 62.99ms（0.0630ms/event） | 88.10ms（0.0881ms/event） | +| 5000 | 303.59ms（0.0607ms/event） | 420.14ms（0.0840ms/event） | + +命令使用独立 Redis metadata backend，并对每档首/中/末 block 做查询校验。该数据只是当前开发机的 +可重复基线，不是 SLA；已有 location 明显更慢也印证了两阶段 HSCAN + targeted HMGET 是下一轮 +I/O 优化重点。线上判断必须结合 `get_io_time_us`、Redis CPU/网络和查询 p99。 + +### 5.2 2026-08-04 GetHostCacheState 纯 local 对照 + +同一台开发机、debug KVCM、真实 meta/admin HTTP 接口、单 reporter/medium 下，分别以 +`kvcm.meta_query.worker_count=1` 和默认 `4` 重启进程运行 +`test_21_get_host_cache_state_local_scaling`。每个请求包含 N 个连续命中 block 和一个尾部 miss；串行数据 +为 3 次 warmup 后 20 次请求，并发数据为 16-way、共 32 次请求。每次响应均校验 prefix： + +| blocks | worker=1 串行 p50/p99 | worker=4 串行 p50/p99 | worker=1 16-way p50/p99 | worker=4 16-way p50/p99 | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 1.83/1.87ms | 1.87/2.71ms | 10.71/23.03ms | 13.68/23.01ms | +| 1000 | 11.26/11.62ms | 6.25/6.31ms | 14.86/23.20ms | 13.24/21.75ms | +| 5000 | 53.35/55.41ms | 27.79/29.60ms | 68.45/102.07ms | 52.59/70.90ms | + +100 block 小于默认 threshold，差异属于噪声且没有并发收益；5000 block 的单请求 p50 下降约 48%， +16-way p99 下降约 31%。这是 debug 单机趋势而非线上 SLA。线上 rollout 仍应先小流量，确认分段 metrics、 +CPU 和 ReportEvent p99；若并发度带来负收益，可直接把 worker_count 回退为 1。 + +### 5.3 2026-08-04 Release/O2 纯 local before/after + +为避免把 Debug 构建开销当成线上结论，在同一台开发机上分别构建性能提交的直接父版本 +`88e29c1` 和当前版本；两者均使用 Release/O2、真实 meta/admin HTTP、纯 local metadata backend。 +当前版本使用默认 `worker_count=4`、`parallel_threshold=256`、`chunk_size=128`。下表选择双方第二轮 +完整运行的数据；每个响应仍逐次校验 host prefix： + +| blocks | 父版本串行 p50/p99 | 当前串行 p50/p99 | 父版本 16-way p50/p99 | 当前 16-way p50/p99 | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.71/0.73ms | 0.68/0.70ms | 5.09/10.27ms | 5.34/11.42ms | +| 1000 | 2.62/2.69ms | 1.80/1.87ms | 9.03/15.78ms | 6.99/11.10ms | +| 5000 | 11.60/11.82ms | 6.95/7.05ms | 29.04/42.10ms | 22.01/28.56ms | +| 20000 | 47.33/51.67ms | 26.02/27.30ms | 142.68/188.26ms | 95.26/123.18ms | + +20k 串行 p50 下降约 45%；这说明轻量 local read、host 投影与有界并行都产生了实际收益。高基数 +16-way 请求即使优化后仍可能超过 100ms，不应只看全局 RT；至少要按 `request_key_count` 分桶。 +一次 5001-key 并发请求的 gauge 快照中，父版本 `prefix_match/get_io/service` 分别为 +25.717/9.620/25.750ms；当前版本为 10.441/6.623/10.471ms，且新增 projection/reduce 分别为 +3.168/0.184ms。Gauge 只是最后一次观测，不是 percentile，不能与上表混用。 + +并发度和 chunk 调优结论： + +- 低并发下 worker 8/16 可继续降低单请求 RT，但 worker 32 收益已明显递减且并发尾延迟回升； +- 20k isolated 热轮中，worker 4/8/16 的串行 p50 约为 26.02/23.05/21.18ms；16-way + p50/p99 代表值分别为 95.26/123.18、73.92/115.82、100.24/133.01ms； +- 同时运行 100-thread ReportEvent ADD 与重复 20k GetHost 时，worker 4 与 8 的 ReportEvent 分别为 + 1399/1435 QPS、p99 186.93/184.97ms；但第二轮 GetHost 16-way p50/p99 从 worker 4 的 + 108.62/184.91ms 恶化到 worker 8 的 132.50/217.67ms； +- chunk 64 在小批次略快但冷并发更抖，chunk 256 在 5000 blocks 略慢，默认 128 更均衡。 + +因此默认仍保持 worker 4/chunk 128。低并发、CPU 余量充足且更关心单请求 RT 的部署可以显式尝试 +worker 8；必须同时观察 ReportEvent p99、GetHost key-count 分桶、CPU 和 executor queue saturation， +不能把本机 isolated 结果直接当作通用默认值。 + +### 5.4 ReportEvent 高基数结论 + +本节与 5.5、5.6 保留的是 fused targeted RMW 落地前的历史基线和决策背景；当前实现状态以 2.5 和 +5.10 为准，不能再把“尚未合并两阶段”当作现状。 + +Release/O2、纯 local 下，当前版本单请求 20k BLOCK_ADD 的 create/update 为 +196.55/239.51ms，约 9.8/12.0us 每 event，整体近似线性。对比 `feature/event_report_4@b776dd4`， +5000-event 单请求及 100-thread ADD、50-thread mixed throughput 均只相差约 0~4%，属于噪声；当前改动 +没有回归 ReportEvent，但也不能宣称改善其高并发尾延迟。 + +一次 20k existing-location update 的详细观测为：客户端 RT 249.59ms，KVCM service/event timer +约 161ms，block RMW 32.18ms，location RMW 63.38ms，local backend get/upsert I/O 仅 +7.33/6.04ms，metadata lock wait 约 1us。也就是说，剩余成本不是 Redis 或全局锁，而是： + +1. HTTP/JSON/protobuf 请求解析与响应边界约 89ms； +2. 两阶段 RMW 内逐 key 的 CPU、拷贝和 modifier 约 95ms，其中真实 backend I/O 约 13ms。 + +这组数据是 5.5 拷贝收敛前的基线；本轮先减少 block-create/targeted-location 两阶段的数据变换，并保持 +串行 RMW。若线上 10k~20k 单批 ReportEvent 在 5.5 的收益后仍需显著低于 100ms，再单独评估 local RMW +shard 并行并增加 parse/fold 指标。并行会触及写入原子性、lifecycle fence 和锁序，不能仅凭单请求 RT +开启；上线前仍可通过限制单请求 event 数或分批上报控制尾延迟。 + +### 5.5 2026-08-05 ReportEvent 拷贝收敛同机 A/B + +使用本节 2.4 的性能改动直接父提交 `8f7d5bc` 和当前工作树分别构建 Release/O2 二进制；两个进程使用 +独立端口、独立纯 local instance，在同一台机器连续运行 +`test_20_large_single_request_delta_scaling` 两轮。下表是两轮平均值，所有请求均校验 committed token 以及 +首/中/末 block 的最终 URI： + +| events/request | 父提交 create/update | 当前 create/update | create/update 降幅 | +| ---: | ---: | ---: | ---: | +| 100 | 1.84/1.89ms | 1.55/1.56ms | 15.8%/17.7% | +| 1000 | 11.78/13.16ms | 9.41/10.16ms | 20.1%/22.8% | +| 5000 | 54.23/63.01ms | 43.88/48.14ms | 19.1%/23.6% | +| 20000 | 217.19/269.40ms | 171.52/202.26ms | 21.0%/24.9% | + +20k 单次请求的两轮原始区间分别为：父提交 create 216.86~217.51ms、update +267.44~271.36ms；当前 create 170.27~172.77ms、update 201.97~202.54ms。趋势随规模增大而扩大， +符合“减少每 event/node 分配与深拷贝”的预期，不是固定开销或单次快样本。 + +同一当前二进制随后运行 GetHostCacheState local benchmark，100/1000/5000/20000 block 串行 p50 为 +0.67/1.78/6.71/25.52ms；20k 的 16-way p50/p99 为 95.65/125.41ms，与 5.3 的优化后基线基本一致， +未观察到通用 move 修复带来的查询回归。以上仍是单机趋势而非 SLA；线上 rollout 应同时观察 +ReportEvent key-count 分桶、RMW 两阶段、lock wait、CPU 与 GetHost p99。 + +### 5.6 2026-08-05 线上 L2 大批次反馈与后续判别 + +本节记录当时尚未落地 fused targeted RMW 的线上反馈。后文关于“下一步做 fused”的表述是历史判断， +最终实现、风险控制与新数据见 2.5、5.10。 + +一轮按目标 QPS 持续发送的线上压测中，客户端全程 `fail/drop/skipped=0`，工作队列通常为 0~1； +RSS 峰值约 159.4MB、payload budget 峰值约 129.2MB。停止边界丢弃的一个任务不计入稳态失败。 +客户端 L2 构包平均约 3.84ms，而 HTTP 约 217ms；Get 构包约 1.28ms，而 HTTP 约 118ms。因此本轮 +瓶颈不在客户端构包、排队或内存预算，HTTP 往返与服务端处理占绝大多数。 + +L2 延迟随单批 block 数近似线性：batch p50 约 7k 时 RT p50 约 158ms，batch p95 约 34.5k 时 +RT p95 约 795ms，两点折算的处理速度都约为 44k blocks/s。大批次期间 Get 出现 200~400ms 尾延迟， +很小的 Heartbeat 也达到约 164ms p99。这些数据足以判断“大 L2 请求的服务端线性工作正在拖慢共享 +资源”，但仅凭客户端 HTTP 时间仍不能区分以下来源各占多少： + +1. meta HTTP 端口上的所有 API 共用同一组 `coro_http_server` I/O worker，业务 handler 又同步进入 + `MetaServiceImpl`/`CacheManager`；长请求可能造成 worker 占用或 CPU 调度排队； +2. `MetaIndexer` 的 RMW 对每个 batch 在 writer shard lock 内依次完成 backend read、modifier、可选的 + persistent-backend 序列化和 upsert/delete；同 shard 的其他写入会等待。纯 local 的 Get 不获取这把 + writer shard lock，但会和 upsert 争用对应 `MetaMemCacheItem` 的 shared/unique mutex；跨 shard 请求 + 仍可能争用 CPU、allocator 和 cache； +3. HTTP body 解析、protobuf/JSON 转换以及响应序列化不在现有 RMW 分段指标内，大 payload 会继续带来 + 线性 CPU 和内存带宽成本。 + +下一轮线上观测应把客户端 HTTP RT 与服务端 `ReportEvent` service timer 对齐，并同时按 batch-size +分桶采集 request parse/fold、block/location RMW、`get_io_time_us`、deserialize/serialize、 +`lock_wait_time_us`、upsert 和 HTTP worker queue/active 数。现有 `lock_wait_time_us` 只覆盖 writer shard +mutex，不覆盖 local item mutex；若要验证 Get 被 upsert 阻塞，需另加 item-lock wait 指标。若 HTTP RT +显著大于 service timer，优先查 HTTP worker 排队与 body 编解码；若 service timer 本身接近 HTTP RT， +再依据 RMW/lock/CPU 分段决定是继续减少 Location 处理成本，还是做 shard-aware 并行。不要从 Heartbeat +p99 单独反推出某一把锁有问题。 + +本节 5.5 的拷贝收敛可把 20k create/update 降低约 21%/25%，属于应先上线验证的常数优化；按本轮 +34.5k p95 批次估算，它不足以单独消除 700ms 级尾延迟。发布侧最直接的保护是限制单次 L2 block 数并 +拆成较小批次（可先以 2k~5k 做压测起点），同时给 Get/Heartbeat 保留独立的并发或队列预算。若拆批后 +服务端总吞吐仍稳定且尾延迟显著下降，再决定是否需要把 ReportEvent 放到独立有界 executor，或按互斥 +metadata shard 做 2~4 路有界并行。两种结构性改动都必须保留 request 内 last-operation-wins、两阶段 +key-count、lifecycle fence 和逐 item 错误语义，并设置过载回退，不能用无限并行掩盖单批过大。 + +随后另一轮混合压测得到：L1P5 ADD 为 1.999 QPS、平均/p99/max RT 为 +7.56/42/146ms；L2 ADD 为 14.998 QPS、149,969 blocks/s，即平均约 10k blocks/request，平均/p99/max +RT 为 224/841/924ms；Get 为 0.099 QPS、平均 8,984 keys/request，平均/p99 RT 为 119/418ms。L2 的 +单请求处理速度约为 44.6k blocks/s，与前一轮约 44k blocks/s 基本相同，进一步确认瓶颈随 block 数线性 +增长，而不是压测器吞吐不足；约 3.36 个 L2 请求的平均并发也解释了为何 aggregate blocks/s 高于单请求 +速度。 + +这轮数据发生在 5.5 的本地性能 patch 推送之前；当时该远端开发分支 head 仍为 `637d3e0`，所以不能把 +它当作 5.5 优化后的线上结果。应在包含 5.5 commit 的新 head 上用相同流量重跑 before/after，再判断 +剩余差距。预期 20%~25% 的 RMW/折叠收益仍不足以完全消除 800ms p99；若 after 仍呈相同斜率，下一项 +若本轮只优化 GetHostCacheState，应先上线 2.3 的紧凑投影、摘要 access log，并在 CPU 有余量的环境用 +worker 4/8 做 A/B；fused targeted RMW 会改变 ReportEvent 写入原语和 key-count 语义，不应混入这次 +低风险查询优化。 + +`GetHostCacheState` 的完整 access-log JSON 问题已按 2.3 收敛。`MakeBatches` 的 `batch_key_size` 仍是 +shard-boundary soft limit，同一 shard 的 keys 不会被硬切；但 writer shard lock 不阻塞纯 local Get。 +若线上 `mutex_shard_num=16`，10k keys 的均匀请求约为 625 keys/shard；增加 128/256 的 lock-hold hard +limit 主要改善写写公平性，必须用混合写 p99 验证，不能把它误报成 Get 尾延迟根因。 + +### 5.7 2026-08-05 GetHostCacheState 紧凑投影与 worker 4/8 A/B + +在 2.3 的 packed presence、EventReport 解析复用和摘要 access log 完成后，使用同一 Release/O2 +二进制、纯 local metadata、真实 HTTP 接口运行 `test_21_get_host_cache_state_local_scaling`。每档都先 +构造连续命中 block，再追加一个尾部 miss；20 次串行与 16-way 请求逐次校验 prefix。结果如下： + +| blocks | worker=4 串行 p50/p99 | worker=8 串行 p50/p99 | worker=4 16-way p50/p99 | worker=8 16-way p50/p99 | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.56/0.59ms | 0.56/0.59ms | 8.47/17.51ms | 10.84/28.22ms | +| 1000 | 1.32/1.35ms | 1.20/1.23ms | 6.47/10.87ms | 6.49/11.22ms | +| 5000 | 4.54/4.72ms | 3.86/3.89ms | 21.12/40.57ms | 20.18/39.99ms | +| 20000 | 16.07/16.18ms | 13.56/13.63ms | 77.87/101.60ms | 103.57/134.09ms | + +对比 5.5 中改动前同一分支的 20k worker=4 基线（串行 25.52ms、16-way 95.65/125.41ms），新实现的 +串行 p50 下降约 37%，16-way p50/p99 下降约 19%/19%。worker=8 在低并发 20k 单请求上比 worker=4 +再快约 16%，符合“CPU 有余量、Get QPS 很低”的部署条件；但 16-way 20k p99 反而增加约 32%。因此代码 +默认值继续保持 4。若线上 Get 约 0.1 QPS 且 CPU 确有余量，可显式配置 worker=8 做小流量 A/B，必须 +同时观察 L2 ReportEvent 压力下的 Get p99 和 executor queue；不能仅凭单请求数据修改全局默认值。 + +### 5.8 2026-08-05 Get 查询 URI 零分配扫描与 histogram 批量提交 + +在 5.7 的 worker=4 版本上继续检查发现两个与 block 数线性相关、且都位于 GetHostCacheState 的热点： + +1. `IsEventReportLocationReadable` 对每个 spec 构造 `DataStorageUri`。解析会复制 URI 的多个 substring，并为 + 每个 query param 分配 `std::map` node；随后 `GetParam` 和 `SnapshotUriInfo` 又复制 32-byte token。查询 + 实际只需要确认 URI 有合法 scheme、`s_version` 不重复且为 32 位十六进制，并与 request snapshot 中的 + committed token 比较。因此改为一次 `string_view` 扫描，malformed/重复 token 仍然 fail closed； +2. `MetaLocalBackend::GetLocationValues` 对每个命中 key 调用 revisit histogram `Observe`。默认 13 个 bucket + 下，一次 10k-key 查询会产生十万级共享 counter 原子 RMW；4 个 query worker 会争抢同一组 cache line。 + 现在每个 executor chunk 先本地聚合，再一次性提交 bucket/count/sum，最终指标值不变。 + +用当前源码、Release/O2、4096 条变化的典型 EventReport URI、100 万次循环做隔离微基准： + +| URI 可见性检查 | 每 spec 分配次数 | 每 spec 累计分配 | 每 spec CPU | +| --- | ---: | ---: | ---: | +| 原完整 `DataStorageUri` parse | 10 | 605.7B | 约 497ns | +| 新 `string_view` 单次扫描 | 0 | 0B | 约 51ns | + +新扫描约快 9.7 倍。按 20k specs 估算，仅这一步减少约 12.1MB 短生命周期 allocator 流量和 8.9ms +单核 CPU；这里的 MB 是累计分配流量，不是常驻 RSS。另一个 4-thread、默认 13 buckets、128-key chunk、 +每轮 10240 observations 的隔离基准中，逐 key 原子更新每轮约 3.9~4.1ms，chunk 聚合约 +0.039~0.041ms，count/sum/buckets 完全一致。该数字只衡量 histogram 自身，不应外推成完整 API 倍数。 + +随后使用与 5.7 相同的 Release/O2 二进制、纯 local metadata、真实 HTTP benchmark，连续运行三次并取 +各项中位数： + +| blocks | 本轮 worker=4 串行 p50/p99 | 5.7 串行 p50/p99 | 本轮 worker=4 16-way p50/p99 | 5.7 16-way p50/p99 | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 0.51/0.54ms | 0.56/0.59ms | 6.26/13.51ms | 8.47/17.51ms | +| 1000 | 1.12/1.15ms | 1.32/1.35ms | 5.91/10.48ms | 6.47/10.87ms | +| 5000 | 3.10/3.17ms | 4.54/4.72ms | 10.32/23.56ms | 21.12/40.57ms | +| 20000 | 10.28/10.47ms | 16.07/16.18ms | 23.95/34.25ms | 77.87/101.60ms | + +20k 串行 p50/p99 下降约 36%/35%，16-way p50/p99 下降约 69%/66%；三次 20k 串行 p50 为 +10.26/10.30/10.28ms，结果稳定。最后一批 20k 并发请求的 gauges 随单请求调度不同落在： +`get_io_time_us=4.7~8.4ms`、`host_projection_time_us=2.2~3.3ms`、外层 +`prefix_match_time_us=8.7~12.9ms`。这证明两项分别降低了 metadata read 内共享原子竞争和 read 后 URI +投影；它不改变 ReportEvent 写入语义，也不缓存每 block 的解析对象。 + +该 benchmark 是空闲服务的 local 对照，不含线上 15 QPS、约 10k blocks/request 的 L2 ReportEvent +混合压力。上线后仍需用同一压测流量重点比较 `meta_indexer.get_io_time_us`、 +`meta_searcher.host_projection_time_us` 和 Get p99；若混合负载仍远高于该基线，再依据分段指标检查 local +LRU/item lock，而不是重新增加无界 worker。 + +### 5.9 2026-08-05 混合压力复核与并行位图组合回归 + +在当前 Release/O2、纯 local metadata、真实 HTTP 服务上继续做两组隔离验证。第一组持续 75 秒，使用 +4 个 reporter 混合 20 QPS ADD（100 blocks/request）、10 QPS DELETE（50 blocks/request）、2 QPS +10k-key Get、5 秒 heartbeat，并在 35 秒周期 snapshot 中主动遗漏 10% 当前数据验证 authoritative cleanup。 +最终 1503 ADD、752 DELETE、8 snapshot、151 Get 和 60 heartbeat 全部成功，影子状态逐事件校验无误； +10k Get 客户端 p50/p95/p99 为 5.43/20.35/25.18ms，最大 101.20ms 出现在大 snapshot 并发窗口。 + +第二组按线上复现参数发送约 15 QPS、约 10k blocks/request 的 L2 ADD，持续 45 秒。单个 Python 进程 +同时承担大 JSON 构包、writer、逐 ADD 的 10k-key 正确性查询和独立 reader 时，客户端统计出现约 2.7s +的 Get p99；但三种小请求的延迟同时抬升，说明该值包含本地 GIL/worker 排队。以服务端 access log 作为 +业务处理边界重新统计，同一阶段结果为： + +| 服务端请求 | count | avg | p95 | p99 | max | +| --- | ---: | ---: | ---: | ---: | ---: | +| GetHostCacheState（全部） | 753 | 7.43ms | 11.35ms | 12.83ms | 26.00ms | +| GetHostCacheState（独立 10k reader） | 46 | 7.11ms | 10.16ms | 10.28ms | 10.47ms | +| ReportEvent | 707 | 73.77ms | 96.97ms | 117.87ms | 143.41ms | + +因此同进程压测器的 HTTP wall time 不能直接当作服务端 Get RT。大写入结束后立即重新运行连续命中 +benchmark，20k Get 串行 p50/p99 为 8.84/8.95ms，16-way p50/p99 为 23.39/35.07ms，未观察到 +allocator/LRU 经高分配压力后的持续退化。 + +另外补充一个确定性参考模型 UT，将此前分别覆盖的两个维度叠加：70 个 candidate host 横跨两个 +64-bit presence word，384 个 key 触发 metadata query executor 并行路径，每个 host 使用不同的预期 +prefix，并同时检查 Eagle-pop 和 medium filter。该用例完成 Release 100 轮、ASAN 20 轮以及完整 +MetaSearcher Release/ASAN 回归，结果一致。它主要防止 packed presence 的跨 word 索引、并行 slice 写入 +或 prefix reduction 在后续优化中发生静默错位。 + +### 5.10 2026-08-06 fused targeted RMW、纯 local 回归与同吞吐分批 A/B + +本轮工作位于 `codex/report-event-write-performance`，开始前已 fetch 并 rebase 到 +`origin/main@ae9cb0dfa593071be47e0a601af05d8159b383b7`；原查询优化分支未改动。部署明确使用纯 local +metadata，因此本节的功能、ASAN 与性能结论只以 local 为交付门槛，不把真实 Redis 结果混入结论。 + +#### 单请求 before/after + +在同一 Release/O2、真实 HTTP、全新 local instance 上运行 +`test_20_large_single_request_delta_scaling`。before 是 rebase 后、fused 改动前的基线；after 是最终链接 +产物三次独立串行复测的中位数。每档均校验 committed token 和首/中/末 block 的最终 URI： + +| events/request | before create/update | after create/update | update 降幅 | +| ---: | ---: | ---: | ---: | +| 100 | 1.57/1.59ms | 1.53/1.40ms | 11.9% | +| 1000 | 9.41/10.16ms | 9.20/8.73ms | 14.1% | +| 5000 | 43.75/48.26ms | 42.34/40.99ms | 15.1% | +| 20000 | 172.00/202.12ms | 169.75/172.09ms | 14.9% | + +create 路径没有旧的第二阶段，after 基本持平，说明新 key-status 语义和 flat bookkeeping 没有引入明显 +回归；收益集中在已有 location。三轮 20k create/update 区间为 `166.81~191.29ms` / +`170.75~189.44ms`，因此应看中位数和线上分桶，不能拿单次最好值当 SLA。20k 的 KVCM +service/access-log timer 从 before create/update `81.642/113.413ms` 变为两轮 after 平均 +`79.28/85.60ms`，约下降 `2.9%/24.5%`。客户端 update 中位数下降约 14.9%，剩余差异主要在 HTTP +JSON/protobuf 边界和 worker 排队，不能继续算作 metadata RMW 收益。 + +#### 相同约 150k blocks/s 的 batch-size A/B + +使用 `tools/scripts/report_event_load.py`、16 reporter、纯 local、真实 HTTP，把总 blocks/s 保持接近， +同时发送 9k-key Get 和 heartbeat。三轮 ADD、Get、heartbeat 均 `failed=0`，每个 ADD 还抽查一个最终 key； +结果如下： + +| ADD 形态 | 成功数 / 实际 QPS | ADD avg/p95/p99 | 9k Get avg/p95/p99 | Heartbeat avg/p99 | +| --- | ---: | ---: | ---: | ---: | +| 10k × 15QPS | 317 / 14.41 | 364.49/1268.12/1661.32ms | 271.45/1050.49/1254.13ms（n=5） | 54.66/489.21ms | +| 5k × 30QPS | 457 / 29.60 | 98.84/197.97/313.58ms | 29.56/150.72/152.65ms（n=16） | 12.55/82.22ms | +| 2k × 75QPS | 1149 / 74.62 | 50.04/123.93/174.86ms | 18.71/56.68/99.07ms（n=16） | 14.36/87.62ms | + +10k 轮的 Get 只有 5 个样本，不能把其 percentile 当成精确 SLA；但 ADD 样本数足够，三个 API 的尾延迟 +又同向变化，足以说明超大 HTTP 请求的长任务、瞬时分配和共享执行资源占用是主要放大器。相同吞吐下， +2k batch 的 ADD p99 比 10k batch 低约 89%，本轮 9k Get p99 也降到约 99ms。当前推荐发布端先以 +`2k blocks/request` 为起点做线上 A/B；5k 可作为降低 QPS/请求数的折中。该建议不是服务端硬限制，也 +不是无条件的 SLA 保证，仍需按线上 CPU、HTTP worker 数和 URI 大小复测。 + +这组数据同时否定了“先在 ReportEvent 内继续加线程”的必要性：外部分批已经在不改变写入语义的前提下 +显著改善公平性；服务端 writer 并行会抢占 Get 使用的 CPU、allocator 与 local LRU/item lock。若线上 +2k 分批后仍不满足，下一轮先补 HTTP parse/serialize/worker queue 和 item-lock wait 指标，再决定是否 +做独立有界 writer executor。 + +#### 本轮验证矩阵 + +- HTTP 功能：`test_report_event_snapshot.py --skip-bench`，36/36 通过，覆盖 ADD/DELETE/SNAPSHOT、 + last-operation-wins、校验无副作用、部分失败/重试、并发 snapshot/delta、首次 delta 与清理； +- Release UT：纯内存 meta/manager 全包 23/23 通过；`CacheManagerTest` 的 10 个 shard 全部通过； +- 生命周期竞态：HOST_DOWN 拦截已 admission 的 delta、旧 lifecycle 不能跨重注册写入，两项连续 50 轮 + 通过。测试 backend 已显式拦截新的 `GetLocationsWithKeyStatus`，不能只 hook 旧 `GetLocations`； +- 容量边界：新 key、已有 key 缺 location、已有 target update、max capacity，以及“已有更新 + 超容量新 + key 位于同一个 internal batch”均通过；最后一种返回 `EC_OK/EC_NOSPC`，`key_count` 与 usage 不漂移； +- ASAN：`meta_indexer_test`、`meta_local_backend_test`、`meta_dummy_backend_test`、 + `meta_storage_backend_manager_test`、`MetaSearcherTest` 全部通过；ReportEvent/HOST_DOWN/并发相关 + `CacheManagerTest` filter 通过； +- 性能与数据正确性：100/1k/5k/20k create/update 单请求通过；10k/5k/2k 同吞吐持续压测全部零失败。 + +新增原语不改变协议、URI 或持久化 schema，回滚可以整体 revert 本轮 ReportEvent commit，恢复原两阶段 +RMW。回滚/后续修改时必须一起处理 `MetaStorageBackend::GetLocationsWithKeyStatus`、manager recovery +路由、`MetaIndexer::ReadModifyWriteTargetLocations` 和 `BatchMergeLocationSpecs`，不能只删除其中一层； +否则最容易出现的是 `key_count` 漂移或 lifecycle fence 窗口被重新打开。 + +### 5.11 2026-08-06 百万 key 纯 local 查询优化 + +本轮位于独立分支 `codex/gethost-million-key-performance`，基于 5.10 的 ReportEvent 分支头创建，未继续 +修改 ReportEvent 写入流程。目标是让连续全命中的百万 key 查询保持有界内存和可取消性，同时收敛纯 +local LRU 上逐 key 加锁的固定成本。实现要点如下： + +1. local backend 提供紧凑结果：一个扁平 `shared_ptr` 数组加每 key offset，避免为 + 百万 key 创建百万个外层 vector/map。其他 backend 使用兼容 fallback，不改变 Redis/cached recovery； +2. `MetaIndexer::VisitLocationValuesForPrefix` 以连续范围渐进读取。首个范围同步完成，用来建立候选 host； + 后续范围才提交共享有界 executor。普通 prefix 或 Mamba 的所有候选已经终止时，通过原子 stop index + 阻止尚未领取的后缀工作； +3. metadata 读取和 host/spec 投影在 visitor 内融合，不再先保存百万 key 的 location 集合、随后做第二次 + 全量遍历。普通查询只保留每 host 的 prefix stop；Mamba 只保留后续 Eagle pop 所需的状态位矩阵； +4. 只有 prefix 内的 `EC_NOENT` 是正常 miss。答案终止位置之前的硬错误仍返回；终止位置之后已经开始的 + speculative 读取即使失败，也不能推翻已经确定的短 prefix； +5. EventReport location id 与 URI 使用只读 `string_view` 解析，并通过透明比较直接查询 snapshot，避免 + 对每个 key 复制 reporter medium/host 和 generation token； +6. advanced cache 增加带默认 fallback 的 batch lookup/release，LRU 实现先按内部 shard 分组，每个 + shard 一次持锁完成该批 lookup 或 release。引用计数、重复 key、被 pin 时 erase、LRU reinsertion 和 + capacity eviction 语义与逐 key API 相同； +7. 纯 local metadata 读取范围固定至少为 4096 key，以摊薄默认 1024 个 LRU shard 的锁获取。这个数与 + projection 的 128-key CPU chunk 解耦。首范围仍有界，因此短 prefix 不会扫描完整百万 key 后缀；代价是 + 极短 miss 最多多读一个 4096-key 窗口。 + +使用 Release/O2、纯 local backend、同一进程直接调用 manager 内部查询链路（不含 HTTP JSON、protobuf +响应序列化和网络），最终结果如下。每项先 warmup，再重复采样并报告 p50/平均值： + +| case | 100k p50/avg | 500k p50/avg | 1M p50/avg | +| --- | ---: | ---: | ---: | +| metadata only | 7.21/7.34ms | 39.90/39.98ms | 84.73/90.02ms | +| 全命中完整 host 投影 | 9.05/9.04ms | 48.61/48.69ms | 108.76/113.54ms | + +同一 benchmark 中，第 1024 个 key 结束 host prefix 的 1M 请求为 p50/avg `0.483/0.497ms`；第 1024 个 +key metadata miss 为 `0.438/0.438ms`。批量 LRU 前的同机 1M metadata/all-hit 约为 +`132.1/155.1ms`，最终分别下降约 36%/30%。这里不能解读成完整 HTTP API 已保证 100ms：1M metadata +读取已低于 100ms，但全命中投影仍约 109ms p50，HTTP 边界还会额外增加延迟。 + +可复现命令： + +```bash +bazelisk test -c opt //kv_cache_manager/manager/test:GetHostCacheStateBenchmark \ + --test_output=streamed --test_arg=--gtest_also_run_disabled_tests +``` + +该 benchmark 默认带 `manual` tag 且测试名为 `DISABLED_...`，不会拖慢常规 CI。修改 compact layout、 +visitor stop/error 规则、LRU batch 引用管理或 4096 窗口时，必须同时运行 LRU、QueryExecutor、 +MetaLocalBackend、MetaIndexer、MetaSearcher 五组回归及本 benchmark；不能只比较全命中吞吐而忽略短 prefix +取消延迟。 + +### 5.12 package 启动脚本预加载 jemalloc + +`package/script/start_server.sh` 默认尝试在启动时预加载 jemalloc，以降低大批量 ReportEvent 与百万 key +查询产生的短生命周期分配对系统 allocator 的压力。它只影响通过 package 启动脚本拉起的进程；直接运行 +Bazel 二进制不会自动启用。控制项和降级规则如下： + +- `KVCM_USE_JEMALLOC=0` 显式禁用；其他值或未设置时尝试启用； +- `KVCM_JEMALLOC_PATH` 非空时优先尝试该路径；否则/失败后，x86_64 依次检查 + `/usr/lib/x86_64-linux-gnu/libjemalloc.so.2`、`/usr/lib64/libjemalloc.so.2`，aarch64 依次检查 + `/usr/lib/aarch64-linux-gnu/libjemalloc.so.2`、`/usr/lib64/libjemalloc.so.2`； +- 找不到可读库或架构不受支持时打印告警并继续使用默认 allocator，不能因此阻止服务启动； +- 已有 `LD_PRELOAD` 会保留，jemalloc 放在最前；若已经包含同一个解析路径则不重复追加； +- `configure_jemalloc` 当前在 `install_kvcm_ops` 之前执行，因此同一启动脚本中的 pip 子进程也会继承 + `LD_PRELOAD`。后续若只想影响 server，应调整调用顺序并重新验证 package 启动，而不是改变变量作用域后 + 假定 `exec` 仍会继承。 + +这项改动只选择 allocator，不构成性能 SLA。上线应同时比较 Get/ReportEvent p50/p99、RSS、CPU 与 allocator +相关崩溃；回滚可设置 `KVCM_USE_JEMALLOC=0`，无需重新打包。最低提交门禁包括 `bash -n`，以及禁用、默认 +探测、自定义路径、已有 preload、缺库/未知架构降级的隔离函数测试。 + +### 5.13 2026-08-06 ReportEvent 请求内热路径收敛 + +本轮位于独立分支 `codex/report-event-hotpath-optimization`，直接基于 +`codex/gethost-million-key-performance@6c44b5025f5aeefbfa663cf94c39c915f4966314` 创建；查询优化分支未被 +修改。本节只讨论纯 local metadata、Release/O2、直接运行 Bazel 二进制（未预加载 jemalloc）的结果， +before/after 使用相同进程形态，因此 allocator 条件一致。 + +#### 设计与实现 + +1. delta fold 使用三个 request-wide 连续数组保存稳定 `(block, location)`、最终 spec mutation 和事件重试 + 依赖，并用一个哈希索引定位 location。常见的一 event/一 spec/一 location block 不再创建三组小 vector； + 最后只排序 location 索引并直接构造 ADD/DELETE task。last-operation-wins、ADD 先于 DELETE 的持久化阶段、 + admission failure 和整组重试闭包语义保持不变； +2. 同一请求的 `medium -> location_id` 只构造一次。location 索引中的指针只指向这个 request-owned intern + map 的 mapped string；`unordered_map` rehash 不会使元素引用失效，且 map 在整个 fold/task 构造阶段都 + 存活。后续不能把该指针改为指向临时字符串或可能搬迁元素的 vector； +3. `DeltaMutationGuard` 缓存并返回 request-owned lease，后续事件只读其 snapshot token/generation，不再为 + 每个 event 复制 32-byte token。lease 仍由 guard 析构时成对结束，generation-pinned lifecycle fence + 窗口不变； +4. 入口完整解析 URI 时同时保存 `size` 和已经验证的 parsed URI；追加 `s_version` 使用 + `ToUriStringWithExtraParam` 直接生成与 `SetParam + ToUriString` 完全相同的 canonical sorted URI，不再 + clone/mutate 参数 map 或使用 `ostringstream`。`StandardUri` 的 query 解析改用 `string_view`，host/port + 也不再先复制一份中间字符串； +5. ReportEvent 内部 task 把已验证 spec 的总 size 传给 `BatchMergeLocationSpecs`，避免在 shard RMW 前再次 + parse 新 URI。只有该内部路径允许设置 `prevalidated_total_size`；普通 caller 留空后仍执行完整的 URI、 + spec name、duplicate name 和 snapshot-version 一致性校验。这个 hint 不能扩展为对外部输入跳过验证； +6. pure-local targeted read 使用 LRU batch lookup/release，把同 shard 的数千次 lookup mutex 获取合并。 + unconditional Upsert 对唯一 key 同样 batch lookup、原地更新、统一 release，再插入 miss；release 必须在 + 插入 miss 前完成，以保留 strict-capacity eviction。公共 backend API 若出现重复 key，会自动退回原逐项 + Upsert，保持“同一 batch 内先创建、后续 partial update 继续 merge”的请求顺序语义； +7. HTTP handler 直接把 request body 的 `string_view` 交给 protobuf JSON parser，删除整份大 body copy。 + parser 在 handler 栈内同步完成，view 不会越过 request body 生命周期。 + +这些改动没有新增 writer 线程、没有把 manager callback 放进 backend item lock，也没有修改协议或 metadata +schema。Local batch lookup 会同时 pin 一批 handle，但只在本次同步投影/更新期间持有；所有 handle 均通过 +`ReleaseBatch` 一次释放。新增 `StandardUri` serializer 用精确输出对照测试锁定 canonical 兼容性，不能仅以 +“URI 语义等价”为由改变输出顺序。 + +#### 纯 local 单请求 before/after + +真实 HTTP benchmark 使用 `test_20_large_single_request_delta_scaling`，每档先创建 block，再对相同 location +执行 update，并检查 committed token 和首/中/末 key。before 为本分支创建后立即记录的基线；after 为最终 +Release 链接产物三个全新 instance 的串行复测中位数： + +| events/request | before create/update | after create/update | create/update 降幅 | +| ---: | ---: | ---: | ---: | +| 100 | 1.53/1.38ms | 1.47/1.34ms | 3.9%/2.9% | +| 1000 | 9.47/9.00ms | 8.21/7.80ms | 13.3%/13.3% | +| 5000 | 43.54/42.57ms | 37.33/35.14ms | 14.3%/17.5% | +| 20000 | 173.87/175.00ms | 145.85/145.87ms | 16.1%/16.6% | + +最后一次安全审查和重新链接后的三轮 20k create 为 `145.27~147.59ms`，update 为 +`145.32~146.53ms`。 +结果仍近似 O(events)，本轮降低了每 event 常数，没有把大 JSON API 变为固定耗时，也不能据此承诺线上 +混合负载 p99。 + +最后一轮 20k existing-location update 的服务端指标为： + +| 指标 | 数值 | +| --- | ---: | +| `service.query_rt_us` | 58,473us | +| `meta_searcher.indexer_read_modify_write_location_time_us` | 35,602us | +| `meta_indexer.rmw_get_io_time_us` | 7,061us | +| `meta_indexer.upsert_io_time_us` | 6,715us | +| `meta_indexer.lock_wait_time_us` | 1us | +| `meta_searcher.index_deserialize_time_us` / `index_serialize_time_us` | 0/0us | + +同一次请求的 Python 客户端 HTTP wall time 为 145.32ms。两者之间约 87ms 包含客户端 JSON 编码、loopback +HTTP、服务端 protobuf JSON parse/response serialization 及统计边界差异，不能全部归到某一个环节；但可以 +确定纯 local 下不是 Redis、location deserialize/serialize 或 metadata shard lock wait 主导。下一项有望 +立竿见影的优化应先补 `http_parse_us/http_serialize_us` 并做同 payload 的 gRPC A/B，而不是继续增加 +ReportEvent writer 并行度。 + +#### 本轮验证矩阵与后续门禁 + +- HTTP 纯内存功能集成 36/36 通过，覆盖 ADD/DELETE/SNAPSHOT、同请求 last-op-wins、部分失败重试闭包、 + snapshot/delta 并发、HostDown/重注册、首次 delta、异常 URI/payload 无副作用； +- Release/O2 的 StandardUri、MetaLocalBackend 和完整 CacheManager 分片测试通过；重复且非相邻 key 的 + Upsert 专项测试验证请求顺序 merge； +- ASAN 下 `StandardUriTest`、`meta_local_backend_test`、`CacheManagerTest`、`SnapshotUriUtilsTest`、 + `MetaSearcherTest`、`ProtoMessageJsonUtilTest` 全部通过； +- 100/1k/5k/20k create/update 三次性能复测全部通过数据校验，没有错误返回或规模拐点。 + +后续若修改连续数组索引、location-id intern、prevalidated size、batch handle 生命周期或 HTTP body view， +至少重跑上述六个 ASAN 目标和 36 项 HTTP 功能套件。当前没有真实 Redis 验证，不能把本节数据外推到 +cached/redis backend；也不要在没有混合 ReportEvent/Get/heartbeat p99 对照前增加内部 writer 并行。 + +### 5.14 2026-08-06 热路径优化后的第二轮正确性审计 + +本节记录 `codex/report-event-hotpath-optimization` 在 5.13 提交后的继续审查结果。它不是另一轮并行化， +而是针对批处理顺序、可信输入边界、整数上溢和借用内存生命周期逐层构造反例。后续 AI 不应只重跑 happy +path benchmark 后删除这些保护。 + +#### 审计中发现并修复的问题 + +1. `MetaLocalBackend::Upsert` 的初版 batch lookup 会先更新所有 hit，再插入所有 miss。在 strict-capacity + local cache 中这会改变可观察的请求顺序：`[new key, existing key]` 原本可以先接纳新 key、再原地扩展旧 + key，重排后却可能让新 key 返回 `EC_NOSPC`。最终实现对重复 key 继续逐项处理；全 hit 和全 miss 保留 + batch fast path；混合 hit/miss 按原下标更新/插入并逐个释放已有 handle。全 miss 不再调用只包含空 + handle 的 `ReleaseBatch`，避免一次无意义的 shard 分桶分配； +2. `MergeLocationSpecsTask::prevalidated_total_size` 最初是公开的 `optional`，任何 caller 都能伪造 + 值并绕过 URI/spec/version 校验。现在它是只能由 `CacheManager` 构造的 capability token；普通 + `MetaSearcher` caller 无法创建 token，必须走严格验证。不要为了测试方便重新开放它的构造函数； +3. `StandardUri(const string&)` 忽略 `Parse()` 返回值。旧 parser 遇到非数字端口时会保留 protocol，导致 + `Valid()==true`，随后追加 snapshot version 又把非法端口静默丢掉。现在端口直接在原字符区间上用 + `from_chars` 严格解析，非数字、负数（包括数值等于 0 的 `-0`）、空端口、前导空白和 `+` 均 + fail-closed；保留仓库已有的 `:0` + 兼容语义。authority 中的 `@`/`:` 只在 path/query 之前解释，query value 中的 callback URL、`/` 和邮箱 + 地址不再污染 user-info/path； +4. 多 spec 的 `size` 以前直接做 `uint64_t` 加法，恶意或损坏输入可以回绕并破坏 storage usage。单个 ADD + event 和完整 SNAPSHOT 在获取 reporter generation/write gate 前按整个输入做 checked sum；跨多个 ADD + event 折叠后若才发生上溢，则不信任预计算 hint，退回 `MetaSearcher` 严格校验并按既有“首次 metadata + 写失败仍复用 generation”的语义返回失败。通用 merge/replace 校验也拒绝上溢，Replace 复用校验得到的 + total，删除一次写阶段重复 URI size 解析。第二轮反例还覆盖“已有 location 的 size 合法、当前 ADD 的 + size 也合法，但两者合并后才上溢”：target-location modifier 在写入前计算 retained + incoming 的 checked + sum，返回 `EC_BADARGS` 并保持原 metadata/usage 不变； +5. HTTP body 的 `string_view` 解析新增非 NUL 结尾、带前后垃圾 backing string 的边界测试，证明 protobuf + parser 严格使用 view 长度且不读取尾部。`req.get_body()` 的 view 仍只在同步 handler/parser 栈内使用， + 不能缓存到异步任务或 response 生命周期之后。 + +#### 新增的模型与反例 + +- 768 个确定性伪随机 ADD/DELETE event、48 个 block、17 个 medium、4 个 spec name，与独立 map 参考模型 + 比较最终 canonical URI、last-op-wins、spec 排序、location 数量、`spec_size` 和总 storage usage。17 个 + medium 会强制 request-owned intern `unordered_map` rehash，用来验证 location-id 指针在 rehash 后仍有效； +- optimized batch Upsert 与逐项 `UpsertForOneKey` 做差分，覆盖全 hit、全 miss、混合、非相邻重复 key，比较 + 每项错误码、location、除动态 `BP#lru_time` 外的全部 properties 及内存计数；另用 1 MiB strict capacity + 分别锁定 `[new, existing] -> [OK, OK]` 和 `[existing, new] -> [OK, NOSPC]`； +- 非法端口、单 event size 上溢、跨 event fold 后 size 上溢、已有 spec 与新 spec 合并后上溢、 + snapshot/Replace size 上溢均验证无错误 metadata 写入。跨 event 上溢还验证失败响应保留可复用 + generation，这与已有 + `TestReportEventFirstDeltaMetadataFailureReportsFailureAndReusesGeneration` 契约一致，不应误改成失败即删除 + generation； +- `PrevalidatedTotalSize` 的私有构造保证 ReportEvent 之外的测试和生产 caller 仍走完整校验，而不是仅依赖 + 注释约定。 + +#### 验证结果和环境限制 + +- Release/O2 完整通过 10 个纯内存目标：`StandardUriTest`、`LruCacheTest`、 + `SnapshotUriUtilsTest`、`EventReportBackendTest`、`meta_local_backend_test`、`meta_indexer_test`、 + `meta_storage_backend_manager_test`、`MetaSearcherTest`、`CacheManagerTest`、 + `ProtoMessageJsonUtilTest`； +- 相同 10 个目标 ASAN 全绿。UBSAN 下 URI、snapshot URI、local backend、MetaSearcher、JSON 边界及定向 + ReportEvent 用例全绿；完整 `CacheManagerTest` 中依赖第三方 `cpp_stub` 改写函数机器码的 3 个测试因 + `external/cpp_stub/stub.h:456` 非对齐写入被 UBSAN 阻止，这发生在 mock 安装阶段，不在本轮生产路径。 + TSAN 在链接 gRPC 时因环境缺失 `/usr/lib64/libtsan.so.0.0.0` 无法启动，不能记录为通过； +- Release 下并发读写、ReportEvent/HostDown 并发、flat fold、容量顺序、折叠/最终合并上溢和写失败传播的 + 关键组合重复 50 轮；`CacheManagerTest` 分片合计执行 500 次，另对并发可见性和失败依赖闭包合计执行 + 200 次，全部通过； +- 最终源码重新构建的新进程上，真实 HTTP、纯 local metadata 的双类型基础套件 20/20、 + snapshot/并发/失败套件 36/36 全绿。随后 10 秒影子状态混合压力完成 401 次 100-block ADD、198 次最多 + 50-block DELETE、201 次 1000-key Get、40 次 heartbeat，所有请求和写后抽样校验零失败；ADD + p50/p99 `2.50/4.69ms`，Get p50/p99 `2.27/10.24ms`； +- 最终源码无并行编译负载时三轮 20k 单请求 create/update 为 `147.26/145.87ms`、`147.27/146.39ms`、 + `145.51/145.16ms`。一次与两个大 Bazel build 重叠的 `315.50/344.60ms` 已明确标记为环境噪声，不用于 + before/after 结论。性能复测前必须确认没有编译器/linker/其他 benchmark 占用 CPU。 + +本轮仍只验证 pure local metadata；按用户部署前提没有启动 Redis。local 结果不能证明 cached/redis 语义， +但本轮也没有修改 Redis 专属实现。若以后改变 mixed Upsert 分支、capability token、checked size、URI parser +或 borrowed HTTP body，必须至少重复本节的差分模型、sanitizer 定向用例和 36 项 HTTP 套件。 + +### 5.15 2026-08-06 TSAN 关闭期审计与写读混合 A/B + +本节是在 5.14 之后继续检查 `codex/report-event-hotpath-optimization` 的结果。测试仍只使用 pure local +metadata。系统补装 `libtsan-10.2.1-3.3.alios7.x86_64` 后，5.14 记录的 TSAN 环境限制已经解除；这只是 +本机测试依赖，不是仓库或发布包改动。 + +#### TSAN 发现的两个关闭期问题 + +1. `EventReportBackend::Close()` 先收集 `LifecycleFence::mutex` 的 `unique_lock`，再清空 + `lifecycle_fences_`。`unique_lock` 不拥有 mutex 对象，原实现可能先析构最后一个 fence，再由 lock vector + 解锁已经释放的 `shared_mutex`。TSAN 在 + `LivenessUnregistersBeforeCleanupAndHeartbeatCannotReviveOldSnapshot` 中稳定报 heap-use-after-free； +2. 仅增加 fence 的强引用仍不够。原 `Close()` 在持有 `lifecycle_fences_mutex_` 时阻塞等待每个 fence 的 + writer lock，能形成真实三线程环：host cleanup 持有 lifecycle read lease 后等待 metadata；已进入的 + metadata RMW 持有 metadata 后查找 lifecycle fence；`Close()` 持有 fence table mutex 后等待 cleanup + 的 lifecycle lease。delta modifier 使用 `try_lock` 只能切断直接的 metadata/lifecycle 两锁环，不能切断 + `Close()` 引入的第三条边； +3. 最终实现只在 `lifecycle_fences_mutex_` 下复制 `shared_ptr`，释放 table mutex 后才等待 + 每个 fence，清空节点状态后再短暂获取 table mutex 清表；显式先销毁 lock vector，再销毁强引用。 + `Close()` 在任何阻塞等待期间都不持有 table mutex； +4. 所有可能在 `Close()` 设置 `retired_` 后才返回或新建 fence 的入口均已逐个审计。它们在获得 fence 后、 + 修改 node/snapshot 状态前再次检查 `Retired()` 或 `AcceptingReports()`。因此 table snapshot 之后创建的 + fence 只会得到关闭错误，不会越过关闭边界写状态。以后若新增 `GetOrCreateLifecycleFence()` caller，必须 + 保留这次二次检查；不能用一次函数入口检查替代。 + +以后修改这里必须同时保持三个不变量：不在持有 fence table mutex 时等待 reporter fence；保留 fence 的 +强引用直到对应 lock 已释放；可能跨过 `Close()` 的 caller 在 fence 内再次检查 retired/available 状态。 + +#### Sanitizer 与 Release 验证 + +- TSAN：完整 `EventReportBackendTest`、8 个 ReportEvent/snapshot/HostDown/Get 跨层并发用例、MetaLocal + batch Upsert/compact read 及 MetaIndexer 多线程/提前终止用例全绿。最初触发 UAF/锁环的 4 个用例最后各 + 重复 20 次；EventReportBackend 20/20，分片后的 CacheManager 合计 200/200，无 data race、UAF 或 + lock-order-inversion； +- ASAN+UBSAN：`LruCacheTest`、`StandardUriTest`、`EventReportBackendTest`、`SnapshotUriUtilsTest`、 + `query_executor_test`、`meta_local_backend_test`、`meta_indexer_test`、`MetaSearcherTest`、 + `ProtoMessageJsonUtilTest` 全量通过；`CacheManagerTest` 10 个 shard 全量通过。当前构建会把同一个 + generated protobuf 常量从两个 DSO 注册给 ASAN，因此使用 ASAN 建议的 + `detect_odr_violation=0`；UBSAN 只 suppress `external/cpp_stub/stub.h` 的 alignment 检查，因为该测试库 + 本来就通过非对齐机器码写入安装函数 stub，仓库自身路径仍为 `halt_on_error=1`； +- Release/O2：上述 9 个目标加完整 `CacheManagerTest`，共 10 个目标全绿；最终重新链接的真实 HTTP + 进程上，snapshot/并发/失败套件 36/36 全绿； +- 本轮 ASAN 首次运行若不关闭 protobuf ODR 检查，会在测试 main 之前退出；完整 CacheManager UBSAN + 若不 suppress `cpp_stub`，会在 mock 安装阶段退出。这两种已知测试基础设施诊断不能当作生产代码失败， + 也不能直接忽略后宣称 sanitizer 通过，必须按上面的精确范围重跑。 + +#### ReportEvent 优化对 Get 的 2×2 A/B + +为了验证写侧优化不会挤压用户更关心的 Get，使用同一台机器、Release/O2、全新 pure-local instance，比较 +父提交 `6c44b5025f5aeefbfa663cf94c39c915f4966314` 与当前实现。每轮 30 秒、8 reporter、15 QPS × 10k +BLOCK_ADD、2 QPS × 10k-key standalone Get、5 秒心跳，所有请求和 shadow-state 校验均零失败。 + +注意 `report_event_load.py` 会在每个 ADD 后用同一批 10k key 再调用一次 Get 并校验完整 prefix，且 `add` +统计从 ReportEvent 开始一直覆盖到该校验结束。因此这里的 `add` 不是纯 ReportEvent HTTP RT；实际查询 +压力约为 15 QPS 的写后 10k-key Get 加 2 QPS standalone Get。这个模型故意比线上 0.1 QPS Get 更苛刻， +适合观察写优化是否伤害查询，但不能用 `add` 数字替代独立 ReportEvent benchmark。 + +| 版本 | ADD 实际 QPS | ADD avg | ADD p50 | Get 实际 QPS | Get avg | Get p50 | Get p95 | Get p99 | Get max | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| before round 1 | 12.60 | 1157.91ms | 489.22ms | 1.70 | 605.96ms | 144.49ms | 1993.64ms | 6175.34ms | 7163.79ms | +| before round 2 | 13.07 | 982.79ms | 406.74ms | 1.74 | 828.08ms | 120.25ms | 3645.72ms | 4322.02ms | 4510.07ms | +| current round 1 | 13.19 | 906.85ms | 314.52ms | 1.78 | 458.93ms | 78.01ms | 2037.09ms | 2939.51ms | 3223.37ms | +| current round 2 | 13.41 | 770.95ms | 343.81ms | 1.81 | 470.10ms | 89.03ms | 1961.27ms | 3176.85ms | 3855.16ms | + +按请求数加权，ADD+写后校验平均从 `1071.20ms` 降到 `838.30ms`，下降 21.7%；standalone Get 平均从 +`715.23ms` 降到 `464.56ms`，下降 35.0%。两轮 current 的 Get p50 都低于两轮 before，p99 也从 +`4.32~6.18s` 收敛到 `2.94~3.18s`。第一组配对的 Get p95 有约 2.2% 反向波动，第二组明显改善；不能从 +约 60 个 Get 样本推导精确 percentile SLA，但 2×2 的平均、p50、p99 和完成吞吐一致表明：在固定目标写 +负载下，当前 ReportEvent 优化对查询是正向的，没有发现以增加 writer 并行度换吞吐、反而抢占 Get 的情况。 + +绝对尾延迟仍不合格：10k 大批次加每写一次完整 10k-key 校验会把服务推入排队区。当前结果支持“优化没有 +伤害查询”，不支持“150k blocks/s 下 Get 已满足 100ms SLA”。线上仍应优先把 ReportEvent 外部分成 +2k/5k block 批次，并按固定 blocks/s 重测 Get p99。 + +#### 最终混合正确性与日志边界 + +最终源码另跑 45 秒混合压力：1357 次 100-block ADD、673 次最多 50-block DELETE、24 次周期 SNAPSHOT +（每次主动遗漏 10% 当前 key 验证权威对账）、227 次 5k-key Get、72 次 heartbeat，全部请求和写后 +shadow-state 校验零失败。Get avg/p50/p95/p99/max 为 +`5.24/3.67/11.70/12.72/86.03ms`；ADD 为 `4.16/2.54/4.33/54.37/237.09ms`，snapshot 平均 +`191.59ms`。 + +压测停止心跳并等待 liveness 清理时，多个 host 会并行扫描同一个 local index。一个 cleanup 的 `Scan` +返回 key 后，另一个 cleanup 可能先删除该 key，使前者的 `GetLocations` 返回包含 `EC_NOENT` 的 +`EC_PARTIAL_OK`。当前 `CleanupLocationsByHost` 会立即把任何 `EC_PARTIAL_OK` 记为 failure，所以外层可能 +记录 `finished with partial failures`，即使后续逐 location 已把 `EC_NOENT/EC_MISMATCH` 当作幂等成功且没有 +底层 delete error。这是并发清理的保守/偏噪声日志，不能仅凭该汇总 warning 判断 metadata 丢失；后续若 +收敛日志，应只把非 `EC_NOENT` 的 per-key read error 计为真实 failure，并补 Scan/Get 竞态测试，不要改变 +cleanup 的 generation lease 或 conditional-delete 语义。 + +本节没有启动 Redis，也没有把 local 结果外推到 cached/Redis backend。后续 AI 至少应保留：36 项 HTTP +功能套件、固定 blocks/s 的写读混合 A/B、TSAN 关闭期复现，以及 sanitizer 对第三方 ODR/stub 的精确处理。 + +### 5.16 2026-08-07 perf/futex 驱动的 pure-local 单 location RMW 优化 + +本节基于 `codex/mu-main` 的 Release/O2 产物重新采集 CPU 与 futex 栈，目标只覆盖用户线上采用的 pure-local +metadata 模式。结论是：大请求确实会在 metadata shard mutex 上形成可见等待，但等待的根因是锁内做了过多 +通用 RMW 构造、local LRU 的逐 key lookup/release 以及 allocator 工作；单纯把 mutex 换成原子变量或增加 +writer 线程既不能保护多字段 RMW，也会破坏 key-count、capacity 和同 key 顺序语义。 + +#### 优化前的证据 + +- 20k `BLOCK_ADD` create/update 的 HTTP wall time 都约为 `145~147ms`； +- CPU 样本中 HTTP JSON -> protobuf 约占 `34~36%`，`CacheManager::ReportEvent` 约占 `22%`， + BatchMerge/RMW 约占 `12%`；glibc `malloc/free/_int_malloc/_int_free` 等 allocator 自身累计接近 `29%`； +- 117 个并发 10k update 请求的 paired futex 样本中，metadata shard mutex 有 284 次等待，总计 + `119.31ms`，平均 `420us`，p95 `1.35ms`，p99 `3.65ms`，最大 `4.136ms`；glibc allocator 另有 + 1158 次等待，总计 `26.59ms`；local LRU futex 等待可忽略； +- `meta_indexer.get_io_time_us` 在 pure-local 下并不是远端 I/O，它包含 local LRU lookup、item shared lock、 + location 投影以及通用容器构造。不能仅凭指标名把它归因到 Redis。 + +#### 最终实现 + +1. `/api/reportEvent` 使用 request-scoped protobuf Arena。request/response 及其嵌套 event/spec 一次性回收， + 不改变 protobuf wire/JSON schema；其他 HTTP API 继续使用原 handler，缩小行为变化范围； +2. 仅当 backend 是**精确的生产 `MetaLocalBackend` 类型**、没有 cache/persistent 双层组合且每个 key 恰好一个 + target location 时，启用扁平 RMW。装饰器、子类、测试 fault backend、cached/Redis 或多 location 请求全部 + 回退原 `ReadModifyWriteTargetLocations`，避免绕过它们覆写的审计、故障注入或恢复语义； +3. 快路径使用扁平 `keys/location-id/location/result` 数组，一次 local batch lookup 同时返回“key 是否存在”和 + “target location 是否存在”，随后在同一组 metadata shard locks 内 merge/upsert。这样去掉每 key 的 + `vector>`、location map 构造和第二次 key existence probe，但仍严格区分新 key 与“已有 key + 缺少该 location”，所以 `max_key_count`、`key_count` 与 sibling locations/properties 都保持正确； +4. local backend 对 LRU 使用 `LookupBatch/ReleaseBatch`，并只替换一个 immutable `CacheLocation` shared_ptr。 + 所有 request-shaped scratch vector 在取得 metadata shard locks 前 reserve；`ScopedBatchLock` 借用稳定的 + shard-index vector，不再为每个 batch 复制一次； +5. duplicate key 在进入快路径前拒绝，防止同批次把一个新 key 重复计数。非法 location/null pointer、backend + 返回 shape mismatch、key/location 状态矛盾、capacity full、modifier skip/fail 和写失败均有 fail-closed + 处理；容量满时已有 key 的 update 仍可写入，新 key 返回 `EC_NOSPC`； +6. metadata write lease 仍在读到旧值之后、实际 merge 前获取，并持续到 RMW 返回。不要为了缩短锁时间把 + lease 提前到读取之前或在写入前释放，否则会破坏 snapshot generation/leader 切换期间的校验闭包。 + +#### Release 结果 + +同机、同配置、无编译负载的真实 HTTP `test_20_large_single_request_delta_scaling`： + +| events/request | 本轮基线 create/update | 最终 create/update | 降幅 | +| ---: | ---: | ---: | ---: | +| 100 | 约 1.5/1.4ms | 1.32~1.37/1.17~1.20ms | 小请求主要受固定 HTTP 开销影响 | +| 1000 | 约 9ms | 7.23~7.35/6.72~6.79ms | 约 18%/24% | +| 5000 | 约 43/42ms | 32.03~32.30/30.21~30.42ms | 约 25%/28% | +| 20000 | 约 146/146ms | 128.01~128.11/119.76~121.40ms | 约 12%/17% | + +10k existing-location update 的并发墙钟时间为：单请求 `52~54ms`；同 reporter x2/x4/x8 分别 +`59.39/57.99/68.48ms`，distinct reporter x2/x4/x8 分别 `55.37/61.21/72.30ms`。x8 总吞吐超过 +`1.1M blocks/s`，说明此时继续拆 metadata lock 或增加 writer pool 没有收益证据。 + +优化后的 futex 采样覆盖 186 个并发大请求，8 个 HTTP worker 没有一次进入 `FUTEX_WAIT`；原来最长的 +metadata shard wait 栈已降到内核休眠采样阈值以下。最新 CPU 样本的 flat-RMW 函数只剩约 `0.59%`，主要 +自耗时转为 protobuf JSON parser、glibc allocator 与 memmove/memcmp：`malloc 7.68%`、`_int_free 6.04%`、 +`_int_malloc 3.52%`、JSON token/string parse `10%+`。生产启动脚本会 preload jemalloc，直接运行 Bazel +binary 的 benchmark 不会，因此这里的 glibc allocator 比例是保守结果。下一轮若继续优化，应先对 HTTP +JSON 与 gRPC 做同 payload A/B；不要在没有新 futex 证据时重写正确性敏感的 RMW 锁协议。 + +最终重新链接产物又做了一轮 20 秒全进程 trace，并在窗口内完成 8 个 10k-key reporter 的建库和 31 个 +existing-location 并发 update。全进程没有任何 `FUTEX_WAIT/FUTEX_WAIT_PRIVATE`；其余 wait-like 事件全部 +是后台线程稳定的 50ms/100ms/200ms/1s/5s timed condition wait，不在 ReportEvent 请求栈。最终 x8 same +reporter/distinct reporter 墙钟为 `67.54/73.37ms`，吞吐为 `1.184M/1.090M blocks/s`。 + +#### 回归门禁 + +- pure-local HTTP snapshot/并发/失败功能套件 36/36、旧版双类型兼容套件 20/20 通过； +- `meta_local_backend_test` 覆盖 flat read 的 key-status、sibling/property 保留、新建/更新、重复 key 顺序、 + malformed/null/empty；`meta_indexer_test` 覆盖精确 capacity、已有 key 缺 target、modifier skip/no-op、 + duplicate rejection； +- `--nocache_test_results` 下 8 个核心 Release Bazel 目标全部通过；完整 `CacheManagerTest` 10 个 shard 全绿， + 并专门验证继承 `MetaLocalBackend` 的 fault backend 会回退通用虚函数路径；MetaSearcher、backend manager、 + HTTP JSON 等相关目标同时通过； +- 本轮没有 Redis 性能结论，也没有修改 Redis 数据路径。以后改动借用的 location-id 生命周期、exact-type + guard、batch lock 范围、key-status 或 capacity 逻辑时，必须重跑上述单测、36 项 HTTP 套件、20k scaling + benchmark 和 paired futex 采样。 + +### 5.17 2026-08-07 ReportEvent JSON/Arena 与 RMW allocator 收敛 + +5.16 的锁内 flat RMW 完成后，Release `perf` 显示通用 protobuf JSON 转换仍占 CPU 的约 34%~36%，glibc +allocator 相关符号累计接近 29%。protobuf 3.13 的通用 JSON 路径会先构造中间表示/二进制字符串，再把它解析 +进 request；默认 Arena block 只有 256B 起步、最大 8KiB。20k event 请求因此仍有大量短命分配。优化目标是 +只收敛 `/api/reportEvent` 的常见 JSON 形态，不改变其他 API、protobuf schema 或异常输入兼容性。 + +#### JSON 与 request allocator + +1. 新增 `ReportEventJsonParser`，使用 RapidJSON 直接把已知 ReportEvent 字段写入最终 protobuf request；request + 位于 handler 的 request-scoped Arena，因此不再经过“JSON -> 临时 protobuf binary -> request”的通用路径； +2. fast parser 支持 protobuf JSON 的 snake_case/lowerCamel 字段名、字符串/数字 enum、全部现有 event oneof、 + map/repeated 字段和未知字段忽略。`null`、未知 enum、重复字段等少见但通用 protobuf parser 可接受的形态会 + 返回 fast-path miss，清空 request 后退回 `ProtoMessageJsonUtil`。这个 fallback 是兼容性边界，不能删除； +3. ASCII 请求先用同一轮 64-bit scan 同时检查 high bit 和原始 NUL。大于 32KiB、全 ASCII 且无原始 NUL 的 + body 只做一次连续 mutable copy，再由 RapidJSON in-situ parse：DOM 字符串直接引用/原地 unescape 该 buffer， + 不再先复制到 DOM pool、随后又复制到 protobuf。buffer 保持到 `ParseRequest` 完成，不跨 parser 生命周期； +4. 非 ASCII 请求仍启用 length-aware 严格 UTF-8 校验；带原始 NUL 的 ASCII body 也保留 length-aware parser， + 防止 C-string 提前结束把畸形 trailing bytes 当成合法 JSON。合法 `\u0000`、Unicode escape、quote/backslash/ + newline escape 均由 in-situ 路径正确解码，并与通用 protobuf parser 做语义差分； +5. 大于 32KiB 的 JSON 按 body 大小配置 RapidJSON pool/stack（pool 单块最多 4MiB，stack 64KiB~1MiB）， + protobuf Arena 使用 64KiB start block、1MiB max block；小 heartbeat 保留小块默认行为。response string 预留 + 512B，避免常见响应的额外增长； +6. parser 只借用 `req.get_body()` 的同步 `string_view`，不会保存指针。测试覆盖非 NUL 结尾 view、前后垃圾 + backing storage、Unicode、非法 UTF-8、unknown field、数字 enum、oneof 和 fallback 与通用 parser 语义一致。 + +#### RMW writer 临时分配 + +1. `Cache::BatchOperationScratch` 保存 LRU batch lookup/release 的 hash、shard offset、cursor 和排序下标；pure-local + single-location RMW 在进入 metadata shard locks 前一次 reserve，并让读、写以及后续内部 batch 重用。普通 + cache API 和其他 backend 仍走原虚函数语义； +2. key view、handle、location、错误码和 upsert 数组提升到 RMW batch 循环外，按最大 batch 一次 reserve。 + 读阶段的错误码 vector 在 modifier 完成后直接承接 writer 返回值，不再另建 request-sized upsert vector； +3. `max_key_count` 满时先把新 key 标记为 `EC_NOSPC`，再原地 compact 已有 key 的 update，删除锁内四个 subset + vector。已有 key 仍照常更新，新 key 不增加 key count；location handle 在每次 backend 调用结束前全部释放； +4. scratch 快路径仍受 5.16 的 exact `MetaLocalBackend`、无 cache backend、single target location guard 保护。 + cached/Redis、装饰 backend、fault backend 和多 location RMW 不会被静态转换或 allocation fast path 绕过。 + +#### Release 性能与最终 profile + +同机、纯 local、8 HTTP worker、无编译负载，真实 HTTP +`test_20_large_single_request_delta_scaling` 的最终结果： + +| events/request | create | update | +| ---: | ---: | ---: | +| 100 | 1.15ms | 1.15ms | +| 1000 | 4.38ms | 4.54ms | +| 5000 | 17.43ms | 18.12ms | +| 20000 | 79.31~82.86ms，五轮均值 80.65ms | 74.21~76.07ms，五轮均值 74.85ms | + +只读 DOM 版本与 in-situ 版本在同机交替五轮均值为 `85.51/79.83ms` 与 `80.65/74.85ms`，in-situ 额外降低 +`5.7%/6.2%`。相对 5.16 同一基线的 `128.11/121.40ms`，最终 create/update 累计下降约 `37.0%/38.3%`。 +8 路同 reporter、每请求 10k existing update 的 1992 个稳态请求为 avg/p50/p95/p99 +`42.32/41.91/51.54/55.52ms`，墙钟吞吐 `1.561M blocks/s`。 + +最终 in-situ `perf record -e cycles:u -F 999 -g --call-graph dwarf` 覆盖 14,272 个用户态样本。ASCII 路径原有的 +RapidJSON UTF-8 validation 热点保持为零，`ParseString` 从只读 DOM profile 的 `8.82%` 降到 `2.52%`。 +剩余 self CPU 以 `malloc 9.14%`、`_int_free 8.28%`、`_int_malloc 6.31%`、`memmove 5.56%` 为主，主要来自 +必须跨请求存活的 URI/LocationSpec、immutable CacheLocation、shared_ptr 和 hash/LRU 节点，而不再是 JSON +临时 string。production 启动脚本还会 preload jemalloc，因此不要仅凭直接运行 Bazel binary 的 glibc self +比例引入无界 thread-local body、全局对象池或 location-id interner。SAX 最多继续消除约 2.5% 的 string parse +self CPU，却会显著扩大 parser 状态机；没有新的端到端收益证据前不建议实施。 + +system-wide futex trace 在 792 个成功的 10k 请求窗口内，按当前 server PID 过滤后只有 9 次后台 +`FUTEX_WAIT_BITSET_PRIVATE|CLOCK_REALTIME` 定时等待和对应 wake；没有请求路径 +`FUTEX_WAIT_PRIVATE`。因此本轮没有继续拆 metadata locks 或增加 writer worker。 + +#### 验证与后续 AI 门禁 + +- Release 定向 `LruCacheTest`、`meta_local_backend_test`、`meta_indexer_test`、 + `ProtoMessageJsonUtilTest` 全绿；最终真实 HTTP snapshot/并发/失败功能套件 36/36 全绿； +- `bazelisk test --config=release --test_output=errors --nocache_test_results //kv_cache_manager/...` 分析 351 个 + target，发现 107 个 test；106 个通过，唯一未执行的 `SdkBufferCheckUtilTest` 由 BUILD 在无 CUDA/MUSA 平台 + 显式标记 incompatible； +- 仓库根 `//...` 会在 analysis 阶段因可选 `//3rdparty/tair_mempool` 引用未声明的 `@tair_mempool` 失败， + 尚未进入任何 test。这是当前 checkout 的可选外部依赖配置限制，不能误报为本轮测试失败或“全仓通过”； +- 后续修改 fast parser 必须保留 generic fallback、非 ASCII 严格校验和 bounded `string_view`；修改 RMW scratch + 必须保证 reserve/clear 在 metadata lock 外、handle 不跨 backend 调用逃逸，并保留 exact-type/capacity/key-count + 门禁。若新 profile 仍以 URI/CacheLocation 为主，应先优化解析结果复用或对象布局，不要回到盲目拆锁。 + +### 5.18 2026-08-07 body 复用、location-id interning 与 fused LRU handle + +5.17 最终 profile 中，10K block 请求体约 1.94MiB，HTTP worker 每次仍要为 in-situ parse 创建 mutable copy； +每个 block 又分别复制相同 reporter/medium location-id。pure-local fused RMW 的读阶段释放 LRU handle 后，写阶段 +还会对同一批 key 再做一次 hash/LookupBatch；替换旧 `CacheLocation` 时，最后一个 shared_ptr 的 URI/容器析构也 +发生在 metadata shard locks 内。本轮只优化这些已经由 profile 证实的重复工作，不改变 Redis/cached backend。 + +#### 实现与边界 + +1. 大 ASCII ReportEvent 使用每 HTTP thread 一个 mutable body buffer。常驻 capacity 上限为 4MiB；超过上限的 + body 使用 request-local buffer，递归/重入解析同样回退本地 buffer。这样 10K 请求不再反复 malloc/free + 约 1.94MiB，同时最多保留 `4MiB * HTTP worker 数`，不能删除该上限； +2. ReportEvent request 内按 medium 创建一个 `shared_ptr` location-id，并贯通 delta ADD/DELETE、 + snapshot replace 与 MetaSearcher task。`CacheLocation` 用 variant 保存 owned 或 interned id：普通调用方仍保留 + owned string，只有 event path 共享。unordered_map rehash 只移动 shared_ptr，不会改变 pointee 地址，因此 + request fold 可安全使用 pointee identity；序列化和值比较仍通过 `id()`，cache charge 继续按每 location + 保守计算完整 id 大小； +3. delta 的 `(block_key, location-id)` fold 从 node-based unordered_map 改为请求内 power-of-two linear-probing + table，正常负载不超过 50%，去掉每个 distinct block 的 hash node allocation。最终仍按 block/id 排序， + last-operation-wins、failure dependency closure 与输出顺序不变； +4. exact `MetaLocalBackend` + pure-local + single-target 快路径的第一次 batch lookup 可把 handle 保留到匹配的 + writer call。writer 使用原 read index 直接 update/insert，跳过第二次 key hash、LRU lookup 和 handle acquire； + skipped hit 会在新 key admission 前先释放，保持 strict-capacity 顺序。scratch 析构和所有 validation/shape/ + capacity early-return 都兜底释放 handle；其他 backend 仍走原路径； +5. writer 消费新 `CacheLocation` shared_ptr 并 move 进 item map，避免一次无意义的 refcount increment/decrement。 + 被替换的旧指针先移入预留好的 retired vector；该 vector 的 guard 在 `ScopedBatchLock` 之后析构，所以 URI、 + spec vector 和 CacheLocation 的最终 free 发生在 metadata locks 外。guard 在 lock 前构造，`continue`/错误分支 + 也遵循“先 unlock、后 clear”的 C++ 逆序析构； +6. 单 location item 用一次字符串比较代替 unordered_map hash+比较；多 location item 保留通用 find。针对 + interned-id variant 的 hot copy constructor 显式分派 owned/shared alternative，避免 libstdc++ 通用 variant + copy 分派。曾尝试直接重建最终 spec vector、跳过旧 LocationSpec 深拷贝，但 20K update 五轮约慢 1ms， + 已撤回，不能在没有新证据时重新引入。 + +这里有意修正 5.17 的旧门禁：handle 现在可以**只在同一次 exact-local fused RMW 的配对 read/write backend +调用之间**保留；不得越过 metadata shard lock 生命周期、请求或 backend。bounded TLS buffer 与 request-scoped +location-id interning 也已有明确上限/所有权，不等同于无界 thread-local 或全局 interner。 + +#### Release 性能 + +同机、pure-local、8 HTTP worker 的 `test_20_large_single_request_delta_scaling`，最终 20K 五轮范围/均值为： + +| 路径 | 5.17 均值 | 本轮范围 | 本轮均值 | 进一步降幅 | +| --- | ---: | ---: | ---: | ---: | +| create | 80.65ms | 76.85~84.18ms | 78.40ms（中位数 77.12ms） | 2.8% | +| existing update | 74.85ms | 69.46~71.43ms | 70.46ms（中位数 70.28ms） | 5.9% | + +8 路同 reporter、每请求 10K existing update 的长跑吞吐在多轮测试中为 `1.53~1.75M blocks/s`；较短且无 +编译干扰的轮次曾达到 `1.82~2.09M blocks/s`。并发结果会明显受同机编译负载和温度影响，因此不把某一轮 +最好值或固定提升百分比作为结论。可确定的结构性收益是第二次 LRU lookup/handle acquire 已消除，旧 +CacheLocation 的析构也已移出 metadata locks；perf 中 retained-handle writer self 约 0.2%，旧的 delta +unordered_map node lookup 已退出主要热点。发布判断仍应使用隔离机器上的相同负载 A/B。 + +#### 必须保留的测试门禁 + +- parser 重复解析测试要覆盖 TLS buffer 复用、escaped NUL/Unicode 与 generic fallback； +- ReportEvent 端到端测试要确认同一请求两个 block 的 `&CacheLocation::id()` 相同，不同 medium 仍隔离； +- retained-handle 单测必须同时覆盖 existing target、existing key/missing target、new key、部分 skip、非法 + read index 和 RAII release；MetaIndexer 的 capacity、duplicate、modifier skip/fail 用例不可删； +- 性能结论只适用于 pure-local。任何把 retained-handle API 扩到 cached/Redis、装饰 backend 或多 target RMW + 的修改，都必须重新证明 recovery、fault injection、capacity 和 lock ordering 语义。 + +### 5.19 2026-08-07 spec/URI 与 request-scoped ownership 收敛 + +5.18 后的同机 Release profile 仍显示 `StandardUri::ParseParams/Parse/ToUriStringWithExtraParam` 合计约 3.8%， +单元素 spec/task 容器、旧 URI 深拷贝和 allocator 仍是主要 CPU 来源。RapidJSON ASCII in-situ 路径已经没有 +UTF-8 validation 栈；剩余 `ParseString` 是 JSON 字符串扫描和反转义，不能通过关闭合法性校验消除。本轮因此 +只收敛已被 profile 证明的 URI、LocationSpec、vector 和 shared ownership 开销，不引入 SAX parser，也不修改 +metadata lock、Redis/cached backend 或查询语义。 + +#### 实现与所有权边界 + +1. 常见单 spec BLOCK_ADD/SNAPSHOT 的校验结果和 `MergeLocationSpecsTask` 使用 inline optional；只有第二个 spec + 到来时才提升为预留好容量的 vector。纯 ADD 请求也不再创建每 block 的一元素 task vector、空 delete task + reserve，最终用 flat task vector + offsets 调用 MetaSearcher；通用嵌套接口和多 spec 语义保持不变； +2. canonical URI 使用 allocation-free string_view 扫描，一次得到 `size` 和 `s_version` 的有序插入位置，再直接 + 生成最终 URI。只有协议、正数 canonical port、显式 `key=value`、严格递增且无重复参数等条件全部满足时 + 才走快路；合法但非 canonical 的历史输入继续回退完整 StandardUri，非法 port、重复内部参数和溢出仍失败； +3. 同一请求的 snapshot token 只做一次严格 ASCII `[0-9A-Fa-f]` 校验。每个已经独立验证过的 spec 随后使用 + prevalidated append；通用 URI helper 仍保留逐次校验。locale `isxdigit` 被显式 ASCII 判断替代，非 ASCII + 字节继续拒绝，不把协议 token 校验与 JSON UTF-8 校验混为一谈； +4. `CacheLocation` 保存一个不序列化的 validated total-size hint。常见“一个旧 spec 被同名新 spec 替换”可直接 + 复用旧总大小并构造最终 immutable location，避免再次解析旧 URI、复制后立即销毁旧 URI。反序列化和任何 + mutable spec 访问都会使 hint 失效并安全回退；多 spec、重复历史 name 和溢出仍走严格校验； +5. ReportEvent 的 medium map 是请求内 location-id 的唯一 owner。delta fold、snapshot entry 和同步 MetaSearcher + task 只借用该 `shared_ptr` 对象，不再为每个 block 做原子 refcount 增减；持久化 CacheLocation 时仍获取正常 + shared ownership。unordered_map rehash 不使 element reference 失效，且所有 borrowed 指针只允许存活到同一 + 次同步 Batch 调用返回，禁止缓存、异步投递或跨请求保存； +6. 没有把“消费 task 并移动 spec”扩展成通用 RMW API：modifier 在不同 backend 上可能重试，贸然消费输入会改变 + retry 语义。当前只在已经严格限定的 one-old/one-new/same-name 情况直接构造最终 immutable value，删除旧 URI + copy 和中间一元素 vector；CacheLocation 对外仍保持 `vector`，不扩大查询侧对象模型。 + +#### Release A/B 与最终 profile + +同机、pure-local、8 HTTP worker、同一 Release 构建方式。修改前 20K 五轮 create/update 均值为 +`77.01/70.31ms`；最终独立五轮范围为 `66.96~69.57/56.12~57.57ms`，均值 `67.92/57.09ms`，分别降低约 +`11.8%/18.8%`。8 路同 reporter、每请求 10K existing update 的 100 轮长跑为 avg/p50/p95/p99 +`23.12/21.87/33.89/36.07ms`，吞吐 `2.572M blocks/s`；本轮初始基线为 avg `31.92ms`、吞吐 +`1.992M blocks/s`，对应平均 RT 降约 `27.6%`、吞吐升约 `29.1%`。共享机器存在频率和编译扰动，生产判断仍 +应在隔离机器做同 payload A/B。 + +提交前另启一个长生命周期 pure-local Release 实例复验：20K 五轮 create/update 为 +`71.16~73.93/57.31~59.44ms`，均值 `73.00/58.41ms`；8x10K、100 轮的两次独立长跑 avg 为 +`22.68~27.67ms`、p99 为 `30.33~42.17ms`、吞吐 `2.328~2.973M blocks/s`。create 对共享机频率和 +allocator 冷热更敏感，但 update 与并发吞吐均保持相对基线的明确改善，且全程无业务错误。 + +重放到包含 P2P host-count 配置的新远端基线后，再次从头构建 Release 二进制：20K create/update 为 +`67.51/55.96ms`；8x10K、100 轮为 avg/p50/p95/p99 `23.07/22.03/33.16/34.85ms`，吞吐 +`2.605M blocks/s`。这组提交前数据说明基线组合没有抵消本轮收益。 + +最终 perf 中 `StandardUri::ParseParams/Parse/ToUriStringWithExtraParam` 和 locale `isxdigit` 已退出热点列表； +中间版本的 shared_ptr add-ref self 从 `3.50%` 降到最终 `1.53%`。剩余主要是 JSON 必需字符串扫描、glibc +allocator、immutable CacheLocation 写入、LRU hash/lock 和最终 URI copy。production 使用 jemalloc，直接 Bazel +binary 的 glibc 比例仍是保守上界；没有证据支持无界 object pool、全局 location-id interner或放宽 JSON 校验。 + +#### 验证门禁 + +- `SnapshotUriUtilsTest` 覆盖 canonical 与 StandardUri 输出等价、非 canonical fallback、无效/重复参数、严格 + ASCII token；`MetaSearcherTest` 覆盖 flat offsets、inline spec、borrowed owner 引用计数和 total-size hint； +- `CacheManagerTest` 10 个 shard 覆盖 ADD/DELETE/SNAPSHOT、last-op-wins、failure closure、capacity 和同请求 + location-id 共享；真实 HTTP snapshot 套件 36/36、双类型兼容套件 20/20 通过； +- reporter lifecycle 定向阻塞回归连续 50/50 通过；全量 `bazel test --config=release + //kv_cache_manager/...` 共发现 107 个目标，106 个可执行目标全部通过，另 1 个 GPU-only 目标因环境不兼容跳过； +- 修改 canonical parser 时必须维持“无法证明 canonical 就 fallback”的 fail-safe 边界；修改 borrowed id 时 + 必须证明 owner 覆盖整个同步 Batch 调用。不得把裸指针写入 backend、队列、cleanup callback 或 response。 + +### 5.20 2026-08-07 顺序流快路、异常路径按需分配与最终锁审计 + +5.19 之后用 production-like `LD_PRELOAD=/lib64/libjemalloc.so.2` 重新采样。常见请求是同一 reporter、同一 +medium、block key 递增且每个 `(block, location)` 只有一个 event；旧实现仍会为这些已经有序且唯一的数据建立 +完整 open-address index、event dependency 数组、排序 permutation 和 ADD/DELETE 两套 failure range。另一个 +剩余热点是通用字段名 helper 的函数调用，以及 canonical decimal 字段走 `from_chars`。本轮只增加能保持任意 +事件顺序、partial failure 和历史 URI 兼容性的按需快路，没有修改 persistent/cached/Redis backend、查询对象 +布局、metadata lock 范围或 lifecycle fencing。 + +#### 实现与正确性边界 + +1. `DeltaMutationGuard` 直接保存本 RPC 唯一的 `ReporterSnapshotKey`、可选 lease 和可选 snapshot-in-progress + failure，不再为每个 delta event 查两张 reporter-key unordered map。一个 ReportEvent request 的 instance、 + host 和 storage 在入口已经固定，因此不存在第二个合法 reporter key；generation adoption 和析构时 + `EndDeltaMutation` 仍只针对成功取得的同一 lease； +2. registration 状态和 interned location-id 合并到 request-scoped medium state。连续相同 medium 用最后一次 + state 指针命中，不重复 hash/probe；location-id 仍按需创建并由 medium state 持有。`NODE_REGISTER` 的全请求 + 预扫描改为第一次真正遇到 register 时才执行，纯 ADD/DELETE 不再额外遍历一遍 protobuf events；多 register、 + malformed register、delta-before-register 和 generation 继承语义由原有回归保留； +3. delta fold 对递增唯一 `(block_key, location-id)` 直接 append。只有出现非相邻 duplicate 或逆序 pair 时才 + 建立 power-of-two index；只有最终 unique-location 顺序确实非递增时才分配并排序 permutation。任意顺序仍 + 使用同一个 last-op-wins fold，随机 768-event reference-model 测试覆盖 48 keys、17 media、4 specs； +4. 每个 location 的第一个 dependency event 内联保存，额外 event 链只在同一 `(block, location)` 第二次出现时 + 分配，索引收窄为 `uint32_t`（protobuf event count 上限是 `INT_MAX`）。ADD/DELETE phase failure 以及 admission + failure 的 retry closure 仍遍历完整逻辑 event 链； +5. 只在实际 materialize ADD/DELETE 后 reserve 对应 phase 数组，ADD-only 不再分配 DELETE capacity，反之亦然。 + 每 block 的 24-byte failure range 也被移除：成功路径不写这份数据，只有 backend 返回错误时才在已排序 + location view 中二分定位 block range。新增乱序三 block fault test 验证只标记实际失败 block，前后成功 block + 仍可查询； +6. canonical URI 的 port/size 与 block key 使用严格、overflow-checked 的手写十进制循环，去掉 generic + `from_chars`。port 继续拒绝 0、符号、前导零和大于 `INT64_MAX`；size 接受完整 `uint64`；block key 继续接受 + signed int64 和 vLLM unsigned uint64 decimal，并保持相同 64-bit pattern。测试覆盖 `INT64_MAX` port、port + overflow、`UINT64_MAX` size、size overflow，以及 block-key 两侧边界和非法符号； +7. canonical spec 不再内嵌构造重量级 `StandardUri`；只有合法但非 canonical 的兼容输入才按需分配 parser。 + `ValidatedEventLocationSpecs::Push`、`PushReportEventSpec` 改为显式右值入口，最终 URI/LocationSpec 继续 move； +8. JSON parser 的字段名比较改成 compile-time string-literal 长度加 `memcmp`，保留 snake_case、camelCase、 + embedded-NUL 和 exact-length 语义。最终 profile 中原先约 1.58% 的 out-of-line `NameIs` 已完全退出热点； +9. task 构造复用入口已验证过的 `requested_type`，不再对每个 block 重复调用 backend virtual getter。snapshot、 + delta 和 query 仍使用同一 routing decision，不改变 L1P5/L2 隔离。 + +#### 同机 A/B 与 perf 结论 + +同一 Release 构建、pure-local、8 HTTP workers、jemalloc、五个 fresh instance 的 20K scaling： + +| 路径 | 修改前五轮均值 | 最终五轮范围/均值 | 降幅 | +| --- | ---: | ---: | ---: | +| create | 65.75ms | 58.38~61.89ms / 59.56ms | 约 9.4% | +| existing update | 57.37ms | 49.20~51.99ms / 50.23ms | 约 12.5% | + +8 路同 reporter、每请求 10K existing update 在共享机器及 perf instrumentation 下为约 +`2.72~3.47M blocks/s`，所有轮次 fail/drop 均为 0。共享机器频率、其他 Bazel 进程和 perf tracing 会造成明显 +漂移，因此并发结果只用于排除回退，生产收益应继续做隔离机 paired A/B；串行五轮是本轮更稳定的比较。 + +最终 `cycles:u` profile 覆盖 57,389 samples、35.92M blocks。主要 self CPU 为 LRU mutex unlock `9.34%`、 +`memmove 5.50%`、shared_ptr add/release `5.46%/3.47%`、RapidJSON `ParseString 4.99%`、ReportEvent orchestration +`4.59%`、targeted RMW `3.60%`、mutex lock `3.01%`、`ParseObject 2.67%`、LRU hash find `2.05%` 和 canonical URI +`1.75%`；jemalloc `malloc` 仅 `1.45%`。`from_chars`、out-of-line `NameIs`、eager fallback `StandardUri`、 +common-path delta hash-table build 和 success-path failure range 都已退出热点。 + +锁需要区分“执行 lock/unlock 指令”与“线程实际睡眠”。15 秒 futex enter/exit 配对覆盖 2,392 个 10K 请求： + +- `FUTEX_WAIT_PRIVATE` 真正睡眠 17,679 次，总计约 2.879s，单次平均约 163us、最大 4.735ms，折合每请求约 + 1.20ms；另有 35,158 次在约 19.1ms 总计内返回 `EAGAIN`； +- CPU call graph 将 mutex 成本定位在 local LRU `LookupBatchWithScratch` 和 retained-handle + `ReleaseBatchWithScratch`；metadata `pthread_rwlock` 没有进入 futex sleep，因此不是线上 80ms 的来源； +- 尝试按 worker 旋转 LRU shard 遍历起点以打散 convoy，真实吞吐连续下降到 `2.21~2.77M blocks/s`，串行也 + 回退，已完整撤销。稳定 shard 顺序的 cache locality 比减少短 wait 更重要； +- retained handle 的 release 当前仍发生在 metadata lock 生命周期内。把它移出锁会改变并发 capacity/LRU + admission 窗口，不能只为约 1.2ms/request 的可消除上界冒险。若以后重写 refs 为 atomic 或 fused shard + callback，必须单独证明 eviction、delete、capacity、query 并发和 lock ordering。 + +剩余 `ParseString` 是 JSON 字符串扫描/反转义，ASCII 路径已没有 UTF-8 validation 栈；继续下降需要 SAX/direct +parser，收益上界约 5% 且兼容性风险明显。shared_ptr/LRU 与最终 URI copy 是下一批结构性候选，但都会触及查询 +共享对象或 cache eviction。没有新的隔离机 profile 与正确性模型前，不应继续用全局 interner、无界对象池、 +扩大 metadata lock batch 或 lock-free refcount 改写换取小优化。 + +#### 最终门禁 + +- 每项结构变化后均运行对应 Release `CacheManagerTest --test_filter=*ReportEvent*`；新增乱序 failure-range 测试； +- `SnapshotUriUtilsTest`、`ProtoMessageJsonUtilTest`、`MetaSearcherTest`、`LruCacheTest`、 + `meta_local_backend_test` 必须保持全绿； +- 提交前必须重新跑 pure-local HTTP snapshot 套件 41/41（36 functional + 5 benchmark）、旧 ReportEvent + 兼容套件 21/21（20 functional + 1 benchmark）、完整 Release + `//kv_cache_manager/...`（无 GPU 环境预期 106 pass + 1 incompatible）； +- 性能测试必须确认启动进程 maps 中加载 jemalloc。实验性 URI prefix cache、直接重建最终 spec vector 和 LRU + shard rotation 都已因无收益或回退撤销，后续 AI 不应在没有新的 paired A/B 证据时重复引入。 + +### 5.21 2026-08-07 收敛复核：allocator、URI 所有权与 item lock 的负向实验 + +5.20 后又在完全相同的 pure-local Release 配置下做了一轮独立复核。本节的目的不是记录“还能想到什么”，而是 +把已经实测无收益的候选、其正确性边界和停止条件固定下来，避免后续仅凭 profile 百分比重复引入更复杂的所有权 +或锁协议。下面所有实验都在独立修改后测试、A/B，未达到门槛的实现均已完整撤销；最终生产代码仍是 5.20 的 +实现。 + +#### 更大样本的 clean profile + +使用 production-like `LD_PRELOAD=/lib64/libjemalloc.so.2`、8 HTTP workers，对干净 HEAD 采集 +`cycles:u -c 100003 -g --call-graph fp`。样本覆盖 198,470 个 samples、792 个 10K-block 请求，共 7.92M +blocks；负载 avg/p50/p95/p99 为 `19.09/18.46/27.00/28.53ms`，吞吐 `3.177M blocks/s`，业务错误为 0。 +主要 self CPU 为： + +| 热点 | self CPU | +| --- | ---: | +| `memmove` | 10.08% | +| LRU `pthread_mutex_unlock` | 6.22% | +| RapidJSON in-situ `ParseString` | 5.19% | +| `CacheManager::ReportEvent` | 4.73% | +| targeted RMW | 3.63% | +| merge modifier | 3.40% | +| parser orchestration/ASCII scan | 3.08% | +| shared_ptr add/release | 2.81% / 2.21% | +| RapidJSON `ParseObject` | 2.77% | +| LRU hash lookup | 2.20% | +| LRU `pthread_mutex_lock` | 1.92% | +| item rwlock read/write/unlock | 1.68% / 1.63% / 1.59% | +| jemalloc `malloc` | 1.52% | + +`memmove` 的主要调用方依次是最终 immutable `LocationSpec` 构造、TLS body copy、protobuf arena string 和 +RapidJSON；它不是一个可整体删除的重复 copy。锁的百分比同样主要是成功的 lock/unlock 指令，不等于线程睡眠： +5.20 的 futex trace 已把实际等待上限量化为约 1.20ms/request。 + +#### 已撤销实验及 A/B + +1. **旧版消费 task URI 实验（历史结果；5.24 已用更窄的 prevalidated-only 实现重新验证并保留）。** + 当时为 pure-local、单 location、flat task 增加了显式 consumable + API，并保留 spec name 供失败映射；create/update、多 location 不消费及错误语义定向 UT 全部通过。但五轮串行 + existing-update 对照为 `50.02ms`，候选为 `50.08ms`，并发也无稳定信号。该 copy 在 profile 中可见，却不是 + 当前 wall-time 瓶颈；消费输入还会扩大 backend retry 语义，故完整撤销。5.18 中“直接重建最终 spec vector” + 曾回退约 1ms，这次不同实现得到相同结论。 +2. **exact-local RMW 跳过逐 item shared read-lock。** 前提审计确认 MetaIndexer shard lock 覆盖本路径的生产写， + 定向 `MetaLocalBackend/MetaIndexer/MetaSearcher/CacheManager` Release UT 全绿。12 轮 fresh-process A/B 中，baseline + update `50.01ms`、候选 `50.28ms`；并发吞吐 baseline `3.255M`、候选 `3.165M blocks/s`。把分支移出 key loop + 后仍为 `50.42ms` 对 `50.48ms`，约 1.8% 的并发差异处于机器噪声内。可见的 rwlock 指令没有形成稳定 RT 收益， + 而特殊“外层锁隐含保护”会增加未来 backend 维护风险，故完整撤销。 +3. **bounded TLS RapidJSON DOM arena + DOM/stack 共用 allocator。** 实测 DOM：`202,157B` body 使用 + `256,160B`、capacity `404,314B`；`1,010,163B` body 使用 `1,280,160B`、capacity `2,020,326B`，即 DOM + 约为 body 的 1.27 倍。第一版 `vector::resize` 首次清零使 create 从 baseline `60.15ms` 回退到 `62.12ms`， + 虽然同连接 update 一度为 `48.93ms` 对 `50.09ms`。改成对齐但不初始化、最大 6MiB 的 TLS storage 后，七轮 + create/update 仍为 `61.52/50.91ms`，相对 baseline `60.15/50.09ms` 均回退。jemalloc 已能有效复用大块； + 单一 arena 的布局/局部性损失超过 1.52% allocator 上限，故完整撤销且不承担每 worker 额外常驻内存。 + +#### 最终停止条件与后续边界 + +- 不实现 SAX/direct JSON parser。剩余 `ParseString` 上界约 5.2%，但必须重新实现 snake/camel aliases、enum + string/numeric、int64/uint64 JSON 表达、unknown/duplicate field、escaped/raw NUL、Unicode/非法 UTF-8 和 generic + fallback 的全部兼容面；收益与上线风险不匹配。 +- 不启用 adaptive LRU mutex，也不把 item unique-lock跨越 modifier。真实 futex wait 很短，而持锁延长或自旋会 + 直接与优先级更高的 GetHostCacheState 争用；“CPU 有余量”不能替代 query p99 的同负载 A/B。 +- 不把**持久化 owner**改成裸指针/全局 interner，不原地修改对查询可见的 CacheLocation。add/ref 与析构是 + immutable query snapshot 的所有权成本；只有 5.22 所述“同步调用内、retained handle + shard lock 双重保护”的 + 旧值借用是例外，借用结束后的 owner 仍是 `shared_ptr`。 +- `EstimateMemUsage`、metrics attach、block-key decimal parse 等单项均低于约 1%；为它们增加 persistent 字段、 + request TLS side channel 或跨层 hint 会增加每 key 内存与协议耦合，不满足“端到端有稳定收益”的门槛。 + +后续只有在隔离机的新 profile 显示热点结构发生变化时，才应重新打开上述方向。下一阶段若要获取超过噪声的收益， +需要独立设计并验证 cache item/immutable location 的表示或真正的 backend shard callback；这属于新的并发协议，必须 +以 Get/ReportEvent 混合负载、eviction/capacity/fault/lifecycle 完整模型作为前置条件。5.22 只完成了 exact-local +同步借用这一条窄路径的证明，不代表可以把相同假设扩展到持久 owner、异步 backend 或调用边界之外。 + +### 5.22 2026-08-07 最终所有权复核：借用旧 location，保留 immutable query snapshot + +5.21 的停止条件之后又用更大的 clean profile 做了源码行和调用栈聚合。结论是不能把全部 +`shared_ptr add/release` 都看成同一个问题：新 `CacheLocation` 持有 interned location id 所产生的一次引用是 +持久化所有权，不能删除；pure-local fused RMW 从 item map 读取旧 `CacheLocation` 时产生的临时引用，仅用于同步 +modifier，生命周期已被 metadata shard lock 和 retained cache handle 覆盖，可以安全消除。本节记录两项先行负向 +实验、最终保留实现及其严格边界。 + +#### 两项已完整撤销的 HTTP/parser 实验 + +1. **canonical BLOCK_ADD 的 RapidJSON SAX parser。** 实现支持 snake/camel alias、字符串/数字 enum、任意字段顺序， + 遇到 mixed event、未知形状、Unicode 或非 canonical 输入即回退现有 DOM parser，相关 JSON UT 全绿。SAX 确实 + 让 DOM `ParseObject`/`Document::String` 退出 profile，但状态机和逐 token protobuf setter 把成本转移到了 SAX + `ParseString`：七轮 20K create/update 为 `60.39/50.26ms`，同机 fresh baseline 为 `60.28/50.25ms`；8 路 + concurrent 10K 为 avg `19.10ms`、`3.508M blocks/s`，baseline 为 `17.96ms`、`3.699M blocks/s`。没有串行 + 收益且并发回退，已完整撤销。 +2. **thread-local reusable protobuf Arena/request graph。** 对 32KiB~2MiB body 复用 worker-local request message， + reentrant/oversize 请求回退到 request-scoped arena，并设置 8MiB hard cap。`Clear()` 仍需遍历全部 message tree， + 保留的 repeated message/string capacity 还增加 cache footprint。三轮 concurrent 10K 只有 + `3.01~3.23M blocks/s`，低于同机 baseline `3.699M`；进程 RSS 约 `267MiB`。实现和常驻内存均已撤销。 + +这两项说明 allocator 百分比不能直接当作可回收 wall time：jemalloc 和 request-scoped protobuf Arena 已能较好 +复用大块，跨请求保留对象反而破坏局部性；direct parser 也必须用端到端 A/B 判断，不能只看某个 DOM symbol 消失。 + +#### 保留实现：RMW 旧值使用同步借用视图 + +pure-local `ReadModifyWriteSingleTargetLocations` 原来把 item map 中的旧 `shared_ptr` 复制到 +batch vector，modifier 构造新 immutable value 时再释放这份临时引用。最终实现改为： + +- backend 在 item shared lock 内只返回 `const CacheLocation *`；接口名显式为 borrowed view，并且**没有** + “不保留 handle”的开关； +- `ScopedBatchLock` 在整个 read/modifier/write 周期持有目标 metadata shards，retained cache handle 保证被 LRU + eviction 摘除的 `MetaMemCacheItem` 也不会析构，item map 自身继续拥有旧 location； +- modifier 只读旧对象，在栈上构造独立的 `shared_ptr` 新值，成功后直接 move 到 upsert vector； + 不原地修改任何查询可见对象； +- writer 仍在 item unique lock 内原子替换 map entry，旧 owner move 到 `retired_locations`，并在 metadata lock + 释放后析构。并发查询仍先在 item shared lock 内复制旧或新 `shared_ptr`，因此继续获得完整 immutable snapshot； +- generic/Redis/cached backend 仍走原有 owning `shared_ptr` 路径。借用接口只在 + `SupportsSingleLocationRmw()` 已确认 exact pure-local backend 时可达。 + +第一版曾额外创建一个 request-sized replacement `shared_ptr` 数组；串行虽有约 2% 信号，但 8 路并发回退 +4%~8%。最终改成每次 modifier 的栈上新值、成功后直接 move，去掉第二个数组后并发回退消失。后续不得重新引入 +双 request-sized location 数组。 + +#### 正确性门禁与性能证据 + +- backend/indexer UT 直接记录旧 owner 的 `use_count`，断言 borrowed read 前后不增加；同时覆盖 hit、key exists but + location miss、key miss、capacity full 时保留 existing-key update、modifier skip、duplicate key、retained handle + subset validation 和旧值延迟析构； +- `TestGetHostCacheStateConcurrentWithReportEventAndHostDown` 在 Release 下重复 200 个 sharded runs 全绿，覆盖查询 + 与连续 immutable replacement、HOST_DOWN 可见性切换并发；定向 local backend、MetaIndexer、CacheManager 测试全绿； +- 最终完整 Release `//kv_cache_manager/...` 为 106 个可执行测试全绿、1 个 GPU-only 测试按预期 skip；最终链接 + HTTP 二进制的 snapshot 套件 36 functional + 5 benchmark 全绿，旧协议/双类型套件 20 functional + 1 benchmark + 全绿。`/proc//environ` 和 `maps` 同时确认加载 `/usr/lib64/libjemalloc.so.2`； +- 八组交错顺序的同机 20K paired A/B：create `61.017 -> 60.911ms`（符合预期，new-key 路径基本中性），existing + update `50.429 -> 49.714ms`，下降约 `1.42%`； +- 三轮固定 800 个 10K 请求的 `perf stat`：cycles 均值 `44.171B -> 41.675B`（约 -5.65%），instructions + `103.277B -> 102.978B`（约 -0.29%），cache misses `211.54M -> 187.13M`（约 -11.54%）。共享机频率会影响 + cycles，稳定结论是“没有新增指令膨胀，旧 owner 的 cache-line/refcount 流量下降”； +- 最终候选 profile 覆盖 548,846 samples。`shared_ptr::_M_release` self share 从 clean baseline 的 `2.21%` 降到 + `1.20%`。剩余 `_M_add_ref_copy` 主要来自每个新 `CacheLocation` 对 interned location id 的持久 owner，不应按本次 + 方法继续删除； +- 8 路 10K update 加 200 次 10K-key GetHostCacheState 的 closed-loop 混合压测无错误。五组候选 Get avg 聚合约 + `10.75ms`，baseline 约 `11.23ms`；p99 范围分别为 `22.13~27.97ms` 与 `14.53~26.84ms`，共享机 tail 有明显 + 抖动且两者重叠，因此只下“查询未出现系统性回退”的结论，不宣称 p99 提升。 + +#### 本轮后的停止线 + +当前 profile 的大项依次是 LRU mutex、最终 string/memmove、RapidJSON string scan、manager/merge、interned-id owner +和 item rwlock。已有 futex trace 证明真正 sleep 远小于 lock/unlock self CPU；SAX、TLS protobuf graph、DOM pool、 +LRU shard rotation 和跳过 item read-lock 均已有负向 A/B。继续消除 interned-id owner 需要 process/instance lifetime +string pool，继续消除 item lock 需要把 modifier 放进 backend critical section，这两者都会扩大查询或回收风险。 +在出现新的隔离机 profile 之前，本分支不再接受以 raw global pool、原地可变 CacheLocation、扩大锁范围或无界 TLS +缓存换取低个位数百分比的改动。 + +### 5.23 2026-08-07 HTTP body 原地解析、SIMD 扫描与 RMW 临时状态收敛 + +5.22 的 clean profile 继续按 `memmove` 调用方拆分后发现一个此前被总占比掩盖的确定重复工作： +`MutableJsonBufferLease` 把 cinatra 已经完整收进 `std::string` 的 HTTP body 再复制一遍，随后才执行 RapidJSON +in-situ parse。该调用方占当时全部 `memmove` samples 的 `23.44%`，折算约占总 CPU `2.07%`。这与最终 +`LocationSpec/URI` 的持久化 copy 不同：后者建立 immutable metadata 所有权，前者只是为获得 mutable buffer +而复制同一请求体，可以在明确 transport 生命周期后删除。 + +#### 保留实现与兼容边界 + +1. `GetArenaHandler` 允许为特定请求注册 `char * + size` parser。当前锁定的 + yalantinglibs/cinatra 0.5.5 在 `coro_http_connection` 中用 mutable `std::string body_` 保存完整 Content-Length + body，`request_.set_body(body_)` 只暴露同步 `string_view`；logger 在 handler 前读取 request，handler 返回后才复用 + connection。因此 ReportEvent 可以在 coroutine 内直接 ParseInsitu，protobuf/DOM 均不保存 body 指针；其他 HTTP + API 继续使用 immutable、length-aware parser。 +2. mutable API 的契约显式要求 `json[size]` 可读且为 `\0`，入口仍做防御检查。小于 32KiB、非 ASCII 或含 raw NUL + 的 body 保留旧 parser；小 heartbeat/register 直接进入旧 parser，不再先做一次随后必然重复的 ASCII scan。 +3. 快速 protobuf converter 若遇到 `null`、未知 enum 等少见但 generic protobuf JSON 接受的形状，不能再使用已被 + in-situ 修改的原文。实现从**完整 DOM**序列化一次 normalized JSON，再调用 generic parser；该分配只发生在兼容 + fallback。JSON 语法错误直接返回 bad request，因为 fast/generic parser 都不应接受它。 +4. ASCII/raw-NUL 预扫描在 x86 上运行时分派 AVX2（32 bytes/iteration），无 AVX2 时使用 SSE2；AArch64 使用 NEON， + 其他平台保留严格 scalar fallback。非 ASCII 路径仍启用 RapidJSON UTF-8 validation，没有用 SIMD 检测替代编码 + 正确性。 +5. `BatchMergeLocationSpecsImpl` 不再同时保存 `incoming_task_sizes` 与 `usage_changes`。incoming size 直接初始化对应 + usage slot，modifier 一次调用内读出后写回 final size；exact pure-local 的一 location/key 路径还直接用 key index， + 不分配 offsets。20K 主路径因此减少约 `160KiB + 160KiB` 临时数组和一次 request-shaped allocation； + multi-location 仍保留 flattened offsets 和完全相同的验证/计量语义。 + +这里的 transport 假设是严格回退边界：若升级 cinatra 后 body 不再由 mutable、NUL-terminated `std::string` 支撑， +必须删除 specialized parser 或恢复 immutable copy，不能仅依赖 `const_cast` 继续运行。UT 直接覆盖 mutable API 的 +escaped quote/backslash/newline、合法 `\u0000`、Unicode escape、raw NUL 拒绝，以及 source 已被修改后 rare fallback +仍与 generic protobuf parser 完全一致。 + +#### A/B 与最终 profile + +- 删除 HTTP TLS body copy 的 8 组交错 20K scaling：create `60.276 -> 59.301ms`（约 `-1.62%`），existing update + `50.035 -> 49.894ms`（接近中性）；8 路 10K steady throughput 约提高 `1.83%`。create 包含旧 TLS buffer 的首次 + allocation/first-touch，warm update 的收益上界本来就较小。 +- SSE2/AVX2 各用 fresh process 做 8 组同长度 payload：`3.258M -> 3.307M blocks/s`，约 `+1.5%`；profile 中 + ASCII scan self share 从约 `2.13%` 降至 `1.69%`。该数据只证明 portable SIMD dispatch 有小幅正收益，不外推 + 为整条 ReportEvent 的固定 SLA。 +- RMW 临时状态收敛单独做 8 组 paired A/B：20K create/update 均值 + `56.755/49.008 -> 56.392/48.648ms`，约 `-0.64%/-0.73%`；8 路 10K 的平均 RT + `18.730 -> 18.281ms`，吞吐 `3.238M -> 3.347M blocks/s`（约 `+3.4%`）。共享机频率会放大低个位数差异， + 稳定结论以“减少 320KiB 临时写流量且没有回退”为主。 +- 最终 `cycles:u` profile 覆盖 14.32M blocks。TLS body-copy caller 已从 `memmove` call graph 完全退出；剩余 + `memmove` 中约 `72.26%` 来自 merge modifier 建立最终 LocationSpec/URI，约 `11.87%` 来自 protobuf arena string。 + flat self CPU 主要为 LRU mutex unlock `8.91%`、RapidJSON ParseString `5.90%`、memmove `5.67%`、interned-id + persistent owner add-ref `5.00%`、single-target RMW `4.86%`、manager `4.06%`、mutex lock `2.94%`。热点结构与 + 5.22 的所有权/锁结论一致，没有出现新的远端 I/O、serialization 或 futex sleep 路径。 + +#### 本轮已撤销实验 + +1. **把 UTF-8 validation 融入一次 RapidJSON parse、删除预扫描。** 定向 UT 通过，但 fresh 20K create/update 从 + `58.78/49.52ms` 回退到 `62.77/52.83ms`，约 6%；branch-heavy codepoint validation 明显慢于 SIMD ASCII scan， + 已撤销。 +2. **按 canonical JSON 字段位置直接取 DOM member。** 兼容 fallback 与 UT 均通过，但 8 组吞吐信号仅约 `+1.4%` + 且噪声较大，`memcmp` self share 仍为 `1.62%`，只是把成本移进 `ParseBlockAdd`，已撤销。5.22 已完整测试并撤销 + SAX parser，本轮没有重复引入。 +3. **ordered unique key 只做一次 duplicate scan。** 理论上少一遍线性比较，但 10 组 20K A/B 为 + `56.554/48.300 -> 56.270/48.792ms`，existing update 反向约 1%，没有端到端证据，已撤销。 +4. **优先访问 `CacheLocation` 的 interned-id variant。** 汇编确实少一个 index branch，但 10 组 ReportEvent + create/update 为 `55.785/48.578 -> 56.656/48.510ms`；两台 fresh process 的 6 组 20K Get 串行均值 + `13.613 -> 13.880ms`，16-way p50/p99 基本重叠。为避免牺牲 owned-id 的通用路径，已撤销。`std::visit` 版本还 + 生成了 out-of-line indirect dispatch，更不应保留。 +5. **targeted RMW shard index 改成连续 counting-sort。** 第一版用一张 flat index 表替换 per-shard vector 和 + batch index copy，却在 count/scatter 两遍重复计算 `HashKey`；8 组交错 20K A/B 为 baseline + `58.454/49.156ms`、candidate `59.020/49.084ms`，create 明确回退。第二版缓存首遍 shard id，定向 Release UT + 通过，并重启两端进程做 10 组交错 A/B；baseline create/update 为 `57.628/48.962ms`，candidate 为 + `57.903/49.062ms`，平均仍分别多 `0.275/0.100ms`。少量 allocator/node 收益被额外的连续 scatter 写流量抵消， + 没有 wall-time 正收益，故代码完整撤销，只保留本记录。 +6. **直接 move 每个 shard 已排序的 index vector 到空 batch。** 典型 20K/16-shard 请求中单个 shard 已超过 soft + batch size，理论上可省掉每 batch 一次 allocation 和约 1250 个 `int32_t` copy，而且不改变 shard/batch 顺序。 + 定向 Release UT 通过后，重启 baseline/candidate 做 12 组交错 A/B：candidate 相对 baseline 的 create 平均 + `+0.113ms`、update `-0.037ms`，paired median 为 `-0.025/-0.085ms`，全部处于噪声内。说明这段 index copy + 已不是端到端瓶颈；为避免增加特殊所有权分支，代码完整撤销。 + +最终源码重新构建后完成以下门禁（均为 pure-local metadata，不依赖 Redis）： + +- Release `//kv_cache_manager/...`：106 个可执行测试通过，1 个 GPU-only 测试按预期 skip； +- snapshot HTTP：36 functional + 5 benchmark 全绿；20K scaling 对 create/update 后逐点查询验证最终 URI； +- 旧协议/双 storage type HTTP：20 functional + 1 mixed benchmark 全绿； +- `TestGetHostCacheStateConcurrentWithReportEventAndHostDown` 连续 200 次通过；最终二进制的 8 writer × 50 个 + 10K update 与 200 个 10K-key Get 混合压测零错误，write avg/p95/p99 为 `15.89/22.80/28.57ms`，Get + avg/p50/p95/p99 为 `10.86/7.39/23.20/24.81ms`； +- 真实 HTTP 额外验证 large rare fallback 与 large Unicode 返回 200，截断 JSON 与 raw NUL 返回 400； +- `/proc//environ` 与 `/proc//maps` 同时确认最终 Release 进程加载 + `/usr/lib64/libjemalloc.so.2`。 + +因此下一步不应继续围绕低于 1% 的 metrics、decimal parse、variant branch 或 URI copy 做局部改写。若线上新 profile +仍以 LRU lock/unlock 与 persistent owner 为主，能够超过噪声的下一阶段已经属于 LRU ref/list 协议或 metadata +representation 重设计，必须先建立 capacity/eviction、Get/ReportEvent 混合负载和 lifecycle fault 的独立正确性模型， +不能作为本次低风险 hot-path patch 顺手合入。 + +### 5.24 2026-08-08 对齐 pure-local 分片锁与消费 prevalidated URI + +5.23 的最终 profile 中，LRU lock/unlock 仍是最大项，且 memmove 的约 72% 来自把 flat task 中已经拥有的 URI +复制到最终 immutable LocationSpec。本轮没有扩大 backend critical section，也没有改变 Get 的 immutable snapshot +协议，而是分别消除两类可以严格证明为冗余的工作。 + +#### 保留实现一：MetaIndexer mutex shard 复用真实 local LRU hash seed + +此前 MetaIndexer 用固定 HashKey seed 把请求按 16 个 metadata mutex shard 分批，而 pure-local LRU 用自身的 +host-specific seed 把同一批 key 分散到默认 1024 个 LRU shard。两个 seed 不同意味着一个 metadata batch 通常又 +横跨大部分 LRU shard；两阶段 lookup/release 因而反复执行数千次 LRU mutex lock/unlock。最终实现： + +- MetaLocalBackend 只读暴露其实际 Cache::GetHashSeed()；MetaStorageBackendManager 仅在 single pure-local + backend 下向 MetaIndexer 提供该值； +- MetaIndexer 初始化完成后保存每实例的 mutex hash seed，所有 mutation/RMW batching 统一通过 + GetMutexShardIndex()，不能让同一个 key 在不同操作中使用不同 mutex； +- Redis、cached 和其他 backend 保留原固定 seed 与原 batching 行为。实现没有给 LRU 设置固定 seed，也没有改变 + LRU 的 key→shard/capacity 分布；只是让外层 metadata mutex 使用 LRU 已经选定的 host seed，因此仍保留跨主机 + hash 独立性； +- 测试不再把两个连续 key 写死为“不同 shard”，而是按 indexer 的真实 seed 选择。新增 UT 直接断言 pure-local + mutex 的低 hash bits 与 LRU 一致；这也修复了更换 seed 后生命周期并发测试自身可能等待闸门造成的假死。 + +原型 perf 在相同 8-way 10K update 下显示：pthread_mutex_unlock self share 7.93% → 1.99%， +pthread_mutex_lock 2.53% → 0.94%，baseline 可见的 futex wait/wake 在候选中降到 0.05% 报告阈值以下。 +动态 seed 最终版的十组交错 20K A/B 为： + +- create 57.780 → 56.048ms（约 -3.0%）； +- existing-location update 49.039 → 47.512ms（约 -3.1%），两项均 10/10 轮更快； +- 两组各 2392 个请求的长窗口 8-way 10K update，吞吐平均约 + 3.226M → 3.686M blocks/s（约 +14.3%），两组 p95/p99 均下降。 + +收益在并发下更大，符合“减少锁指令和 cache-line 争用、没有增加单 key 业务逻辑”的预期。该优化不得扩展为 +固定全局 LRU seed，也不得让 read/write 选择不同 metadata seed。 + +#### 保留实现二：只消费 CacheManager-prevalidated flat task 的 URI + +ReportEvent 在进入 MetaSearcher 前已经完成 URI 解析、snapshot metadata 校验和 size 汇总，且 flat task 在同步调用 +结束后不再重试。旧实现仍把其中的 LocationSpec 复制到新 CacheLocation，长 URI 因而产生一次额外 allocation + +memmove。最终实现将边界收窄为： + +- BatchMergeLocationSpecsFlat 接收 mutable task vector，但只有携带 + CacheManager::PrevalidatedTotalSize 的 task 可以转移 URI 所有权；普通/non-prevalidated flat task 和 nested + 通用 API 仍保持完整 copy/retry 语义； +- 转移前复制并在 source task 中恢复 spec name。metadata write 即使失败，CacheManager 仍能用 + (location_id, spec_name) 精确回填每个原始 event；source URI 可以为空，因为失败映射从不读取它； +- allocation、reserve、类型/size/duplicate-name 校验都在消费前完成。最终对象继续是独立 immutable + shared_ptr，没有原地修改查询可见值； +- 多 spec、legacy duplicate name、new key、existing location、lease failure、capacity/error alignment 均继续走同一 + modifier/write 结果模型。UT 同时断言 prevalidated task 的 URI 被消费、name 保留，non-prevalidated task 不变。 + +相对“已含动态 shard seed、尚未消费 URI”的二进制，十二组交错 20K A/B 为： + +- create 56.289 → 54.570ms（约 -3.05%）； +- existing update 47.674 → 46.950ms（约 -1.52%）； +- 两组长窗口 8-way 10K update 吞吐分别 + 2.956M → 3.385M（+14.5%）和 2.951M → 3.290M blocks/s（+11.5%），p99 均下降约 3~4ms。 + +5.21 记录的旧 consumable API 在当时代码形态和五轮短 A/B 中没有稳定收益，因此被正确撤销。本轮是在后续 +HTTP body copy、RMW scratch、borrowed old owner 都已经收敛后重新按调用栈定位，并以现有 prevalidated marker +作为严格消费凭据；历史负向数据保留，后续不应恢复更宽泛的“所有 flat task 都可消费”版本。 + +#### 本轮撤销实验与剩余停止线 + +- 尝试在“旧 location 只有一个 spec、incoming 同名一个 spec”时更早折叠 update，跳过通用 old-name/duplicate + 检查。定向 Release MetaSearcher/CacheManager UT 全绿，但 12 轮 fresh-process paired A/B 的 create/update + 仅 -0.102/-0.104ms，update median 为 0ms，没有稳定收益，已完整撤销。 +- 不把 persistent interned location-id owner 改成 raw pointer/immortal global pool。剩余 + shared_ptr::_M_add_ref_copy 是每个缓存 CacheLocation 的真实跨线程生命周期所有权；移除它会把收益建立在 + backend/reporter 永不销毁的隐含假设上。 +- 不跳过 RapidJSON string/UTF-8 语义。当前 ASCII SIMD 预扫描后，ParseString 是输入字节本身的线性扫描； + SAX、TLS protobuf graph、DOM pool 和直接 UTF-8 parse 都已有负向 A/B。除非协议改成 protobuf/gRPC 或建立新的 + direct parser 兼容模型，否则不能用放松 JSON/Unicode 校验换性能。 +- 不继续增加 ReportEvent worker 或延长 item/LRU 锁。当前优化已从锁次数入手；GetHostCacheState 优先级更高， + 后续任何锁范围/并行度变化都必须先通过 mixed Get p99 门禁。 + +#### 最终正确性与端到端门禁 + +全部测试使用 pure-local metadata，不依赖 Redis： + +- Release bazel test //kv_cache_manager/...：106 个可执行测试通过，1 个 GPU-only 测试按预期 skip； +- 最终 Release HTTP 二进制：snapshot 36 functional + 5 benchmark 全绿；旧协议/双 storage type + 20 functional + 1 benchmark 全绿； +- snapshot 20K scaling 在同一最终进程中为 create/update 59.25/48.70ms，随后逐点查询校验最终 URI； + GetHostCacheState 20K local serial p50/p99/avg 为 13.83/13.98/13.85ms； +- 8 writer × 50 个 10K update 与 200 个 10K-key Get 的混合压测零错误，write avg/p95/p99 + 14.34/21.11/23.17ms，Get avg/p95/p99 11.08/21.15/22.41ms。共享机数据只作为无功能/查询回退门禁， + 不声明线上固定 SLA。 + +本轮之后，仍能看到的主要成本是协议 JSON scan、最终 URI 的第一次持久化所有权、interned-id owner 和必要的 +immutable object allocation。它们不再是可通过局部 move、换容器或跳过锁安全删除的重复工作。后续若继续优化， +应从新 profile 重新建立证据，优先考虑协议边界或 metadata representation 的独立设计，不要重复 5.20~5.24 +已经完整撤销的微优化。 + +### 5.25 2026-08-10 功能正确性复核与大批量部分失败门禁 + +本轮不再改动生产逻辑，重点复核 5.24 优化后容易被性能测试遗漏的失败闭包、生命周期并发和 HTTP 大请求解析。 +新增或加强以下自动化门禁： + +- fast JSON parser 使用完整事件类型矩阵构造大于 40 KiB 的请求，强制进入 HTTP mutable in-situ 路径，并与 + protobuf generic JSON parser 做 message 等价比较；同时断言输入 buffer 确实被原地消费，防止测试误走小请求 + compatibility 路径； +- `max_key_count=1` 下同批更新已有 key 并插入新 key，验证 fused writer 只拒绝新 key、已有 key 更新仍提交，且 + 已消费 URI 的 source task 仍能把 `EC_NOSPC` 精确映射到原 event index； +- HTTP 端到端一次提交 512 个 ADD，按固定间隔混入 31 个非法 URI，请求体大于 32 KiB。验证 + `item_results` 与全部 512 个输入严格对齐、合法项可查询、非法项无副作用；随后只重试失败项，验证 generation + 不变、`snapshot_required` 清除且 metadata 最终收敛。 + +pure-local Release 验证结果： + +- snapshot HTTP 功能集（包含上述大批量用例）37/37、legacy/dual-storage HTTP 功能集 20/20 全绿； +- 5 个 reporter lifecycle/query 并发用例各重复 30 次，共 150 次通过；3 个 snapshot atomicity/lease/fused-RMW + 用例各重复 50 次，共 150 次通过； +- `CacheManagerTest` 全量重跑通过；其中一次并行构建压力下已有的 `TestFilterWriteCache_StaleSuffix` 报 + `std::future_error: Broken promise`，该用例隔离重复 100 次以及随后全量重跑均未复现，因此未把偶发结果误判为 + 本轮生产缺陷或静默修改无关代码； +- `bazel test --config=release //kv_cache_manager/...` 共 106 个可执行测试通过，1 个 GPU-only 测试按预期跳过。 + +同一 Release 进程的功能后性能门禁仍保持：20K ReportEvent create/update 为 58.96/50.38ms；20K-key +GetHostCacheState local serial p50/p99/avg 为 14.92/16.03/15.11ms。该共享机数据只证明本轮测试加强没有引入明显 +性能回退，不承诺线上固定 SLA。 + +### 5.26 2026-08-10 parser 差分与跨请求多 reporter 状态机复核 + +本轮从“快路径必须与原实现语义等价”出发补充差分测试，发现并修复一个此前用例未覆盖的真实兼容问题： +heartbeat 的 `system_status` JSON object 若包含重复 key，protobuf `JsonStringToMessage` 会拒绝请求，而 specialized +parser 原先通过 protobuf map 的 `operator[]` 静默采用最后一个值。这样同一 payload 会因为是否进入快速 parser +而产生不同结果。现在写入 map 前显式检查已存在 key；重复 key 让 fast conversion 失败，再由 generic parser 给出 +与原协议完全一致的拒绝结果。该检查只位于低频 heartbeat map 解析，不进入 BLOCK_ADD/DELETE、metadata RMW 或 +查询路径，也不改变任何锁范围。 + +新增 parser 兼容语料同时覆盖 canonical/snake/camel 字段、字符串和数字 enum、known field 为 null、未来 enum、 +数字形式的 string 字段、字段 alias 重复、map key 重复、多个 oneof member、unknown nested value、Unicode surrogate +pair/未配对 surrogate、错误字段类型以及 null repeated/oneof entry。每个样例都执行两套比较： + +1. immutable `ReportEventJsonParser::FromJson` 与 protobuf generic parser 的成功/失败和 message equality； +2. 追加未知 padding 形成大于 40 KiB 的 body，强制走 HTTP mutable in-situ 路径，再与 generic parser 做相同比较。 + +ReportEvent 状态模型也从单请求、单 reporter 扩展到跨请求场景：两个 reporter 交替执行 8 轮、每轮 96 个事件， +在 32 个 key、5 个 medium、3 个 spec 上做确定性随机 ADD/DELETE。每轮末尾强制更新两个 reporter 共用 block 的 +各自 location，专门验证 fused targeted RMW 只替换目标 `(block_key, location_id)`，不会覆盖同 key 的另一 reporter。 +每轮提交后均重新核对： + +- 全部物理 location id、spec name、完整 versioned URI 和 location type； +- query-visible URI 必须来自 reference model，空 key 不能产生伪命中； +- 两个 reporter 的 committed generation 独立且跨 delta 保持不变； +- 删除最后一个 spec 后 location/key 收敛，所有 replacement/delete 后的 storage usage 与 reference 精确相等。 + +pure-local Release 验证结果： + +- parser 完整测试进程重复 100 次通过；上述跨请求模型、单请求 flat-fold reference model 和容量部分失败映射分别 + 重复 20 次通过； +- 真实 HTTP snapshot 功能集 37/37、legacy/dual-storage 功能集 20/20 全绿；另外直接发送保留重复 JSON member + 的小 heartbeat 和大于 40 KiB heartbeat，均返回 HTTP 400，证明 handler 两条 parser 路径行为一致； +- `bazel test --config=release //kv_cache_manager/... --nocache_test_results`：106 个可执行测试通过，1 个 + GPU-only 测试按预期跳过； +- 同一 Release + jemalloc 进程的 20K ReportEvent create/update 为 55.86/48.20ms；20K-key GetHostCacheState + local serial p50/p99/avg 为 14.15/14.74/14.17ms，16-way p50/p99 为 35.88/50.53ms。数据表明本轮兼容修复和测试 + 扩展没有造成 ADD/Get 性能回退，但仍只作为共享开发机门禁，不是线上 SLA。 + +### 5.27 2026-08-10 GetHostCacheState 渐进读取、可见性纠偏与百万 key 收敛 + +本轮以 pure-local metadata 为唯一生产目标，重新从 GetHostCacheState 的 100 万 key Release benchmark、代码语义和 +`cycles:u` profile 交叉检查。基线的普通 host-prefix 路径虽然 metadata compact read 本身约 84ms，却在读完全部 key +后为每个 key 构造 `map>`；同机 4-worker 的 100K/500K/1M 全命中 p50 约为 +47.94/294.72/597.15ms。更严重的是，把第 1024 个 key 换成其他 host 或 metadata miss 时仍分别耗时约 +582.97/178.21ms，证明此前的并行投影不能取消已经无意义的百万 key 后缀。 + +代码复核还发现两个比性能更优先的正确性问题：host projection 没有过滤 `CLS_WRITING/CLS_DELETING/CLS_NEW` 等 +非 serving 状态，`StartWriteCache` 尚未 `FinishWriteCache` 的 location 会被当成命中；普通与 Mamba 路径还把 +非 `EC_NOENT` backend 错误当作 prefix 结束并返回成功。这两点都可能直接造成 KVCM 统计命中与实际引擎可读命中 +不一致，因此本轮先建立失败用例，再实现优化。 + +#### 最终实现与并发边界 + +1. `p2p_host_count=0` 使用真正的渐进式 compact projection。先同步读取 4096 key，利用第 0 个 key 建立有序候选 + host；每个候选维护 atomic prefix stop。只要所有候选均已停止，visitor 就降低全局 stop，尚未领取的后缀任务 + 直接取消。首窗口内的 host miss/no-entry 因而只读取 4096 key，而不是先物化整个请求。 +2. 首窗口成功后使用 16384-key 后续批次并由现有有界 QueryExecutor 调度。默认 local LRU 有 1024 shard，旧 4096 + 批次会在百万 key 请求中重复约 24 万次 lookup/release shard lock；增大后续批次可摊薄 lock/unlock，同时首窗口 + 仍保持早停上界。16384 相对 4096 的同机 A/B 将 1M metadata/all-hit p50 从约 109.3/114.9ms 降至 + 94.3/103.7ms；32768 没有进一步改善 all-hit 且放大后缀过读，已撤销。 +3. `p2p_host_count>0` 使用 peer-aware ordered reduction。普通 prefix 只为最终 top-N host 保留 peer 交集和精确 + fetched-key 计数；Mamba 最多用三个有界 pass 分别求 local 状态、各 full group 的 prefix peer 与 state coverage + peer、最终合并结果；没有选中实际 peer plan 时跳过最终 pass。local backend 不再保留逐 key 的 host/spec 图； + 常驻状态随请求内 distinct host 数、top-N plan 和协议要求的 fetched-key 去重向量增长，而不是随 + `key × host × spec` 组合数增长。不支持 progressive read 的 + backend 保留一次 compact batch，后续 pass 只重放该 batch，避免重复远端 I/O。 +4. request-specific checker 除返回可见/不可见外，同时借用返回已经解析的 reporter medium/host，并声明 EventReport URI + 已通过 generation fence 校验。projection 因而不再对同一 location 重复拆 location id、解析 URI 和提取 host； + medium filter 使用借用的 `string_view` hash set，generic URI path 也不再复制 path string。 +5. 每个 location 必须先满足 `CLS_SERVING`，再执行 backend liveness/generation checker。`EC_NOENT` 是正常 prefix + 终止；`EC_TIMEOUT/EC_MISMATCH/EC_ERROR` 等只要发生在仍需要的 prefix 内就原样返回。若所有候选已在更早位置停止， + 后续错误不再影响结果，这与 prefix 查询的最小必要读取语义一致。 +6. ordinary projection 用候选 host 位图而不是 per-key ordered map/set。Mamba 将 `(key, host)` state byte matrix 压为 + host-major 64-bit words，1M key/host 从约 1MiB 降到约 125KiB，并按 word 反向查找最后一个 state；read callback + 边界固定按 64 key 对齐，保证不同 worker 写入互不重叠。 +7. MetaIndexer 和 MetaLocalBackend 各自使用可重入保护的 bounded thread-local scratch，复用 compact offsets/value、 + key view、handle 和 LRU batch hash/shard 数组的 capacity。上限与后续批次同为 16384；每次 visitor 返回后立即清空 + `shared_ptr` values，只保留 capacity，避免把 scratch 变成旧 CacheLocation 的隐式对象缓存。 +8. 流水化之后重新校准观测语义：`meta_indexer.get_io_time_us` 统计并行 backend call interval 的墙钟并集，不再把 + visitor CPU 混入 I/O；`host_projection_time_us` 同样统计 projection callback interval 的墙钟并集，而不是把多个 + worker 时间相加。两者会重叠，均包含在 `indexer_get_time_us`/`prefix_match_time_us` 内，不能相加。 + +没有为这一优化扩大 item shared-lock 生命周期，也没有绕过 immutable `shared_ptr` 快照。把 raw +location 指针带出 item lock 虽可少一次 owner add-ref，但 ReportEvent 可在锁释放后替换 map entry，会形成真实 UAF； +因此该方向明确不采用。最终 profile 中 URI visibility/location-id parse 已降为低个位数占比，`__lll_lock_wait` 约 +0.15%，没有长时间 futex sleep；剩余主要是每 key 必需的 item rwlock、LRU lookup/release 和 immutable owner copy。 + +#### Release 性能与门禁 + +一次同机 `perf record -e cycles:u -F 499 -g --call-graph dwarf` 的最终纯内存运行得到： + +| 场景 | 4-worker p50 | 8-worker p50 | 16-worker p50 | +| --- | ---: | ---: | ---: | +| 1M metadata only | 92.19ms | 71.63ms | 63.38ms | +| 1M ordinary all-hit | 99.61ms | 71.38ms | 66.10ms | +| 第 1024 key host stop | 0.48ms | - | - | +| 第 1024 key metadata miss | 0.44ms | - | - | + +同一运行的 100K/500K ordinary all-hit p50 为 9.60/45.50ms。共享开发机另几轮 4-worker 1M all-hit 在 +约 104~123ms 波动，因此不能把单次 99.61ms 当成线上 SLA；但相对约 597ms 的全量 map/set 基线和约 583ms 的 +无效早停基线，量级收益稳定。默认 worker_count 仍保持 4：8-worker 对 isolated、CPU 充足的百万 key 查询明确有益， +但 5.3 的混合负载已证明盲目增加 worker 可能恶化 Get p99，上线应按同负载 A/B 配置而不是在代码中改默认值。 + +当前 Release HTTP 二进制、pure-local metadata 的真实 ReportEvent → GetHostCacheState benchmark 为：100/1K/5K/20K +串行 p50 0.49/0.87/2.49/8.30ms；20K 的 16-way p50/p99 为 15.04/27.45ms。每次响应均校验目标 host 的完整 prefix， +该数据包含 HTTP JSON parse/serialize，但仍只作当前机器回归门禁。 + +最终验证均不依赖 Redis： + +- serving/WRITING/DELETING/NEW/NOT_FOUND 全状态矩阵在 p2p=0 和 p2p>0 下结果一致；Manager 端到端验证 + StartWrite 在 Finish 前不可见； +- ordinary/Mamba 覆盖 first-window stop、第二并行 range timeout、较早 visitor stop 屏蔽无关后续 timeout、70 host + 多 presence word、75 个 required spec 跨 word，以及 40K key 跨 4096/16384 边界；极端 + `chunk_size=SIZE_MAX` 也会被安全收敛为单个 suffix range，不发生整数回绕； +- 上述并行/生命周期重点用例 20 轮重复通过；MetaSearcherTest、CacheManagerTest 全量通过； +- `bazel test --config=release --nocache_test_results //kv_cache_manager/...` 为 106 个测试通过，1 个 GPU-only 测试 + 按预期 skip； +- 当前 Release server 的 snapshot HTTP 功能集 37/37 全绿，覆盖 ReportEvent ADD/DELETE/SNAPSHOT、heartbeat/ + host-down、部分失败、并发 generation 与 GetHostCacheState 可见性。 + +后续停止线：pure-local、无 P2P 的查询已经没有证据支持继续删除锁或 URI 校验。下一项真正可能带来量级收益的是修改 +LRU ref/list 或 metadata item representation，但这会同时改变 eviction、ReportEvent writer 和对象生命周期，必须作为 +独立设计并建立 mixed ReportEvent/Get p99、capacity eviction、host-down/generation 与 fault-injection 门禁，不能继续 +作为本分支的低风险局部优化。 + +#### 最终混合负载复核与附带健壮性修复 + +在最终全量测试中，`EventReportBackendTest.CloseInterruptsLongLivenessWait` 曾低概率等待完整的 60 秒 tick。 +`Close()` 原先只更新 atomic predicate 后调用 condition-variable notify，没有用等待侧的 mutex 串行化 predicate 更新； +因此 waiter 在“检查 predicate”和“真正睡眠”之间存在丢唤醒窗口。现在关闭路径在同一 mutex 下清除 running flag，再 +notify/join，不改变运行期 heartbeat、ReportEvent 或查询锁范围。该用例 Release 连续重复 100 次通过，随后全量测试也 +不再出现长等待。 + +local compact read 还去掉了一个确定的逐 key 冗余：未配置 revisit histogram 时不再读取 +`last_access_time` atomic；访问时间仍按原语义更新，启用 histogram 时采样逻辑完全不变。另修正混合压测器读取已经 +废弃的 `prefix_match_blocks` 字段所造成的假失败，改为校验当前协议的 `local` 字段。 + +最终 Release、pure-local 真实 HTTP 复核： + +- snapshot ReportEvent/GetHostCacheState 功能集 37/37 通过；20K 单请求 ADD create/update 为 + 57.00/50.34ms； +- 最终二进制的 GetHostCacheState 100/1K/5K/20K 串行 p50 为 0.49/0.89/2.57/8.52ms，20K 的 + 16-way p50/p99 为 17.41/31.36ms； +- 10 秒混合负载以约 15 QPS × 10K blocks（约 150K blocks/s）持续 L2 ADD，同时执行 10 QPS × 10K-key Get + 和 heartbeat。151 次 ADD、101 次 Get、20 次 heartbeat 全部成功且逐事件抽样校验一致；ADD avg/p95/p99 为 + 63.41/102.64/112.04ms，Get avg/p95/p99/max 为 17.07/51.35/65.47/75.36ms； +- 最后两次 1M-key 内部基准中，4-worker metadata p50 为 91.01~91.39ms、all-hit 为 + 107.97~113.74ms；8/16-worker all-hit 分别在 86.88~93.43ms/83.88~93.49ms，首 1024 key + host stop/metadata miss 为 0.43~0.47ms/0.42~0.44ms。共享机波动仍然明显，因此保留可配置 worker 数和默认 4， + 不把单机数字声明为 SLA。 + +最终 `bazel test --config=release --nocache_test_results //kv_cache_manager/...` 为 106 个可执行测试全部通过，1 个 +GPU-only 测试按预期跳过。上述 mixed 结果也说明渐进 query 没有通过扩大 LRU/item 锁范围把 ReportEvent 写路径拖慢。 + +### 5.28 2026-08-10 恢复数据 URI 校验、渐进取消差分与最终门禁 + +本轮继续以 GetHostCacheState 的“长命中性能不能用放松校验换取”为停止线，对渐进 visitor、EventReport URI +可见性和 metadata mutation 生命周期做反例审查。审查发现一个真实的 fail-open：恢复或外部注入的 +`event_report://host:not-a-port/mem` 过去会通过 allocation-free visibility scanner，而完整 `StandardUri` +会拒绝它。若 reporter/generation 其余条件满足，这类损坏 metadata 会被错误计为命中。 + +最终实现同时保住恢复数据安全和 pure-local 热路径： + +1. 未知来源的 URI 在原字符串上严格检查 authority 里的 textual port，空端口、负数、`+`、非数字和 + `int64` 上溢均 fail-closed；`:0`、前导零、user-info 里的冒号以及 path/query value 中的冒号继续与 + `StandardUri` 保持兼容。该检查不构造 URI component、参数 map 或临时字符串； +2. `CacheLocation::validated_total_size_` 的含义收紧为“全部当前 spec URI 已严格验证”的非序列化证明及 size + aggregate。任意 spec mutator 都清除证明；Replace 仅在完整校验成功后设置，Delete 仅从已有证明推导，Merge + 只有在所有保留旧 spec 也验证通过时才恢复。反序列化/恢复值始终没有证明，必须重新检查； +3. 特别覆盖“恢复值含坏端口，随后 merge 一个不同名字的正常 spec”：坏 spec 仍被保留时不能错误恢复证明，查询 + 继续 fail-closed；只有把最后一个坏 spec 按名替换掉后，最终 vector 才重新获得证明。这样 query 只对可信的 + pure-local ReportEvent 值跳过重复 authority/port 扫描，scheme/query 位置和每次请求的 `s_version` generation + 仍逐 spec 检查； +4. ordinary 与 Mamba visitor 在一个已经读取的 callback range 内，一旦所有候选 host 的单调 prefix stop 都不晚于 + 当前 key，就立即停止剩余 location projection。它不取消仍可能决定更短 prefix 的早期 range，也不屏蔽仍在任一 + host 必需区间内的 backend error； +5. 新增 6000-key、7-host 的确定性随机模型，组合 medium、L1.5/L2、全部非 serving 状态、非法 URI、Eagle pop + 和 Mamba spec group；用独立的 materialized `map/set` 参考算法逐项比较 ordinary/Mamba 的 host、local、P2P + fetched-key 去重数和 total match，并覆盖 top-N 为 0/1/3/7。另用 33000 key、两个不同 stop 的 host 验证： + 所有候选 stop 之后的 speculative timeout 可以忽略，但仍处于较长 host prefix 内的同类 timeout 必须原样返回； +6. 全量门禁顺便暴露两个测试自身的不确定假设并已固定：MigrationManager 的 capacity partial-failure 用例不再假设 + 固定 key 在动态 hash seed 下必然属于不同 shard；两个直接清空 executor queue 的 stale-cache 用例会先停止 + reclaimer supervisor，避免销毁它正在等待的 `packaged_task` 后随机抛出 `broken promise`。两项均只修改测试， + 不改变生产锁或调度语义。 + +最终 Release、pure-local 1M-key 内部基准（同一进程 p50）为：4-worker metadata-only/all-hit +89.12/106.32ms，8-worker为 64.48/84.38ms；16-worker为 64.85/93.28ms，说明当前共享机继续增加线程已经没有 +稳定收益。100K/500K 的 all-hit 为 9.44/48.57ms；首窗口 host stop 和 metadata miss 分别为 0.46/0.44ms。 +在校验证明接入前的一轮同机运行中，4-worker 1M metadata-only/all-hit 为 120.68/166.79ms；机器本身有明显波动， +因此不能直接把两轮总耗时相减，但完整投影相对同轮 metadata 的增量从约 46ms 收敛到约 17ms，证明没有让恢复 +数据的端口校验重新成为长前缀主热点。以上仍是共享开发机回归数据，不承诺线上固定 SLA。 + +本轮最终门禁： + +- Snapshot URI、MetaSearcher、CacheManager、MigrationManager 四个定向 Release 目标全绿；随机差分/错误边界 + 连续 20 轮通过，ReportEvent/Get/host-down 并发生命周期用例在 Bazel 10-shard 配置下 300 个 run 全部通过； +- MigrationManager 完整测试进程连续 50 轮通过，两个 stale-cache 用例各 500 个 shard-run 通过； +- 使用当前工作树独立 Bazel output base 执行 + `bazel test --config=release --nocache_test_results --test_output=errors --jobs=8 //kv_cache_manager/...`： + 106 个可执行测试全部通过，1 个 GPU-only 测试按预期跳过。工作树迁移后复用旧绝对路径 output base 曾导致 7 个 + 用例在 0ms 内因 runfiles 缺失失败；独立重建后这些用例全部通过，未把基础设施假失败当作代码缺陷。 + +### 5.29 2026-08-11 P2P 有界归约 + +更新后的基线已经解决 `p2p_host_count=0` 的渐进读取，但 P2P 路径仍为每个 key 构造两份 +`map>`。本轮删除这套逐 key 对象图：host 在请求内映射为 dense id，spec group 编码为多 word +bitmask；普通 prefix 在线维护最终 top-N host 的 peer 交集，Mamba 通过最多三个有界 ordered pass 保留原来的 +“full group 独立选 prefix peer、state group 统一选 coverage peer”语义；没有选中实际 peer plan 时跳过最终 pass。 +非 local backend 仍只读取一次 compact batch，后续 pass 重放该 batch。 + +同一 Release/O2 二进制、4 query workers、pure-local metadata、1M key、在第 1024 key 制造一个可由 Vineyard +peer 补齐的 local gap，对更新前后的 `mu-main` 使用完全相同的 benchmark： + +| 场景 | 更新前 p50 | 本轮 p50 | 变化 | +| --- | ---: | ---: | ---: | +| ordinary P2P | 632.07ms | 162.81ms | -74.2% | +| Mamba P2P | 635.87ms | 370.01ms | -41.8% | +| 进程峰值 RSS | 1,340,012KiB | 586,120KiB | -56.3%（约 -736MiB） | + +数字来自共享开发机，只用于同机回归，不是线上 SLA。门禁覆盖动态 top-N 淘汰、重复 block key 的 fetched count +去重、多个 full group 分别选 peer 后合并、非 local backend 单次读取、跨 64 个 spec 的 bitmask、ordered visitor +窗口/stop、空 spec group，以及 P2P 前缀内硬错误传播。Mamba 的 pure-local 路径最多会用三遍有界扫描换取不随 +`key * host * spec` 增长的中间对象图；后续 pass 会重复命中 local LRU，但最后的 LRU 顺序/访问时间与单遍扫描一致。 diff --git a/docs/design/report_event_snapshot_uri_version.md b/docs/design/report_event_snapshot_uri_version.md index 3f0698e07..54890cdd3 100644 --- a/docs/design/report_event_snapshot_uri_version.md +++ b/docs/design/report_event_snapshot_uri_version.md @@ -33,8 +33,10 @@ ADD 或 DELETE 直接建立一个进程内 version，然后执行写入。不要 进程内发生 HOST_DOWN 或 grace cleanup 后会留下 tombstone，必须由 REGISTER 明确清除， 防止迟到数据事件复活已下线节点。KVCM 重启会清空该进程内 tombstone。 -每次成功 REGISTER 同时开启一个新的 lifecycle generation：重复请求会合并 medium 并返回 -成功，但会取消更早 lifecycle 中尚未进入最终 metadata 写入阶段的 mutation/cleanup。因此 +每次至少包含一个合法 REGISTER 的成功请求至多开启一个新的 lifecycle generation：同一请求 +中的多个合法 REGISTER 会先分别校验，再合并 medium，只执行一次实际注册，并给每个合法 item +返回相同的注册结果；非法 REGISTER 只影响自身 item。跨请求的重复 REGISTER 会再次开启新 +generation，并取消更早 lifecycle 中尚未进入最终 metadata 写入阶段的 mutation/cleanup。因此 REGISTER 是启动/重建边界，不是 HEARTBEAT 的替代品；调用方不应高频发送或与普通数据请求 无序并发。 @@ -374,12 +376,16 @@ snapshot replace + commit/abort - commit 后等待 delta 使用新 generation； - abort 后等待 delta 使用旧 generation；若此前没有旧 generation，则创建一个新 generation； - Close、Unregister、HOST_DOWN 唤醒 waiter；等待 active delta 另有可配置超时，不会无界阻塞； -- metadata read-modify-write 在最终写阶段持有 lifecycle generation lease；旧请求不能在 - HOST_DOWN、重新 REGISTER 和新 snapshot 后恢复写入； -- mutation 在已经持有 metadata 锁时只做非阻塞的 per-reporter lifecycle lease 获取；若同一 - reporter 的 HOST_DOWN/REGISTER lifecycle writer 已经开始等待，则旧 mutation 立即失败， - 避免与 cleanup 的 `lifecycle -> metadata` 顺序形成锁序反转；不同 reporter 使用独立 - fence，不会因其他 host 的 HEARTBEAT/REGISTER 产生假失败； +- metadata read-modify-write 在每个 RMW 阶段完成读、准备进入最终写时，只获取一次 + lifecycle generation lease，并持有到该阶段结束；同一阶段不再按 key 重复查 fence 或分配 + shared lock； +- lease 不能提前到 metadata read 之前获取。这样旧请求阻塞在 metadata I/O 时，HOST_DOWN/ + REGISTER 仍可先取得 lifecycle writer；旧请求恢复后使用非阻塞获取立即失败，不能跨越新 + lifecycle 写入；BatchMerge 的 block-create 与 targeted-location 两个 RMW 阶段之间同样释放并 + 重新获取 lease； +- mutation 在已经持有 metadata 锁时只做上述非阻塞的 per-reporter lifecycle lease 获取， + 避免与 cleanup 的 `lifecycle -> metadata` 顺序形成锁序反转；不同 reporter 使用独立 fence， + 不会因其他 host 的 HEARTBEAT/REGISTER 产生假失败； - liveness unregister 的 generation 比较与节点删除在同一把锁内完成； - 显式 HOST_DOWN 的 generation 捕获与节点删除同样在同一把锁内完成，Heartbeat/REGISTER 只能在线性化的 HOST_DOWN 之前或之后生效，不能在中间恢复后又被旧请求删除； @@ -456,6 +462,8 @@ reporter -> location 反向索引，而不是继续提高全量频率。 - snapshot commit 后 delta 刷新 mixed-generation location 时，旧 cleanup 不删除新写； - snapshot cleanup 只删除 metadata，不调用外部 URI backend； - 成功 snapshot 最终清理旧数据，失败 snapshot 不触发专用清理； +- backend requested-spec 使用 any-of 语义，在 peer 选择前检查 location 的所有 specs；Prefix 在 + spec gap 停止，Coverage 跳过 gap，最终响应投影保持 `spec_size == location_specs.size()`； - snapshot 限流、storage type 隔离和 liveness 竞态。 ### 12.2 集成测试 @@ -473,10 +481,32 @@ reporter -> location 反向索引，而不是继续提高全量频率。 - snapshot partial failure、完整重试； - snapshot commit 后立即到达的 delta 在异步 cleanup 后仍可查询； - reporter host 或 medium 含 `#` 时 fail closed 且无写入副作用； +- Bazel `--runs_per_test` 并发重复时，每个 test action 的可变 worker 目录必须位于其私有 + `TEST_TMPDIR`，不得在共享 runfiles/source tree 中清理或复制；至少用 20 路重复验证目录隔离； +- heartbeat/grace 计时用例必须使用独立的短超时 storage/instance group，不能缩短功能与容量 + 用例共享 storage 的生命周期窗口，否则异步 cleanup 验证会被 liveness cleanup 交叉干扰； - ASAN/TSAN 或等价并发检测（仅记录实际执行结果，不能由普通 CI 结果推断）。 Snapshot 与重启 HTTP 测试 target 带 `manual` 标签，不属于默认 GitHub CI；需要显式执行并单独记录结果。 +### 12.3 Vineyard 跨仓集成镜像 + +Vineyard 的 ReportEvent 集成不能继续使用不含 `event_report` protobuf 的旧 KVCM 镜像；否则 +`addStorage` 会返回 `missing or invalid fields: {StorageConfig: {storage_spec}}`，后续用例全部是 +同一前置失败的连锁结果。 + +需要验证 KVCM 分支时，手动运行 `.github/workflows/build-dev-image.yml` 并选择 +`flavor=integration`。该 flavor 只构建 CI 所需的 `linux/amd64` 生产镜像，并发布唯一的 +`integration--` tag；把该精确 tag 写入 Vineyard 的 +`.aoneci/v6d-pytest-integration.yaml`，禁止使用 `latest`。合并前至少确认 Vineyard 的 +group-aware 三节点用例、green、KVCM fault 和 PACE fault 都实际执行，且从失败制品中的 +pytest 汇总判断结果，不能根据 Aone 对自由脚本显示的 `NOT_RUN` 状态推断。 + +该 workflow 在 dev 容器内完成 Bazel 构建后，也必须在容器仍存活时把 server tar 复制到 +Docker build context，并把文件 owner 改回 runner 用户。`bazel-bin` 可能指向容器内的 +`/root/.cache/bazel`；容器退出后再从宿主 runner 读取该 symlink 会得到 `Permission denied` +或断链，不能据此误判为编译失败。 + ## 13. 接受的取舍 本方案优先 cache availability，不提供原子 snapshot 查询视图： diff --git a/docs/design/service_discovery_framework.md b/docs/design/service_discovery_framework.md index 7165cf30f..97f6a5e04 100644 --- a/docs/design/service_discovery_framework.md +++ b/docs/design/service_discovery_framework.md @@ -504,6 +504,18 @@ extra config 中透传这些 Manager Client 参数，包括 `request_timeout_seconds`；未配置时普通 API 请求默认超时为 `1.0` 秒。 Leader 查询使用独立的 5 秒超时，不受该参数影响。 +Manager Client 的构造和关闭是资源事务：服务发现 factory、初始 endpoint 解析、discovery +type 读取以及 route-refresh 线程构造/启动中的任意一步失败时，已创建的 HTTP session 与服务 +发现 client 都会回滚；`close()` 等待刷新线程最多 5 秒。若线程仍在 Leader/Spectrum +请求中，HTTP session 可以立即关闭，但服务发现 client 必须由刷新线程退出时延迟关闭，不能与 +正在进行的 endpoint refresh 并发释放。重复 `close()` 对两类资源都只执行一次。 + +响应异常按以下契约分类，供上层熔断器稳定判断：连接、超时和 JSON 解码沿用 +`requests.RequestException`；HTTP 非 200 抛 `KvCacheManagerHTTPError`；HTTP 200 但缺少合法 +`header.status.code` 抛 `KvCacheManagerProtocolError`。后两者同时兼容旧的 `AssertionError` +捕获方式。Manager 明确返回非 `OK` 的业务状态仍抛原有 `AssertionError`。即使调用方传入 +`check_response=False`，也只跳过业务状态检查，不能跳过 HTTP 与公共 envelope 校验。 + 开源构建可直接使用 `static://`；需要内部实现的 scheme 仍由对应的 `stub_source` 实现提供，通用 Client 不依赖具体服务发现类型。 diff --git a/docs/prometheus-en_US.md b/docs/prometheus-en_US.md index 32ab4109f..47418a890 100644 --- a/docs/prometheus-en_US.md +++ b/docs/prometheus-en_US.md @@ -131,7 +131,11 @@ every `kvcm.metrics.report_interval_ms`, default 20s). | `manager.prefix_match_len` | gauge | Prefix match length | | `manager.get_cache_location_query_block_counter` | counter | Total blocks queried via GetCacheLocation (cumulative) | | `manager.get_cache_location_hit_block_counter` | counter | Total blocks hit via GetCacheLocation (cumulative) | -| `manager.prefix_match_time_us` | gauge | Prefix match latency (us) | +| `manager.prefix_match_time_us` | gauge | Outer total latency for GetHostCacheState-style prefix matching (us) | +| `meta_searcher.indexer_get_time_us` | gauge | Wall time spent reading metadata through MetaIndexer (us) | +| `meta_indexer.get_io_time_us` | gauge | Wall-time union of metadata-backend call intervals; local mode includes LRU/locks/copies, excludes parallel projection, and does not imply Redis I/O | +| `meta_searcher.host_projection_time_us` | gauge | Wall-time union of GetHostCacheState visibility and host/spec projection callback intervals (us) | +| `meta_searcher.host_prefix_reduce_time_us` | gauge | GetHostCacheState normal/Mamba host-prefix reduction time (us) | | `meta_indexer.search_cache_hit_ratio` | gauge | Search cache hit ratio | | `data_storage.create_keys_counter` | counter | Total created keys | @@ -151,6 +155,13 @@ every `kvcm.metrics.report_interval_ms`, default 20s). The full list depends on the active `MetricsReporter` type. The `kmonitor` reporter populates the most complete set of metrics. +The GetHostCacheState phase metrics above are nested: +`meta_indexer.get_io_time_us` is inside `meta_searcher.indexer_get_time_us`, and +the indexer/projection/reduction phases are inside `manager.prefix_match_time_us`. +Progressive local queries pipeline backend reads with projection, so these +intervals can overlap and must not be added together. `get_io_time_us` is a +historical name; with `storage_type=local` it contains no Redis network operation. + ## Mapping to KMonitor Metrics KVCacheManager simultaneously exports metrics via KMonitor and via diff --git a/docs/prometheus-zh_CN.md b/docs/prometheus-zh_CN.md index 465638b33..77957f386 100644 --- a/docs/prometheus-zh_CN.md +++ b/docs/prometheus-zh_CN.md @@ -128,7 +128,11 @@ kvcm_data_storage_storage_usage_ratio{type="nfs",unique_name="store_02"} 0.3 | `manager.prefix_match_len` | gauge | 前缀匹配长度 | | `manager.get_cache_location_query_block_counter` | counter | GetCacheLocation 查询的 Block 总数（累计） | | `manager.get_cache_location_hit_block_counter` | counter | GetCacheLocation 命中的 Block 总数（累计） | -| `manager.prefix_match_time_us` | gauge | 前缀匹配延迟（微秒） | +| `manager.prefix_match_time_us` | gauge | GetHostCacheState 等前缀匹配的外层总延迟（微秒） | +| `meta_searcher.indexer_get_time_us` | gauge | MetaSearcher 调用 MetaIndexer 读取 metadata 的墙钟时间（微秒） | +| `meta_indexer.get_io_time_us` | gauge | metadata backend 调用区间的墙钟并集；local 模式包含 LRU/锁/复制，不包含并行 projection，也不表示 Redis I/O | +| `meta_searcher.host_projection_time_us` | gauge | GetHostCacheState location 可见性及 host/spec projection 回调区间的墙钟并集（微秒） | +| `meta_searcher.host_prefix_reduce_time_us` | gauge | GetHostCacheState 普通/Mamba host 前缀归约时间（微秒） | | `meta_indexer.search_cache_hit_ratio` | gauge | 搜索缓存命中率 | | `data_storage.create_keys_counter` | counter | 已创建 key 总数 | @@ -148,6 +152,11 @@ kvcm_data_storage_storage_usage_ratio{type="nfs",unique_name="store_02"} 0.3 完整指标列表取决于当前使用的 `MetricsReporter` 类型。`kmonitor` 类型的 reporter 会填充最完整的指标集。 +上述 GetHostCacheState 分段指标是嵌套关系：`meta_indexer.get_io_time_us` 位于 +`meta_searcher.indexer_get_time_us` 内，后者与 projection/reduce 又位于 +`manager.prefix_match_time_us` 内。渐进 local 查询会流水化 backend read 与 projection，两者可能重叠，排障时 +不能把它们相加。`get_io_time_us` 是历史命名；当实例使用 `storage_type=local` 时没有 Redis 网络调用。 + ## 与 KMonitor 指标对照 KVCacheManager 同时通过 KMonitor 与 Prometheus `/metrics` 端点导出 diff --git a/integration_test/meta_service/http_interface_test.py b/integration_test/meta_service/http_interface_test.py index e0548c931..a585bb8a8 100644 --- a/integration_test/meta_service/http_interface_test.py +++ b/integration_test/meta_service/http_interface_test.py @@ -5,14 +5,15 @@ class MetaServiceHttpClient(cases.MetaServiceClientBase): """HTTP client for MetaService API endpoints""" - def __init__(self, base_url): + def __init__(self, base_url, admin_url=None): self.base_url = base_url + self.admin_url = admin_url or base_url self.session = requests.Session() self.headers = {'Accept': 'application/json', 'Content-Type': 'application/json'} - def _make_request(self, method, endpoint, data=None): + def _make_request(self, method, endpoint, data=None, base_url=None): """Helper method to make HTTP requests to the service""" - url = self.base_url + endpoint + url = (base_url or self.base_url) + endpoint if method == 'POST': response = self.session.post(url, json=data, headers=self.headers) @@ -23,9 +24,9 @@ def _make_request(self, method, endpoint, data=None): return response - def _make_api_request(self, endpoint, data=None, check_response=True): + def _make_api_request(self, endpoint, data=None, check_response=True, base_url=None): """Helper method to make POST requests to API endpoints and optionally validate response""" - response = self._make_request('POST', endpoint, data) + response = self._make_request('POST', endpoint, data, base_url=base_url) if response.status_code != 200: raise AssertionError(f"Request to {endpoint} failed with status code {response.status_code}") try: @@ -72,6 +73,30 @@ def get_cluster_info(self, data, check_response=True): """Get cluster info (leader discovery)""" return self._make_api_request('/api/getClusterInfo', data, check_response) + def report_event(self, data, check_response=True): + """Report EventReport node/block lifecycle events.""" + return self._make_api_request('/api/reportEvent', data, check_response) + + def get_cache_locations_by_backend(self, data, check_response=True): + """Query locations with explicit backend selection.""" + return self._make_api_request('/api/getCacheLocationsByBackend', data, check_response) + + def _make_admin_api_request(self, endpoint, data=None, check_response=True): + return self._make_api_request( + endpoint, + data, + check_response, + base_url=self.admin_url, + ) + + def add_storage(self, data, check_response=True): + """Register storage through the admin HTTP endpoint.""" + return self._make_admin_api_request('/api/addStorage', data, check_response) + + def create_instance_group(self, data, check_response=True): + """Create an instance group through the admin HTTP endpoint.""" + return self._make_admin_api_request('/api/createInstanceGroup', data, check_response) + def close(self): """Close the HTTP session""" self.session.close() @@ -81,9 +106,194 @@ class MetaServiceHttpTest(cases.MetaServiceTestBase): """HTTP version of the MetaService tests""" def _get_manager_client(self): - self._http_port = self.worker_manager.get_worker(0).env.http_port + worker_env = self.worker_manager.get_worker(0).env + self._http_port = worker_env.http_port self._http_url = "http://localhost:%d" % self._http_port - return MetaServiceHttpClient(self._http_url) + self._admin_http_url = "http://localhost:%d" % worker_env.admin_http_port + return MetaServiceHttpClient(self._http_url, self._admin_http_url) + + @staticmethod + def _event_report_storage(storage_name): + return { + "global_unique_name": storage_name, + "storage_type": "ST_EVENT_REPORT_L2", + "event_report": { + "heartbeat_timeout_ms": 30000, + "cleanup_grace_ms": 30000, + "liveness_check_interval_ms": 1000, + }, + "check_storage_available_when_open": False, + } + + @staticmethod + def _event_report_instance_group(group_name, storage_name): + return { + "name": group_name, + "storage_candidates": ["nfs_01"], + "global_quota_group_name": "default_quota_group", + "max_instance_count": 10, + "quota": { + "capacity": 10737418240, + "quota_config": [{"storage_type": 4, "capacity": 10737418240}], + }, + "cache_config": { + "reclaim_strategy": { + "storage_unique_name": "nfs_01", + "reclaim_policy": 1, + "trigger_strategy": {"used_size": 1073741824, "used_percentage": 0.8}, + "trigger_period_seconds": 60, + "reclaim_step_size": 1073741824, + "reclaim_step_percentage": 10, + }, + "data_storage_strategy": 2, + "meta_indexer_config": { + "max_key_count": 10000, + "mutex_shard_num": 16, + "batch_key_size": 16, + "meta_storage_backend_config": {"storage_type": "local", "storage_uri": ""}, + "meta_cache_policy_config": {"type": "LRU", "capacity": 10000}, + }, + }, + "event_report_storage_candidates": [storage_name], + "version": 1, + } + + @staticmethod + def _report_events(instance_id, host, events, trace_id): + return { + "trace_id": trace_id, + "instance_id": instance_id, + "host_ip_port": host, + "storage_type": "ST_EVENT_REPORT_L2", + "events": events, + } + + @staticmethod + def _node_and_block_events(host, spec_name, block_keys): + events = [{ + "event_type": "EVENT_NODE_REGISTER", + "node_register": {"mediums": ["mem"]}, + }] + events.extend({ + "event_type": "EVENT_BLOCK_ADD", + "block_add": { + "block_key": str(block_key), + "medium": "mem", + "specs": [{ + "name": spec_name, + "uri": f"vineyard://{host}/mem", + }], + }, + } for block_key in block_keys) + return events + + def test_event_report_requested_spec_filters_before_peer_selection(self): + """Exercise ReportEvent -> HTTP query with the adversarial peer layout. + + The full-only peer has better raw coverage. Both cross-key strategies + must nevertheless select the linear peer when linear_1 is requested. + """ + storage_name = "event_report_l2_http_spec_filter" + group_name = "event_report_http_spec_filter_group" + instance_id = "event_report_http_spec_filter_instance" + full_host = "10.10.0.1:9600" + linear_host = "10.10.0.2:9600" + block_keys = [82000, 82001] + + self._client.add_storage({ + "trace_id": "event_report_add_storage", + "storage": self._event_report_storage(storage_name), + }) + self._client.create_instance_group({ + "trace_id": "event_report_create_group", + "instance_group": self._event_report_instance_group(group_name, storage_name), + }) + self._client.register_instance({ + "trace_id": "event_report_register_instance", + "instance_group": group_name, + "instance_id": instance_id, + "block_size": 128, + "model_deployment": self._get_test_model_deployment(), + "location_spec_infos": [ + {"name": "full_0", "size": 1024}, + {"name": "linear_1", "size": 1024}, + ], + "location_spec_groups": [ + {"name": "full_0", "spec_names": ["full_0"]}, + {"name": "linear_1", "spec_names": ["linear_1"]}, + ], + }) + self._client.report_event(self._report_events( + instance_id, + full_host, + self._node_and_block_events(full_host, "full_0", block_keys), + "event_report_full_peer", + )) + self._client.report_event(self._report_events( + instance_id, + linear_host, + self._node_and_block_events(linear_host, "linear_1", block_keys[:1]), + "event_report_linear_peer", + )) + + for strategy in ("LSS_V6D_PREFIX", "LSS_V6D_COVERAGE"): + response = self._client.get_cache_locations_by_backend({ + "trace_id": f"event_report_query_{strategy}", + "instance_id": instance_id, + "query_type": "QT_BATCH_GET", + "block_keys": block_keys, + "block_mask": {"offset": 0}, + "location_spec_names": ["linear_1"] * len(block_keys), + "backend_selectors": [{ + "backend_type": "ST_EVENT_REPORT_L2", + "strategy": strategy, + }], + }) + key_locations = response.get("key_locations", []) + self.assertEqual(2, len(key_locations), response) + first_locations = key_locations[0].get("locations", []) + self.assertEqual(1, len(first_locations), response) + first_specs = first_locations[0].get("location_specs", []) + self.assertEqual(["linear_1"], [spec.get("name") for spec in first_specs], response) + self.assertIn(linear_host, first_specs[0].get("uri", ""), response) + self.assertEqual([], key_locations[1].get("locations", []), response) + + unknown_response = self._client.get_cache_locations_by_backend({ + "trace_id": "event_report_query_unknown_spec", + "instance_id": instance_id, + "query_type": "QT_BATCH_GET", + "block_keys": block_keys, + "block_mask": {"offset": 0}, + "location_spec_names": ["unknown_spec"] * len(block_keys), + "backend_selectors": [{ + "backend_type": "ST_EVENT_REPORT_L2", + "strategy": "LSS_V6D_PREFIX", + }], + }) + self.assertEqual( + [[], []], + [item.get("locations", []) for item in unknown_response.get("key_locations", [])], + unknown_response, + ) + + # Protobuf enums are open on the wire. 263 would truncate to the + # internal uint8 value 7 (L1P5) if the service used a direct cast. + invalid_selector = self._client.get_cache_locations_by_backend({ + "trace_id": "event_report_query_unknown_backend", + "instance_id": instance_id, + "query_type": "QT_BATCH_GET", + "block_keys": block_keys, + "block_mask": {"offset": 0}, + "backend_selectors": [{ + "backend_type": 263, + "strategy": "LSS_WEIGHTED_RANDOM", + }], + }, check_response=False) + self.assertEqual( + "INVALID_ARGUMENT", + invalid_selector.get("header", {}).get("status", {}).get("code"), + invalid_selector, + ) if __name__ == "__main__": diff --git a/integration_test/meta_service/test_report_event.py b/integration_test/meta_service/test_report_event.py index 2e4f449cc..e3cd54181 100644 --- a/integration_test/meta_service/test_report_event.py +++ b/integration_test/meta_service/test_report_event.py @@ -352,6 +352,14 @@ def _ensure_instance_registered(cls): }, "location_spec_infos": [ {"name": "tp0", "size": 1024}, + {"name": "mem_spec", "size": 1024}, + {"name": "disk_spec", "size": 1024}, + {"name": "spec_4096", "size": 1024}, + {"name": "spec_8192", "size": 1024}, + {"name": "keep_spec", "size": 1024}, + {"name": "drop_spec", "size": 1024}, + {"name": "l1p5_spec", "size": 1024}, + {"name": "l2_spec", "size": 1024}, ], }) except Exception as e: @@ -1014,9 +1022,13 @@ def test_18_get_host_cache_state_dual_type_prefix_match(self): "10.0.0.3:8080": 1, } actual = { - h["host_ip_port"]: int(h["prefix_match_blocks"]) + h["host_ip_port"]: int(h["local"]) for h in resp.get("hosts", []) } + self.assertNotIn("p2p_1_hit_count", resp) + for host_match in resp.get("hosts", []): + self.assertIn("p2p_1_fetch", host_match) + self.assertIn("p2p_1_total_match", host_match) for host, prefix in expected.items(): self.assertIn(host, actual, f"host {host} not found in response") self.assertEqual(actual[host], prefix, f"host {host}: expected prefix={prefix}, got {actual[host]}") diff --git a/integration_test/meta_service/test_report_event_restart.py b/integration_test/meta_service/test_report_event_restart.py index ab9d98826..b5b7039e4 100644 --- a/integration_test/meta_service/test_report_event_restart.py +++ b/integration_test/meta_service/test_report_event_restart.py @@ -138,7 +138,11 @@ def verify(args, client, checks): checks.assertEqual(accepted["header"]["status"]["code"], "OK") active_version = accepted["committed_snapshot_version"] checks.assertEqual(len(active_version), 32) - checks.assertFalse(accepted.get("snapshot_required")) + # A generation-creating delta is accepted immediately, but its own + # response keeps snapshot_required=true so a snapshot-capable caller sees + # that the request arrived without a current-process generation. The next + # delta reuses the generation and clears the hint. + checks.assertTrue(accepted.get("snapshot_required")) delta_specs = report_event._wait_for_block_spec_names( client, args.instance_id, @@ -160,6 +164,26 @@ def verify(args, client, checks): "restart_verify_delta_keeps_untouched_historical_cache", ) + followup = client.report_event( + report_event._make_request( + args.instance_id, + HOST, + [report_event._ev_block_add( + BLOCK_KEY + 2, + "gpu", + report_event._make_single_spec( + "gpu_0", + report_event._build_event_report_uri(HOST, "gpu"), + ), + )], + trace_id="restart_verify_followup_delta", + ) + ) + checks.assertEqual( + followup.get("committed_snapshot_version"), active_version + ) + checks.assertFalse(followup.get("snapshot_required")) + after_uri = report_event._build_event_report_uri( HOST, "mem", {"source": "after_restart"} ) diff --git a/integration_test/meta_service/test_report_event_snapshot.py b/integration_test/meta_service/test_report_event_snapshot.py index aeae0aabf..f59f839d2 100644 --- a/integration_test/meta_service/test_report_event_snapshot.py +++ b/integration_test/meta_service/test_report_event_snapshot.py @@ -33,8 +33,6 @@ INSTANCE_ID = "event_report_cluster_0" SKIP_BENCH = False ONLY_BENCH = False -# Requires small heartbeat_timeout_ms/cleanup_grace_ms in addStorage spec. -ENABLE_LIVENESS_TIMING_TESTS = False HEARTBEAT_TIMEOUT_MS = 1000 CLEANUP_GRACE_MS = 2000 SNAPSHOT_MIN_INTERVAL_MS = 1000 @@ -82,6 +80,16 @@ def add_storage(self, data): raise AssertionError(f"addStorage failed: {json.dumps(body)}") return body + def list_storage(self, data): + url = f"{self.admin_url}/api/listStorage" + resp = self.session.post(url, json=data) + resp.raise_for_status() + body = resp.json() + code = body.get("header", {}).get("status", {}).get("code") + if code != "OK": + raise AssertionError(f"listStorage failed: {json.dumps(body)}") + return body + def create_instance_group(self, data): url = f"{self.admin_url}/api/createInstanceGroup" resp = self.session.post(url, json=data) @@ -391,32 +399,60 @@ def setUpClass(cls): @classmethod def _ensure_event_report_storage_registered(cls): - try: - cls.client.add_storage({ - "trace_id": "setup_storage", - "storage": { - "global_unique_name": cls.EVENT_REPORT_STORAGE_NAME, - "storage_type": "ST_EVENT_REPORT_L2", - "event_report": { - "heartbeat_timeout_ms": ( - HEARTBEAT_TIMEOUT_MS - if ENABLE_LIVENESS_TIMING_TESTS else 30000 - ), - "cleanup_grace_ms": ( - CLEANUP_GRACE_MS - if ENABLE_LIVENESS_TIMING_TESTS else 300000 - ), - "liveness_check_interval_ms": ( - 100 if ENABLE_LIVENESS_TIMING_TESTS else 5000 - ), - "snapshot_min_interval_ms": SNAPSHOT_MIN_INTERVAL_MS, - }, - "check_storage_available_when_open": False, + cls._ensure_storage_registered( + "setup_storage", + { + "global_unique_name": cls.EVENT_REPORT_STORAGE_NAME, + "storage_type": "ST_EVENT_REPORT_L2", + "event_report": { + # Functional and capacity cases share this backend and + # intentionally do not turn data events into implicit + # heartbeats. Keep their lifecycle window independent + # from the fast timing fixture below; otherwise a cleanup + # test can pass and then lose its reporter while observing + # asynchronous snapshot cleanup. + "heartbeat_timeout_ms": 30000, + "cleanup_grace_ms": 300000, + "liveness_check_interval_ms": 5000, + "snapshot_min_interval_ms": SNAPSHOT_MIN_INTERVAL_MS, }, + "check_storage_available_when_open": False, + }, + ) + + @classmethod + def _ensure_storage_registered(cls, trace_id, expected_storage): + storage_name = expected_storage["global_unique_name"] + listed = cls.client.list_storage({"trace_id": f"{trace_id}_list"}) + existing = next( + ( + storage + for storage in listed.get("storage", []) + if storage.get("global_unique_name") == storage_name + ), + None, + ) + if existing is None: + cls.client.add_storage({ + "trace_id": trace_id, + "storage": expected_storage, }) - print(f"[SETUP] Event report storage '{cls.EVENT_REPORT_STORAGE_NAME}' registered") - except Exception as e: - print(f"[WARN] addStorage failed (may already exist): {e}") + print(f"[SETUP] Event report storage '{storage_name}' registered") + return + + if existing.get("storage_type") != expected_storage.get("storage_type"): + raise AssertionError( + f"Storage {storage_name!r} has unexpected type: {existing}" + ) + actual_spec = existing.get("event_report", {}) + for name, expected_value in expected_storage.get("event_report", {}).items(): + actual_value = actual_spec.get(name) + if str(actual_value) != str(expected_value): + raise AssertionError( + f"Storage {storage_name!r} has {name}={actual_value!r}; " + f"expected {expected_value!r}" + ) + print(f"[SETUP] Event report storage '{storage_name}' reused") @classmethod def _ensure_instance_group_created(cls): @@ -516,9 +552,9 @@ def _ensure_liveness_fixture(cls): Keeping the small timeout on a dedicated instance group prevents the long functional/benchmark cases from timing out their shared reporters. """ - cls.client.add_storage({ - "trace_id": "setup_liveness_storage", - "storage": { + cls._ensure_storage_registered( + "setup_liveness_storage", + { "global_unique_name": cls.LIVENESS_STORAGE_NAME, "storage_type": "ST_EVENT_REPORT_L2", "event_report": { @@ -529,7 +565,7 @@ def _ensure_liveness_fixture(cls): }, "check_storage_available_when_open": False, }, - }) + ) existing = cls.client.get_instance_group({ "trace_id": "setup_get_liveness_ig", @@ -979,7 +1015,7 @@ def test_11_mixed_batch(self): "block_cache_keys": [block_key], }) matches = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in host_state.get("hosts", []) } self.assertEqual(matches.get(host), 1) @@ -1131,7 +1167,7 @@ def test_16a_heartbeat_timeout_then_recovery(self): "block_cache_keys": [block_key], }) target_prefixes = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in target_host_state.get("hosts", []) } self.assertEqual(target_prefixes.get(host), 1) @@ -1248,7 +1284,7 @@ def test_16a_heartbeat_timeout_then_recovery(self): "block_cache_keys": [block_key], }) hidden_prefixes = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in hidden_host_state.get("hosts", []) } self.assertEqual( @@ -1352,7 +1388,7 @@ def test_16a_heartbeat_timeout_then_recovery(self): "block_cache_keys": [block_key], }) recovered_prefixes = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in recovered_host_state.get("hosts", []) } self.assertEqual(recovered_prefixes.get(host), 1) @@ -1635,14 +1671,14 @@ def test_16_get_host_cache_state(self): "block_cache_keys": [10000, 10001, 10002, 10003, 10004], }) - # 3. Verify prefix_match_blocks per host + # 3. Verify local per host expected = { "10.0.0.1:8080": 2, "10.0.0.2:8080": 4, "10.0.0.3:8080": 1, } actual = { - h["host_ip_port"]: int(h["prefix_match_blocks"]) + h["host_ip_port"]: int(h["local"]) for h in resp.get("hosts", []) } for host, prefix in expected.items(): @@ -1652,6 +1688,28 @@ def test_16_get_host_cache_state(self): f"host {host}: expected prefix={prefix}, got {actual[host]}", ) + # Cross the default parallel threshold and verify that chunked local + # reads plus parallel projection preserve prefix and output ordering. + large_keys = [10000, 10001, 10002, 10003] * 96 + large_resp = self.client.get_host_cache_state({ + "trace_id": "t16_large_query", + "instance_id": instance_id, + "query_type": "QT_PREFIX_MATCH", + "block_cache_keys": large_keys, + }) + large_hosts = large_resp.get("hosts", []) + self.assertEqual( + [item["host_ip_port"] for item in large_hosts], + ["10.0.0.1:8080", "10.0.0.2:8080", "10.0.0.3:8080"], + ) + large_actual = { + item["host_ip_port"]: int(item["local"]) + for item in large_hosts + } + self.assertEqual(large_actual["10.0.0.1:8080"], 2) + self.assertEqual(large_actual["10.0.0.2:8080"], len(large_keys)) + self.assertEqual(large_actual["10.0.0.3:8080"], 1) + # 17. Snapshot is a complete reconciliation barrier and supports empty clear. def test_17_snapshot_reconciliation_contract(self): host = "192.168.1.240:8080" @@ -3173,7 +3231,7 @@ def test_26_unregistered_errors_remain_distinct_from_delta_only_mode(self): "block_cache_keys": [25_000_250], }) matches = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in host_state.get("hosts", []) } self.assertEqual(matches.get(host), 1) @@ -3364,7 +3422,7 @@ def send_first_delta(writer): ], }) matches = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in host_state.get("hosts", []) } self.assertEqual(matches.get(host), writer_count) @@ -3478,7 +3536,7 @@ def test_31_first_delete_without_snapshot_creates_reusable_generation(self): "block_cache_keys": [block_key], }) empty_matches = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in empty_host_state.get("hosts", []) } self.assertNotIn(host, empty_matches) @@ -3524,7 +3582,7 @@ def test_31_first_delete_without_snapshot_creates_reusable_generation(self): "block_cache_keys": [block_key], }) visible_matches = { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in visible_host_state.get("hosts", []) } self.assertEqual(visible_matches.get(host), 1) @@ -3912,6 +3970,132 @@ def assert_single_invalid(body): baseline_version, ) + def test_34_large_partial_batch_preserves_item_alignment_and_retry(self): + host = f"large-partial-{time.time_ns()}:8080" + base_key = 34_000_000_000 + time.time_ns() % 1_000_000_000 + event_count = 512 + invalid_indices = set(range(0, event_count, 17)) + + self.client.report_event( + _make_request( + self.instance_id, + host, + [_ev_node_register(["mem"])], + trace_id="t34_register", + ) + ) + + events = [] + raw_uris = {} + for index in range(event_count): + block_key = base_key + index + raw_uri = _build_event_report_uri( + host, + "mem", + {"size": str(index + 1), "source": f"large_{index}"}, + ) + raw_uris[index] = raw_uri + events.append( + _ev_block_add( + block_key, + "mem", + _make_single_spec( + "tp0", + "not-a-uri" if index in invalid_indices else raw_uri, + ), + ) + ) + + payload = _make_request( + self.instance_id, + host, + events, + trace_id="t34_large_partial", + ) + self.assertGreater(len(json.dumps(payload)), 32 * 1024) + partial = self.client.report_event(payload, check_ok=False) + self.assertEqual( + partial.get("header", {}).get("status", {}).get("code"), + "INVALID_ARGUMENT", + partial, + ) + self.assertEqual( + partial.get("item_results"), + [ + "INVALID_ARGUMENT" if index in invalid_indices else "OK" + for index in range(event_count) + ], + ) + generation = partial.get("committed_snapshot_version", "") + self.assertEqual(len(generation), 32) + self.assertTrue(partial.get("snapshot_required")) + + valid_samples = [1, event_count // 2 + 1, event_count - 1] + invalid_samples = [0, 17, max(invalid_indices)] + for index in valid_samples: + specs = _query_block_specs( + self.client, + self.instance_id, + base_key + index, + f"t34_valid_{index}", + ) + self.assertEqual(len(specs), 1) + self.assertEqual(specs[0].get("name"), "tp0") + self.assertEqual( + _uri_identity_without_snapshot_version(specs[0]["uri"]), + _uri_identity_without_snapshot_version(raw_uris[index]), + ) + self.assertEqual( + _snapshot_version_from_uri(self, specs[0]["uri"]), + generation, + ) + for index in invalid_samples: + self.assertEqual( + _query_block_specs( + self.client, + self.instance_id, + base_key + index, + f"t34_invalid_absent_{index}", + ), + [], + ) + + retry_events = [ + _ev_block_add( + base_key + index, + "mem", + _make_single_spec("tp0", raw_uris[index]), + ) + for index in sorted(invalid_indices) + ] + retry = self.client.report_event( + _make_request( + self.instance_id, + host, + retry_events, + trace_id="t34_retry_failed_only", + ) + ) + self.assertEqual(retry.get("item_results", []), []) + self.assertEqual(retry.get("committed_snapshot_version"), generation) + self.assertFalse(retry.get("snapshot_required")) + for index in invalid_samples: + specs = _query_block_specs( + self.client, + self.instance_id, + base_key + index, + f"t34_retry_visible_{index}", + ) + self.assertEqual(len(specs), 1) + self.assertEqual( + _uri_identity_without_snapshot_version(specs[0]["uri"]), + _uri_identity_without_snapshot_version(raw_uris[index]), + ) + self.assertEqual( + _snapshot_version_from_uri(self, specs[0]["uri"]), + generation, + ) + # --------------------------------------------------------------------------- # Bench tests # --------------------------------------------------------------------------- @@ -3923,6 +4107,14 @@ class EventReportBenchTest(unittest.TestCase): def setUpClass(cls): cls.client = KVCMClient(BASE_URL, ADMIN_URL) cls.instance_id = INSTANCE_ID + # --only-bench does not load EventReportFunctionalTest, so benchmark + # setup must not depend on that class having run first. + fixture = EventReportFunctionalTest + fixture.client = cls.client + fixture.instance_id = cls.instance_id + fixture._ensure_event_report_storage_registered() + fixture._ensure_instance_group_created() + fixture._ensure_instance_registered() @classmethod def tearDownClass(cls): @@ -4167,6 +4359,243 @@ def test_19_ten_reporters_full_snapshot_capacity(self): ) print(f" Latency max: {max(ordered_latencies):.2f}ms") + # 20. One ReportEvent carrying many small-block deltas. This benchmark + # tracks the workload that previously amplified reporter-node locking and + # lifecycle lease acquisition once per event/key. + def test_20_large_single_request_delta_scaling(self): + host = "192.168.2.200:8080" + self._ensure_host_registered(self.client, self.instance_id, host) + measurements = [] + + for batch_index, event_count in enumerate((100, 1000, 5000, 20_000)): + # Keep every scale isolated even when larger cases are appended. + base_key = 40_000_000 + batch_index * 100_000 + create_events = [ + _ev_block_add( + base_key + offset, + "mem", + _make_single_spec( + "spec_4096", + _build_event_report_uri( + host, + "mem", + { + "block": str(base_key + offset), + "phase": "create", + }, + ), + ), + ) + for offset in range(event_count) + ] + start = time.monotonic() + response = self.client.report_event( + _make_request( + self.instance_id, + host, + create_events, + trace_id=f"bench_large_delta_{event_count}", + ) + ) + create_elapsed_ms = (time.monotonic() - start) * 1000 + version = response.get("committed_snapshot_version", "") + self.assertEqual(len(version), 32) + self.assertFalse(response.get("snapshot_required")) + + # Report the same blocks again to exercise the existing-location + # path, including BatchMerge's block lookup and targeted merge + # RMW phases. + update_events = [ + _ev_block_add( + base_key + offset, + "mem", + _make_single_spec( + "spec_4096", + _build_event_report_uri( + host, + "mem", + { + "block": str(base_key + offset), + "phase": "update", + }, + ), + ), + ) + for offset in range(event_count) + ] + start = time.monotonic() + update_response = self.client.report_event( + _make_request( + self.instance_id, + host, + update_events, + trace_id=f"bench_large_delta_update_{event_count}", + ) + ) + update_elapsed_ms = (time.monotonic() - start) * 1000 + self.assertEqual( + update_response.get("committed_snapshot_version", ""), + version, + ) + measurements.append( + (event_count, create_elapsed_ms, update_elapsed_ms) + ) + + for offset in (0, event_count // 2, event_count - 1): + block_key = base_key + offset + specs = _query_block_specs( + self.client, + self.instance_id, + block_key, + f"bench_large_delta_query_{event_count}_{offset}", + ) + self.assertEqual(len(specs), 1) + self.assertEqual(specs[0].get("name"), "spec_4096") + _assert_reporter_scope( + self, + specs[0]["uri"], + _build_event_report_uri( + host, + "mem", + {"block": str(block_key), "phase": "update"}, + ), + self.instance_id, + host, + "mem", + version, + ) + + print("\n[BENCH] Large single-request BLOCK_ADD scaling:") + for event_count, create_elapsed_ms, update_elapsed_ms in measurements: + print( + f" Events: {event_count:5d}, " + f"create: {create_elapsed_ms:9.2f}ms " + f"({create_elapsed_ms / event_count:.4f}ms/event), " + f"update: {update_elapsed_ms:9.2f}ms " + f"({update_elapsed_ms / event_count:.4f}ms/event)" + ) + + # 21. GetHostCacheState scaling for the pure local metadata path. There is + # no latency assertion because debug/release builds and test hosts differ; + # correctness is checked on every measured response and the output is a + # reproducible before/after baseline. + def test_21_get_host_cache_state_local_scaling(self): + host = "192.168.2.210:8080" + self._ensure_host_registered(self.client, self.instance_id, host) + measurements = [] + + for batch_index, block_count in enumerate((100, 1000, 5000, 20_000)): + # Keep every scale isolated even when larger cases are appended. + base_key = 50_000_000 + batch_index * 100_000 + block_keys = [base_key + offset for offset in range(block_count)] + events = [ + _ev_block_add( + block_key, + "mem", + _make_single_spec( + "spec_4096", + _build_event_report_uri( + host, "mem", {"block": str(block_key)} + ), + ), + ) + for block_key in block_keys + ] + self.client.report_event( + _make_request( + self.instance_id, + host, + events, + trace_id=f"bench_host_state_setup_{block_count}", + ) + ) + query_keys = block_keys + [base_key + block_count + 1] + payload = { + "trace_id": f"bench_host_state_{block_count}", + "instance_id": self.instance_id, + "query_type": "QT_PREFIX_MATCH", + "block_cache_keys": query_keys, + "medium": ["mem"], + } + + def assert_response(response): + prefixes = { + item["host_ip_port"]: int(item["local"]) + for item in response.get("hosts", []) + } + self.assertEqual(prefixes.get(host), block_count) + + for _ in range(3): + assert_response(self.client.get_host_cache_state(payload)) + + latencies = [] + for _ in range(20): + start = time.monotonic() + response = self.client.get_host_cache_state(payload) + latencies.append((time.monotonic() - start) * 1000) + assert_response(response) + + concurrent_latencies = [] + errors = [] + + def query_worker(worker_index): + session = requests.Session() + session.headers.update({ + "Content-Type": "application/json", + "Accept": "application/json", + }) + started = time.monotonic() + try: + response = session.post( + f"{BASE_URL}/api/getHostCacheState", json=payload + ) + response.raise_for_status() + body = response.json() + code = body.get("header", {}).get("status", {}).get("code") + if code != "OK": + raise AssertionError( + f"worker {worker_index}: status={code}, body={body}" + ) + prefixes = { + item["host_ip_port"]: int(item["local"]) + for item in body.get("hosts", []) + } + if prefixes.get(host) != block_count: + raise AssertionError( + f"worker {worker_index}: prefixes={prefixes}" + ) + return (time.monotonic() - started) * 1000 + except Exception as error: + errors.append(str(error)) + return None + finally: + session.close() + + with ThreadPoolExecutor(max_workers=16) as pool: + futures = [pool.submit(query_worker, index) for index in range(32)] + for future in as_completed(futures): + latency = future.result() + if latency is not None: + concurrent_latencies.append(latency) + self.assertEqual(errors, []) + self.assertEqual(32, len(concurrent_latencies)) + measurements.append( + (block_count, sorted(latencies), sorted(concurrent_latencies)) + ) + + print("\n[BENCH] GetHostCacheState local metadata scaling:") + for block_count, serial, concurrent in measurements: + print( + f" Blocks: {block_count:5d}, " + f"serial p50/p99/avg: " + f"{self._percentile(serial, 50):8.2f}/" + f"{self._percentile(serial, 99):8.2f}/" + f"{statistics.mean(serial):8.2f}ms, " + f"16-way p50/p99: " + f"{self._percentile(concurrent, 50):8.2f}/" + f"{self._percentile(concurrent, 99):8.2f}ms" + ) + def main(): parser = argparse.ArgumentParser(description="Event Report ReportEvent HTTP integration tests") @@ -4183,11 +4612,19 @@ def main(): default=[], help="Run only the named EventReportFunctionalTest method; repeatable.", ) + parser.add_argument( + "--bench-test", + action="append", + default=[], + help="Run only the named EventReportBenchTest method; repeatable.", + ) parser.add_argument( "--enable-liveness-timing-tests", action="store_true", - help=("Run heartbeat/cleanup timing tests. Requires the Event report storage to be opened with " - "small heartbeat_timeout_ms / cleanup_grace_ms (defaults to 1000ms / 2000ms here)."), + help=( + "Compatibility flag. Heartbeat/cleanup timing tests use their own " + "isolated fast-liveness storage and always run." + ), ) parser.add_argument("--heartbeat-timeout-ms", type=int, default=1000) parser.add_argument("--cleanup-grace-ms", type=int, default=2000) @@ -4202,11 +4639,12 @@ def main(): ) args, _ = parser.parse_known_args() + if args.functional_test and args.bench_test: + parser.error("--functional-test and --bench-test are mutually exclusive") admin_port = args.admin_http_port or args.http_port global BASE_URL, ADMIN_URL, INSTANCE_ID, SKIP_BENCH, ONLY_BENCH - global ENABLE_LIVENESS_TIMING_TESTS global HEARTBEAT_TIMEOUT_MS, CLEANUP_GRACE_MS global SNAPSHOT_MIN_INTERVAL_MS, META_STORAGE_URI BASE_URL = f"http://{args.host}:{args.http_port}" @@ -4214,7 +4652,6 @@ def main(): INSTANCE_ID = args.instance_id SKIP_BENCH = args.skip_bench ONLY_BENCH = args.only_bench - ENABLE_LIVENESS_TIMING_TESTS = args.enable_liveness_timing_tests HEARTBEAT_TIMEOUT_MS = args.heartbeat_timeout_ms CLEANUP_GRACE_MS = args.cleanup_grace_ms SNAPSHOT_MIN_INTERVAL_MS = args.snapshot_min_interval_ms @@ -4226,9 +4663,12 @@ def main(): if args.functional_test: for test_name in args.functional_test: suite.addTest(EventReportFunctionalTest(test_name)) + elif args.bench_test: + for test_name in args.bench_test: + suite.addTest(EventReportBenchTest(test_name)) elif not ONLY_BENCH: suite.addTests(loader.loadTestsFromTestCase(EventReportFunctionalTest)) - if not args.functional_test and not SKIP_BENCH: + if not args.functional_test and not args.bench_test and not SKIP_BENCH: suite.addTests(loader.loadTestsFromTestCase(EventReportBenchTest)) runner = unittest.TextTestRunner(verbosity=2) diff --git a/integration_test/testlib/BUILD b/integration_test/testlib/BUILD index 7dd60536c..2cf9e902b 100644 --- a/integration_test/testlib/BUILD +++ b/integration_test/testlib/BUILD @@ -17,3 +17,10 @@ py_library( imports = [".."], deps = [], ) + +py_test( + name = "TestBaseTest", + srcs = ["test_base_test.py"], + main = "test_base_test.py", + deps = [":test_base"], +) diff --git a/integration_test/testlib/test_base.py b/integration_test/testlib/test_base.py index 60054cf07..77bb1089d 100644 --- a/integration_test/testlib/test_base.py +++ b/integration_test/testlib/test_base.py @@ -63,6 +63,13 @@ def get_hash_range(self, hash_str): return range_from, range_to def get_workdir(self): + # Bazel gives every test action (including each --runs_per_test + # repetition) a private TEST_TMPDIR. Keep mutable test state there so + # concurrently executing copies cannot delete or overwrite each + # other's worker tree under the shared runfiles directory. + test_tmpdir = os.environ.get('TEST_TMPDIR') + if test_tmpdir: + return os.path.join(os.path.abspath(test_tmpdir), self._testMethodName) return os.path.join(os.path.abspath(os.path.join(os.path.dirname(__file__), '../')), self._testMethodName) def clean_workdir(self): @@ -72,7 +79,10 @@ def clean_workdir(self): def _init_dirs(self, work_dir): self.workdir = work_dir if work_dir is not None else self.get_workdir() - self.path_root = os.path.abspath(os.path.join(self.workdir, '../')) + # The packaged binary is a read-only runfile. It must not be resolved + # relative to TEST_TMPDIR now that the per-test worker tree lives + # there. + self.path_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '../')) self.global_install_root = os.path.join(self.path_root, 'install_root') self.worker_install_root = os.path.join(self.workdir, 'install_root') diff --git a/integration_test/testlib/test_base_test.py b/integration_test/testlib/test_base_test.py new file mode 100644 index 000000000..24937a37f --- /dev/null +++ b/integration_test/testlib/test_base_test.py @@ -0,0 +1,44 @@ +import os +import tempfile +import unittest +from unittest import mock + +from integration_test.testlib.module_base import ModuleBase +from integration_test.testlib.test_base import TestBase + + +class TestBaseWorkdirTest(unittest.TestCase): + def setUp(self): + self.harness = TestBase() + self.harness._testMethodName = "concurrent_case" + + def test_bazel_tmpdir_isolates_mutable_workdir_from_runfiles(self): + with tempfile.TemporaryDirectory() as test_tmpdir: + with mock.patch.dict(os.environ, {"TEST_TMPDIR": test_tmpdir}): + expected_workdir = os.path.join(test_tmpdir, "concurrent_case") + self.assertEqual(expected_workdir, self.harness.get_workdir()) + + with mock.patch.object(ModuleBase, "create_symlink") as create_symlink: + self.harness._init_dirs(None) + + expected_source_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../") + ) + self.assertEqual(expected_workdir, self.harness.workdir) + self.assertEqual(expected_source_root, self.harness.path_root) + create_symlink.assert_called_once_with( + os.path.join(expected_source_root, "install_root"), + os.path.join(expected_workdir, "install_root"), + ) + + def test_non_bazel_run_preserves_source_relative_workdir(self): + with mock.patch.dict(os.environ, {}, clear=True): + expected_workdir = os.path.join( + os.path.abspath(os.path.join(os.path.dirname(__file__), "../")), + "concurrent_case", + ) + self.assertEqual(expected_workdir, self.harness.get_workdir()) + + +if __name__ == "__main__": + unittest.main() diff --git a/kv_cache_manager/common/cache/advanced_cache.h b/kv_cache_manager/common/cache/advanced_cache.h index 7c8c21b10..9c21c583c 100644 --- a/kv_cache_manager/common/cache/advanced_cache.h +++ b/kv_cache_manager/common/cache/advanced_cache.h @@ -13,6 +13,7 @@ #include #include #include +#include #include "kv_cache_manager/common/cache/cache.h" @@ -43,6 +44,16 @@ class Cache { // Opaque handle to an entry stored in the cache. struct Handle {}; + // Caller-owned workspace for allocation-free repeated batch lookup and + // release. The fields are intentionally generic storage owned by Cache; + // callers only prepare and pass the object back to the same cache. + struct BatchOperationScratch { + std::vector hashes; + std::vector shard_offsets; + std::vector cursors; + std::vector ordered_indices; + }; + public: // types hidden from Cache implementation // Pointer to cached object of unspecified type. (This type alias is // provided for clarity, not really for type checking.) @@ -300,6 +311,31 @@ class Cache { Priority priority = Priority::LOW, Statistics *stats = nullptr) = 0; + // Batch form of a basic primary-cache lookup. Every non-null output handle + // has the same lifetime contract as Lookup() and must be released exactly + // once. Implementations may group keys by internal shard; the default + // preserves behavior by issuing independent lookups. + virtual void LookupBatch(const std::string_view *keys, size_t count, Handle **out_handles) { + for (size_t i = 0; i < count; ++i) { + out_handles[i] = Lookup(keys[i]); + } + } + + // Reserve implementation-specific batch workspace before entering a + // caller's critical section. The default backend needs no workspace. + virtual void PrepareBatchOperationScratch(size_t /*max_count*/, BatchOperationScratch * /*scratch*/) {} + + // Scratch-aware variants preserve the ordinary Handle lifetime contract. + // Backends without a specialized implementation delegate to the existing + // methods, so decorators and alternate cache implementations keep their + // behavior unchanged. + virtual void LookupBatchWithScratch(const std::string_view *keys, + size_t count, + Handle **out_handles, + BatchOperationScratch * /*scratch*/) { + LookupBatch(keys, count, out_handles); + } + // Convenience wrapper when secondary cache not supported inline Handle *BasicLookup(const std::string_view &key, Statistics *stats) { return Lookup(key, nullptr, nullptr, Priority::LOW, stats); @@ -324,6 +360,20 @@ class Cache { // REQUIRES: handle must have been returned by a method on *this. virtual bool Release(Handle *handle, bool erase_if_last_ref = false) = 0; + // Releases handles returned by LookupBatch(). Null entries are ignored. + // The default implementation preserves the ordinary Release semantics. + virtual void ReleaseBatch(Handle *const *handles, size_t count) { + for (size_t i = 0; i < count; ++i) { + if (handles[i] != nullptr) { + Release(handles[i]); + } + } + } + + virtual void ReleaseBatchWithScratch(Handle *const *handles, size_t count, BatchOperationScratch * /*scratch*/) { + ReleaseBatch(handles, count); + } + // Return the object assiciated with a handle returned by a successful // Lookup(). For historical reasons, this is also known at the "value" // associated with the key. @@ -640,6 +690,21 @@ class CacheWrapper : public Cache { return target_->Lookup(key, helper, create_context, priority, stats); } + void LookupBatch(const std::string_view *keys, size_t count, Handle **out_handles) override { + target_->LookupBatch(keys, count, out_handles); + } + + void PrepareBatchOperationScratch(size_t max_count, BatchOperationScratch *scratch) override { + target_->PrepareBatchOperationScratch(max_count, scratch); + } + + void LookupBatchWithScratch(const std::string_view *keys, + size_t count, + Handle **out_handles, + BatchOperationScratch *scratch) override { + target_->LookupBatchWithScratch(keys, count, out_handles, scratch); + } + bool Ref(Handle *handle) override { return target_->Ref(handle); } using Cache::Release; @@ -647,6 +712,12 @@ class CacheWrapper : public Cache { return target_->Release(handle, erase_if_last_ref); } + void ReleaseBatch(Handle *const *handles, size_t count) override { target_->ReleaseBatch(handles, count); } + + void ReleaseBatchWithScratch(Handle *const *handles, size_t count, BatchOperationScratch *scratch) override { + target_->ReleaseBatchWithScratch(handles, count, scratch); + } + ObjectPtr Value(Handle *handle) override { return target_->Value(handle); } bool Exists(const std::string_view &key) override { return target_->Exists(key); } diff --git a/kv_cache_manager/common/cache/lru_cache.cc b/kv_cache_manager/common/cache/lru_cache.cc index eba0f3886..387152c6b 100644 --- a/kv_cache_manager/common/cache/lru_cache.cc +++ b/kv_cache_manager/common/cache/lru_cache.cc @@ -9,6 +9,7 @@ #include "kv_cache_manager/common/cache/lru_cache.h" +#include #include #include #include @@ -475,6 +476,27 @@ LRUHandle *LRUCacheShard::Lookup(const std::string_view &key, return e; } +void LRUCacheShard::LookupBatch(const std::string_view *keys, + const uint32_t *hashes, + const size_t *ordered_indices, + size_t count, + Cache::Handle **out_handles) { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < count; ++i) { + const size_t index = ordered_indices[i]; + LRUHandle *entry = table_.Lookup(keys[index], hashes[index]); + if (entry != nullptr) { + assert(entry->InCache()); + if (!entry->HasRefs()) { + LRU_Remove(entry); + } + entry->Ref(); + entry->SetHit(); + } + out_handles[index] = static_cast(entry); + } +} + bool LRUCacheShard::Ref(LRUHandle *e) { std::lock_guard l(mutex_); // To create another reference - entry must be already externally referenced. @@ -563,6 +585,39 @@ bool LRUCacheShard::Release(LRUHandle *e, bool /*useful*/, bool erase_if_last_re return must_free; } +void LRUCacheShard::ReleaseBatch(Cache::Handle *const *handles, const size_t *ordered_indices, size_t count) { + autovector free_handles; + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < count; ++i) { + auto *entry = static_cast(handles[ordered_indices[i]]); + if (entry == nullptr) { + continue; + } + bool must_free = entry->Unref(); + const bool was_in_cache = entry->InCache(); + if (must_free && was_in_cache) { + if (!no_evict_on_insert_ && usage_ > capacity_) { + assert(lru_.next == &lru_); + table_.Remove(entry->key(), entry->hash); + entry->SetInCache(false); + } else { + LRU_Insert(entry); + must_free = false; + } + } + if (must_free) { + assert(usage_ >= entry->total_charge); + usage_ -= entry->total_charge; + free_handles.push_back(entry); + } + } + } + for (auto *entry : free_handles) { + entry->Free(table_.GetAllocator()); + } +} + LRUHandle *LRUCacheShard::CreateHandle(const std::string_view &key, uint32_t hash, Cache::ObjectPtr value, @@ -786,6 +841,124 @@ const Cache::CacheItemHelper *LRUCache::GetCacheItemHelper(Handle *handle) const return h->helper; } +void LRUCache::LookupBatch(const std::string_view *keys, size_t count, Handle **out_handles) { + BatchOperationScratch scratch; + PrepareBatchOperationScratch(count, &scratch); + LookupBatchWithScratch(keys, count, out_handles, &scratch); +} + +void LRUCache::PrepareBatchOperationScratch(size_t max_count, BatchOperationScratch *scratch) { + if (!scratch) { + return; + } + scratch->hashes.reserve(max_count); + scratch->ordered_indices.reserve(max_count); + const size_t shard_slots = GetNumShards() + 1; + scratch->shard_offsets.reserve(shard_slots); + scratch->cursors.reserve(shard_slots); +} + +void LRUCache::LookupBatchWithScratch(const std::string_view *keys, + size_t count, + Handle **out_handles, + BatchOperationScratch *scratch) { + if (count == 0) { + return; + } + if (!scratch) { + LookupBatch(keys, count, out_handles); + return; + } + std::fill(out_handles, out_handles + count, nullptr); + + const size_t shard_count = GetNumShards(); + scratch->hashes.resize(count); + scratch->shard_offsets.assign(shard_count + 1, 0); + auto &hashes = scratch->hashes; + auto &shard_offsets = scratch->shard_offsets; + for (size_t i = 0; i < count; ++i) { + hashes[i] = LRUCacheShard::ComputeHash(keys[i], hash_seed_); + const size_t shard = LRUCacheShard::HashPieceForSharding(hashes[i]) & shard_mask_; + ++shard_offsets[shard + 1]; + } + for (size_t shard = 0; shard < shard_count; ++shard) { + shard_offsets[shard + 1] += shard_offsets[shard]; + } + + scratch->cursors.assign(shard_offsets.begin(), shard_offsets.end()); + scratch->ordered_indices.resize(count); + auto &cursors = scratch->cursors; + auto &ordered_indices = scratch->ordered_indices; + for (size_t i = 0; i < count; ++i) { + const size_t shard = LRUCacheShard::HashPieceForSharding(hashes[i]) & shard_mask_; + ordered_indices[cursors[shard]++] = i; + } + for (size_t shard = 0; shard < shard_count; ++shard) { + const size_t begin = shard_offsets[shard]; + const size_t end = shard_offsets[shard + 1]; + if (begin == end) { + continue; + } + GetShard(hashes[ordered_indices[begin]]) + .LookupBatch(keys, hashes.data(), ordered_indices.data() + begin, end - begin, out_handles); + } +} + +void LRUCache::ReleaseBatch(Handle *const *handles, size_t count) { + BatchOperationScratch scratch; + PrepareBatchOperationScratch(count, &scratch); + ReleaseBatchWithScratch(handles, count, &scratch); +} + +void LRUCache::ReleaseBatchWithScratch(Handle *const *handles, size_t count, BatchOperationScratch *scratch) { + if (count == 0) { + return; + } + if (!scratch) { + ReleaseBatch(handles, count); + return; + } + + const size_t shard_count = GetNumShards(); + scratch->shard_offsets.assign(shard_count + 1, 0); + auto &shard_offsets = scratch->shard_offsets; + size_t non_null_count = 0; + for (size_t i = 0; i < count; ++i) { + if (handles[i] == nullptr) { + continue; + } + const auto *entry = static_cast(handles[i]); + const size_t shard = LRUCacheShard::HashPieceForSharding(entry->GetHash()) & shard_mask_; + ++shard_offsets[shard + 1]; + ++non_null_count; + } + for (size_t shard = 0; shard < shard_count; ++shard) { + shard_offsets[shard + 1] += shard_offsets[shard]; + } + + scratch->cursors.assign(shard_offsets.begin(), shard_offsets.end()); + scratch->ordered_indices.resize(non_null_count); + auto &cursors = scratch->cursors; + auto &ordered_indices = scratch->ordered_indices; + for (size_t i = 0; i < count; ++i) { + if (handles[i] == nullptr) { + continue; + } + const auto *entry = static_cast(handles[i]); + const size_t shard = LRUCacheShard::HashPieceForSharding(entry->GetHash()) & shard_mask_; + ordered_indices[cursors[shard]++] = i; + } + for (size_t shard = 0; shard < shard_count; ++shard) { + const size_t begin = shard_offsets[shard]; + const size_t end = shard_offsets[shard + 1]; + if (begin == end) { + continue; + } + const auto *entry = static_cast(handles[ordered_indices[begin]]); + GetShard(entry->GetHash()).ReleaseBatch(handles, ordered_indices.data() + begin, end - begin); + } +} + void LRUCache::ApplyToHandle( Cache *cache, Handle *handle, diff --git a/kv_cache_manager/common/cache/lru_cache.h b/kv_cache_manager/common/cache/lru_cache.h index 49b0a8270..5c8f12eb4 100644 --- a/kv_cache_manager/common/cache/lru_cache.h +++ b/kv_cache_manager/common/cache/lru_cache.h @@ -325,8 +325,14 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase { Cache::CreateContext *create_context, Cache::Priority priority, Statistics *stats); + void LookupBatch(const std::string_view *keys, + const uint32_t *hashes, + const size_t *ordered_indices, + size_t count, + Cache::Handle **out_handles); bool Release(LRUHandle *handle, bool useful, bool erase_if_last_ref); + void ReleaseBatch(Cache::Handle *const *handles, const size_t *ordered_indices, size_t count); bool Ref(LRUHandle *handle); bool Erase(const std::string_view &key, uint32_t hash); bool Exists(const std::string_view &key, uint32_t hash); @@ -504,6 +510,15 @@ class LRUCache size_t GetCharge(Handle *handle) const override; const CacheItemHelper *GetCacheItemHelper(Handle *handle) const override; + void LookupBatch(const std::string_view *keys, size_t count, Handle **out_handles) override; + void PrepareBatchOperationScratch(size_t max_count, BatchOperationScratch *scratch) override; + void LookupBatchWithScratch(const std::string_view *keys, + size_t count, + Handle **out_handles, + BatchOperationScratch *scratch) override; + void ReleaseBatch(Handle *const *handles, size_t count) override; + void ReleaseBatchWithScratch(Handle *const *handles, size_t count, BatchOperationScratch *scratch) override; + void ApplyToHandle(Cache *cache, Handle *handle, const std::function namespace kv_cache_manager { @@ -28,30 +28,41 @@ bool StandardUri::Parse(const std::string &uri) { protocol_ = uri.substr(0, pos_protocol_end); size_t authority_start = pos_protocol_end + 3; // skip :// + // Locate the end of authority before interpreting '@' or ':'. Delimiter + // characters in the path/query belong to their values, not user-info or + // host/port (for example callback URLs and email addresses). + size_t pos_path_start = uri.find('/', authority_start); + size_t pos_query_start = uri.find('?', authority_start); + size_t host_end = std::min((pos_path_start != std::string::npos ? pos_path_start : uri.size()), + (pos_query_start != std::string::npos ? pos_query_start : uri.size())); size_t host_start = authority_start; size_t pos_at = uri.find('@', authority_start); - if (pos_at != std::string::npos) { + if (pos_at != std::string::npos && pos_at < host_end) { user_info_ = uri.substr(authority_start, pos_at - authority_start); host_start = pos_at + 1; // hostname 开始位置 } - // 找 hostname 结束的位置（可能有 port） - size_t pos_path_start = uri.find('/', authority_start); - size_t pos_query_start = uri.find('?', authority_start); - size_t host_end = std::min((pos_path_start != std::string::npos ? pos_path_start : uri.size()), - (pos_query_start != std::string::npos ? pos_query_start : uri.size()) - - ); - // 分离 hostname 和 port - std::string host_port = uri.substr(host_start, host_end - host_start); - size_t colon_pos = host_port.find(':'); - if (colon_pos == std::string::npos) { - hostname_ = host_port; + // 分离 hostname 和 port。直接在输入中定位，避免为每个 URI 先复制 + // 一份 host:port 临时字符串。 + size_t colon_pos = uri.find(':', host_start); + if (colon_pos == std::string::npos || colon_pos >= host_end) { + hostname_ = uri.substr(host_start, host_end - host_start); } else { - hostname_ = host_port.substr(0, colon_pos); - std::string port_str = host_port.substr(colon_pos + 1); + hostname_ = uri.substr(host_start, colon_pos - host_start); int64_t tmp_port = 0; - if (!StringUtil::StrToInt64(port_str.c_str(), tmp_port)) { + const char *port_begin = uri.data() + colon_pos + 1; + const char *port_end = uri.data() + host_end; + const auto [parsed_end, parse_ec] = std::from_chars(port_begin, port_end, tmp_port); + if (port_begin == port_end || *port_begin == '-' || parse_ec != std::errc{} || parsed_end != port_end) { + // Parse() is also used through the direct string constructor, + // whose caller observes validity rather than the return value. + // Do not leave a partially parsed object looking valid. + protocol_.clear(); + user_info_.clear(); + hostname_.clear(); + port_ = 0; + path_.clear(); + params_.clear(); return false; } else { port_ = tmp_port; @@ -59,23 +70,22 @@ bool StandardUri::Parse(const std::string &uri) { } // 提取 path 和 query - if (pos_path_start != std::string::npos && pos_path_start < uri.size()) { + if (pos_path_start != std::string::npos && + (pos_query_start == std::string::npos || pos_path_start < pos_query_start)) { if (pos_query_start != std::string::npos && pos_path_start < pos_query_start) { path_ = uri.substr(pos_path_start, pos_query_start - pos_path_start); - std::string query_str = uri.substr(pos_query_start + 1); - ParseParams(query_str); + ParseParams(std::string_view(uri).substr(pos_query_start + 1)); } else { path_ = uri.substr(pos_path_start); } } else if (pos_query_start != std::string::npos && pos_query_start < uri.size()) { - std::string query_str = uri.substr(pos_query_start + 1); - ParseParams(query_str); + ParseParams(std::string_view(uri).substr(pos_query_start + 1)); } return true; } -bool StandardUri::ParseParams(const std::string &uri_params) { - auto start = 0; +bool StandardUri::ParseParams(std::string_view uri_params) { + size_t start = 0; while (start < uri_params.size()) { auto end = uri_params.find('&', start); if (end == std::string::npos) { @@ -83,12 +93,12 @@ bool StandardUri::ParseParams(const std::string &uri_params) { } auto eq_pos = uri_params.find('=', start); if (eq_pos != std::string::npos && eq_pos < end) { - std::string key = uri_params.substr(start, eq_pos - start); - std::string value = uri_params.substr(eq_pos + 1, end - eq_pos - 1); + std::string key(uri_params.substr(start, eq_pos - start)); + std::string value(uri_params.substr(eq_pos + 1, end - eq_pos - 1)); params_[key] = value; } else { // key但无value，value空字符串 - std::string key = uri_params.substr(start, end - start); + std::string key(uri_params.substr(start, end - start)); params_[key] = ""; } start = end + 1; @@ -125,6 +135,53 @@ std::string StandardUri::ToUriString() const { return ss.str(); } +std::string StandardUri::ToUriStringWithExtraParam(const std::string &key, const std::string &value) const { + if (!Valid() || key.empty() || HasParam(key)) { + return ""; + } + + size_t estimated_size = + protocol_.size() + user_info_.size() + hostname_.size() + path_.size() + key.size() + value.size() + 8; + for (const auto &[param_key, param_value] : params_) { + estimated_size += param_key.size() + param_value.size() + 2; + } + std::string result; + result.reserve(estimated_size); + result.append(protocol_).append("://"); + if (!user_info_.empty()) { + result.append(user_info_).push_back('@'); + } + result.append(hostname_); + if (port_ > 0) { + result.push_back(':'); + result.append(std::to_string(port_)); + } + result.append(path_); + result.push_back('?'); + + bool first = true; + bool extra_written = false; + auto append_param = [&result, &first](const std::string ¶m_key, const std::string ¶m_value) { + if (!first) { + result.push_back('&'); + } + result.append(param_key).push_back('='); + result.append(param_value); + first = false; + }; + for (const auto &[param_key, param_value] : params_) { + if (!extra_written && key < param_key) { + append_param(key, value); + extra_written = true; + } + append_param(param_key, param_value); + } + if (!extra_written) { + append_param(key, value); + } + return result; +} + StandardUri StandardUri::FromUri(const std::string &source) { StandardUri result; if (!result.Parse(source)) { diff --git a/kv_cache_manager/common/standard_uri.h b/kv_cache_manager/common/standard_uri.h index a5c0776ba..a592d4d79 100644 --- a/kv_cache_manager/common/standard_uri.h +++ b/kv_cache_manager/common/standard_uri.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace kv_cache_manager { @@ -14,6 +15,10 @@ class StandardUri { public: bool Parse(const std::string &Uri); std::string ToUriString() const; + // Serialize the URI as if one new query parameter had been inserted, + // without cloning/mutating the parameter map. The output keeps the same + // sorted canonical form as SetParam() followed by ToUriString(). + std::string ToUriStringWithExtraParam(const std::string &key, const std::string &value) const; bool Valid() const { return !protocol_.empty(); } const std::string &GetProtocol() const { return protocol_; } @@ -39,10 +44,11 @@ class StandardUri { std::string GetParam(const std::string &key) const; template void GetParamAs(const std::string &key, T &t) const { - std::string val = GetParam(key); - if (val.empty()) { + const auto it = params_.find(key); + if (it == params_.end() || it->second.empty()) { return; } + const std::string &val = it->second; T result; auto [ptr, ec] = std::from_chars(val.data(), val.data() + val.size(), result); if (ec == std::errc{} && ptr == val.data() + val.size()) { @@ -62,7 +68,7 @@ class StandardUri { static std::string ToUri(const StandardUri &source); private: - bool ParseParams(const std::string &Uri_params); + bool ParseParams(std::string_view Uri_params); private: std::string protocol_; diff --git a/kv_cache_manager/common/test/lru_cache_test.cc b/kv_cache_manager/common/test/lru_cache_test.cc index 657184d9d..b8872bebf 100644 --- a/kv_cache_manager/common/test/lru_cache_test.cc +++ b/kv_cache_manager/common/test/lru_cache_test.cc @@ -160,6 +160,50 @@ TEST_F(LRUCacheTest, BasicLRU) { ValidateLRUList({"e", "z", "d", "u", "v"}, 0, 5); } +TEST_F(LRUCacheTest, BatchLookupAndReleasePreserveReferencesAndLruOrder) { + NewCache(5); + Insert("a"); + Insert("b"); + Insert("c"); + + const std::string_view keys[] = {"c", "missing", "a", "c"}; + const uint32_t hashes[] = {0, 0, 0, 0}; + const size_t indices[] = {0, 1, 2, 3}; + Cache::Handle *handles[4] = {}; + cache_->LookupBatch(keys, hashes, indices, 4, handles); + ASSERT_NE(nullptr, handles[0]); + EXPECT_EQ(nullptr, handles[1]); + ASSERT_NE(nullptr, handles[2]); + EXPECT_EQ(handles[0], handles[3]); + // Referenced entries are temporarily absent from the eviction list. + ValidateLRUList({"b"}, 0, 1); + + cache_->ReleaseBatch(handles, indices, 4); + // Release order is the request order. The duplicate c handle is inserted + // only when its final reference is released. + ValidateLRUList({"b", "a", "c"}, 0, 3); +} + +TEST_F(LRUCacheTest, BatchReleaseFreesEntryErasedWhilePinned) { + NewCache(5); + Insert("a"); + Insert("b"); + + const std::string_view keys[] = {"a", "b"}; + const uint32_t hashes[] = {0, 0}; + const size_t indices[] = {0, 1}; + Cache::Handle *handles[2] = {}; + cache_->LookupBatch(keys, hashes, indices, 2, handles); + ASSERT_NE(nullptr, handles[0]); + ASSERT_NE(nullptr, handles[1]); + cache_->Erase("a", 0); + cache_->ReleaseBatch(handles, indices, 2); + + EXPECT_FALSE(Lookup("a")); + EXPECT_TRUE(Lookup("b")); + ValidateLRUList({"b"}, 0, 1); +} + TEST_F(LRUCacheTest, LowPriorityMidpointInsertion) { // Allocate 2 cache entries to high-pri pool and 3 to low-pri pool. NewCache(5, /* high_pri_pool_ratio */ 0.40, /* low_pri_pool_ratio */ 0.60); diff --git a/kv_cache_manager/common/test/standard_uri_test.cc b/kv_cache_manager/common/test/standard_uri_test.cc index d7b257bea..7c682071a 100644 --- a/kv_cache_manager/common/test/standard_uri_test.cc +++ b/kv_cache_manager/common/test/standard_uri_test.cc @@ -369,6 +369,42 @@ TEST_F(StandardUriTest, TestFileUri) { } } +TEST_F(StandardUriTest, TestSerializeWithExtraParamMatchesCanonicalMutation) { + for (const std::string &raw_uri : { + "event_report://host:8080/mem", + "event_report://host:8080/mem?block=7&phase=add", + "event_report://user@host:8080/mem?z=last&a=first", + "file://host/path?empty=&flag", + }) { + StandardUri uri = StandardUri::FromUri(raw_uri); + ASSERT_TRUE(uri.Valid()); + StandardUri expected = uri; + expected.SetParam("s_version", "0123456789abcdef0123456789abcdef"); + EXPECT_EQ(expected.ToUriString(), + uri.ToUriStringWithExtraParam("s_version", "0123456789abcdef0123456789abcdef")); + } + + StandardUri uri = StandardUri::FromUri("event_report://host:8080/mem?s_version=existing"); + ASSERT_TRUE(uri.Valid()); + EXPECT_TRUE(uri.ToUriStringWithExtraParam("s_version", "replacement").empty()); + EXPECT_TRUE(uri.ToUriStringWithExtraParam("", "value").empty()); + EXPECT_TRUE(StandardUri().ToUriStringWithExtraParam("key", "value").empty()); +} + +TEST_F(StandardUriTest, TestQueryDelimitersDoNotChangeAuthorityOrPath) { + const std::string raw_uri = "event_report://cache-host?callback=http://peer/path&owner=user@example.com"; + StandardUri uri(raw_uri); + ASSERT_TRUE(uri.Valid()); + EXPECT_TRUE(uri.GetUserInfo().empty()); + EXPECT_EQ("cache-host", uri.GetHostName()); + EXPECT_TRUE(uri.GetPath().empty()); + EXPECT_EQ("http://peer/path", uri.GetParam("callback")); + EXPECT_EQ("user@example.com", uri.GetParam("owner")); + EXPECT_EQ(raw_uri, uri.ToUriString()); + EXPECT_EQ("event_report://cache-host?callback=http://peer/path&owner=user@example.com&s_version=token", + uri.ToUriStringWithExtraParam("s_version", "token")); +} + TEST_F(StandardUriTest, TestInvalidPort) { { std::string redis_uri_str = "redis://user:pw@127.0.0.1"; @@ -378,10 +414,21 @@ TEST_F(StandardUriTest, TestInvalidPort) { ASSERT_EQ("127.0.0.1", redis_uri.GetHostName()); ASSERT_EQ(0, redis_uri.GetPort()); // default 0 } - { - std::string redis_uri_str = "redis://user:pw@127.0.0.1:abcd/"; - StandardUri redis_uri = StandardUri::FromUri(redis_uri_str); + for (const std::string &invalid_uri : { + "redis://user:pw@127.0.0.1:abcd/", + "redis://user:pw@127.0.0.1:-1/", + "redis://user:pw@127.0.0.1:-0/", + "redis://user:pw@127.0.0.1:+6379/", + "redis://user:pw@127.0.0.1: 6379/", + "redis://user:pw@127.0.0.1:/", + }) { + StandardUri redis_uri = StandardUri::FromUri(invalid_uri); ASSERT_FALSE(redis_uri.Valid()); + + StandardUri directly_constructed(invalid_uri); + EXPECT_FALSE(directly_constructed.Valid()); + EXPECT_TRUE(directly_constructed.ToUriString().empty()); + EXPECT_TRUE(directly_constructed.ToUriStringWithExtraParam("key", "value").empty()); } } } // namespace kv_cache_manager diff --git a/kv_cache_manager/config/registry_manager.cc b/kv_cache_manager/config/registry_manager.cc index dd29e8ce9..521e3436a 100644 --- a/kv_cache_manager/config/registry_manager.cc +++ b/kv_cache_manager/config/registry_manager.cc @@ -291,6 +291,9 @@ ErrorCode RegistryManager::RegisterInstance(RequestContext *request_context, const auto &existing = it->second; auto mismatched = existing->MismatchFields( block_size, location_spec_infos, model_deployment, location_spec_groups, default_query_type); + if (existing->instance_group_name() != instance_group) { + mismatched.insert(mismatched.begin(), "instance_group_name"); + } if (!mismatched.empty()) { auto mismatched_str = StringUtil::Join(mismatched, ", "); request_context->error_tracer()->AddErrorMsg( diff --git a/kv_cache_manager/config/test/instance_group_test.cc b/kv_cache_manager/config/test/instance_group_test.cc index 08414bc53..673f10c45 100644 --- a/kv_cache_manager/config/test/instance_group_test.cc +++ b/kv_cache_manager/config/test/instance_group_test.cc @@ -228,6 +228,31 @@ TEST_F(InstanceGroupTest, EventReportStorageSpecProtoRoundTripPreservesSnapshotS EXPECT_EQ(90, restored_spec->liveness_check_interval_ms()); EXPECT_EQ(4321, restored_spec->snapshot_min_interval_ms()); EXPECT_EQ(8765, restored_spec->snapshot_delta_drain_timeout_ms()); + + proto::admin::StorageConfig invalid_proto_config; + invalid_proto_config.set_global_unique_name("invalid_event_report"); + invalid_proto_config.set_storage_type(proto::admin::ST_EVENT_REPORT_L2); + invalid_proto_config.mutable_event_report()->set_snapshot_delta_drain_timeout_ms(-1); + StorageConfig invalid_restored; + ProtoConvert::StorageFromProto(&invalid_proto_config, invalid_restored); + auto invalid_spec = std::dynamic_pointer_cast(invalid_restored.storage_spec()); + ASSERT_NE(nullptr, invalid_spec); + EXPECT_EQ(-1, invalid_spec->snapshot_delta_drain_timeout_ms()); + std::string invalid_fields; + EXPECT_FALSE(invalid_restored.ValidateRequiredFields(invalid_fields)); + EXPECT_NE(std::string::npos, invalid_fields.find("snapshot_delta_drain_timeout_ms")); +} + +TEST_F(InstanceGroupTest, UnknownProtoStorageTypeFailsClosed) { + DataStorageType storage_type = DataStorageType::DATA_STORAGE_TYPE_NFS; + ProtoConvert::DataStorageTypeFromProto(static_cast(263), storage_type); + EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_UNKNOWN, storage_type); + + proto::admin::CacheLocation proto_location; + proto_location.set_type(static_cast(263)); + CacheLocation location; + ProtoConvert::CacheLocationFromProto(&proto_location, location); + EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_UNKNOWN, location.type()); } } // namespace kv_cache_manager diff --git a/kv_cache_manager/config/test/registry_manager_local_backend_test.cc b/kv_cache_manager/config/test/registry_manager_local_backend_test.cc index 0de6368ab..f18acda81 100644 --- a/kv_cache_manager/config/test/registry_manager_local_backend_test.cc +++ b/kv_cache_manager/config/test/registry_manager_local_backend_test.cc @@ -246,6 +246,25 @@ TEST_F(RegistryManagerLocalBackendTest, TestInstanceRegistration) { ASSERT_EQ(1, instance_infos_after_remove.size()); } +TEST_F(RegistryManagerLocalBackendTest, DuplicateInstanceCannotMoveToAnotherGroup) { + std::string local_path = GetPrivateTestRuntimeDataPath() + "_registry_local_backend_group_mismatch"; + ASSERT_TRUE(InitRegistryManager("local://" + local_path + "?cluster_name=test")); + AddNfsStorage("storage1", 1); + CreateInstanceGroup("group1"); + CreateInstanceGroup("group2"); + RegisterInstance("group1", "shared_instance"); + + LocationSpecInfo info; + ModelDeployment model_deployment; + EXPECT_EQ(EC_DUPLICATE_ENTITY, + registry_manager_->RegisterInstance( + request_context_.get(), "group2", "shared_instance", 1024, {info}, model_deployment)); + const auto existing = registry_manager_->GetInstanceInfo(request_context_.get(), "shared_instance"); + ASSERT_NE(nullptr, existing); + EXPECT_EQ("group1", existing->instance_group_name()); + EXPECT_NE(std::string::npos, request_context_->error_tracer()->ToJsonString().find("instance_group_name")); +} + TEST_F(RegistryManagerLocalBackendTest, TestAccountManagement) { std::string local_path = GetPrivateTestRuntimeDataPath() + "_registry_local_backend_account_test"; std::string uri = "local://" + local_path + "?cluster_name=test"; diff --git a/kv_cache_manager/data_storage/BUILD b/kv_cache_manager/data_storage/BUILD index ba14a7151..5356c36c3 100644 --- a/kv_cache_manager/data_storage/BUILD +++ b/kv_cache_manager/data_storage/BUILD @@ -29,7 +29,6 @@ cc_library( "event_report_backend.h", "hf3fs_backend.h", "nfs_backend.h", - "snapshot_uri_utils.h", ] + select({ ":enable_mooncake": ["mooncake_backend.h"], "//conditions:default": [], @@ -46,6 +45,7 @@ cc_library( # "//3rdparty/3fs:hf3fs", "//kv_cache_manager/common/hash", # for xxph3 ":common_define", + ":snapshot_uri_utils", # Internal implementations "//stub_source/kv_cache_manager/data_storage:tair_mempool", ] + select({ @@ -57,6 +57,16 @@ cc_library( }), ) +cc_library( + name = "snapshot_uri_utils", + hdrs = [ + "snapshot_uri_utils.h", + ], + deps = [ + ":common_define", + ], +) + cc_library( name = "common_define", srcs = [ diff --git a/kv_cache_manager/data_storage/data_storage_backend.h b/kv_cache_manager/data_storage/data_storage_backend.h index 7d8a4b654..e565ebf2f 100644 --- a/kv_cache_manager/data_storage/data_storage_backend.h +++ b/kv_cache_manager/data_storage/data_storage_backend.h @@ -25,7 +25,7 @@ class DataStorageBackend { virtual double GetStorageUsageRatio(const std::string &trace_id) const = 0; inline bool IsOpen() const { return is_open_.load(std::memory_order_relaxed); } inline void SetOpen(bool open) { is_open_.store(open, std::memory_order_relaxed); } - inline void SetAvailable(bool available) { is_available_.store(available, std::memory_order_relaxed); } + virtual void SetAvailable(bool available) { is_available_.store(available, std::memory_order_release); } std::shared_ptr GetMetricsCollector() { return metrics_collector_; } virtual const StorageConfig &GetStorageConfig() { return config_; } @@ -72,7 +72,7 @@ class DataStorageBackend { } protected: - inline bool IsAvailable() const { return is_available_.load(std::memory_order_relaxed); } + inline bool IsAvailable() const { return is_available_.load(std::memory_order_acquire); } protected: StorageConfig config_; diff --git a/kv_cache_manager/data_storage/event_report_backend.cc b/kv_cache_manager/data_storage/event_report_backend.cc index 7841a1a83..24690a1f7 100644 --- a/kv_cache_manager/data_storage/event_report_backend.cc +++ b/kv_cache_manager/data_storage/event_report_backend.cc @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -74,9 +76,29 @@ DataStorageType EventReportBackend::GetType() { return config_.type(); } bool EventReportBackend::Available() { return IsOpen() && IsAvailable(); } +void EventReportBackend::SetAvailable(bool available) { + DataStorageBackend::SetAvailable(available); + if (!available) { + snapshot_state_cv_.notify_all(); + } +} + double EventReportBackend::GetStorageUsageRatio(const std::string & /*trace_id*/) const { return 1.0; } +ErrorCode EventReportBackend::Open(const StorageConfig &config, const std::string &trace_id) { + if (IsOpen() || Retired()) { + KVCM_LOG_WARN("trace_id [%s] | EventReportBackend::Open: backend objects cannot be opened twice or reused " + "after Close", + trace_id.c_str()); + return EC_ERROR; + } + return DataStorageBackend::Open(config, trace_id); +} + ErrorCode EventReportBackend::DoOpen(const StorageConfig &config, const std::string &trace_id) { + if (IsOpen() || Retired()) { + return EC_ERROR; + } auto spec = std::dynamic_pointer_cast(config.storage_spec()); if (!spec) { KVCM_LOG_WARN("trace_id [%s] | EventReportBackend::DoOpen: unexpected config type, storage config: [%s]", @@ -101,7 +123,25 @@ ErrorCode EventReportBackend::DoOpen(const StorageConfig &config, const std::str SetAvailable(true); liveness_checker_running_.store(true, std::memory_order_relaxed); - liveness_checker_thread_ = std::thread(&EventReportBackend::LivenessCheckerLoop, this); + try { + liveness_checker_thread_ = std::thread(&EventReportBackend::LivenessCheckerLoop, this); + } catch (const std::exception &e) { + liveness_checker_running_.store(false, std::memory_order_release); + SetAvailable(false); + SetOpen(false); + KVCM_LOG_ERROR("trace_id [%s] | EventReportBackend::DoOpen: start liveness checker failed: [%s]", + trace_id.c_str(), + e.what()); + return EC_ERROR; + } catch (...) { + liveness_checker_running_.store(false, std::memory_order_release); + SetAvailable(false); + SetOpen(false); + KVCM_LOG_ERROR("trace_id [%s] | EventReportBackend::DoOpen: start liveness checker failed with unknown " + "exception", + trace_id.c_str()); + return EC_ERROR; + } KVCM_LOG_INFO("trace_id [%s] | EventReportBackend opened, storage: [%s], type: [%s], hb_timeout=%ldms, " "cleanup_grace=%ldms, check_interval=%ldms, snapshot_min_interval=%ldms, " @@ -118,21 +158,42 @@ ErrorCode EventReportBackend::DoOpen(const StorageConfig &config, const std::str } ErrorCode EventReportBackend::Close() { + retired_.store(true, std::memory_order_release); SetOpen(false); SetAvailable(false); - liveness_checker_running_.store(false, std::memory_order_relaxed); + // Serialize the predicate update with wait_for(). An atomic predicate is + // not sufficient to prevent a lost notification when the waiter is + // between checking the predicate and actually sleeping. + { + std::lock_guard wait_guard(liveness_wait_mutex_); + liveness_checker_running_.store(false, std::memory_order_release); + } + liveness_wait_cv_.notify_all(); if (liveness_checker_thread_.joinable()) { liveness_checker_thread_.join(); } - std::lock_guard fences_guard(lifecycle_fences_mutex_); - std::vector> fence_locks; - fence_locks.reserve(lifecycle_fences_.size()); - for (const auto &entry : lifecycle_fences_) { - const auto &fence = entry.second; - if (fence) { - fence_locks.emplace_back(fence->mutex); + std::vector> fence_refs; + { + std::lock_guard fences_guard(lifecycle_fences_mutex_); + fence_refs.reserve(lifecycle_fences_.size()); + for (const auto &entry : lifecycle_fences_) { + if (entry.second) { + fence_refs.push_back(entry.second); + } } } + + // Never wait for a lifecycle fence while holding lifecycle_fences_mutex_. + // Cleanup deliberately takes lifecycle -> metadata, while a metadata RMW + // may already hold metadata when it briefly looks up and try-locks its + // lifecycle fence. Close holding the table mutex while waiting for the + // cleanup lease would complete a three-lock cycle. The strong references + // also keep each shared_mutex alive until its unique_lock is released. + std::vector> fence_locks; + fence_locks.reserve(fence_refs.size()); + for (const auto &fence : fence_refs) { + fence_locks.emplace_back(fence->mutex); + } { std::unique_lock lock(nodes_mutex_); instance_nodes_.clear(); @@ -140,7 +201,12 @@ ErrorCode EventReportBackend::Close() { snapshot_versions_.clear(); snapshot_token_owners_.clear(); } - lifecycle_fences_.clear(); + { + std::lock_guard fences_guard(lifecycle_fences_mutex_); + lifecycle_fences_.clear(); + } + fence_locks.clear(); + fence_refs.clear(); snapshot_state_cv_.notify_all(); { std::lock_guard lock(cleanup_cb_mutex_); @@ -153,6 +219,11 @@ ErrorCode EventReportBackend::Close() { void EventReportBackend::SetCleanupCallback(CleanupCallback cb) { std::lock_guard lock(cleanup_cb_mutex_); + if (Retired()) { + cleanup_callback_ = nullptr; + cleanup_cb_set_.store(false, std::memory_order_release); + return; + } cleanup_callback_ = std::move(cb); cleanup_cb_set_.store(cleanup_callback_ != nullptr, std::memory_order_release); } @@ -166,15 +237,21 @@ ErrorCode EventReportBackend::RegisterNode(const std::string &instance_id, })) { return EC_BADARGS; } + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } const ReporterSnapshotKey reporter_key{instance_id, host_ip_port}; const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); std::unique_lock lifecycle_lock(lifecycle_fence->mutex); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } std::unique_lock lock(nodes_mutex_); + auto &host_map = instance_nodes_[instance_id]; + auto it = host_map.find(host_ip_port); ++node_generation_[instance_id][host_ip_port]; lifecycle_fence->generation = node_generation_[instance_id][host_ip_port]; lifecycle_fence->registered = true; - auto &host_map = instance_nodes_[instance_id]; - auto it = host_map.find(host_ip_port); int64_t now_ms = NowMillis(); if (it != host_map.end()) { auto &info = *it->second; @@ -225,6 +302,9 @@ ErrorCode EventReportBackend::EnsureNodeRegistered(const std::string &instance_i })) { return EC_BADARGS; } + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } auto merge_mediums = [&mediums](NodeInfo &info) { for (const auto &medium : mediums) { @@ -234,12 +314,38 @@ ErrorCode EventReportBackend::EnsureNodeRegistered(const std::string &instance_i } }; - // The common path only enriches an existing node. It must not wait for - // the lifecycle write lock: an in-flight metadata mutation intentionally - // holds a shared lifecycle lease, and new deltas still need to reach the - // bounded snapshot gate instead of blocking here indefinitely. + // Repeated reports normally carry a medium that is already known. Keep + // that path read-only so it does not serialize all reporters on the node + // table's exclusive lock. A missing medium is rechecked under the unique + // lock before it is merged (lock-based double check; the map itself must + // never be read without nodes_mutex_). + { + std::shared_lock lock(nodes_mutex_); + auto instance_it = instance_nodes_.find(instance_id); + if (instance_it != instance_nodes_.end()) { + auto node_it = instance_it->second.find(host_ip_port); + if (node_it != instance_it->second.end() && node_it->second) { + const auto &known_mediums = node_it->second->mediums; + const bool all_known = + std::all_of(mediums.begin(), mediums.end(), [&known_mediums](const auto &medium) { + return std::find(known_mediums.begin(), known_mediums.end(), medium) != known_mediums.end(); + }); + if (all_known) { + return EC_OK; + } + } + } + } + + // Enriching an existing node must not wait for the lifecycle write lock: + // an in-flight metadata mutation intentionally holds a shared lifecycle + // lease, and new deltas still need to reach the bounded snapshot gate + // instead of blocking here indefinitely. { std::unique_lock lock(nodes_mutex_); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } auto instance_it = instance_nodes_.find(instance_id); if (instance_it != instance_nodes_.end()) { auto node_it = instance_it->second.find(host_ip_port); @@ -253,6 +359,9 @@ ErrorCode EventReportBackend::EnsureNodeRegistered(const std::string &instance_i const ReporterSnapshotKey reporter_key{instance_id, host_ip_port}; const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); std::unique_lock lifecycle_lock(lifecycle_fence->mutex); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } std::unique_lock lock(nodes_mutex_); auto &host_map = instance_nodes_[instance_id]; if (auto it = host_map.find(host_ip_port); it != host_map.end()) { @@ -294,6 +403,9 @@ ErrorCode EventReportBackend::UnregisterNode(const std::string &instance_id, con const ReporterSnapshotKey reporter_key{instance_id, host_ip_port}; const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); std::unique_lock lifecycle_lock(lifecycle_fence->mutex); + if (Retired()) { + return EC_INSTANCE_NOT_EXIST; + } std::unique_lock lock(nodes_mutex_); const ErrorCode ec = UnregisterNodeLocked(instance_id, host_ip_port); lifecycle_fence->generation = node_generation_[instance_id][host_ip_port]; @@ -307,6 +419,9 @@ ErrorCode EventReportBackend::UnregisterNodeForHostDown(const std::string &insta const ReporterSnapshotKey reporter_key{instance_id, host_ip_port}; const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); std::unique_lock lifecycle_lock(lifecycle_fence->mutex); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } std::unique_lock lock(nodes_mutex_); out_generation = node_generation_[instance_id][host_ip_port]; lifecycle_fence->generation = out_generation; @@ -326,6 +441,9 @@ ErrorCode EventReportBackend::UnregisterNodeIfGeneration(const std::string &inst const ReporterSnapshotKey reporter_key{instance_id, host_ip_port}; const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); std::unique_lock lifecycle_lock(lifecycle_fence->mutex); + if (Retired()) { + return EC_INSTANCE_NOT_EXIST; + } std::unique_lock lock(nodes_mutex_); const auto generation_it = node_generation_.find(instance_id); uint64_t current_generation = 0; @@ -394,6 +512,9 @@ ErrorCode EventReportBackend::OnHeartbeat(const std::string &instance_id, const ReporterSnapshotKey reporter_key{instance_id, host_ip_port}; const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); std::unique_lock lifecycle_lock(lifecycle_fence->mutex); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } std::unique_lock lock(nodes_mutex_); auto &host_map = instance_nodes_[instance_id]; auto it = host_map.find(host_ip_port); @@ -442,25 +563,46 @@ ErrorCode EventReportBackend::OnHeartbeat(const std::string &instance_id, } lifecycle_fence->generation = node_generation_[instance_id][host_ip_port]; lifecycle_fence->registered = true; - { - std::lock_guard status_lock(info.status_mutex); - info.last_system_status = system_status; - } + std::unique_lock status_lock(info.status_mutex); + const std::map previous_system_status = info.last_system_status; + info.last_system_status = system_status; const auto metrics_tags = info.metrics_tags; - // Keep the node lifecycle lock until the gauges are published. - // UnregisterNodeLocked removes the same tagged gauges while holding this - // lock; releasing it earlier would allow HOST_DOWN to remove the node and - // an older heartbeat to recreate ghost metrics afterwards. + // The per-reporter lifecycle writer keeps NodeInfo alive and prevents + // HOST_DOWN/REGISTER from crossing gauge publication. The status lock + // serializes SetNodeUnavailable's gauge reset. Release the global node + // table lock so metric work for one reporter does not block all others. + lock.unlock(); if (metrics_registry_) { - auto prefix = "event_report."; - for (const auto &kv : system_status) { - const auto &s = kv.second; - if (s.empty()) - continue; + const auto parse_gauge = [](const std::string &value, double &out) { + if (value.empty()) { + return false; + } char *end = nullptr; - double val = std::strtod(s.c_str(), &end); - if (end == s.c_str() + s.size()) { + out = std::strtod(value.c_str(), &end); + return end == value.c_str() + value.size() && std::isfinite(out); + }; + const std::string prefix = "event_report."; + // system_status is a full heartbeat snapshot, not a patch. Remove a + // prior numeric gauge when the next heartbeat omits it or changes it + // to a non-numeric value; otherwise stale values would survive and a + // later unregister could no longer discover the omitted key. + for (const auto &[name, previous_value] : previous_system_status) { + double ignored_previous = 0.0; + if (!parse_gauge(previous_value, ignored_previous)) { + continue; + } + const auto current_it = system_status.find(name); + double ignored_current = 0.0; + if (current_it == system_status.end() || !parse_gauge(current_it->second, ignored_current)) { + if (auto data = metrics_registry_->GetMetricsData(prefix + name)) { + data->RemoveByTags(metrics_tags); + } + } + } + for (const auto &kv : system_status) { + double val = 0.0; + if (parse_gauge(kv.second, val)) { REPORT_DYNAMIC_GAUGE_(metrics_registry_, prefix + kv.first, metrics_tags, val); } } @@ -594,14 +736,6 @@ void EventReportBackend::LivenessCheckerLoop() { cb_copy = cleanup_callback_; } for (const auto &entry : to_cleanup) { - KVCM_LOG_WARN("EventReportBackend: node [%s] instance [%s] passed cleanup_grace_ms, " - "triggering cleanup (gen=%" PRIu64 ")", - entry.host.c_str(), - entry.instance_id.c_str(), - entry.gen); - if (cb_copy) { - cb_copy(entry.instance_id, entry.host, entry.gen); - } const ErrorCode unregister_ec = UnregisterNodeIfGeneration(entry.instance_id, entry.host, entry.gen); if (unregister_ec == EC_MISMATCH) { const uint64_t current_gen = GetNodeGeneration(entry.instance_id, entry.host); @@ -610,11 +744,38 @@ void EventReportBackend::LivenessCheckerLoop() { entry.host.c_str(), entry.gen, current_gen); + continue; + } + if (unregister_ec != EC_OK) { + KVCM_LOG_WARN("EventReportBackend: failed to unregister expired node [%s] instance [%s], " + "ec=%d (gen=%" PRIu64 ")", + entry.host.c_str(), + entry.instance_id.c_str(), + unregister_ec, + entry.gen); + continue; + } + + // Unregister is the liveness-expiry linearization point. It + // must happen before cleanup is dispatched: otherwise cleanup + // can delete metadata while a heartbeat waits behind its + // lifecycle lease, then let that heartbeat revive the old + // committed snapshot without requiring reconciliation. + KVCM_LOG_WARN("EventReportBackend: node [%s] instance [%s] passed cleanup_grace_ms, " + "unregistered and triggering cleanup (gen=%" PRIu64 ")", + entry.host.c_str(), + entry.instance_id.c_str(), + entry.gen); + if (cb_copy) { + cb_copy(entry.instance_id, entry.host, entry.gen); } } } - std::this_thread::sleep_for(std::chrono::milliseconds(liveness_check_interval_ms_)); + std::unique_lock wait_lock(liveness_wait_mutex_); + liveness_wait_cv_.wait_for(wait_lock, std::chrono::milliseconds(liveness_check_interval_ms_), [this] { + return !liveness_checker_running_.load(std::memory_order_acquire) || !IsOpen(); + }); } } @@ -639,6 +800,9 @@ std::vector EventReportBackend::Exist(const std::vector &s } std::vector EventReportBackend::MightExist(const std::vector &storage_uris) { + if (!AcceptingReports()) { + return std::vector(storage_uris.size(), false); + } std::vector result; result.reserve(storage_uris.size()); std::shared_lock lock(nodes_mutex_); @@ -696,9 +860,33 @@ std::string EventReportBackend::BuildLocationId(const std::string &medium, const bool EventReportBackend::ParseLocationId(const std::string &location_id, std::string &out_medium, std::string &out_host_ip_port) const { - std::string storage_type; - return SnapshotUriUtils::ParseEventReportLocationId(location_id, storage_type, out_medium, out_host_ip_port) && - storage_type == ToString(config_.type()); + std::string_view medium; + std::string_view host_ip_port; + if (!ParseLocationIdView(location_id, medium, host_ip_port)) { + out_medium.clear(); + out_host_ip_port.clear(); + return false; + } + out_medium.assign(medium.data(), medium.size()); + out_host_ip_port.assign(host_ip_port.data(), host_ip_port.size()); + return true; +} + +bool EventReportBackend::ParseLocationIdView(std::string_view location_id, + std::string_view &out_medium, + std::string_view &out_host_ip_port) const noexcept { + std::string_view storage_type; + if (!SnapshotUriUtils::ParseEventReportLocationIdView(location_id, storage_type, out_medium, out_host_ip_port)) { + return false; + } + switch (config_.type()) { + case DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5: + return storage_type == "event_report_l1p5"; + case DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2: + return storage_type == "event_report_l2"; + default: + return false; + } } std::string EventReportBackend::HostSuffix(const std::string &host_ip_port) const { return "#" + host_ip_port; } @@ -718,15 +906,24 @@ ErrorCode EventReportBackend::BeginDeltaMutation(const ReporterSnapshotKey &repo return EC_BADARGS; } std::unique_lock lock(nodes_mutex_); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } const int64_t snapshot_wait_timeout_ms = snapshot_delta_drain_timeout_ms_; const bool snapshot_finished = snapshot_state_cv_.wait_for(lock, std::chrono::milliseconds(snapshot_wait_timeout_ms), [&] { auto it = snapshot_versions_.find(reporter_key); - return it == snapshot_versions_.end() || it->second.in_flight.empty(); + return !AcceptingReports() || it == snapshot_versions_.end() || it->second.in_flight.empty(); }); if (!snapshot_finished) { return EC_SNAPSHOT_IN_PROGRESS; } + // Close()/dynamic disable can wake this waiter by clearing the snapshot + // state. Admission was checked before wait_for() released nodes_mutex_, + // so check again before creating or incrementing any mutation state. + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } auto state_it = snapshot_versions_.find(reporter_key); if (state_it == snapshot_versions_.end() || state_it->second.committed.empty()) { const auto instance_it = instance_nodes_.find(reporter_key.instance_id); @@ -767,9 +964,22 @@ ErrorCode EventReportBackend::BeginDeltaMutation(const ReporterSnapshotKey &repo return EC_OK; } -void EventReportBackend::EndDeltaMutation(const ReporterSnapshotKey &reporter_key, uint64_t lifecycle_generation) { +void EventReportBackend::EndDeltaMutation(const ReporterSnapshotKey &reporter_key, + uint64_t lifecycle_generation, + const std::string &expected_snapshot_version) { std::unique_lock lock(nodes_mutex_); auto it = snapshot_versions_.find(reporter_key); + if (it != snapshot_versions_.end() && !expected_snapshot_version.empty() && + it->second.committed != expected_snapshot_version) { + // HOST_DOWN removes snapshot state before an already-admitted delta + // necessarily reaches its final metadata lease. If the reporter is + // then registered again, a new delta can recreate state at the same + // key. The old guard must not drain that newer lifecycle's admission. + KVCM_LOG_DEBUG("EventReportBackend: ignoring stale delta mutation lease for instance [%s] host [%s]", + reporter_key.instance_id.c_str(), + reporter_key.host_ip_port.c_str()); + return; + } if (it == snapshot_versions_.end() || it->second.active_delta_mutations == 0) { const auto generation_it = node_generation_.find(reporter_key.instance_id); const auto node_it = instance_nodes_.find(reporter_key.instance_id); @@ -809,12 +1019,23 @@ ErrorCode EventReportBackend::BeginSnapshot(const ReporterSnapshotKey &reporter_ if (reporter_key.instance_id.empty() || reporter_key.host_ip_port.empty()) { return EC_BADARGS; } + // The in-flight token/attempt epoch is the snapshot-cleanup fence. Change + // it while holding the reporter lifecycle writer so an older cleanup can + // either acquire its read lease first and finish, or observe the newer + // epoch after this transition; it can never delete across the boundary. + const auto lifecycle_fence = GetOrCreateLifecycleFence(reporter_key); + std::unique_lock lifecycle_lock(lifecycle_fence->mutex); std::unique_lock lock(nodes_mutex_); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } const auto instance_it = instance_nodes_.find(reporter_key.instance_id); if (instance_it == instance_nodes_.end() || - instance_it->second.find(reporter_key.host_ip_port) == instance_it->second.end()) { + instance_it->second.find(reporter_key.host_ip_port) == instance_it->second.end() || + !lifecycle_fence->registered) { return EC_SNAPSHOT_REQUIRED; } + const uint64_t admitted_lifecycle_generation = lifecycle_fence->generation; auto &state = snapshot_versions_[reporter_key]; if (!state.in_flight.empty()) { return EC_SNAPSHOT_IN_PROGRESS; @@ -842,13 +1063,32 @@ ErrorCode EventReportBackend::BeginSnapshot(const ReporterSnapshotKey &reporter_ // deltas. Deltas arriving from this point wait until commit or abort. ++state.attempt_epoch; state.in_flight = out_candidate_version; + // Do not retain the lifecycle writer while draining already-admitted + // deltas: their final metadata phase needs a lifecycle read lease before + // it can end its delta admission and wake this waiter. + lifecycle_lock.unlock(); const int64_t delta_drain_timeout_ms = snapshot_delta_drain_timeout_ms_; const bool deltas_drained = snapshot_state_cv_.wait_for(lock, std::chrono::milliseconds(delta_drain_timeout_ms), [&] { auto it = snapshot_versions_.find(reporter_key); - return it == snapshot_versions_.end() || it->second.in_flight != out_candidate_version || - it->second.active_delta_mutations == 0; + return !AcceptingReports() || it == snapshot_versions_.end() || + it->second.in_flight != out_candidate_version || it->second.active_delta_mutations == 0; }); + // The wait releases nodes_mutex_. If the backend was retired or disabled + // meanwhile, do not return a usable candidate. Close() has already + // cleared the state; for a dynamic disable, reopen the reporter write gate + // explicitly so a later re-enable is not stuck behind this abandoned + // candidate. + if (!AcceptingReports()) { + auto it = snapshot_versions_.find(reporter_key); + if (it != snapshot_versions_.end() && it->second.in_flight == out_candidate_version) { + it->second.in_flight.clear(); + } + out_candidate_version.clear(); + lock.unlock(); + snapshot_state_cv_.notify_all(); + return EC_INSTANCE_NOT_EXIST; + } if (!deltas_drained) { auto it = snapshot_versions_.find(reporter_key); const uint64_t active_delta_mutations = it == snapshot_versions_.end() ? 0 : it->second.active_delta_mutations; @@ -873,7 +1113,7 @@ ErrorCode EventReportBackend::BeginSnapshot(const ReporterSnapshotKey &reporter_ return EC_SNAPSHOT_REQUIRED; } if (out_lifecycle_generation) { - *out_lifecycle_generation = node_generation_[reporter_key.instance_id][reporter_key.host_ip_port]; + *out_lifecycle_generation = admitted_lifecycle_generation; } return EC_OK; } @@ -893,6 +1133,9 @@ ErrorCode EventReportBackend::AcquireLifecycleMutationLease(const ReporterSnapsh // (lifecycle -> metadata), so blocking here would deadlock. return EC_NODE_NOT_REGISTERED; } + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } if (!lifecycle_fence->registered || lifecycle_fence->generation != expected_generation) { return EC_NODE_NOT_REGISTERED; } @@ -900,6 +1143,28 @@ ErrorCode EventReportBackend::AcquireLifecycleMutationLease(const ReporterSnapsh return EC_OK; } +ErrorCode EventReportBackend::CommitSnapshotVersionIfGeneration(const ReporterSnapshotKey &reporter_key, + const std::string &version, + uint64_t expected_generation) { + const auto lifecycle_fence = FindLifecycleFence(reporter_key); + if (!lifecycle_fence) { + return EC_NODE_NOT_REGISTERED; + } + // Commit runs after the metadata RMW has released its shard locks, so it + // can safely wait for a transient HEARTBEAT writer. Using the mutation + // path's try-lock here would turn harmless lock contention into a failed + // snapshot. REGISTER/HOST_DOWN still serialize first and are rejected by + // the generation/registered check below. + std::shared_lock lifecycle_lease(lifecycle_fence->mutex); + if (!AcceptingReports()) { + return EC_INSTANCE_NOT_EXIST; + } + if (!lifecycle_fence->registered || lifecycle_fence->generation != expected_generation) { + return EC_NODE_NOT_REGISTERED; + } + return CommitSnapshotVersion(reporter_key, version) ? EC_OK : EC_ERROR; +} + ErrorCode EventReportBackend::AcquireLifecycleCleanupLease(const ReporterSnapshotKey &reporter_key, uint64_t expected_generation, LifecycleMutationLease &out_lease) const { @@ -909,7 +1174,44 @@ ErrorCode EventReportBackend::AcquireLifecycleCleanupLease(const ReporterSnapsho return EC_MISMATCH; } auto lease = std::make_shared>(lifecycle_fence->mutex); - if (lifecycle_fence->generation != expected_generation) { + // Host cleanup is only valid after HOST_DOWN/liveness has atomically + // unregistered this exact reporter generation. Checking `registered` + // under the retained lifecycle lease prevents an accidental cleanup caller + // from deleting metadata that still belongs to an active reporter, while + // the generation check fences a concurrent re-registration. + if (!AcceptingReports() || lifecycle_fence->registered || lifecycle_fence->generation != expected_generation) { + return EC_MISMATCH; + } + out_lease = std::move(lease); + return EC_OK; +} + +ErrorCode EventReportBackend::AcquireSnapshotCleanupLease(const ReporterSnapshotKey &reporter_key, + uint64_t expected_generation, + const std::string &expected_snapshot_version, + uint64_t expected_attempt_epoch, + LifecycleMutationLease &out_lease) const { + out_lease.reset(); + const auto lifecycle_fence = FindLifecycleFence(reporter_key); + if (!lifecycle_fence) { + return EC_MISMATCH; + } + // Cleanup acquires this lease before taking metadata locks. It can block + // behind a short-lived HEARTBEAT/REGISTER writer without creating the + // lifecycle->metadata / metadata->lifecycle inversion that forces delta + // mutations to use try_lock. + auto lease = std::make_shared>(lifecycle_fence->mutex); + if (!AcceptingReports() || !lifecycle_fence->registered || lifecycle_fence->generation != expected_generation) { + return EC_MISMATCH; + } + + // BeginSnapshot publishes a new attempt epoch under the lifecycle writer + // before releasing it. Holding the read lease here therefore makes this + // validation atomic with respect to every later snapshot admission. + std::shared_lock lock(nodes_mutex_); + const auto state_it = snapshot_versions_.find(reporter_key); + if (state_it == snapshot_versions_.end() || state_it->second.committed != expected_snapshot_version || + (expected_attempt_epoch != 0 && state_it->second.attempt_epoch != expected_attempt_epoch)) { return EC_MISMATCH; } out_lease = std::move(lease); @@ -981,6 +1283,9 @@ bool EventReportBackend::GetQueryVisibilityState(const ReporterSnapshotKey &repo std::string &out_committed) const { out_strict = false; out_committed.clear(); + if (!AcceptingReports()) { + return false; + } std::shared_lock lock(nodes_mutex_); const auto instance_it = instance_nodes_.find(reporter_key.instance_id); if (instance_it == instance_nodes_.end()) { @@ -999,6 +1304,31 @@ bool EventReportBackend::GetQueryVisibilityState(const ReporterSnapshotKey &repo return true; } +void EventReportBackend::GetQueryVisibilitySnapshot(const std::string &instance_id, + QueryVisibilitySnapshot &out_snapshot) const { + out_snapshot.clear(); + if (!AcceptingReports()) { + return; + } + std::shared_lock lock(nodes_mutex_); + const auto instance_it = instance_nodes_.find(instance_id); + if (instance_it == instance_nodes_.end()) { + return; + } + for (const auto &[host_ip_port, node] : instance_it->second) { + if (!node || !node->available.load(std::memory_order_relaxed)) { + continue; + } + QueryVisibilityState state; + const auto version_it = snapshot_versions_.find({instance_id, host_ip_port}); + if (version_it != snapshot_versions_.end()) { + state.strict = version_it->second.strict_query_visibility; + state.committed_version = version_it->second.committed; + } + out_snapshot.emplace(host_ip_port, std::move(state)); + } +} + uint64_t EventReportBackend::GetSnapshotAttemptEpoch(const ReporterSnapshotKey &reporter_key) const { std::shared_lock lock(nodes_mutex_); const auto it = snapshot_versions_.find(reporter_key); diff --git a/kv_cache_manager/data_storage/event_report_backend.h b/kv_cache_manager/data_storage/event_report_backend.h index a4c8a4eda..317360582 100644 --- a/kv_cache_manager/data_storage/event_report_backend.h +++ b/kv_cache_manager/data_storage/event_report_backend.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,14 @@ class EventReportBackend : public DataStorageBackend { public: using CleanupCallback = std::function; + struct QueryVisibilityState { + bool strict = false; + std::string committed_version; + }; + // Transparent ordering lets GetHostCacheState probe the request snapshot + // with a string_view borrowed from CacheLocation::id without allocating a + // reporter-host string for every block. + using QueryVisibilitySnapshot = std::map>; EventReportBackend() = delete; explicit EventReportBackend(std::shared_ptr metrics_registry); @@ -33,7 +42,12 @@ class EventReportBackend : public DataStorageBackend { // --- DataStorageBackend interface --- DataStorageType GetType() override; bool Available() override; + // Dynamic disable is also an admission cancellation event. Wake snapshot + // and delta waiters so they fail immediately instead of waiting for the + // configured drain timeout before observing the unavailable state. + void SetAvailable(bool available) override; double GetStorageUsageRatio(const std::string &trace_id) const override; + ErrorCode Open(const StorageConfig &config, const std::string &trace_id) override; ErrorCode DoOpen(const StorageConfig &config, const std::string &trace_id) override; ErrorCode Close() override; @@ -84,6 +98,9 @@ class EventReportBackend : public DataStorageBackend { std::string BuildLocationId(const std::string &medium, const std::string &host_ip_port) const; bool ParseLocationId(const std::string &location_id, std::string &out_medium, std::string &out_host_ip_port) const; + bool ParseLocationIdView(std::string_view location_id, + std::string_view &out_medium, + std::string_view &out_host_ip_port) const noexcept; std::string HostSuffix(const std::string &host_ip_port) const; // A delta lease pins the committed token until every metadata mutation in // that ReportEvent request has completed. If this reporter is replacing @@ -93,7 +110,9 @@ class EventReportBackend : public DataStorageBackend { std::string &out_committed_version, uint64_t *out_lifecycle_generation = nullptr, bool *out_created_generation = nullptr); - void EndDeltaMutation(const ReporterSnapshotKey &reporter_key, uint64_t lifecycle_generation = 0); + void EndDeltaMutation(const ReporterSnapshotKey &reporter_key, + uint64_t lifecycle_generation = 0, + const std::string &expected_snapshot_version = {}); ErrorCode BeginSnapshot(const ReporterSnapshotKey &reporter_key, std::string &out_candidate_version, uint64_t &out_retry_after_ms, @@ -105,6 +124,19 @@ class EventReportBackend : public DataStorageBackend { ErrorCode AcquireLifecycleCleanupLease(const ReporterSnapshotKey &reporter_key, uint64_t expected_generation, LifecycleMutationLease &out_lease) const; + // Atomically fences a stale-snapshot cleanup against both reporter + // lifecycle changes and the start of a later snapshot attempt. The lease + // must be retained through the metadata compare-and-delete. + ErrorCode AcquireSnapshotCleanupLease(const ReporterSnapshotKey &reporter_key, + uint64_t expected_generation, + const std::string &expected_snapshot_version, + uint64_t expected_attempt_epoch, + LifecycleMutationLease &out_lease) const; + // Retains a lifecycle read lease through candidate validation and commit, + // so REGISTER/HOST_DOWN cannot cross the final publication boundary. + ErrorCode CommitSnapshotVersionIfGeneration(const ReporterSnapshotKey &reporter_key, + const std::string &version, + uint64_t expected_generation); bool CommitSnapshotVersion(const ReporterSnapshotKey &reporter_key, const std::string &version); void AbortSnapshotVersion(const ReporterSnapshotKey &reporter_key, const std::string &version); std::string GetSnapshotVersion(const ReporterSnapshotKey &reporter_key) const; @@ -119,12 +151,23 @@ class EventReportBackend : public DataStorageBackend { bool GetQueryVisibilityState(const ReporterSnapshotKey &reporter_key, bool &out_strict, std::string &out_committed) const; + // Captures every currently queryable reporter for one instance under one + // node-table read lock. GetHostCacheState uses this request-level snapshot + // instead of repeating registry/backend lookups and nodes_mutex_ acquisition + // for every (block, location). + void GetQueryVisibilitySnapshot(const std::string &instance_id, QueryVisibilitySnapshot &out_snapshot) const; uint64_t GetSnapshotAttemptEpoch(const ReporterSnapshotKey &reporter_key) const; void SetSnapshotMinIntervalMsForTest(int64_t interval_ms); void SetSnapshotDeltaDrainTimeoutMsForTest(int64_t timeout_ms); DataStorageType GetStorageType() const; private: + // Unit-level state-machine tests also exercise a never-opened backend. + // Treat that state as usable, while permanently fencing operations after + // Close and respecting DataStorageManager disablement for opened storage. + bool AcceptingReports() const { return !retired_.load(std::memory_order_acquire) && (!IsOpen() || IsAvailable()); } + bool Retired() const { return retired_.load(std::memory_order_acquire); } + struct LifecycleFence { mutable std::shared_mutex mutex; uint64_t generation = 0; @@ -198,6 +241,9 @@ class EventReportBackend : public DataStorageBackend { std::thread liveness_checker_thread_; std::atomic liveness_checker_running_{false}; + std::atomic retired_{false}; + std::mutex liveness_wait_mutex_; + std::condition_variable liveness_wait_cv_; int64_t heartbeat_timeout_ms_ = EventReportStorageSpec::kDefaultHeartbeatTimeoutMs; int64_t cleanup_grace_ms_ = EventReportStorageSpec::kDefaultCleanupGraceMs; diff --git a/kv_cache_manager/data_storage/snapshot_uri_utils.h b/kv_cache_manager/data_storage/snapshot_uri_utils.h index 7f005565c..440918b93 100644 --- a/kv_cache_manager/data_storage/snapshot_uri_utils.h +++ b/kv_cache_manager/data_storage/snapshot_uri_utils.h @@ -1,8 +1,12 @@ #pragma once -#include +#include +#include #include +#include #include +#include +#include #include "kv_cache_manager/data_storage/data_storage_uri.h" @@ -34,6 +38,12 @@ struct SnapshotUriInfo { std::string version; }; +struct CanonicalSnapshotUriAppendInfo { + size_t insertion_offset = 0; + std::uint64_t size = 0; + bool has_query = false; +}; + class SnapshotUriUtils { public: inline static constexpr const char *kSnapshotVersionParam = "s_version"; @@ -72,7 +82,99 @@ class SnapshotUriUtils { return false; } for (const unsigned char ch : version) { - if (!std::isxdigit(ch)) { + if (!IsAsciiHexDigit(ch)) { + return false; + } + } + return true; + } + + // GetHostCacheState only needs to know whether an EventReport URI is + // structurally usable and which snapshot generation it belongs to. A full + // DataStorageUri parse would copy every URI component and allocate one + // std::map node per query parameter for every (key, spec) visited. Scan the + // immutable URI in place instead. An empty out_version means legacy + // metadata without s_version; malformed or duplicate s_version parameters + // fail closed. + // + // The returned view borrows uri_text and must not outlive it. + static bool InspectSnapshotUriForVisibility(std::string_view uri_text, + std::string_view &out_version, + bool uri_structure_prevalidated = false) noexcept { + out_version = {}; + + // Match the observable validity conditions used by StandardUri + // without rebuilding its strings and parameter map. In addition to a + // non-empty protocol, a textual port must be a non-negative int64. + // Host/path/query contents otherwise remain intentionally permissive, + // just like StandardUri::Parse. + const size_t protocol_end = uri_text.find("://"); + if (protocol_end == std::string_view::npos || protocol_end == 0) { + return false; + } + + const size_t authority_begin = protocol_end + 3; + const size_t query_begin = uri_text.find('?'); + if (query_begin != std::string_view::npos && query_begin < authority_begin) { + return false; + } + if (!uri_structure_prevalidated) { + const size_t path_begin = uri_text.find('/', authority_begin); + const size_t authority_end = + std::min(path_begin == std::string_view::npos ? uri_text.size() : path_begin, + query_begin == std::string_view::npos ? uri_text.size() : query_begin); + size_t host_begin = authority_begin; + const size_t user_info_end = uri_text.find('@', authority_begin); + if (user_info_end != std::string_view::npos && user_info_end < authority_end) { + host_begin = user_info_end + 1; + } + const size_t port_separator = uri_text.find(':', host_begin); + if (port_separator != std::string_view::npos && port_separator < authority_end) { + std::uint64_t port = 0; + if (!ParseDecimalUint64(uri_text.substr(port_separator + 1, authority_end - port_separator - 1), + static_cast(std::numeric_limits::max()), + port)) { + return false; + } + } + } + + if (query_begin == std::string_view::npos) { + return true; + } + + bool found_version = false; + size_t begin = query_begin + 1; + while (begin <= uri_text.size()) { + size_t end = uri_text.find('&', begin); + if (end == std::string_view::npos) { + end = uri_text.size(); + } + const size_t equals = uri_text.find('=', begin); + const size_t key_end = equals != std::string_view::npos && equals < end ? equals : end; + constexpr std::string_view version_key{kSnapshotVersionParam}; + if (key_end - begin == version_key.size() && + uri_text.compare(begin, version_key.size(), version_key) == 0) { + if (found_version || equals == std::string_view::npos || equals >= end) { + return false; + } + found_version = true; + out_version = uri_text.substr(equals + 1, end - equals - 1); + } + if (end == uri_text.size()) { + break; + } + begin = end + 1; + } + + if (!found_version) { + return true; + } + if (out_version.size() != 32) { + return false; + } + for (const unsigned char ch : out_version) { + if (!IsAsciiHexDigit(ch)) { return false; } } @@ -83,6 +185,132 @@ class SnapshotUriUtils { return uri.HasParam(kSnapshotVersionParam); } + // Allocation-free fast parser for an already canonical URI. It accepts + // exactly the textual form StandardUri::ToUriString() produces: a valid + // positive canonical port, explicit `key=value` query entries, and unique + // keys in strict lexical order. Noncanonical-but-valid input returns false + // so callers can use the full StandardUri fallback without changing wire + // compatibility. The returned offset inserts s_version in sorted order. + static bool ParseCanonicalUriForSnapshotAppend(std::string_view uri, CanonicalSnapshotUriAppendInfo &out) noexcept { + out = {}; + const size_t protocol_end = uri.find("://"); + if (protocol_end == std::string_view::npos || protocol_end == 0) { + return false; + } + const size_t authority_start = protocol_end + 3; + const size_t path_start = uri.find('/', authority_start); + const size_t query_start = uri.find('?', authority_start); + if (path_start != std::string_view::npos && query_start != std::string_view::npos && query_start < path_start) { + return false; + } + const size_t host_end = std::min(path_start == std::string_view::npos ? uri.size() : path_start, + query_start == std::string_view::npos ? uri.size() : query_start); + size_t host_start = authority_start; + const size_t user_info_end = uri.find('@', authority_start); + if (user_info_end != std::string_view::npos && user_info_end < host_end) { + // StandardUri omits an empty user-info when serializing, so this + // raw spelling is valid but not canonical. + if (user_info_end == authority_start) { + return false; + } + host_start = user_info_end + 1; + } + const size_t port_separator = uri.find(':', host_start); + if (port_separator != std::string_view::npos && port_separator < host_end) { + const std::string_view port_text = uri.substr(port_separator + 1, host_end - port_separator - 1); + std::uint64_t port = 0; + if (port_text.empty() || port_text.front() == '0' || + !ParseDecimalUint64( + port_text, static_cast(std::numeric_limits::max()), port)) { + return false; + } + } + + out.insertion_offset = uri.size(); + if (query_start == std::string_view::npos) { + return true; + } + out.has_query = true; + size_t param_start = query_start + 1; + if (param_start == uri.size()) { + return false; + } + std::string_view previous_key; + bool has_previous_key = false; + while (param_start < uri.size()) { + size_t param_end = uri.find('&', param_start); + if (param_end == std::string_view::npos) { + param_end = uri.size(); + } + const size_t equals = uri.find('=', param_start); + if (equals == std::string_view::npos || equals >= param_end) { + return false; + } + const std::string_view key = uri.substr(param_start, equals - param_start); + const std::string_view value = uri.substr(equals + 1, param_end - equals - 1); + if ((has_previous_key && !(previous_key < key)) || key == kSnapshotVersionParam) { + return false; + } + if (out.insertion_offset == uri.size() && std::string_view(kSnapshotVersionParam) < key) { + out.insertion_offset = param_start; + } + if (key == "size") { + std::uint64_t parsed_size = 0; + if (ParseDecimalUint64(value, std::numeric_limits::max(), parsed_size)) { + out.size = parsed_size; + } + } + previous_key = key; + has_previous_key = true; + if (param_end == uri.size()) { + break; + } + param_start = param_end + 1; + if (param_start == uri.size()) { + return false; + } + } + return true; + } + + static bool AddSnapshotVersionToCanonicalUri(std::string_view uri, + const CanonicalSnapshotUriAppendInfo &info, + const std::string &version, + std::string &out_uri) { + out_uri.clear(); + if (!IsValidSnapshotVersionToken(version)) { + return false; + } + return AddPrevalidatedSnapshotVersionToCanonicalUri(uri, info, version, out_uri); + } + + // ReportEvent obtains one KVCM-generated, already validated generation + // token and appends it to tens of thousands of independently validated + // specs. Keep the public checked helper above for general callers; this + // variant avoids rescanning the same 32-byte token for every block. + static bool AddPrevalidatedSnapshotVersionToCanonicalUri(std::string_view uri, + const CanonicalSnapshotUriAppendInfo &info, + const std::string &version, + std::string &out_uri) { + out_uri.clear(); + if (info.insertion_offset > uri.size()) { + return false; + } + out_uri.reserve(uri.size() + std::char_traits::length(kSnapshotVersionParam) + version.size() + 2); + if (info.insertion_offset == uri.size()) { + out_uri.assign(uri.data(), uri.size()); + out_uri.push_back(info.has_query ? '&' : '?'); + out_uri.append(kSnapshotVersionParam).push_back('='); + out_uri.append(version); + return true; + } + out_uri.assign(uri.data(), info.insertion_offset); + out_uri.append(kSnapshotVersionParam).push_back('='); + out_uri.append(version).push_back('&'); + out_uri.append(uri.data() + info.insertion_offset, uri.size() - info.insertion_offset); + return true; + } + // DataStorageUri stores query parameters in a map, so duplicate keys from // the original text are no longer observable here. Raw metadata and // protocol input must use the string overload below, which enforces that @@ -115,20 +343,32 @@ class SnapshotUriUtils { CountUriParam(raw_uri, kSnapshotVersionParam) != 0) { return false; } - uri.SetParam(kSnapshotVersionParam, version); - out_uri = uri.ToUriString(); + return AddSnapshotVersionToUri(std::move(uri), version, out_uri); + } + + // ReportEvent validates and parses every URI before it acquires the + // snapshot/delta fence. Reusing that parsed value avoids parsing the same + // URI again merely to append KVCM's internal generation token. + static bool AddSnapshotVersionToUri(DataStorageUri uri, const std::string &version, std::string &out_uri) { + out_uri.clear(); + if (!uri.Valid() || !IsValidSnapshotVersionToken(version) || HasEventReportInternalUriMetadata(uri)) { + return false; + } + out_uri = uri.ToUriStringWithExtraParam(kSnapshotVersionParam, version); return !out_uri.empty(); } - static bool ParseEventReportLocationId(const std::string &location_id, - std::string &out_storage_type, - std::string &out_medium, - std::string &out_host_ip_port) { - out_storage_type.clear(); - out_medium.clear(); - out_host_ip_port.clear(); - constexpr const char *root_prefix = "kvs#"; - constexpr size_t root_prefix_size = 4; + // Zero-copy parser for the per-block EventReport location id hot path. + // Returned views borrow location_id and must not outlive it. + static bool ParseEventReportLocationIdView(std::string_view location_id, + std::string_view &out_storage_type, + std::string_view &out_medium, + std::string_view &out_host_ip_port) noexcept { + out_storage_type = {}; + out_medium = {}; + out_host_ip_port = {}; + constexpr std::string_view root_prefix{"kvs#"}; + constexpr size_t root_prefix_size = root_prefix.size(); if (location_id.size() <= root_prefix_size || location_id.compare(0, root_prefix_size, root_prefix) != 0) { return false; } @@ -136,7 +376,7 @@ class SnapshotUriUtils { if (type_end == std::string::npos || type_end == root_prefix_size) { return false; } - const std::string storage_type = location_id.substr(root_prefix_size, type_end - root_prefix_size); + const std::string_view storage_type = location_id.substr(root_prefix_size, type_end - root_prefix_size); if (storage_type != "event_report_l1p5" && storage_type != "event_report_l2") { return false; } @@ -145,8 +385,8 @@ class SnapshotUriUtils { if (separator == std::string::npos || separator == medium_begin || separator + 1 >= location_id.size()) { return false; } - const std::string medium = location_id.substr(medium_begin, separator - medium_begin); - const std::string host_ip_port = location_id.substr(separator + 1); + const std::string_view medium = location_id.substr(medium_begin, separator - medium_begin); + const std::string_view host_ip_port = location_id.substr(separator + 1); if (host_ip_port.empty() || host_ip_port.find('#') != std::string::npos) { return false; } @@ -156,6 +396,25 @@ class SnapshotUriUtils { return true; } + static bool ParseEventReportLocationId(const std::string &location_id, + std::string &out_storage_type, + std::string &out_medium, + std::string &out_host_ip_port) { + std::string_view storage_type; + std::string_view medium; + std::string_view host_ip_port; + if (!ParseEventReportLocationIdView(location_id, storage_type, medium, host_ip_port)) { + out_storage_type.clear(); + out_medium.clear(); + out_host_ip_port.clear(); + return false; + } + out_storage_type.assign(storage_type.data(), storage_type.size()); + out_medium.assign(medium.data(), medium.size()); + out_host_ip_port.assign(host_ip_port.data(), host_ip_port.size()); + return true; + } + static bool ParseEventReportLocationId(const std::string &location_id, std::string &out_medium, std::string &out_host_ip_port) { std::string storage_type; @@ -163,6 +422,29 @@ class SnapshotUriUtils { } private: + static bool ParseDecimalUint64(std::string_view text, std::uint64_t limit, std::uint64_t &out) noexcept { + if (text.empty()) { + return false; + } + std::uint64_t value = 0; + for (const unsigned char ch : text) { + if (ch < '0' || ch > '9') { + return false; + } + const std::uint64_t digit = ch - '0'; + if (value > (limit - digit) / 10) { + return false; + } + value = value * 10 + digit; + } + out = value; + return true; + } + + static constexpr bool IsAsciiHexDigit(unsigned char ch) noexcept { + return (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); + } + SnapshotUriUtils() = delete; }; diff --git a/kv_cache_manager/data_storage/test/BUILD b/kv_cache_manager/data_storage/test/BUILD index 3432f48d4..1d00360ba 100644 --- a/kv_cache_manager/data_storage/test/BUILD +++ b/kv_cache_manager/data_storage/test/BUILD @@ -78,3 +78,13 @@ cc_test( ], ) +cc_test( + name = "SnapshotUriUtilsTest", + srcs = [ + "snapshot_uri_utils_test.cc", + ], + deps = [ + "//kv_cache_manager/data_storage:snapshot_uri_utils", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/kv_cache_manager/data_storage/test/event_report_backend_test.cc b/kv_cache_manager/data_storage/test/event_report_backend_test.cc index 68957422b..95378ad5b 100644 --- a/kv_cache_manager/data_storage/test/event_report_backend_test.cc +++ b/kv_cache_manager/data_storage/test/event_report_backend_test.cc @@ -127,17 +127,32 @@ TEST_F(EventReportBackendTest, OpenStartsLivenessLoopAndCloseStops) { ASSERT_TRUE(backend.liveness_checker_running_.load()); ASSERT_TRUE(backend.liveness_checker_thread_.joinable()); - backend.SetAvailable(false); + DataStorageBackend &backend_base = backend; + backend_base.SetAvailable(false); ASSERT_FALSE(backend.Available()); - backend.SetAvailable(true); + backend_base.SetAvailable(true); ASSERT_TRUE(backend.Available()); + EXPECT_NE(EC_OK, backend.Open(MakeConfig(), "duplicate_open")); + EXPECT_TRUE(backend.Available()); ASSERT_EQ(EC_OK, backend.Close()); ASSERT_FALSE(backend.Available()); + EXPECT_NE(EC_OK, backend.Open(MakeConfig(), "reopen_retired_backend")); + EXPECT_FALSE(backend.Available()); ASSERT_FALSE(backend.liveness_checker_running_.load()); ASSERT_EQ(EC_OK, backend.Close()); } +TEST_F(EventReportBackendTest, CloseInterruptsLongLivenessWait) { + EventReportBackend backend(metrics_registry_); + ASSERT_EQ(EC_OK, backend.Open(MakeConfig(/*hb*/ 5000, /*grace*/ 10000, /*tick*/ 60000), "trace")); + + const auto close_begin = std::chrono::steady_clock::now(); + ASSERT_EQ(EC_OK, backend.Close()); + const auto close_elapsed = std::chrono::steady_clock::now() - close_begin; + EXPECT_LT(close_elapsed, 2s); +} + // (2) RegisterNode / UnregisterNode TEST_F(EventReportBackendTest, RegisterNodeWithMediums) { EventReportBackend backend(metrics_registry_); @@ -217,6 +232,12 @@ TEST_F(EventReportBackendTest, MightExistTracksRegisteredNodeAvailability) { ASSERT_EQ(1u, result.size()); EXPECT_TRUE(result[0]); + backend.SetAvailable(false); + EXPECT_EQ((std::vector{false}), backend.MightExist({available_uri})); + EXPECT_EQ(EC_INSTANCE_NOT_EXIST, backend.EnsureNodeRegistered(instance_id, available_host, {"mem"})); + backend.SetAvailable(true); + EXPECT_EQ((std::vector{true}), backend.MightExist({available_uri})); + ASSERT_EQ(EC_OK, backend.Close()); } @@ -303,8 +324,8 @@ TEST_F(EventReportBackendTest, LivenessLoopHealthyToUnavailableToCleanup) { std::atomic cleanup_calls{0}; std::string cleanup_host; backend.SetCleanupCallback([&](const std::string & /*instance_id*/, const std::string &host, uint64_t /*gen*/) { - ++cleanup_calls; cleanup_host = host; + cleanup_calls.fetch_add(1, std::memory_order_release); }); ASSERT_EQ(EC_OK, backend.RegisterNode("test_inst", "10.0.0.4:8080", {"mem"})); @@ -319,12 +340,13 @@ TEST_F(EventReportBackendTest, LivenessLoopHealthyToUnavailableToCleanup) { EXPECT_EQ(cleanup_calls.load(), 0); const auto cleanup_deadline = std::chrono::steady_clock::now() + 1s; - while (backend.IsNodeRegistered("test_inst", "10.0.0.4:8080") && + while ((backend.IsNodeRegistered("test_inst", "10.0.0.4:8080") || + cleanup_calls.load(std::memory_order_acquire) == 0) && std::chrono::steady_clock::now() < cleanup_deadline) { std::this_thread::yield(); } ASSERT_FALSE(backend.IsNodeRegistered("test_inst", "10.0.0.4:8080")); - EXPECT_GE(cleanup_calls.load(), 1); + EXPECT_GE(cleanup_calls.load(std::memory_order_acquire), 1); EXPECT_EQ(cleanup_host, "10.0.0.4:8080"); ASSERT_EQ(EC_OK, backend.Close()); @@ -351,7 +373,7 @@ TEST_F(EventReportBackendTest, HeartbeatWithinGraceWindowRecovers) { ASSERT_EQ(EC_OK, backend.Close()); } -TEST_F(EventReportBackendTest, HeartbeatRecoveryFencesCleanupAlreadySelectedByLivenessLoop) { +TEST_F(EventReportBackendTest, LivenessUnregistersBeforeCleanupAndHeartbeatCannotReviveOldSnapshot) { EventReportBackend backend(metrics_registry_); ASSERT_EQ(EC_OK, backend.Open(MakeConfig(/*hb*/ 100, /*grace*/ 50, /*tick*/ 5), "trace")); backend.SetSnapshotMinIntervalMsForTest(0); @@ -388,19 +410,22 @@ TEST_F(EventReportBackendTest, HeartbeatRecoveryFencesCleanupAlreadySelectedByLi EXPECT_FALSE(backend.IsNodeAvailable(instance_id, host)); EXPECT_EQ((std::vector{false}), backend.MightExist({DataStorageUri(uri)})); - // The callback has selected the old generation but has not started - // deleting. A successful heartbeat must invalidate that cleanup before - // it is allowed to continue. - EXPECT_EQ(EC_OK, backend.OnHeartbeat(instance_id, host, {})); - EXPECT_GT(backend.GetNodeGeneration(instance_id, host), initial_generation); - EXPECT_TRUE(backend.IsNodeAvailable(instance_id, host)); - EXPECT_EQ((std::vector{true}), backend.MightExist({DataStorageUri(uri)})); + // Expiry is linearized before the callback can delete metadata. A heartbeat + // arriving while cleanup is running must observe the tombstone instead of + // reviving a committed version whose metadata may already be gone. + EXPECT_FALSE(backend.IsNodeRegistered(instance_id, host)); + EXPECT_EQ(initial_generation, backend.GetNodeGeneration(instance_id, host)); + EXPECT_TRUE(backend.GetSnapshotVersion(reporter_key).empty()); + EXPECT_EQ(EC_NODE_NOT_REGISTERED, backend.OnHeartbeat(instance_id, host, {})); release_cleanup.set_value(); ASSERT_EQ(std::future_status::ready, cleanup_returned.get_future().wait_for(1s)); - EXPECT_TRUE(backend.IsNodeRegistered(instance_id, host)); - EXPECT_TRUE(backend.IsNodeAvailable(instance_id, host)); - EXPECT_EQ((std::vector{true}), backend.MightExist({DataStorageUri(uri)})); + EXPECT_FALSE(backend.IsNodeRegistered(instance_id, host)); + EXPECT_EQ((std::vector{false}), backend.MightExist({DataStorageUri(uri)})); + + ASSERT_EQ(EC_OK, backend.RegisterNode(instance_id, host, {"mem"})); + EXPECT_GT(backend.GetNodeGeneration(instance_id, host), initial_generation); + EXPECT_TRUE(backend.GetSnapshotVersion(reporter_key).empty()); ASSERT_EQ(EC_OK, backend.Close()); } @@ -453,6 +478,12 @@ TEST_F(EventReportBackendTest, CleanupLeaseFencesReregisterThroughFinalDeleteSta const uint64_t cleanup_generation = backend.GetNodeGeneration(reporter_key.instance_id, reporter_key.host_ip_port); EventReportBackend::LifecycleMutationLease cleanup_lease; + EXPECT_EQ(EC_MISMATCH, backend.AcquireLifecycleCleanupLease(reporter_key, cleanup_generation, cleanup_lease)); + uint64_t unregistered_generation = 0; + ASSERT_EQ(EC_OK, + backend.UnregisterNodeForHostDown( + reporter_key.instance_id, reporter_key.host_ip_port, unregistered_generation)); + ASSERT_EQ(cleanup_generation, unregistered_generation); ASSERT_EQ(EC_OK, backend.AcquireLifecycleCleanupLease(reporter_key, cleanup_generation, cleanup_lease)); auto reregister = std::async(std::launch::async, [&] { return backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"disk"}); @@ -474,6 +505,11 @@ TEST_F(EventReportBackendTest, LifecycleCleanupLeaseDoesNotBlockUnrelatedReporte ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_b.instance_id, reporter_b.host_ip_port, {"mem"})); const uint64_t generation_a = backend.GetNodeGeneration(reporter_a.instance_id, reporter_a.host_ip_port); + uint64_t unregistered_generation_a = 0; + ASSERT_EQ( + EC_OK, + backend.UnregisterNodeForHostDown(reporter_a.instance_id, reporter_a.host_ip_port, unregistered_generation_a)); + ASSERT_EQ(generation_a, unregistered_generation_a); EventReportBackend::LifecycleMutationLease cleanup_lease_a; ASSERT_EQ(EC_OK, backend.AcquireLifecycleCleanupLease(reporter_a, generation_a, cleanup_lease_a)); @@ -497,6 +533,35 @@ TEST_F(EventReportBackendTest, EnsureNodeRegisteredMergesNewMediums) { ASSERT_EQ(2u, backend.instance_nodes_["medium-merge"]["10.0.0.91:8080"]->mediums.size()); } +TEST_F(EventReportBackendTest, EnsureNodeRegisteredHandlesConcurrentKnownAndNewMediums) { + EventReportBackend backend(metrics_registry_); + const std::string instance_id = "medium-concurrency"; + const std::string host = "10.0.0.95:8080"; + ASSERT_EQ(EC_OK, backend.EnsureNodeRegistered(instance_id, host, {"mem"})); + const uint64_t generation = backend.GetNodeGeneration(instance_id, host); + + std::atomic failures{0}; + std::vector workers; + for (size_t worker = 0; worker < 12; ++worker) { + workers.emplace_back([&, worker] { + const std::string medium = worker % 2 == 0 ? "mem" : "disk"; + for (size_t iteration = 0; iteration < 200; ++iteration) { + if (backend.EnsureNodeRegistered(instance_id, host, {medium}) != EC_OK) { + failures.fetch_add(1, std::memory_order_relaxed); + } + } + }); + } + for (auto &worker : workers) { + worker.join(); + } + + EXPECT_EQ(0u, failures.load(std::memory_order_relaxed)); + EXPECT_EQ(generation, backend.GetNodeGeneration(instance_id, host)); + const auto &mediums = backend.instance_nodes_[instance_id][host]->mediums; + EXPECT_EQ((std::set{"disk", "mem"}), (std::set(mediums.begin(), mediums.end()))); +} + // (7) Re-registration after cleanup TEST_F(EventReportBackendTest, RegisterAfterCleanupCreatesNewEntry) { EventReportBackend backend(metrics_registry_); @@ -641,6 +706,19 @@ TEST_F(EventReportBackendTest, OnHeartbeatPublishesMetricsGauges) { auto new_gauge = new_data->GetOrCreateGauge(expected_tags); ASSERT_DOUBLE_EQ(42.0, new_gauge.Get()); ASSERT_DOUBLE_EQ(0.90, gauge.Get()); + EXPECT_FALSE(leases_data->GetGauge(expected_tags).has_value()); + + // A non-numeric replacement is also a full-snapshot removal rather than + // leaving the prior numeric sample visible forever. + ASSERT_EQ(EC_OK, backend.OnHeartbeat("test_inst", "10.0.0.10:9600", {{"hit_rate", "unknown"}})); + EXPECT_FALSE(hit_rate_data->GetGauge(expected_tags).has_value()); + + // strtod accepts these spellings, but non-finite values are not valid + // operational gauges and must not poison downstream metric aggregation. + ASSERT_EQ(EC_OK, + backend.OnHeartbeat("test_inst", "10.0.0.10:9600", {{"nan_metric", "nan"}, {"inf_metric", "inf"}})); + EXPECT_EQ(nullptr, metrics_registry_->GetMetricsData("event_report.nan_metric")); + EXPECT_EQ(nullptr, metrics_registry_->GetMetricsData("event_report.inf_metric")); ASSERT_EQ(EC_OK, backend.Close()); } @@ -871,6 +949,119 @@ TEST(EventReportBackendSnapshotTest, SnapshotCommitPublishesOpaqueToken) { backend.EndDeltaMutation(scope); } +TEST(EventReportBackendSnapshotTest, SnapshotCommitRejectsChangedLifecycleGeneration) { + EventReportBackend backend(nullptr); + backend.SetSnapshotMinIntervalMsForTest(0); + const ReporterSnapshotKey reporter_key{"instance-commit-fence", "10.0.0.71:8080"}; + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + + std::string candidate; + uint64_t retry_after_ms = 0; + uint64_t admitted_generation = 0; + ASSERT_EQ(EC_OK, backend.BeginSnapshot(reporter_key, candidate, retry_after_ms, &admitted_generation)); + ASSERT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken(candidate)); + ASSERT_NE(0u, admitted_generation); + + // An explicit REGISTER is a lifecycle boundary. A snapshot admitted by + // the previous lifecycle must not publish after that boundary even if its + // metadata phase already completed. + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + ASSERT_NE(admitted_generation, backend.GetNodeGeneration(reporter_key.instance_id, reporter_key.host_ip_port)); + EXPECT_EQ(EC_NODE_NOT_REGISTERED, + backend.CommitSnapshotVersionIfGeneration(reporter_key, candidate, admitted_generation)); + EXPECT_TRUE(backend.GetSnapshotVersion(reporter_key).empty()); + backend.AbortSnapshotVersion(reporter_key, candidate); +} + +TEST(EventReportBackendSnapshotTest, SnapshotCleanupLeaseFencesLaterAttemptAdmission) { + EventReportBackend backend(nullptr); + backend.SetSnapshotMinIntervalMsForTest(0); + const ReporterSnapshotKey reporter_key{"instance-cleanup-fence", "10.0.0.72:8080"}; + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + + std::string committed; + uint64_t retry_after_ms = 0; + ASSERT_EQ(EC_OK, backend.BeginSnapshot(reporter_key, committed, retry_after_ms)); + ASSERT_TRUE(backend.CommitSnapshotVersion(reporter_key, committed)); + const uint64_t cleanup_generation = backend.GetNodeGeneration(reporter_key.instance_id, reporter_key.host_ip_port); + const uint64_t cleanup_attempt_epoch = backend.GetSnapshotAttemptEpoch(reporter_key); + + EventReportBackend::LifecycleMutationLease cleanup_lease; + ASSERT_EQ(EC_OK, + backend.AcquireSnapshotCleanupLease( + reporter_key, cleanup_generation, committed, cleanup_attempt_epoch, cleanup_lease)); + + std::promise attempt_started; + std::string next_candidate; + auto next_attempt = std::async(std::launch::async, [&] { + attempt_started.set_value(); + uint64_t retry_ms = 0; + return backend.BeginSnapshot(reporter_key, next_candidate, retry_ms); + }); + attempt_started.get_future().wait(); + EXPECT_EQ(std::future_status::timeout, next_attempt.wait_for(20ms)); + + // Releasing the old cleanup's final-delete lease lets the next attempt + // publish its epoch. The same cleanup identity must then be rejected even + // though the reporter lifecycle generation itself has not changed. + cleanup_lease.reset(); + ASSERT_EQ(std::future_status::ready, next_attempt.wait_for(1s)); + ASSERT_EQ(EC_OK, next_attempt.get()); + ASSERT_GT(backend.GetSnapshotAttemptEpoch(reporter_key), cleanup_attempt_epoch); + + EventReportBackend::LifecycleMutationLease stale_cleanup_lease; + EXPECT_EQ(EC_MISMATCH, + backend.AcquireSnapshotCleanupLease( + reporter_key, cleanup_generation, committed, cleanup_attempt_epoch, stale_cleanup_lease)); + EXPECT_FALSE(stale_cleanup_lease); + backend.AbortSnapshotVersion(reporter_key, next_candidate); +} + +TEST(EventReportBackendSnapshotTest, SnapshotCommitAndCleanupWaitForTransientLifecycleWriter) { + EventReportBackend backend(nullptr); + backend.SetSnapshotMinIntervalMsForTest(0); + const ReporterSnapshotKey reporter_key{"instance-transient-writer", "10.0.0.74:8080"}; + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + + std::string candidate; + uint64_t retry_after_ms = 0; + uint64_t lifecycle_generation = 0; + ASSERT_EQ(EC_OK, backend.BeginSnapshot(reporter_key, candidate, retry_after_ms, &lifecycle_generation)); + const auto lifecycle_fence = backend.GetOrCreateLifecycleFence(reporter_key); + ASSERT_TRUE(lifecycle_fence); + + { + std::unique_lock transient_writer(lifecycle_fence->mutex); + std::promise call_started; + auto commit = std::async(std::launch::async, [&] { + call_started.set_value(); + return backend.CommitSnapshotVersionIfGeneration(reporter_key, candidate, lifecycle_generation); + }); + call_started.get_future().wait(); + EXPECT_EQ(std::future_status::timeout, commit.wait_for(20ms)); + transient_writer.unlock(); + ASSERT_EQ(std::future_status::ready, commit.wait_for(1s)); + EXPECT_EQ(EC_OK, commit.get()); + } + + const uint64_t attempt_epoch = backend.GetSnapshotAttemptEpoch(reporter_key); + { + std::unique_lock transient_writer(lifecycle_fence->mutex); + std::promise call_started; + auto cleanup = std::async(std::launch::async, [&] { + call_started.set_value(); + EventReportBackend::LifecycleMutationLease lease; + return backend.AcquireSnapshotCleanupLease( + reporter_key, lifecycle_generation, candidate, attempt_epoch, lease); + }); + call_started.get_future().wait(); + EXPECT_EQ(std::future_status::timeout, cleanup.wait_for(20ms)); + transient_writer.unlock(); + ASSERT_EQ(std::future_status::ready, cleanup.wait_for(1s)); + EXPECT_EQ(EC_OK, cleanup.get()); + } +} + TEST(EventReportBackendSnapshotTest, QueryVisibilityIsStrictOnlyAfterSuccessfulSnapshot) { EventReportBackend backend(nullptr); backend.SetSnapshotMinIntervalMsForTest(0); @@ -912,6 +1103,44 @@ TEST(EventReportBackendSnapshotTest, QueryVisibilityIsStrictOnlyAfterSuccessfulS EXPECT_EQ(recovered, committed); } +TEST(EventReportBackendSnapshotTest, QueryVisibilitySnapshotIsInstanceScopedAndExcludesUnavailableReporters) { + EventReportBackend backend(nullptr); + backend.SetSnapshotMinIntervalMsForTest(0); + const ReporterSnapshotKey soft_reporter{"instance-a", "10.0.0.1:8080"}; + const ReporterSnapshotKey strict_reporter{"instance-a", "10.0.0.2:8080"}; + const ReporterSnapshotKey other_instance{"instance-b", "10.0.0.3:8080"}; + ASSERT_EQ(EC_OK, backend.RegisterNode(soft_reporter.instance_id, soft_reporter.host_ip_port, {"mem"})); + ASSERT_EQ(EC_OK, backend.RegisterNode(strict_reporter.instance_id, strict_reporter.host_ip_port, {"mem"})); + ASSERT_EQ(EC_OK, backend.RegisterNode(other_instance.instance_id, other_instance.host_ip_port, {"mem"})); + + std::string soft_version; + ASSERT_EQ(EC_OK, backend.BeginDeltaMutation(soft_reporter, soft_version)); + backend.EndDeltaMutation(soft_reporter); + std::string strict_version; + uint64_t retry_after_ms = 0; + ASSERT_EQ(EC_OK, backend.BeginSnapshot(strict_reporter, strict_version, retry_after_ms)); + ASSERT_TRUE(backend.CommitSnapshotVersion(strict_reporter, strict_version)); + + EventReportBackend::QueryVisibilitySnapshot snapshot; + backend.GetQueryVisibilitySnapshot("instance-a", snapshot); + ASSERT_EQ(2u, snapshot.size()); + EXPECT_FALSE(snapshot.at(soft_reporter.host_ip_port).strict); + EXPECT_EQ(soft_version, snapshot.at(soft_reporter.host_ip_port).committed_version); + EXPECT_TRUE(snapshot.at(strict_reporter.host_ip_port).strict); + EXPECT_EQ(strict_version, snapshot.at(strict_reporter.host_ip_port).committed_version); + EXPECT_EQ(0u, snapshot.count(other_instance.host_ip_port)); + + backend.SetNodeUnavailable(soft_reporter.instance_id, soft_reporter.host_ip_port); + backend.GetQueryVisibilitySnapshot("instance-a", snapshot); + ASSERT_EQ(1u, snapshot.size()); + EXPECT_EQ(0u, snapshot.count(soft_reporter.host_ip_port)); + EXPECT_EQ(1u, snapshot.count(strict_reporter.host_ip_port)); + + ASSERT_EQ(EC_OK, backend.UnregisterNode(strict_reporter.instance_id, strict_reporter.host_ip_port)); + backend.GetQueryVisibilitySnapshot("instance-a", snapshot); + EXPECT_TRUE(snapshot.empty()); +} + TEST(EventReportBackendSnapshotTest, SnapshotTokensAreNeverReusedAcrossAttempts) { EventReportBackend backend(nullptr); backend.SetSnapshotMinIntervalMsForTest(0); @@ -1509,6 +1738,32 @@ TEST(EventReportBackendSnapshotTest, UnregisterThenReregisterLetsFirstDeltaCreat EXPECT_TRUE(backend.CommitSnapshotVersion(scope, second_token)); } +TEST(EventReportBackendSnapshotTest, StaleDeltaEndCannotDrainReregisteredLifecycle) { + EventReportBackend backend(nullptr); + const ReporterSnapshotKey reporter_key{"delta-incarnation", "10.0.0.73:8080"}; + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + + std::string old_token; + uint64_t old_generation = 0; + ASSERT_EQ(EC_OK, backend.BeginDeltaMutation(reporter_key, old_token, &old_generation)); + ASSERT_EQ(1u, backend.snapshot_versions_[reporter_key].active_delta_mutations); + + ASSERT_EQ(EC_OK, backend.UnregisterNode(reporter_key.instance_id, reporter_key.host_ip_port)); + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + std::string new_token; + uint64_t new_generation = 0; + ASSERT_EQ(EC_OK, backend.BeginDeltaMutation(reporter_key, new_token, &new_generation)); + ASSERT_NE(old_token, new_token); + ASSERT_NE(old_generation, new_generation); + ASSERT_EQ(1u, backend.snapshot_versions_[reporter_key].active_delta_mutations); + + backend.EndDeltaMutation(reporter_key, old_generation, old_token); + EXPECT_EQ(1u, backend.snapshot_versions_[reporter_key].active_delta_mutations); + + backend.EndDeltaMutation(reporter_key, new_generation, new_token); + EXPECT_EQ(0u, backend.snapshot_versions_[reporter_key].active_delta_mutations); +} + TEST(EventReportBackendSnapshotTest, StableLocationIdHasNoSnapshotGeneration) { EventReportBackend backend(nullptr); ASSERT_EQ(EC_OK, backend.Open(EventReportBackendTest::MakeConfig(), "snapshot_location_test")); @@ -1854,7 +2109,7 @@ TEST(EventReportBackendSnapshotTest, CloseUnblocksSnapshotAndDeltaWaiters) { ASSERT_EQ(std::future_status::timeout, waiting_snapshot.wait_for(0ms)); ASSERT_EQ(EC_OK, backend.Close()); ASSERT_EQ(std::future_status::ready, waiting_snapshot.wait_for(1s)); - EXPECT_EQ(EC_SNAPSHOT_REQUIRED, waiting_snapshot.get()); + EXPECT_EQ(EC_INSTANCE_NOT_EXIST, waiting_snapshot.get()); backend.EndDeltaMutation(reporter_key); } @@ -1881,10 +2136,61 @@ TEST(EventReportBackendSnapshotTest, CloseUnblocksSnapshotAndDeltaWaiters) { ASSERT_EQ(std::future_status::timeout, waiting_delta.wait_for(20ms)); ASSERT_EQ(EC_OK, backend.Close()); ASSERT_EQ(std::future_status::ready, waiting_delta.wait_for(1s)); - EXPECT_EQ(EC_SNAPSHOT_REQUIRED, waiting_delta.get()); + EXPECT_EQ(EC_INSTANCE_NOT_EXIST, waiting_delta.get()); } } +TEST_F(EventReportBackendTest, DisableWhileSnapshotDrainsAbortsCandidateAndReopensGate) { + EventReportBackend backend(metrics_registry_); + ASSERT_EQ(EC_OK, + backend.Open(MakeConfig(/*hb*/ 5000, + /*grace*/ 10000, + /*tick*/ 60000, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + /*snapshot_min_interval*/ 0), + "disable_while_snapshot_drains")); + backend.SetSnapshotMinIntervalMsForTest(0); + const ReporterSnapshotKey reporter_key{"instance-disable", "10.0.0.72:8080"}; + ASSERT_EQ(EC_OK, backend.RegisterNode(reporter_key.instance_id, reporter_key.host_ip_port, {"mem"})); + std::string first; + uint64_t retry_after_ms = 0; + ASSERT_EQ(EC_OK, BeginSnapshotForRegisteredReporter(backend, reporter_key, first, retry_after_ms)); + ASSERT_TRUE(backend.CommitSnapshotVersion(reporter_key, first)); + std::string committed; + ASSERT_EQ(EC_OK, backend.BeginDeltaMutation(reporter_key, committed)); + + std::string candidate; + auto waiting_snapshot = std::async(std::launch::async, [&] { + uint64_t retry_ms = 0; + return backend.BeginSnapshot(reporter_key, candidate, retry_ms); + }); + std::string observed_committed; + std::string observed_in_flight; + const auto in_flight_deadline = std::chrono::steady_clock::now() + 1s; + do { + backend.GetSnapshotVersionTokens(reporter_key, observed_committed, observed_in_flight); + if (!observed_in_flight.empty()) { + break; + } + std::this_thread::yield(); + } while (std::chrono::steady_clock::now() < in_flight_deadline); + ASSERT_FALSE(observed_in_flight.empty()); + + DataStorageBackend &backend_base = backend; + backend_base.SetAvailable(false); + ASSERT_EQ(std::future_status::ready, waiting_snapshot.wait_for(1s)); + EXPECT_EQ(EC_INSTANCE_NOT_EXIST, waiting_snapshot.get()); + EXPECT_TRUE(candidate.empty()); + + backend_base.SetAvailable(true); + backend.EndDeltaMutation(reporter_key); + std::string after_reenable; + ASSERT_EQ(EC_OK, backend.BeginDeltaMutation(reporter_key, after_reenable)); + EXPECT_EQ(first, after_reenable); + backend.EndDeltaMutation(reporter_key); + ASSERT_EQ(EC_OK, backend.Close()); +} + TEST(EventReportBackendSnapshotTest, SnapshotUriUtilitiesHandleExactParameterBoundaries) { const std::string token = "00112233445566778899aabbccddeeff"; const std::string raw_uri = "event_report://10.0.0.1:8080/mem?size=7&user_s_version=kept&s_version_hint=kept"; diff --git a/kv_cache_manager/data_storage/test/snapshot_uri_utils_test.cc b/kv_cache_manager/data_storage/test/snapshot_uri_utils_test.cc new file mode 100644 index 000000000..501cf9cdd --- /dev/null +++ b/kv_cache_manager/data_storage/test/snapshot_uri_utils_test.cc @@ -0,0 +1,223 @@ +#include +#include +#include +#include +#include + +#include "kv_cache_manager/data_storage/snapshot_uri_utils.h" + +namespace kv_cache_manager { + +class SnapshotUriUtilsTest : public ::testing::Test {}; + +TEST_F(SnapshotUriUtilsTest, SnapshotVersionTokenUsesStrictAsciiHex) { + EXPECT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken("0123456789abcdefABCDEF0123456789")); + EXPECT_FALSE(SnapshotUriUtils::IsValidSnapshotVersionToken("0123456789abcdefABCDEF012345678g")); + EXPECT_FALSE( + SnapshotUriUtils::IsValidSnapshotVersionToken(std::string("0123456789abcdefABCDEF01234567") + "\xc3\xa9")); + EXPECT_FALSE(SnapshotUriUtils::IsValidSnapshotVersionToken("0123456789abcdefABCDEF012345678")); +} + +TEST_F(SnapshotUriUtilsTest, InspectSnapshotUriForVisibilityReturnsBorrowedVersion) { + constexpr std::string_view token = "0123456789abcdefABCDEF0123456789"; + const std::string uri = "event_report://10.0.0.1:8080/mem?rank=0&s_version=" + std::string(token) + "&size=4096"; + + std::string_view version; + ASSERT_TRUE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version)); + EXPECT_EQ(token, version); + EXPECT_EQ(uri.data() + uri.find(token), version.data()); +} + +TEST_F(SnapshotUriUtilsTest, InspectSnapshotUriForVisibilityAcceptsLegacyAndUnrelatedParams) { + const std::vector uris = { + "event_report://10.0.0.1:8080/mem", + "event_report://host:0/mem", + "event_report://host:08080/mem", + "event_report://host:9223372036854775807/mem", + "event_report://user:secret@host:8080/mem", + "event_report://host/path:with:colon?callback=http://peer:9000/path", + "event_report://10.0.0.1:8080/mem?rank=0&size=4096", + "event_report://10.0.0.1:8080/mem?xs_version=ignored&s_version_suffix=ignored", + "event_report://10.0.0.1:8080/mem?&rank=0&&", + }; + + for (const auto &uri : uris) { + std::string_view version = "must be cleared"; + EXPECT_TRUE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version)) << uri; + EXPECT_TRUE(version.empty()) << uri; + version = "must be cleared"; + EXPECT_TRUE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version, true)) << uri; + EXPECT_TRUE(version.empty()) << uri; + } +} + +TEST_F(SnapshotUriUtilsTest, InspectSnapshotUriForVisibilityRejectsMalformedUri) { + const std::vector uris = { + "", + "event_report:/10.0.0.1/mem", + "://10.0.0.1/mem", + "event?bad_report://10.0.0.1/mem", + "event_report://host:/mem", + "event_report://host:-0/mem", + "event_report://host:-1/mem", + "event_report://host:+1/mem", + "event_report://host:not-a-port/mem", + "event_report://host:9223372036854775808/mem", + "event_report://user:secret@host:not-a-port/mem", + }; + + for (const auto &uri : uris) { + std::string_view version = "must be cleared"; + EXPECT_FALSE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version)) << uri; + EXPECT_TRUE(version.empty()) << uri; + } +} + +TEST_F(SnapshotUriUtilsTest, InspectSnapshotUriForVisibilityRejectsInvalidOrDuplicateVersion) { + constexpr const char *token = "0123456789abcdef0123456789abcdef"; + const std::vector uris = { + "event_report://host/mem?s_version", + "event_report://host/mem?s_version=", + "event_report://host/mem?s_version=0123456789abcdef0123456789abcde", + "event_report://host/mem?s_version=0123456789abcdef0123456789abcdef0", + "event_report://host/mem?s_version=0123456789abcdef0123456789abcdeg", + std::string("event_report://host/mem?s_version=") + token + "&s_version=" + token, + std::string("event_report://host/mem?s_version=") + token + "&s_version", + std::string("event_report://host/mem?s_version&rank=0&s_version=") + token, + }; + + for (const auto &uri : uris) { + std::string_view version = "must be cleared"; + EXPECT_FALSE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version)) << uri; + } +} + +TEST_F(SnapshotUriUtilsTest, InspectSnapshotUriForVisibilityAcceptsVersionAtParamBoundaries) { + constexpr const char *token = "abcdef0123456789abcdef0123456789"; + const std::vector uris = { + std::string("event_report://host/mem?s_version=") + token, + std::string("event_report://host/mem?&s_version=") + token, + std::string("event_report://host/mem?rank=0&s_version=") + token + "&", + std::string("event_report://host/mem?rank=0&s_version=") + token + "&size=1", + }; + + for (const auto &uri : uris) { + std::string_view version; + ASSERT_TRUE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version)) << uri; + EXPECT_EQ(token, version) << uri; + version = {}; + ASSERT_TRUE(SnapshotUriUtils::InspectSnapshotUriForVisibility(uri, version, true)) << uri; + EXPECT_EQ(token, version) << uri; + } +} + +TEST_F(SnapshotUriUtilsTest, PrevalidatedStructureStillRejectsProtocolQueryAndBadVersionMetadata) { + std::string_view version = "must be cleared"; + EXPECT_FALSE(SnapshotUriUtils::InspectSnapshotUriForVisibility("event?bad_report://host/mem", version, true)); + EXPECT_TRUE(version.empty()); + + EXPECT_FALSE(SnapshotUriUtils::InspectSnapshotUriForVisibility( + "event_report://host/mem?s_version=not-a-version", version, true)); +} + +TEST_F(SnapshotUriUtilsTest, AddSnapshotVersionRejectsInvalidPortWithoutCanonicalizingItAway) { + constexpr const char *token = "0123456789abcdef0123456789abcdef"; + for (const std::string &invalid_uri : { + "event_report://physical-cache:not-a-port/mem?size=1", + "event_report://physical-cache:-1/mem?size=1", + "event_report://physical-cache:9223372036854775808/mem?size=1", + }) { + std::string out_uri = "stale"; + EXPECT_FALSE(SnapshotUriUtils::AddSnapshotVersionToUri(invalid_uri, token, out_uri)); + EXPECT_TRUE(out_uri.empty()); + } +} + +TEST_F(SnapshotUriUtilsTest, CanonicalSnapshotAppendMatchesStandardUriWithoutAllocatingParseState) { + constexpr const char *token = "0123456789abcdef0123456789abcdef"; + const std::vector> cases = { + {"event_report://host/mem", 0}, + {"event_report://user@host:8080/mem?a=1&size=4096&z=last", 4096}, + {"event_report://host:9223372036854775807/mem?size=18446744073709551615", + std::numeric_limits::max()}, + {"event_report://host/mem?a=1&s=before&size=7", 7}, + {"event_report://host/mem?size=invalid&z=last", 0}, + {"event_report://host/mem?size=18446744073709551616&z=last", 0}, + {"scheme://host?empty=&rank=0", 0}, + }; + + for (const auto &[uri, expected_size] : cases) { + CanonicalSnapshotUriAppendInfo info; + ASSERT_TRUE(SnapshotUriUtils::ParseCanonicalUriForSnapshotAppend(uri, info)) << uri; + EXPECT_EQ(expected_size, info.size) << uri; + + std::string fast_uri; + ASSERT_TRUE(SnapshotUriUtils::AddSnapshotVersionToCanonicalUri(uri, info, token, fast_uri)) << uri; + std::string prevalidated_uri; + ASSERT_TRUE(SnapshotUriUtils::AddPrevalidatedSnapshotVersionToCanonicalUri(uri, info, token, prevalidated_uri)) + << uri; + EXPECT_EQ(fast_uri, prevalidated_uri) << uri; + DataStorageUri parsed(uri); + ASSERT_TRUE(parsed.Valid()) << uri; + std::string standard_uri; + ASSERT_TRUE(SnapshotUriUtils::AddSnapshotVersionToUri(std::move(parsed), token, standard_uri)) << uri; + EXPECT_EQ(standard_uri, fast_uri) << uri; + } +} + +TEST_F(SnapshotUriUtilsTest, CanonicalSnapshotAppendDefersNoncanonicalUrisToStandardParser) { + const std::vector noncanonical_uris = { + "event_report://host:08080/mem?size=1", + "event_report://host:0/mem?size=1", + "event_report://@host/mem?size=1", + "event_report://host/mem?", + "event_report://host/mem?size=1&", + "event_report://host/mem?size", + "event_report://host/mem?z=1&a=2", + "event_report://host/mem?a=1&a=2", + "event_report://host/mem?s_version=0123456789abcdef0123456789abcdef", + "event_report://host?query=before/path", + }; + for (const auto &uri : noncanonical_uris) { + CanonicalSnapshotUriAppendInfo info; + EXPECT_FALSE(SnapshotUriUtils::ParseCanonicalUriForSnapshotAppend(uri, info)) << uri; + } +} + +TEST_F(SnapshotUriUtilsTest, ParseEventReportLocationIdViewBorrowsComponents) { + const std::string location_id = "kvs#event_report_l2#hbm-cache#10.0.0.1:8080"; + std::string_view storage_type; + std::string_view medium; + std::string_view host; + ASSERT_TRUE(SnapshotUriUtils::ParseEventReportLocationIdView(location_id, storage_type, medium, host)); + EXPECT_EQ("event_report_l2", storage_type); + EXPECT_EQ("hbm-cache", medium); + EXPECT_EQ("10.0.0.1:8080", host); + EXPECT_EQ(location_id.data() + location_id.find(storage_type), storage_type.data()); + EXPECT_EQ(location_id.data() + location_id.find(medium), medium.data()); + EXPECT_EQ(location_id.data() + location_id.find(host), host.data()); +} + +TEST_F(SnapshotUriUtilsTest, ParseEventReportLocationIdViewRejectsMalformedValuesAndClearsOutputs) { + const std::vector location_ids = { + "", + "kvs#", + "bad#event_report_l2#mem#host:8080", + "kvs#unknown#mem#host:8080", + "kvs#event_report_l2##host:8080", + "kvs#event_report_l2#mem#", + "kvs#event_report_l2#mem#host:8080#extra", + }; + for (const auto &location_id : location_ids) { + std::string_view storage_type = "stale"; + std::string_view medium = "stale"; + std::string_view host = "stale"; + EXPECT_FALSE(SnapshotUriUtils::ParseEventReportLocationIdView(location_id, storage_type, medium, host)) + << location_id; + EXPECT_TRUE(storage_type.empty()) << location_id; + EXPECT_TRUE(medium.empty()) << location_id; + EXPECT_TRUE(host.empty()) << location_id; + } +} + +} // namespace kv_cache_manager diff --git a/kv_cache_manager/manager/cache_manager.cc b/kv_cache_manager/manager/cache_manager.cc index 1209f4220..1cb7cc897 100644 --- a/kv_cache_manager/manager/cache_manager.cc +++ b/kv_cache_manager/manager/cache_manager.cc @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -135,58 +137,54 @@ class DeltaMutationGuard { std::string snapshot_version; uint64_t lifecycle_generation = 0; }; - explicit DeltaMutationGuard(std::shared_ptr backend) : backend_(std::move(backend)) {} + DeltaMutationGuard(std::shared_ptr backend, ReporterSnapshotKey reporter_key) + : backend_(std::move(backend)), reporter_key_(std::move(reporter_key)) {} DeltaMutationGuard(const DeltaMutationGuard &) = delete; DeltaMutationGuard &operator=(const DeltaMutationGuard &) = delete; ~DeltaMutationGuard() { - for (const auto &entry : versions_) { - backend_->EndDeltaMutation(entry.first, entry.second.lifecycle_generation); + if (lease_) { + backend_->EndDeltaMutation(reporter_key_, lease_->lifecycle_generation, lease_->snapshot_version); } } - ErrorCode Acquire(const ReporterSnapshotKey &reporter_key, - std::string &out_committed_version, - uint64_t &out_lifecycle_generation, - bool &out_created_generation) { - const auto failure_it = snapshot_wait_failures_.find(reporter_key); - if (failure_it != snapshot_wait_failures_.end()) { - out_committed_version.clear(); - out_lifecycle_generation = 0; + ErrorCode Acquire(const LeaseInfo *&out_lease, bool &out_created_generation) { + out_lease = nullptr; + if (snapshot_wait_failure_) { out_created_generation = false; - return failure_it->second; + return *snapshot_wait_failure_; } - const auto it = versions_.find(reporter_key); - if (it != versions_.end()) { - out_committed_version = it->second.snapshot_version; - out_lifecycle_generation = it->second.lifecycle_generation; + if (lease_) { + out_lease = &*lease_; out_created_generation = false; return EC_OK; } + LeaseInfo lease; const ErrorCode ec = backend_->BeginDeltaMutation( - reporter_key, out_committed_version, &out_lifecycle_generation, &out_created_generation); + reporter_key_, lease.snapshot_version, &lease.lifecycle_generation, &out_created_generation); if (ec != EC_OK) { out_created_generation = false; if (ec == EC_SNAPSHOT_IN_PROGRESS) { - snapshot_wait_failures_.emplace(reporter_key, ec); + snapshot_wait_failure_ = ec; } return ec; } - versions_.emplace(reporter_key, LeaseInfo{out_committed_version, out_lifecycle_generation}); + lease_.emplace(std::move(lease)); + out_lease = &*lease_; return EC_OK; } - void AdoptLifecycleGeneration(const ReporterSnapshotKey &reporter_key, uint64_t lifecycle_generation) { - const auto it = versions_.find(reporter_key); - if (it != versions_.end()) { - it->second.lifecycle_generation = lifecycle_generation; + void AdoptLifecycleGeneration(uint64_t lifecycle_generation) { + if (lease_) { + lease_->lifecycle_generation = lifecycle_generation; } } private: std::shared_ptr backend_; - std::unordered_map versions_; - std::unordered_map snapshot_wait_failures_; + ReporterSnapshotKey reporter_key_; + std::optional lease_; + std::optional snapshot_wait_failure_; }; // 共享 helper：收集目标 storage 上指定 status 的 location 联合覆盖的 spec name 集合。 @@ -418,6 +416,7 @@ CacheManager::~CacheManager() { } ClearEventCleanupCallbacks(); StopRecoverRetryLoop(); + DeactivateEventCleanupCallbacks(); if (write_location_manager_) { write_location_manager_->Stop(); write_location_manager_.reset(); @@ -440,6 +439,9 @@ bool CacheManager::Init(int32_t schedule_plan_executor_thread_count, uint32_t cache_reclaimer_worker_size, CacheReclaimerAsyncDeleteConfig cache_reclaimer_async_delete_config, uint32_t schedule_plan_migration_worker_budget, + uint32_t meta_query_worker_count, + std::size_t meta_query_parallel_threshold, + std::size_t meta_query_chunk_size, CacheGarbageCollector::Config cache_gc_config) { if (schedule_plan_executor_thread_count <= 1 || schedule_plan_migration_worker_budget == 0 || schedule_plan_migration_worker_budget >= static_cast(schedule_plan_executor_thread_count)) { @@ -448,6 +450,16 @@ bool CacheManager::Init(int32_t schedule_plan_executor_thread_count, schedule_plan_migration_worker_budget); return false; } + if (meta_query_worker_count == 0 || meta_query_worker_count > 64 || meta_query_parallel_threshold == 0 || + meta_query_chunk_size == 0 || meta_query_chunk_size > meta_query_parallel_threshold || + !meta_indexer_manager_->ConfigureQueryExecutor( + meta_query_worker_count, meta_query_parallel_threshold, meta_query_chunk_size)) { + KVCM_LOG_ERROR("invalid meta query executor config: workers=%u threshold=%zu chunk_size=%zu", + meta_query_worker_count, + meta_query_parallel_threshold, + meta_query_chunk_size); + return false; + } schedule_plan_executor_ = std::make_shared(schedule_plan_executor_thread_count, meta_indexer_manager_, registry_manager_->data_storage_manager(), @@ -549,6 +561,9 @@ CacheManager::RegisterInstance(RequestContext *request_context, model_deployment, location_spec_groups, static_cast(default_query_type)); + if (instance_info->instance_group_name() != instance_group) { + mismatched.insert(mismatched.begin(), "instance_group_name"); + } if (!mismatched.empty()) { auto mismatched_str = StringUtil::Join(mismatched, ", "); request_context->error_tracer()->AddErrorMsg( @@ -893,6 +908,26 @@ CacheManager::GetCacheLocationsByBackend(RequestContext *request_context, query_keys = GenKeyVector(tokens, block_size); } + const bool has_implicit_empty_mask = + std::holds_alternative(block_mask) && std::get(block_mask).empty(); + if (!has_implicit_empty_mask && !IsBlockMaskValid(block_mask, query_keys.size())) { + request_context->error_tracer()->AddErrorMsg("block_mask must match the number of query keys"); + RETURN_IF_EC_NOT_OK_WITH_TYPE_LOG( + WARN, EC_BADARGS, BatchLocationsView, "block_mask must match the number of query keys"); + } + + if (!location_spec_names.empty()) { + if (location_spec_names.size() != query_keys.size() || + std::any_of(location_spec_names.begin(), location_spec_names.end(), [](const std::string &name) { + return name.empty(); + })) { + request_context->error_tracer()->AddErrorMsg( + "location_spec_names must be empty or contain one non-empty name per query key"); + RETURN_IF_EC_NOT_OK_WITH_TYPE_LOG( + WARN, EC_BADARGS, BatchLocationsView, "invalid per-key location_spec_names"); + } + } + auto query_scope = KVCM_METRICS_COLLECTOR_CHRONO_SCOPE(service_metrics_collector, ManagerBatchGet); KVCM_METRICS_COLLECTOR_SET_METRICS(service_metrics_collector, manager, request_key_count, query_keys.size()); @@ -900,10 +935,44 @@ CacheManager::GetCacheLocationsByBackend(RequestContext *request_context, request_context->error_tracer()->AddErrorMsg("backend_selectors must not be empty"); RETURN_IF_EC_NOT_OK_WITH_TYPE_LOG(WARN, EC_BADARGS, BatchLocationsView, "backend_selectors must not be empty"); } + std::unordered_set selected_backend_types; + for (const auto &selector : backend_selectors) { + const auto backend_index = ToIndex(selector.backend_type); + if (selector.backend_type == DataStorageType::DATA_STORAGE_TYPE_UNKNOWN || + backend_index >= ToIndex(DataStorageType::COUNT)) { + request_context->error_tracer()->AddErrorMsg("backend selector has invalid backend_type"); + RETURN_IF_EC_NOT_OK_WITH_TYPE_LOG( + WARN, EC_BADARGS, BatchLocationsView, "backend selector has invalid backend_type"); + } + if (!selected_backend_types.insert(selector.backend_type).second) { + request_context->error_tracer()->AddErrorMsg("backend selector contains duplicate backend_type"); + RETURN_IF_EC_NOT_OK_WITH_TYPE_LOG( + WARN, EC_BADARGS, BatchLocationsView, "backend selector contains duplicate backend_type"); + } + switch (selector.strategy) { + case LocationSelectStrategy::LSS_WEIGHTED_RANDOM: + break; + case LocationSelectStrategy::LSS_V6D_PREFIX: + case LocationSelectStrategy::LSS_V6D_COVERAGE: + if (selector.backend_type == DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2) { + break; + } + [[fallthrough]]; + default: + request_context->error_tracer()->AddErrorMsg("backend selector has invalid strategy for backend_type"); + RETURN_IF_EC_NOT_OK_WITH_TYPE_LOG( + WARN, EC_BADARGS, BatchLocationsView, "backend selector has invalid strategy for backend_type"); + } + } LocationsPerKey locations_per_key; - ec = meta_searcher->BatchGetBestLocationByBackend( - request_context, query_keys, locations_per_key, policy.get(), backend_selectors); + ec = meta_searcher->BatchGetBestLocationByBackend(request_context, + query_keys, + locations_per_key, + policy.get(), + backend_selectors, + location_spec_names, + block_mask); query_scope = ChronoScopeGuard{}; // prefix_match_len: count keys with at least one hit (non-empty id). // Miss keys have empty CacheLocation objects with no id. @@ -929,8 +998,10 @@ CacheManager::GetCacheLocationsByBackend(RequestContext *request_context, for (auto &key_locs : locations_per_key) { FillEmptyLocationSpecs(instance_info->location_spec_infos(), key_locs); } - for (auto &key_locs : locations_per_key) { - FilterLocationSpecByName(key_locs, location_spec_names); + if (!location_spec_names.empty()) { + for (size_t i = 0; i < locations_per_key.size(); ++i) { + FilterLocationSpecByName(locations_per_key[i], {location_spec_names[i]}); + } } auto cache_get_event = std::make_shared(instance_id); @@ -1433,6 +1504,22 @@ ErrorCode CacheManager::TrimCache(RequestContext *request_context, void CacheManager::PauseReclaimer() { cache_reclaimer_->Pause(); } void CacheManager::ResumeReclaimer() { cache_reclaimer_->Resume(); } +ErrorCode CacheManager::StartCacheGarbageCollector() { + return cache_garbage_collector_ ? cache_garbage_collector_->Start() : EC_ERROR; +} + +void CacheManager::RequestStopCacheGarbageCollector() { + if (cache_garbage_collector_) { + cache_garbage_collector_->RequestStop(); + } +} + +void CacheManager::JoinCacheGarbageCollector() { + if (cache_garbage_collector_) { + cache_garbage_collector_->Join(); + } +} + void CacheManager::StartMigrationManager() { migration_manager_->Start(); } void CacheManager::StopMigrationManager() { migration_manager_->Stop(); } @@ -1531,22 +1618,6 @@ CacheManager::MigrateCacheResult CacheManager::MigrateCache(RequestContext *requ return result; } -ErrorCode CacheManager::StartCacheGarbageCollector() { - return cache_garbage_collector_ ? cache_garbage_collector_->Start() : EC_ERROR; -} - -void CacheManager::RequestStopCacheGarbageCollector() { - if (cache_garbage_collector_) { - cache_garbage_collector_->RequestStop(); - } -} - -void CacheManager::JoinCacheGarbageCollector() { - if (cache_garbage_collector_) { - cache_garbage_collector_->Join(); - } -} - void CacheManager::FilterLocationSpecByName(CacheLocationVector &locations, const std::vector &location_spec_names) { if (location_spec_names.empty()) { @@ -1568,6 +1639,7 @@ void CacheManager::FilterLocationSpecByName(CacheLocationVector &locations, // COW: copy, modify, replace auto new_loc = std::make_shared(*loc_ptr); new_loc->set_location_specs(std::move(new_specs)); + new_loc->set_spec_size(new_loc->location_specs().size()); loc_ptr = std::move(new_loc); } } @@ -2319,7 +2391,8 @@ namespace { std::shared_ptr LookupEventReportBackend(const std::shared_ptr ®istry_manager, const std::string &instance_id, - DataStorageType requested_type) { + DataStorageType requested_type, + bool require_available = false) { if (!registry_manager || !registry_manager->data_storage_manager()) { return nullptr; } @@ -2338,29 +2411,142 @@ std::shared_ptr LookupEventReportBackend(const std::shared_p for (const auto &candidate_name : ig->event_report_storage_candidates()) { auto backend = dsm->GetDataStorageBackend(candidate_name); auto *event_backend = dynamic_cast(backend.get()); - if (event_backend && event_backend->GetStorageType() == requested_type) { + if (event_backend && event_backend->GetStorageType() == requested_type && + (!require_available || event_backend->Available())) { return backend; } } return nullptr; } +bool IsCurrentEventReportBackend(const std::shared_ptr ®istry_manager, + const std::string &instance_id, + DataStorageType requested_type, + const std::shared_ptr &expected_backend) { + if (!expected_backend || expected_backend->GetStorageType() != requested_type) { + return false; + } + // Match the same first-available-candidate rule used by ReportEvent and + // GetHostCacheState. Merely remaining in the candidate list is not enough: + // an older backend incarnation must never clean metadata owned by the + // backend that currently wins routing for this storage tier. + const auto current = LookupEventReportBackend(registry_manager, instance_id, requested_type, true); + return current.get() == expected_backend.get(); +} + bool ParseInt64(const std::string &s, int64_t &out) { - try { - size_t consumed = 0; - uint64_t v = std::stoull(s, &consumed); - if (consumed != s.size()) { + if (s.empty() || s.front() == '+') { + return false; + } + + const bool negative = s.front() == '-'; + const char *begin = s.data() + (negative ? 1 : 0); + const char *end = s.data() + s.size(); + if (begin == end) { + return false; + } + + uint64_t magnitude = 0; + constexpr uint64_t kMaxMagnitude = std::numeric_limits::max(); + for (const char *cursor = begin; cursor != end; ++cursor) { + const unsigned char ch = static_cast(*cursor); + if (ch < '0' || ch > '9') { return false; } - if (v <= static_cast(std::numeric_limits::max())) { - out = static_cast(v); - } else { - out = std::numeric_limits::min() + static_cast(v - (uint64_t{1} << 63)); + const uint64_t digit = ch - '0'; + if (magnitude > (kMaxMagnitude - digit) / 10) { + return false; + } + magnitude = magnitude * 10 + digit; + } + + constexpr uint64_t kSignBit = uint64_t{1} << 63; + if (negative) { + if (magnitude > kSignBit) { + return false; + } + out = magnitude == kSignBit ? std::numeric_limits::min() : -static_cast(magnitude); + return true; + } + + // vLLM serializes its external block hash as an unsigned uint64 decimal. + // Preserve that 64-bit pattern when KVCM stores and queries signed int64 + // keys instead of rejecting values above INT64_MAX. + out = magnitude < kSignBit ? static_cast(magnitude) + : std::numeric_limits::min() + static_cast(magnitude - kSignBit); + return true; +} + +struct ValidatedEventLocationSpec { + std::string_view name; + std::string_view raw_uri; + // Canonical ReportEvent URIs never need the heavyweight StandardUri + // object. Allocate it only for the compatibility fallback so the common + // validation result stays small and cheap to move through inline storage. + std::unique_ptr parsed_uri; + CanonicalSnapshotUriAppendInfo canonical_uri; + std::uint64_t size = 0; + std::string versioned_uri; + bool is_canonical_uri = false; + + bool AddPrevalidatedSnapshotVersion(const std::string &version) { + if (is_canonical_uri) { + return SnapshotUriUtils::AddPrevalidatedSnapshotVersionToCanonicalUri( + raw_uri, canonical_uri, version, versioned_uri); + } + if (!parsed_uri) { + return false; } + versioned_uri = parsed_uri->ToUriStringWithExtraParam(SnapshotUriUtils::kSnapshotVersionParam, version); + return !versioned_uri.empty(); + } +}; + +bool ValidateEventLocationSpec(const proto::meta::LocationSpec &spec, ValidatedEventLocationSpec &out) { + out = {}; + if (spec.name().empty()) { + return false; + } + out.name = spec.name(); + out.raw_uri = spec.uri(); + if (SnapshotUriUtils::ParseCanonicalUriForSnapshotAppend(out.raw_uri, out.canonical_uri)) { + out.is_canonical_uri = true; + out.size = out.canonical_uri.size; return true; - } catch (...) { return false; } + } + out.parsed_uri = std::make_unique(spec.uri()); + if (!out.parsed_uri->Valid() || SnapshotUriUtils::HasEventReportInternalUriMetadata(*out.parsed_uri)) { + return false; + } + out.parsed_uri->GetParamAs("size", out.size); + return true; } +class ValidatedEventLocationSpecs { +public: + void Push(ValidatedEventLocationSpec &&spec, size_t max_spec_count) { + if (many_.empty() && !one_) { + one_.emplace(std::move(spec)); + return; + } + if (one_) { + many_.reserve(max_spec_count < 2 ? 2 : max_spec_count); + many_.push_back(std::move(*one_)); + one_.reset(); + } + many_.push_back(std::move(spec)); + } + + [[nodiscard]] size_t Size() const noexcept { return many_.size() + (one_ ? 1 : 0); } + [[nodiscard]] ValidatedEventLocationSpec &At(size_t index) noexcept { + return index < many_.size() ? many_[index] : *one_; + } + +private: + std::optional one_; + std::vector many_; +}; + bool IsSnapshotLocationStale(const EventReportBackend *event_backend, const std::string &instance_id, const CacheLocation &location, @@ -2416,27 +2602,16 @@ bool IsEventReportLocationReadable(const CacheLocation &location, return false; } bool contains_readable_version = !strict_query_visibility; + const bool uri_structure_prevalidated = location.HasValidatedLocationSpecs(); for (const auto &spec : location.location_specs()) { - const DataStorageUri uri(spec.uri()); - if (!uri.Valid()) { - return false; - } - const size_t version_param_count = - SnapshotUriUtils::CountUriParam(spec.uri(), SnapshotUriUtils::kSnapshotVersionParam); - if (version_param_count > 1) { + std::string_view snapshot_version; + if (!SnapshotUriUtils::InspectSnapshotUriForVisibility( + spec.uri(), snapshot_version, uri_structure_prevalidated)) { return false; } - if (version_param_count == 1) { - SnapshotUriInfo info; - // Count on the raw text first so duplicate parameters remain - // observable, then reuse the parsed URI instead of parsing it a - // second time on every query. - if (!SnapshotUriUtils::ParseSnapshotUriInfo(uri, info)) { - return false; - } - if (strict_query_visibility && info.version == committed_version) { - contains_readable_version = true; - } + if (!snapshot_version.empty() && strict_query_visibility && + snapshot_version == std::string_view(committed_version)) { + contains_readable_version = true; } } // Delta merge is spec-granular. A post-snapshot ADD may refresh one spec @@ -2498,23 +2673,33 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, return EC_BADARGS; } - DataStorageType requested_type = static_cast(request->storage_type()); - if (requested_type == DataStorageType::DATA_STORAGE_TYPE_UNKNOWN) { + DataStorageType requested_type = DataStorageType::DATA_STORAGE_TYPE_UNKNOWN; + switch (request->storage_type()) { + case proto::meta::ST_EVENT_REPORT_L1P5: + requested_type = DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5; + break; + case proto::meta::ST_EVENT_REPORT_L2: + requested_type = DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2; + break; + case proto::meta::ST_UNSPECIFIED: KVCM_LOG_WARN("trace_id [%s] | ReportEvent: storage_type is required but not specified", trace_id.c_str()); response_status->set_code(proto::meta::INVALID_ARGUMENT); response_status->set_message("storage_type is required"); return EC_BADARGS; - } - if (!IsEventReportStorageType(requested_type)) { + default: + // Do not cast the open protobuf enum directly to DataStorageType (whose + // underlying type is uint8_t): an unknown wire value such as 263 would + // truncate to 7 and be misrouted to the L1P5 backend. KVCM_LOG_WARN("trace_id [%s] | ReportEvent: unsupported event-report storage_type [%d]", trace_id.c_str(), - static_cast(requested_type)); + static_cast(request->storage_type())); response_status->set_code(proto::meta::INVALID_ARGUMENT); - response_status->set_message("unsupported event-report storage_type: " + ToString(requested_type)); + response_status->set_message("unsupported event-report storage_type: " + + std::to_string(static_cast(request->storage_type()))); return EC_BADARGS; } - auto event_backend_holder = LookupEventReportBackend(registry_manager_, instance_id, requested_type); + auto event_backend_holder = LookupEventReportBackend(registry_manager_, instance_id, requested_type, true); auto event_backend = std::dynamic_pointer_cast(event_backend_holder); if (!event_backend) { KVCM_LOG_WARN("trace_id [%s] | ReportEvent: EventReportBackend not found for instance [%s] type [%d]", @@ -2548,13 +2733,49 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, refresh_snapshot_response(false); if (!event_backend->IsCleanupCallbackSet()) { - event_backend->SetCleanupCallback([this, requested_type](const std::string &cleanup_instance, - const std::string &down_host, - uint64_t generation) { - assert(this->schedule_plan_executor_); - this->schedule_plan_executor_->SubmitTask([this, cleanup_instance, down_host, generation, requested_type] { - this->CleanupHostLocations(cleanup_instance, down_host, generation, requested_type); - }); + const std::weak_ptr expected_backend = event_backend; + const auto callback_state = event_cleanup_callback_state_; + event_backend->SetCleanupCallback([this, requested_type, expected_backend, callback_state]( + const std::string &cleanup_instance, + const std::string &down_host, + uint64_t generation) { + std::shared_lock callback_lease(callback_state->mutex); + if (!callback_state->accepting) { + return; + } + const uint64_t callback_epoch = callback_state->epoch; + auto cleanup_backend = expected_backend.lock(); + if (!cleanup_backend) { + return; + } + const auto cleanup = [this, + cleanup_instance, + down_host, + generation, + requested_type, + cleanup_backend, + callback_state, + callback_epoch] { + // A queued task may start after the backend callback that + // submitted it has returned. Hold the same lifetime lease for + // the whole cleanup so DoCleanup/destruction drains running + // tasks and makes tasks that are still queued harmless. + std::shared_lock task_lease(callback_state->mutex); + if (!callback_state->accepting || callback_state->epoch != callback_epoch) { + return; + } + this->CleanupHostLocations(cleanup_instance, down_host, generation, requested_type, cleanup_backend); + }; + if (!this->schedule_plan_executor_ || !this->schedule_plan_executor_->SubmitTask(cleanup)) { + // This callback runs on EventReportBackend's liveness thread. + // Running inline could enter DataStorageManager while an + // unregister operation holds its write lock and waits for this + // backend thread to join. The node is still removed from the + // visibility table below, so dropping best-effort physical + // metadata cleanup during executor shutdown is fail-closed. + KVCM_LOG_WARN("ReportEvent liveness cleanup queue unavailable; skipping metadata cleanup for host [%s]", + down_host.c_str()); + } }); } @@ -2568,50 +2789,130 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, return EC_INSTANCE_NOT_EXIST; } + auto instance_info = registry_manager_->GetInstanceInfo(request_context, instance_id); + if (!instance_info) { + KVCM_LOG_WARN("trace_id [%s] | ReportEvent: instance info not found for instance [%s]", + trace_id.c_str(), + instance_id.c_str()); + response_status->set_code(proto::meta::INSTANCE_NOT_EXIST); + response_status->set_message("instance info not found for instance: " + instance_id); + return EC_INSTANCE_NOT_EXIST; + } + std::unordered_set registered_spec_names; + registered_spec_names.reserve(instance_info->location_spec_infos().size()); + for (const auto &spec_info : instance_info->location_spec_infos()) { + registered_spec_names.insert(spec_info.name()); + } + const int events_size = request->events_size(); std::vector per_item_ec(events_size, EC_OK); - DeltaMutationGuard delta_mutations(event_backend); + DeltaMutationGuard delta_mutations(event_backend, reporter_key); uint64_t mutation_lifecycle_generation = 0; + bool delta_snapshot_version_validated = false; bool has_heartbeat = false; bool has_host_down = false; + std::vector valid_heartbeat_indices; std::vector register_mediums; std::map heartbeat_status; - for (const auto &item : request->events()) { - if (item.event_type() != proto::meta::EVENT_NODE_REGISTER || !item.has_node_register()) { - continue; - } - for (const auto &medium : item.node_register().mediums()) { - if (std::find(register_mediums.begin(), register_mediums.end(), medium) == register_mediums.end()) { - register_mediums.push_back(medium); - } - } - } + bool registration_items_prepared = false; bool registration_applied = false; ErrorCode register_ec = EC_OK; - - struct BlockAddEntry { - std::string location_id; - std::vector specs; - int event_index; + bool node_registration_ensured = false; + struct RequestMediumState { + InternedLocationId location_id; + bool registration_ensured = false; }; - struct BlockAddMergeEntry { - std::string location_id; - std::vector specs; - std::vector event_indices; + // A report overwhelmingly uses one medium. Keep registration state and + // the interned location id in one table, and retain the last node so the + // common run of thousands of `mem` events performs no repeated string + // hash or table probe. References to unordered_map elements survive a + // rehash and every resulting CacheLocation retains the shared id. + std::unordered_map medium_states; + medium_states.reserve(register_mediums.size() + 1); + std::string_view last_medium; + RequestMediumState *last_medium_state = nullptr; + auto get_medium_state = [&](const std::string &medium) -> RequestMediumState & { + if (last_medium_state != nullptr && last_medium == medium) { + return *last_medium_state; + } + auto [it, inserted] = medium_states.try_emplace(medium); + (void)inserted; + last_medium = it->first; + last_medium_state = &it->second; + return *last_medium_state; }; - struct BlockDelEntry { - std::string location_id; - std::vector spec_names; - int event_index; + auto get_location_id_for_state = [&](const std::string &medium, + RequestMediumState &medium_state) -> const InternedLocationId & { + if (!medium_state.location_id) { + medium_state.location_id = + std::make_shared(event_backend->BuildLocationId(medium, host_ip_port)); + } + return medium_state.location_id; + }; + auto get_location_id = [&](const std::string &medium) -> const InternedLocationId & { + return get_location_id_for_state(medium, get_medium_state(medium)); + }; + auto prepare_registration_items = [&] { + if (registration_items_prepared) { + return; + } + registration_items_prepared = true; + for (int event_index = 0; event_index < events_size; ++event_index) { + const auto &event = request->events(event_index); + if (event.event_type() != proto::meta::EVENT_NODE_REGISTER || !event.has_node_register()) { + continue; + } + const bool valid_mediums = std::all_of( + event.node_register().mediums().begin(), + event.node_register().mediums().end(), + [](const std::string &medium) { return SnapshotUriUtils::IsValidLocationIdComponent(medium); }); + if (!valid_mediums) { + per_item_ec[event_index] = EC_BADARGS; + continue; + } + for (const auto &medium : event.node_register().mediums()) { + if (std::find(register_mediums.begin(), register_mediums.end(), medium) == register_mediums.end()) { + register_mediums.push_back(medium); + } + } + } + medium_states.reserve(register_mediums.size() + 1); }; - struct BlockDelMergeEntry { - std::string location_id; - std::vector spec_names; - std::vector event_indices; + auto ensure_node_medium = [&](const std::string &medium, RequestMediumState &medium_state) { + if (medium_state.registration_ensured) { + return EC_OK; + } + const ErrorCode ec = event_backend->EnsureNodeRegistered(instance_id, host_ip_port, {medium}); + if (ec == EC_OK) { + node_registration_ensured = true; + medium_state.registration_ensured = true; + } + return ec; }; + auto ensure_node_mediums = [&](const std::vector &mediums) { + std::vector missing_mediums; + missing_mediums.reserve(mediums.size()); + for (const auto &medium : mediums) { + if (!get_medium_state(medium).registration_ensured) { + missing_mediums.push_back(medium); + } + } + if (node_registration_ensured && missing_mediums.empty()) { + return EC_OK; + } + const ErrorCode ec = event_backend->EnsureNodeRegistered(instance_id, host_ip_port, missing_mediums); + if (ec == EC_OK) { + node_registration_ensured = true; + for (const auto &medium : missing_mediums) { + get_medium_state(medium).registration_ensured = true; + } + } + return ec; + }; + struct SnapshotReplaceEntry { - std::string location_id; + const InternedLocationId *location_id = nullptr; std::vector specs; int event_index; }; @@ -2624,11 +2925,167 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, struct DeltaSpecMutation { bool is_add = false; LocationSpec spec; - std::vector event_indices; + std::uint64_t size = 0; + size_t next = std::numeric_limits::max(); + }; + struct DeltaEventMutation { + std::uint32_t event_index = 0; + std::uint32_t next = std::numeric_limits::max(); + bool materialized = false; + }; + struct DeltaLocationMutation { + int64_t block_key = 0; + const InternedLocationId *location_id = nullptr; + size_t first_spec = std::numeric_limits::max(); + size_t last_spec = std::numeric_limits::max(); + size_t spec_count = 0; + // Keep the overwhelmingly common sole event inline. Only repeated + // mutations of one block/location need nodes in the extra-event list. + // ADD and DELETE are persisted in separate phases; if either phase + // fails, every related event must still be retried together. + std::uint32_t first_event_index = std::numeric_limits::max(); + std::uint32_t first_extra_event = std::numeric_limits::max(); + std::uint32_t last_extra_event = std::numeric_limits::max(); + bool first_event_materialized = false; + }; + constexpr size_t kInvalidDeltaIndex = std::numeric_limits::max(); + constexpr std::uint32_t kInvalidDeltaEventIndex = std::numeric_limits::max(); + std::vector delta_locations; + std::vector delta_spec_mutations; + std::vector delta_event_mutations; + delta_locations.reserve(events_size); + delta_spec_mutations.reserve(events_size); + bool has_materialized_delta_add = false; + bool has_materialized_delta_delete = false; + + // Reporters normally emit unique block keys in ascending order. Fold that + // stream directly into the contiguous location vector: allocating and + // clearing an O(events) hash table only to prove every key is new costs + // hundreds of KiB per large request. On the first non-adjacent duplicate + // or out-of-order pair, lazily index all locations accumulated so far and + // retain the same last-op-wins semantics for arbitrary event order. + const size_t event_count = static_cast(events_size); + size_t delta_slot_mask = 0; + std::vector delta_location_slots; + bool delta_locations_sorted = true; + auto delta_location_hash = [](int64_t block_key, const InternedLocationId &location_id) { + size_t hash = std::hash{}(block_key); + hash ^= std::hash{}(location_id.get()) + 0x9e3779b9U + (hash << 6) + (hash >> 2); + return hash; + }; + auto find_delta_location = [&](int64_t block_key, const InternedLocationId &location_id) { + size_t slot = delta_location_hash(block_key, location_id) & delta_slot_mask; + while (delta_location_slots[slot] != kInvalidDeltaIndex) { + const size_t location_index = delta_location_slots[slot]; + const auto &location = delta_locations[location_index]; + if (location.block_key == block_key && location.location_id->get() == location_id.get()) { + return std::pair{slot, location_index}; + } + slot = (slot + 1) & delta_slot_mask; + } + return std::pair{slot, kInvalidDeltaIndex}; + }; + auto build_delta_location_index = [&] { + size_t delta_slot_count = 8; + const size_t max_delta_slot_count = std::numeric_limits::max() / 2; + while (event_count > delta_slot_count / 2 && delta_slot_count <= max_delta_slot_count) { + delta_slot_count <<= 1; + } + delta_location_slots.assign(delta_slot_count, kInvalidDeltaIndex); + delta_slot_mask = delta_slot_count - 1; + for (size_t location_index = 0; location_index < delta_locations.size(); ++location_index) { + const auto &location = delta_locations[location_index]; + size_t slot = delta_location_hash(location.block_key, *location.location_id) & delta_slot_mask; + while (delta_location_slots[slot] != kInvalidDeltaIndex) { + slot = (slot + 1) & delta_slot_mask; + } + delta_location_slots[slot] = location_index; + } + }; + auto append_delta_location = [&](int64_t block_key, const InternedLocationId &location_id) { + if (!delta_locations.empty()) { + const auto &previous = delta_locations.back(); + if (block_key < previous.block_key || + (block_key == previous.block_key && *location_id < **previous.location_id)) { + delta_locations_sorted = false; + } + } + const size_t location_index = delta_locations.size(); + delta_locations.push_back(DeltaLocationMutation{block_key, &location_id}); + return location_index; }; - std::map> block_to_add; - std::map> block_to_del; - std::map>> delta_spec_mutations; + auto record_delta_event = + [&](int64_t block_key, const InternedLocationId &location_id, int event_index) -> std::pair { + size_t location_index = kInvalidDeltaIndex; + if (delta_locations.empty()) { + location_index = append_delta_location(block_key, location_id); + } else { + const auto &last = delta_locations.back(); + if (last.block_key == block_key && last.location_id->get() == location_id.get()) { + location_index = delta_locations.size() - 1; + } else if (delta_location_slots.empty() && + (last.block_key < block_key || + (last.block_key == block_key && **last.location_id < *location_id))) { + location_index = append_delta_location(block_key, location_id); + } else { + if (delta_location_slots.empty()) { + build_delta_location_index(); + } + auto [slot, indexed_location] = find_delta_location(block_key, location_id); + location_index = indexed_location; + if (location_index == kInvalidDeltaIndex) { + location_index = append_delta_location(block_key, location_id); + delta_location_slots[slot] = location_index; + } + } + } + auto &location = delta_locations[location_index]; + if (location.first_event_index == kInvalidDeltaEventIndex) { + location.first_event_index = static_cast(event_index); + return {location_index, kInvalidDeltaIndex}; + } + if (delta_event_mutations.empty()) { + // A request contains at most INT_MAX protobuf events, so uint32_t + // indexes are sufficient. Delay this allocation until a repeated + // block/location actually needs dependency closure. + delta_event_mutations.reserve(event_count); + } + const auto event_mutation_index = static_cast(delta_event_mutations.size()); + delta_event_mutations.push_back( + DeltaEventMutation{static_cast(event_index), kInvalidDeltaEventIndex, false}); + if (location.first_extra_event == kInvalidDeltaEventIndex) { + location.first_extra_event = event_mutation_index; + } else { + delta_event_mutations[location.last_extra_event].next = event_mutation_index; + } + location.last_extra_event = event_mutation_index; + return {location_index, event_mutation_index}; + }; + auto apply_delta_spec = + [&](DeltaLocationMutation &location, bool is_add, LocationSpec spec, std::uint64_t size = 0) { + has_materialized_delta_add = has_materialized_delta_add || is_add; + has_materialized_delta_delete = has_materialized_delta_delete || !is_add; + for (size_t index = location.first_spec; index != kInvalidDeltaIndex; + index = delta_spec_mutations[index].next) { + auto &mutation = delta_spec_mutations[index]; + if (mutation.spec.name() != spec.name()) { + continue; + } + mutation.is_add = is_add; + mutation.spec = std::move(spec); + mutation.size = size; + return; + } + const size_t mutation_index = delta_spec_mutations.size(); + delta_spec_mutations.push_back(DeltaSpecMutation{is_add, std::move(spec), size, kInvalidDeltaIndex}); + if (location.first_spec == kInvalidDeltaIndex) { + location.first_spec = mutation_index; + } else { + delta_spec_mutations[location.last_spec].next = mutation_index; + } + location.last_spec = mutation_index; + ++location.spec_count; + }; std::map> snapshot_to_replace; std::vector snapshot_commit_tasks; std::string request_snapshot_version; @@ -2637,16 +3094,24 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, const auto &item = request->events(i); switch (item.event_type()) { case proto::meta::EVENT_NODE_REGISTER: { + prepare_registration_items(); if (!item.has_node_register()) { per_item_ec[i] = EC_BADARGS; break; } + if (per_item_ec[i] != EC_OK) { + break; + } if (!registration_applied) { register_ec = event_backend->RegisterNode(instance_id, host_ip_port, register_mediums); registration_applied = true; if (register_ec == EC_OK) { + node_registration_ensured = true; + for (const auto &medium : register_mediums) { + get_medium_state(medium).registration_ensured = true; + } mutation_lifecycle_generation = event_backend->GetNodeGeneration(instance_id, host_ip_port); - delta_mutations.AdoptLifecycleGeneration(reporter_key, mutation_lifecycle_generation); + delta_mutations.AdoptLifecycleGeneration(mutation_lifecycle_generation); } } per_item_ec[i] = register_ec; @@ -2658,6 +3123,7 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, break; } has_heartbeat = true; + valid_heartbeat_indices.push_back(i); heartbeat_status.clear(); for (const auto &kv : item.heartbeat().system_status()) { heartbeat_status[kv.first] = kv.second; @@ -2684,59 +3150,84 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, break; } - std::vector specs; - specs.reserve(params.specs_size()); - std::unordered_set seen_spec_names; + ValidatedEventLocationSpecs specs; + std::unordered_set seen_spec_names; + if (params.specs_size() > 1) { + seen_spec_names.reserve(params.specs_size()); + } + std::uint64_t event_total_size = 0; for (const auto &spec : params.specs()) { - const DataStorageUri parsed_uri(spec.uri()); - if (spec.name().empty() || !seen_spec_names.insert(spec.name()).second || !parsed_uri.Valid() || - SnapshotUriUtils::HasEventReportInternalUriMetadata(parsed_uri)) { + if (registered_spec_names.find(spec.name()) == registered_spec_names.end() || + (params.specs_size() > 1 && !seen_spec_names.insert(std::string_view(spec.name())).second)) { + per_item_ec[i] = EC_BADARGS; + break; + } + ValidatedEventLocationSpec validated_spec; + if (!ValidateEventLocationSpec(spec, validated_spec) || + validated_spec.size > std::numeric_limits::max() - event_total_size) { per_item_ec[i] = EC_BADARGS; break; } - specs.emplace_back(spec.name(), spec.uri()); + event_total_size += validated_spec.size; + specs.Push(std::move(validated_spec), params.specs_size()); } if (per_item_ec[i] != EC_OK) { break; } - const ErrorCode ensure_node_ec = - event_backend->EnsureNodeRegistered(instance_id, host_ip_port, {params.medium()}); + auto &medium_state = get_medium_state(params.medium()); + const auto &location_id = get_location_id_for_state(params.medium(), medium_state); + // Record the retry dependency as soon as the event is + // structurally valid. Admission may still fail before a mutation + // is materialized (for example, a tombstoned reporter followed by + // REGISTER and a later related mutation). In that case a later + // success must also be retried, otherwise retrying only the early + // failed item could reverse the request's final operation order. + const auto [location_mutation_index, event_mutation_index] = record_delta_event(block_key, location_id, i); + auto &location_mutation = delta_locations[location_mutation_index]; + const ErrorCode ensure_node_ec = ensure_node_medium(params.medium(), medium_state); if (ensure_node_ec != EC_OK) { per_item_ec[i] = ensure_node_ec; break; } - std::string committed_version; + const DeltaMutationGuard::LeaseInfo *lease = nullptr; bool created_generation = false; - const ErrorCode fence_ec = delta_mutations.Acquire( - reporter_key, committed_version, mutation_lifecycle_generation, created_generation); + const ErrorCode fence_ec = delta_mutations.Acquire(lease, created_generation); if (fence_ec != EC_OK) { per_item_ec[i] = fence_ec; break; } + mutation_lifecycle_generation = lease->lifecycle_generation; if (created_generation) { request_created_generation = true; } - for (auto &spec : specs) { - std::string versioned_uri; - if (!SnapshotUriUtils::AddSnapshotVersionToUri(spec.uri(), committed_version, versioned_uri)) { + if (!delta_snapshot_version_validated) { + if (!SnapshotUriUtils::IsValidSnapshotVersionToken(lease->snapshot_version)) { + per_item_ec[i] = EC_BADARGS; + break; + } + delta_snapshot_version_validated = true; + } + for (size_t spec_index = 0; spec_index < specs.Size(); ++spec_index) { + if (!specs.At(spec_index).AddPrevalidatedSnapshotVersion(lease->snapshot_version)) { per_item_ec[i] = EC_BADARGS; break; } - spec.set_uri(std::move(versioned_uri)); } if (per_item_ec[i] != EC_OK) { break; } - const std::string location_id = event_backend->BuildLocationId(params.medium(), host_ip_port); - auto &mutations_by_name = delta_spec_mutations[block_key][location_id]; - for (auto &spec : specs) { - auto &mutation = mutations_by_name[spec.name()]; - mutation.is_add = true; - mutation.spec = std::move(spec); - if (mutation.event_indices.empty() || mutation.event_indices.back() != i) { - mutation.event_indices.push_back(i); - } + for (size_t spec_index = 0; spec_index < specs.Size(); ++spec_index) { + auto &spec = specs.At(spec_index); + LocationSpec versioned_spec; + versioned_spec.set_name_view(spec.name); + versioned_spec.set_uri(std::move(spec.versioned_uri)); + apply_delta_spec(location_mutation, true, std::move(versioned_spec), spec.size); + } + if (event_mutation_index == kInvalidDeltaIndex) { + location_mutation.first_event_materialized = true; + } else { + delta_event_mutations[event_mutation_index].materialized = true; } break; } @@ -2752,45 +3243,48 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, per_item_ec[i] = EC_BADARGS; break; } - std::vector spec_names; - spec_names.reserve(params.spec_names_size()); - std::unordered_set seen_spec_names; + std::unordered_set seen_spec_names; + if (params.spec_names_size() > 1) { + seen_spec_names.reserve(params.spec_names_size()); + } for (const auto &spec_name : params.spec_names()) { - if (spec_name.empty() || !seen_spec_names.insert(spec_name).second) { + if (spec_name.empty() || registered_spec_names.find(spec_name) == registered_spec_names.end() || + (params.spec_names_size() > 1 && !seen_spec_names.insert(std::string_view(spec_name)).second)) { per_item_ec[i] = EC_BADARGS; break; } - spec_names.push_back(spec_name); } if (per_item_ec[i] != EC_OK) { break; } - const ErrorCode ensure_node_ec = - event_backend->EnsureNodeRegistered(instance_id, host_ip_port, {params.medium()}); + auto &medium_state = get_medium_state(params.medium()); + const auto &location_id = get_location_id_for_state(params.medium(), medium_state); + const auto [location_mutation_index, event_mutation_index] = record_delta_event(block_key, location_id, i); + auto &location_mutation = delta_locations[location_mutation_index]; + const ErrorCode ensure_node_ec = ensure_node_medium(params.medium(), medium_state); if (ensure_node_ec != EC_OK) { per_item_ec[i] = ensure_node_ec; break; } - std::string committed_version; + const DeltaMutationGuard::LeaseInfo *lease = nullptr; bool created_generation = false; - const ErrorCode fence_ec = delta_mutations.Acquire( - reporter_key, committed_version, mutation_lifecycle_generation, created_generation); + const ErrorCode fence_ec = delta_mutations.Acquire(lease, created_generation); if (fence_ec != EC_OK) { per_item_ec[i] = fence_ec; break; } + mutation_lifecycle_generation = lease->lifecycle_generation; if (created_generation) { request_created_generation = true; } - const std::string location_id = event_backend->BuildLocationId(params.medium(), host_ip_port); - auto &mutations_by_name = delta_spec_mutations[block_key][location_id]; - for (auto &spec_name : spec_names) { - auto &mutation = mutations_by_name[spec_name]; - mutation.is_add = false; - if (mutation.event_indices.empty() || mutation.event_indices.back() != i) { - mutation.event_indices.push_back(i); - } + for (const auto &spec_name : params.spec_names()) { + apply_delta_spec(location_mutation, false, LocationSpec(spec_name, "")); + } + if (event_mutation_index == kInvalidDeltaIndex) { + location_mutation.first_event_materialized = true; + } else { + delta_event_mutations[event_mutation_index].materialized = true; } break; } @@ -2803,11 +3297,12 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, struct ValidatedBlock { int64_t block_key; std::string medium; - std::vector specs; + ValidatedEventLocationSpecs specs; }; std::vector validated_blocks; validated_blocks.reserve(params.blocks_size()); std::unordered_set seen_blocks; + std::uint64_t snapshot_total_size = 0; for (const auto &block : params.blocks()) { int64_t block_key = 0; const std::string &block_medium = block.medium().empty() ? params.medium() : block.medium(); @@ -2821,17 +3316,25 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, per_item_ec[i] = EC_BADARGS; break; } - std::vector specs; - specs.reserve(block.specs_size()); - std::unordered_set seen_spec_names; + ValidatedEventLocationSpecs specs; + std::unordered_set seen_spec_names; + if (block.specs_size() > 1) { + seen_spec_names.reserve(block.specs_size()); + } for (const auto &spec : block.specs()) { - const DataStorageUri parsed_uri(spec.uri()); - if (spec.name().empty() || !seen_spec_names.insert(spec.name()).second || !parsed_uri.Valid() || - SnapshotUriUtils::HasEventReportInternalUriMetadata(parsed_uri)) { + if (registered_spec_names.find(spec.name()) == registered_spec_names.end() || + (block.specs_size() > 1 && !seen_spec_names.insert(std::string_view(spec.name())).second)) { per_item_ec[i] = EC_BADARGS; break; } - specs.emplace_back(spec.name(), spec.uri()); + ValidatedEventLocationSpec validated_spec; + if (!ValidateEventLocationSpec(spec, validated_spec) || + validated_spec.size > std::numeric_limits::max() - snapshot_total_size) { + per_item_ec[i] = EC_BADARGS; + break; + } + snapshot_total_size += validated_spec.size; + specs.Push(std::move(validated_spec), block.specs_size()); } if (per_item_ec[i] != EC_OK) { break; @@ -2849,8 +3352,7 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, snapshot_mediums.push_back(block.medium); } } - const ErrorCode ensure_node_ec = - event_backend->EnsureNodeRegistered(instance_id, host_ip_port, snapshot_mediums); + const ErrorCode ensure_node_ec = ensure_node_mediums(snapshot_mediums); if (ensure_node_ec != EC_OK) { per_item_ec[i] = ensure_node_ec; break; @@ -2868,22 +3370,32 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, } break; } + if (!SnapshotUriUtils::IsValidSnapshotVersionToken(request_snapshot_version)) { + per_item_ec[i] = EC_BADARGS; + event_backend->AbortSnapshotVersion(reporter_key, request_snapshot_version); + request_snapshot_version.clear(); + break; + } for (auto &block : validated_blocks) { - for (auto &spec : block.specs) { - std::string versioned_uri; - if (!SnapshotUriUtils::AddSnapshotVersionToUri( - spec.uri(), request_snapshot_version, versioned_uri)) { + std::vector versioned_specs; + versioned_specs.reserve(block.specs.Size()); + for (size_t spec_index = 0; spec_index < block.specs.Size(); ++spec_index) { + auto &spec = block.specs.At(spec_index); + if (!spec.AddPrevalidatedSnapshotVersion(request_snapshot_version)) { per_item_ec[i] = EC_BADARGS; break; } - spec.set_uri(std::move(versioned_uri)); + LocationSpec versioned_spec; + versioned_spec.set_name_view(spec.name); + versioned_spec.set_uri(std::move(spec.versioned_uri)); + versioned_specs.push_back(std::move(versioned_spec)); } if (per_item_ec[i] != EC_OK) { break; } - snapshot_to_replace[block.block_key].push_back(SnapshotReplaceEntry{ - event_backend->BuildLocationId(block.medium, host_ip_port), std::move(block.specs), i}); + snapshot_to_replace[block.block_key].push_back( + SnapshotReplaceEntry{&get_location_id(block.medium), std::move(versioned_specs), i}); } if (per_item_ec[i] != EC_OK) { event_backend->AbortSnapshotVersion(reporter_key, request_snapshot_version); @@ -2901,160 +3413,260 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, } } - // A request is an ordered event stream. Resolve repeated mutations of the - // same (block, stable location, spec name) by keeping the last operation, - // then batch the disjoint final ADD and DELETE sets. All original events - // absorbed into one final mutation share that mutation's write result: - // none of the intermediate operations is persisted independently. This - // preserves request order without giving up batched metadata writes. - for (auto &[block_key, mutations_by_location] : delta_spec_mutations) { - for (auto &[location_id, mutations_by_name] : mutations_by_location) { - for (auto &[spec_name, mutation] : mutations_by_name) { - for (const int event_index : mutation.event_indices) { - if (mutation.is_add) { - block_to_add[block_key].push_back(BlockAddEntry{location_id, {mutation.spec}, event_index}); + // A request is an ordered event stream. Specs and event dependencies are + // linked through request-wide contiguous arrays, so the common one-spec, + // one-event block does not allocate three tiny vectors of its own. + // Preserve deterministic block/location task ordering for arbitrary input + // without allocating an identity permutation for the already-sorted fast + // path. + std::vector ordered_delta_location_indices; + if (!delta_locations_sorted) { + ordered_delta_location_indices.resize(delta_locations.size()); + for (size_t i = 0; i < delta_locations.size(); ++i) { + ordered_delta_location_indices[i] = i; + } + std::sort(ordered_delta_location_indices.begin(), + ordered_delta_location_indices.end(), + [&delta_locations](size_t lhs_index, size_t rhs_index) { + const auto &lhs = delta_locations[lhs_index]; + const auto &rhs = delta_locations[rhs_index]; + return lhs.block_key != rhs.block_key ? lhs.block_key < rhs.block_key + : **lhs.location_id < **rhs.location_id; + }); + } + auto delta_location_index_at = [&ordered_delta_location_indices](size_t position) { + return ordered_delta_location_indices.empty() ? position : ordered_delta_location_indices[position]; + }; + + KeyVector add_keys_aggr; + std::vector merge_tasks; + std::vector merge_task_offsets{0}; + KeyVector del_keys_aggr; + std::vector> delete_tasks; + if (has_materialized_delta_add) { + add_keys_aggr.reserve(delta_locations.size()); + merge_tasks.reserve(delta_locations.size()); + merge_task_offsets.reserve(delta_locations.size() + 1); + } + if (has_materialized_delta_delete) { + del_keys_aggr.reserve(delta_locations.size()); + delete_tasks.reserve(delta_locations.size()); + } + + for (size_t block_begin = 0; block_begin < delta_locations.size();) { + const int64_t block_key = delta_locations[delta_location_index_at(block_begin)].block_key; + size_t block_end = block_begin + 1; + while (block_end < delta_locations.size() && + delta_locations[delta_location_index_at(block_end)].block_key == block_key) { + ++block_end; + } + std::vector block_delete_tasks; + const size_t add_task_begin = merge_tasks.size(); + for (size_t location_position = block_begin; location_position < block_end; ++location_position) { + auto &location = delta_locations[delta_location_index_at(location_position)]; + MetaSearcher::MergeLocationSpecsTask add_task{ + {}, + requested_type, + CacheLocationStatus::CLS_SERVING, + {}, + }; + add_task.borrowed_interned_location_id = location.location_id; + std::uint64_t add_task_total_size = 0; + bool add_task_size_overflow = false; + MetaSearcher::DeleteLocationSpecsTask delete_task{{}, {}}; + delete_task.borrowed_interned_location_id = location.location_id; + for (size_t mutation_index = location.first_spec; mutation_index != kInvalidDeltaIndex; + mutation_index = delta_spec_mutations[mutation_index].next) { + auto &mutation = delta_spec_mutations[mutation_index]; + if (mutation.is_add) { + if (mutation.size > std::numeric_limits::max() - add_task_total_size) { + add_task_size_overflow = true; } else { - block_to_del[block_key].push_back(BlockDelEntry{location_id, {spec_name}, event_index}); + add_task_total_size += mutation.size; + } + add_task.PushReportEventSpec(std::move(mutation.spec), location.spec_count); + } else { + if (delete_task.spec_names.empty()) { + delete_task.spec_names.reserve(location.spec_count); } + delete_task.spec_names.push_back(mutation.spec.name()); } } - } - } - - if (has_heartbeat) { - const ErrorCode ec = event_backend->OnHeartbeat(instance_id, host_ip_port, heartbeat_status); - if (ec != EC_OK) { - for (int i = 0; i < events_size; ++i) { - if (request->events(i).event_type() == proto::meta::EVENT_HEARTBEAT) { - per_item_ec[i] = ec; + if (add_task.specs.size() > 1) { + std::sort(add_task.specs.begin(), add_task.specs.end(), [](const auto &lhs, const auto &rhs) { + return lhs.name() < rhs.name(); + }); + } + std::sort(delete_task.spec_names.begin(), delete_task.spec_names.end()); + if (!add_task.SpecsEmpty()) { + // A pathological overflow falls back to the strict validator, + // which rejects the task without trusting a wrapped total. + if (!add_task_size_overflow) { + add_task.prevalidated_total_size = MetaSearcher::PrevalidatedTotalSize(add_task_total_size); } + merge_tasks.push_back(std::move(add_task)); } + if (!delete_task.spec_names.empty()) { + if (block_delete_tasks.empty()) { + block_delete_tasks.reserve(block_end - block_begin); + } + block_delete_tasks.push_back(std::move(delete_task)); + } + } + if (merge_tasks.size() != add_task_begin) { + add_keys_aggr.push_back(block_key); + merge_task_offsets.push_back(merge_tasks.size()); } + if (!block_delete_tasks.empty()) { + del_keys_aggr.push_back(block_key); + delete_tasks.push_back(std::move(block_delete_tasks)); + } + block_begin = block_end; } - if (!block_to_add.empty()) { - KeyVector add_keys_aggr; - std::vector> add_merged_entries_aggr; - add_keys_aggr.reserve(block_to_add.size()); - add_merged_entries_aggr.reserve(block_to_add.size()); - for (const auto &kv : block_to_add) { - add_keys_aggr.push_back(kv.first); - - std::map> specs_by_location; - std::map> event_indices_by_location; - for (const auto &entry : kv.second) { - auto &specs_by_name = specs_by_location[entry.location_id]; - for (const auto &spec : entry.specs) { - specs_by_name[spec.name()] = spec; - } - event_indices_by_location[entry.location_id].push_back(entry.event_index); + // Failure propagation is exceptional. Avoid writing a 24-byte range for + // every successful block; locate the affected range in the already-sorted + // location view only if a backend error actually occurs. + auto find_delta_block_range = [&](int64_t block_key) { + size_t lower = 0; + size_t upper = delta_locations.size(); + while (lower < upper) { + const size_t middle = lower + (upper - lower) / 2; + if (delta_locations[delta_location_index_at(middle)].block_key < block_key) { + lower = middle + 1; + } else { + upper = middle; } - - auto &merged_entries = add_merged_entries_aggr.emplace_back(); - merged_entries.reserve(specs_by_location.size()); - for (auto &[location_id, specs_by_name] : specs_by_location) { - BlockAddMergeEntry merged_entry; - auto event_indices_it = event_indices_by_location.find(location_id); - if (event_indices_it != event_indices_by_location.end()) { - merged_entry.event_indices = std::move(event_indices_it->second); - } - merged_entry.location_id = location_id; - merged_entry.specs.reserve(specs_by_name.size()); - for (auto &[spec_name, spec] : specs_by_name) { - merged_entry.specs.push_back(std::move(spec)); - } - merged_entries.push_back(std::move(merged_entry)); + } + const size_t block_begin = lower; + upper = delta_locations.size(); + while (lower < upper) { + const size_t middle = lower + (upper - lower) / 2; + if (delta_locations[delta_location_index_at(middle)].block_key <= block_key) { + lower = middle + 1; + } else { + upper = middle; } } + assert(block_begin < lower && delta_locations[delta_location_index_at(block_begin)].block_key == block_key); + return std::pair{block_begin, lower}; + }; - std::vector> merge_tasks(add_keys_aggr.size()); - for (size_t i = 0; i < add_keys_aggr.size(); ++i) { - const auto &entries = add_merged_entries_aggr[i]; - merge_tasks[i].reserve(entries.size()); - for (const auto &entry : entries) { - merge_tasks[i].push_back(MetaSearcher::MergeLocationSpecsTask{ - entry.location_id, - event_backend->GetStorageType(), - CacheLocationStatus::CLS_SERVING, - entry.specs, + auto visit_delta_events = [&delta_event_mutations, kInvalidDeltaEventIndex](const DeltaLocationMutation &location, + const auto &visitor) { + if (location.first_event_index != kInvalidDeltaEventIndex && + !visitor(static_cast(location.first_event_index), location.first_event_materialized)) { + return false; + } + for (std::uint32_t event_index = location.first_extra_event; event_index != kInvalidDeltaEventIndex; + event_index = delta_event_mutations[event_index].next) { + const auto &event = delta_event_mutations[event_index]; + if (!visitor(static_cast(event.event_index), event.materialized)) { + return false; + } + } + return true; + }; + auto mark_delta_phase_failure = + [&per_item_ec, request, &delta_location_index_at, &delta_locations, &visit_delta_events]( + size_t location_begin, size_t location_end, ErrorCode ec, const auto &spec_participates) { + for (size_t location_position = location_begin; location_position < location_end; ++location_position) { + const auto &location = delta_locations[delta_location_index_at(location_position)]; + visit_delta_events(location, [&](int event_index, bool materialized) { + if (!materialized || per_item_ec[event_index] != EC_OK) { + return true; + } + const auto &event = request->events(event_index); + bool participates = false; + if (event.event_type() == proto::meta::EVENT_BLOCK_ADD && event.has_block_add()) { + participates = std::any_of(event.block_add().specs().begin(), + event.block_add().specs().end(), + [&spec_participates, &location](const auto &spec) { + return spec_participates(**location.location_id, spec.name()); + }); + } else if (event.event_type() == proto::meta::EVENT_BLOCK_DELETE && event.has_block_delete()) { + participates = std::any_of(event.block_delete().spec_names().begin(), + event.block_delete().spec_names().end(), + [&spec_participates, &location](const auto &spec_name) { + return spec_participates(**location.location_id, spec_name); + }); + } + if (participates) { + per_item_ec[event_index] = ec; + } + return true; }); } + }; + + if (has_heartbeat) { + const ErrorCode ec = event_backend->OnHeartbeat(instance_id, host_ip_port, heartbeat_status); + if (ec != EC_OK) { + for (const int event_index : valid_heartbeat_indices) { + per_item_ec[event_index] = ec; + } + } else { + // Recovering an unavailable reporter advances its lifecycle to + // fence cleanup selected for the old generation. Mutations + // admitted by this same RPC belong to the recovered lifecycle; + // retain that generation through metadata writes and lease drain. + mutation_lifecycle_generation = event_backend->GetNodeGeneration(instance_id, host_ip_port); + delta_mutations.AdoptLifecycleGeneration(mutation_lifecycle_generation); } + } + // MetaSearcher calls this once after the fused target-location read and + // before its first mutation. This removes the old per-key lock/allocation + // amplification while preserving the window in which HOST_DOWN can fence + // a request that is stalled in metadata I/O. + auto acquire_write_lease = [event_backend, reporter_key, mutation_lifecycle_generation] { + EventReportBackend::LifecycleMutationLease lease; + const ErrorCode ec = + event_backend->AcquireLifecycleMutationLease(reporter_key, mutation_lifecycle_generation, lease); + return std::make_pair(ec, std::static_pointer_cast(std::move(lease))); + }; + + if (!add_keys_aggr.empty()) { std::vector per_key_ec; - auto acquire_write_lease = [event_backend, reporter_key, mutation_lifecycle_generation] { - EventReportBackend::LifecycleMutationLease lease; - const ErrorCode ec = - event_backend->AcquireLifecycleMutationLease(reporter_key, mutation_lifecycle_generation, lease); - return std::make_pair(ec, std::static_pointer_cast(std::move(lease))); - }; - meta_searcher->BatchMergeLocationSpecs( - request_context, add_keys_aggr, merge_tasks, per_key_ec, acquire_write_lease); + meta_searcher->BatchMergeLocationSpecsFlat( + request_context, add_keys_aggr, merge_task_offsets, merge_tasks, per_key_ec, acquire_write_lease); for (size_t k = 0; k < add_keys_aggr.size(); ++k) { ErrorCode key_ec = (k < per_key_ec.size()) ? per_key_ec[k] : EC_ERROR; if (key_ec == EC_OK) { continue; } - for (const auto &entry : add_merged_entries_aggr[k]) { - for (int event_index : entry.event_indices) { - if (per_item_ec[event_index] == EC_OK) { - per_item_ec[event_index] = key_ec; + const auto tasks_begin = merge_tasks.begin() + merge_task_offsets[k]; + const auto tasks_end = merge_tasks.begin() + merge_task_offsets[k + 1]; + const auto [location_begin, location_end] = find_delta_block_range(add_keys_aggr[k]); + mark_delta_phase_failure( + location_begin, + location_end, + key_ec, + [tasks_begin, tasks_end](const std::string &location_id, const std::string &spec_name) { + const auto task = std::find_if(tasks_begin, tasks_end, [&location_id](const auto &candidate) { + return candidate.ResolvedLocationId() == location_id; + }); + if (task == tasks_end) { + return false; } - } - } + for (size_t spec_index = 0; spec_index < task->SpecCount(); ++spec_index) { + if (task->SpecAt(spec_index).name() == spec_name) { + return true; + } + } + return false; + }); } } - if (!block_to_del.empty()) { - KeyVector del_keys_aggr; - std::vector> del_merged_entries_aggr; - del_keys_aggr.reserve(block_to_del.size()); - del_merged_entries_aggr.reserve(block_to_del.size()); - for (const auto &kv : block_to_del) { - del_keys_aggr.push_back(kv.first); - - std::map> spec_names_by_location; - std::map> event_indices_by_location; - for (const auto &entry : kv.second) { - event_indices_by_location[entry.location_id].push_back(entry.event_index); - auto &spec_names = spec_names_by_location[entry.location_id]; - spec_names.insert(entry.spec_names.begin(), entry.spec_names.end()); - } - - auto &merged_entries = del_merged_entries_aggr.emplace_back(); - merged_entries.reserve(event_indices_by_location.size()); - for (auto &[location_id, event_indices] : event_indices_by_location) { - BlockDelMergeEntry merged_entry; - merged_entry.location_id = location_id; - merged_entry.event_indices = std::move(event_indices); - const auto &spec_names = spec_names_by_location[location_id]; - merged_entry.spec_names.assign(spec_names.begin(), spec_names.end()); - merged_entries.push_back(std::move(merged_entry)); - } - } - + if (!del_keys_aggr.empty()) { std::vector> per_location_ec; std::vector> missing_delete_targets; - std::vector> delete_tasks(del_keys_aggr.size()); size_t delete_target_count = 0; - for (size_t i = 0; i < del_keys_aggr.size(); ++i) { - const auto &entries = del_merged_entries_aggr[i]; - delete_tasks[i].reserve(entries.size()); - for (const auto &entry : entries) { - delete_tasks[i].push_back(MetaSearcher::DeleteLocationSpecsTask{ - entry.location_id, - entry.spec_names, - }); - } - delete_target_count += delete_tasks[i].size(); + for (const auto &tasks : delete_tasks) { + delete_target_count += tasks.size(); } - auto acquire_write_lease = [event_backend, reporter_key, mutation_lifecycle_generation] { - EventReportBackend::LifecycleMutationLease lease; - const ErrorCode ec = - event_backend->AcquireLifecycleMutationLease(reporter_key, mutation_lifecycle_generation, lease); - return std::make_pair(ec, std::static_pointer_cast(std::move(lease))); - }; meta_searcher->BatchDeleteLocationSpecs(request_context, del_keys_aggr, delete_tasks, @@ -3064,7 +3676,7 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, for (size_t k = 0; k < del_keys_aggr.size(); ++k) { ErrorCode key_ec = EC_OK; - if (k < per_location_ec.size()) { + if (k < per_location_ec.size() && per_location_ec[k].size() == delete_tasks[k].size()) { for (const auto &loc_ec : per_location_ec[k]) { if (loc_ec != EC_OK && loc_ec != EC_NOENT) { key_ec = loc_ec; @@ -3077,13 +3689,20 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, if (key_ec == EC_OK) { continue; } - for (const auto &entry : del_merged_entries_aggr[k]) { - for (int event_index : entry.event_indices) { - if (per_item_ec[event_index] == EC_OK) { - per_item_ec[event_index] = key_ec; - } - } - } + const auto &tasks = delete_tasks[k]; + const auto [location_begin, location_end] = find_delta_block_range(del_keys_aggr[k]); + mark_delta_phase_failure( + location_begin, + location_end, + key_ec, + [&tasks](const std::string &location_id, const std::string &spec_name) { + const auto task = std::find_if(tasks.begin(), tasks.end(), [&location_id](const auto &candidate) { + return candidate.ResolvedLocationId() == location_id; + }); + return task != tasks.end() && + std::find(task->spec_names.begin(), task->spec_names.end(), spec_name) != + task->spec_names.end(); + }); } size_t missing_block_count = 0; for (const auto &missing_per_key : missing_delete_targets) { @@ -3117,23 +3736,19 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, tasks.reserve(entries.size()); indices.reserve(entries.size()); for (auto &entry : entries) { - tasks.push_back(MetaSearcher::ReplaceLocationSpecsTask{ - entry.location_id, - event_backend->GetStorageType(), + MetaSearcher::ReplaceLocationSpecsTask task{ + {}, + requested_type, CacheLocationStatus::CLS_SERVING, std::move(entry.specs), - }); + }; + task.borrowed_interned_location_id = entry.location_id; + tasks.push_back(std::move(task)); indices.push_back(entry.event_index); } } std::vector per_key_ec; - auto acquire_write_lease = [event_backend, reporter_key, mutation_lifecycle_generation] { - EventReportBackend::LifecycleMutationLease lease; - const ErrorCode ec = - event_backend->AcquireLifecycleMutationLease(reporter_key, mutation_lifecycle_generation, lease); - return std::make_pair(ec, std::static_pointer_cast(std::move(lease))); - }; meta_searcher->BatchReplaceLocationSpecs( request_context, snapshot_keys, replace_tasks, per_key_ec, acquire_write_lease); for (size_t key_index = 0; key_index < snapshot_keys.size(); ++key_index) { @@ -3156,12 +3771,18 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, // a snapshot commits after every persistent write has been accepted // by the async backend and mirrored into the local cache; it does not // wait for Redis consumers to flush their queues. - if (!snapshot_failed && !event_backend->CommitSnapshotVersion(task.reporter_key, task.version)) { - KVCM_LOG_ERROR("trace_id [%s] | EVENT_BLOCK_SNAPSHOT: failed to publish host [%s] token [%s]", - trace_id.c_str(), - task.reporter_key.host_ip_port.c_str(), - task.version.c_str()); - snapshot_failed = true; + if (!snapshot_failed) { + const ErrorCode commit_ec = event_backend->CommitSnapshotVersionIfGeneration( + task.reporter_key, task.version, mutation_lifecycle_generation); + if (commit_ec != EC_OK) { + KVCM_LOG_ERROR("trace_id [%s] | EVENT_BLOCK_SNAPSHOT: failed to publish host [%s] token [%s], ec [%d]", + trace_id.c_str(), + task.reporter_key.host_ip_port.c_str(), + task.version.c_str(), + commit_ec); + per_item_ec[task.event_index] = commit_ec; + snapshot_failed = true; + } } if (snapshot_failed) { if (per_item_ec[task.event_index] == EC_OK) { @@ -3170,14 +3791,29 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, event_backend->AbortSnapshotVersion(task.reporter_key, task.version); } else if (schedule_plan_executor_) { const auto cleanup_backend = event_backend; + const auto cleanup_state = event_cleanup_callback_state_; + uint64_t cleanup_epoch = 0; + { + std::shared_lock cleanup_lease(cleanup_state->mutex); + if (cleanup_state->accepting) { + cleanup_epoch = cleanup_state->epoch; + } + } if (!schedule_plan_executor_->SubmitTask([this, reporter_key = task.reporter_key, version = task.version, attempt_epoch = task.attempt_epoch, + lifecycle_generation = mutation_lifecycle_generation, requested_type, - cleanup_backend] { + cleanup_backend, + cleanup_state, + cleanup_epoch] { + std::shared_lock cleanup_lease(cleanup_state->mutex); + if (!cleanup_state->accepting || cleanup_state->epoch != cleanup_epoch) { + return; + } this->CleanupStaleSnapshotLocations( - reporter_key, version, requested_type, cleanup_backend, attempt_epoch); + reporter_key, version, requested_type, cleanup_backend, attempt_epoch, lifecycle_generation); })) { KVCM_LOG_WARN("trace_id [%s] | EVENT_BLOCK_SNAPSHOT: failed to submit stale-data scan for host [%s]", trace_id.c_str(), @@ -3188,26 +3824,80 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, if (has_host_down) { uint64_t gen_at_trigger = 0; - event_backend->UnregisterNodeForHostDown(instance_id, host_ip_port, gen_at_trigger); - assert(schedule_plan_executor_); - const bool cleanup_scheduled = - schedule_plan_executor_->SubmitTask([this, instance_id, host_ip_port, gen_at_trigger, requested_type] { - this->CleanupHostLocations(instance_id, host_ip_port, gen_at_trigger, requested_type); - }); - if (!cleanup_scheduled) { - KVCM_LOG_WARN("trace_id [%s] | HOST_DOWN: failed to submit metadata cleanup for host [%s], " - "instance [%s], gen=%" PRIu64, + const ErrorCode host_down_ec = + event_backend->UnregisterNodeForHostDown(instance_id, host_ip_port, gen_at_trigger); + if (host_down_ec != EC_OK) { + per_item_ec[0] = host_down_ec; + } + bool cleanup_dispatched = false; + if (host_down_ec == EC_OK) { + const auto cleanup_state = event_cleanup_callback_state_; + uint64_t cleanup_epoch = 0; + { + std::shared_lock cleanup_lease(cleanup_state->mutex); + if (cleanup_state->accepting) { + cleanup_epoch = cleanup_state->epoch; + } + } + const auto cleanup = [this, + instance_id, + host_ip_port, + gen_at_trigger, + requested_type, + event_backend, + cleanup_state, + cleanup_epoch] { + std::shared_lock cleanup_lease(cleanup_state->mutex); + if (!cleanup_state->accepting || cleanup_state->epoch != cleanup_epoch) { + return; + } + this->CleanupHostLocations(instance_id, host_ip_port, gen_at_trigger, requested_type, event_backend); + }; + cleanup_dispatched = schedule_plan_executor_ && schedule_plan_executor_->SubmitTask(cleanup); + if (!cleanup_dispatched) { + KVCM_LOG_WARN("trace_id [%s] | HOST_DOWN: cleanup queue unavailable for host [%s], " + "instance [%s], gen=%" PRIu64 "; running inline", + trace_id.c_str(), + host_ip_port.c_str(), + instance_id.c_str(), + gen_at_trigger); + cleanup(); + cleanup_dispatched = true; + } + KVCM_LOG_INFO("trace_id [%s] | HOST_DOWN: host [%s] removed from node table, cleanup_dispatched=%s " + "(gen=%" PRIu64 ")", trace_id.c_str(), host_ip_port.c_str(), - instance_id.c_str(), + cleanup_dispatched ? "true" : "false", gen_at_trigger); + } else { + KVCM_LOG_WARN("trace_id [%s] | HOST_DOWN: failed to end lifecycle for host [%s], " + "instance [%s], ec=%d", + trace_id.c_str(), + host_ip_port.c_str(), + instance_id.c_str(), + host_down_ec); } - KVCM_LOG_INFO("trace_id [%s] | HOST_DOWN: host [%s] removed from node table, cleanup_scheduled=%s " - "(gen=%" PRIu64 ")", - trace_id.c_str(), - host_ip_port.c_str(), - cleanup_scheduled ? "true" : "false", - gen_at_trigger); + } + + for (const auto &location : delta_locations) { + ErrorCode group_failure = EC_OK; + visit_delta_events(location, [&](int event_index, bool /*materialized*/) { + if (per_item_ec[event_index] != EC_OK) { + group_failure = per_item_ec[event_index]; + return false; + } + return true; + }); + if (group_failure == EC_OK) { + continue; + } + visit_delta_events(location, [&](int event_index, bool /*materialized*/) { + if (per_item_ec[event_index] == EC_OK) { + per_item_ec[event_index] = group_failure; + } + return true; + }); } bool any_failure = false; @@ -3291,21 +3981,25 @@ ErrorCode CacheManager::ReportEvent(RequestContext *request_context, void CacheManager::CleanupHostLocations(const std::string &instance_id, const std::string &host_ip_port, uint64_t cleanup_generation, - DataStorageType storage_type) { - auto event_backend_holder = LookupEventReportBackend(registry_manager_, instance_id, storage_type); - auto *event_backend = dynamic_cast(event_backend_holder.get()); - - if (event_backend) { - uint64_t current_gen = event_backend->GetNodeGeneration(instance_id, host_ip_port); - if (current_gen != cleanup_generation) { - KVCM_LOG_INFO("CleanupHostLocations: skipping stale cleanup for host [%s] instance [%s] " - "(trigger_gen=%" PRIu64 ", current_gen=%" PRIu64 " — node re-registered)", - host_ip_port.c_str(), - instance_id.c_str(), - cleanup_generation, - current_gen); - return; - } + DataStorageType storage_type, + const std::shared_ptr &expected_backend) { + if (!IsCurrentEventReportBackend(registry_manager_, instance_id, storage_type, expected_backend)) { + KVCM_LOG_INFO("CleanupHostLocations: skipping stale backend incarnation for host [%s] instance [%s]", + host_ip_port.c_str(), + instance_id.c_str()); + return; + } + const auto &event_backend = expected_backend; + + uint64_t current_gen = event_backend->GetNodeGeneration(instance_id, host_ip_port); + if (current_gen != cleanup_generation) { + KVCM_LOG_INFO("CleanupHostLocations: skipping stale cleanup for host [%s] instance [%s] " + "(trigger_gen=%" PRIu64 ", current_gen=%" PRIu64 " — node re-registered)", + host_ip_port.c_str(), + instance_id.c_str(), + cleanup_generation, + current_gen); + return; } MetaSearcher *meta_searcher = meta_searcher_manager_->GetMetaSearcher(instance_id); @@ -3315,21 +4009,26 @@ void CacheManager::CleanupHostLocations(const std::string &instance_id, } RequestContext cleanup_ctx("cleanup_host_" + host_ip_port); - const std::string host_suffix = event_backend ? event_backend->HostSuffix(host_ip_port) : ("#" + host_ip_port); + const std::string host_suffix = event_backend->HostSuffix(host_ip_port); - auto abort_if_reregistered = [event_backend, instance_id, host_ip_port, cleanup_generation]() -> bool { - if (!event_backend) { - return false; - } - return event_backend->GetNodeGeneration(instance_id, host_ip_port) != cleanup_generation; + auto backend_is_current = [registry_manager = registry_manager_, instance_id, storage_type, event_backend] { + return IsCurrentEventReportBackend(registry_manager, instance_id, storage_type, event_backend); }; - auto acquire_cleanup_lease = [event_backend, instance_id, host_ip_port, cleanup_generation] { + auto abort_if_reregistered = + [event_backend, instance_id, host_ip_port, cleanup_generation, backend_is_current]() -> bool { + return !backend_is_current() || + event_backend->GetNodeGeneration(instance_id, host_ip_port) != cleanup_generation; + }; + auto acquire_cleanup_lease = [event_backend, instance_id, host_ip_port, cleanup_generation, backend_is_current] { EventReportBackend::LifecycleMutationLease lease; - if (!event_backend) { - return std::make_pair(EC_OK, MetaSearcher::MetadataWriteLease{}); + if (!backend_is_current()) { + return std::make_pair(EC_MISMATCH, MetaSearcher::MetadataWriteLease{}); } const ErrorCode ec = event_backend->AcquireLifecycleCleanupLease({instance_id, host_ip_port}, cleanup_generation, lease); + // Keep the global lock order DataStorageManager -> backend lifecycle. + // Re-entering backend_is_current() while holding this lease would + // invert UnRegisterStorage's DataStorageManager -> Close order. return std::make_pair(ec, std::static_pointer_cast(std::move(lease))); }; @@ -3355,7 +4054,11 @@ ErrorCode CacheManager::CleanupStaleSnapshotLocations(const ReporterSnapshotKey const std::string &snapshot_version, DataStorageType storage_type, const std::shared_ptr &event_backend, - uint64_t snapshot_attempt_epoch) { + uint64_t snapshot_attempt_epoch, + uint64_t lifecycle_generation) { + if (!IsCurrentEventReportBackend(registry_manager_, reporter_key.instance_id, storage_type, event_backend)) { + return EC_OK; + } if (!event_backend || snapshot_version.empty() || event_backend->GetStorageType() != storage_type || event_backend->GetSnapshotVersion(reporter_key) != snapshot_version || (snapshot_attempt_epoch != 0 && @@ -3384,15 +4087,39 @@ ErrorCode CacheManager::CleanupStaleSnapshotLocations(const ReporterSnapshotKey IsSnapshotLocationStale( event_backend.get(), reporter_key.instance_id, location, /*preserve_in_flight=*/true); }; - auto should_abort = [event_backend, reporter_key, snapshot_attempt_epoch] { - return snapshot_attempt_epoch != 0 && - event_backend->GetSnapshotAttemptEpoch(reporter_key) != snapshot_attempt_epoch; + auto backend_is_current = [registry_manager = registry_manager_, reporter_key, storage_type, event_backend] { + return IsCurrentEventReportBackend(registry_manager, reporter_key.instance_id, storage_type, event_backend); + }; + auto should_abort = [event_backend, reporter_key, snapshot_attempt_epoch, backend_is_current] { + return !backend_is_current() || (snapshot_attempt_epoch != 0 && event_backend->GetSnapshotAttemptEpoch( + reporter_key) != snapshot_attempt_epoch); + }; + if (lifecycle_generation == 0) { + lifecycle_generation = event_backend->GetNodeGeneration(reporter_key.instance_id, reporter_key.host_ip_port); + } + auto acquire_cleanup_lease = [event_backend, + reporter_key, + snapshot_version, + snapshot_attempt_epoch, + lifecycle_generation, + backend_is_current] { + EventReportBackend::LifecycleMutationLease lease; + if (!backend_is_current()) { + return std::make_pair(EC_MISMATCH, MetaSearcher::MetadataWriteLease{}); + } + const ErrorCode ec = event_backend->AcquireSnapshotCleanupLease( + reporter_key, lifecycle_generation, snapshot_version, snapshot_attempt_epoch, lease); + // Do not query DataStorageManager again while holding the backend + // lifecycle lease; storage unregister takes those locks in the + // opposite (global-manager then backend) order. + return std::make_pair(ec, std::static_pointer_cast(std::move(lease))); }; const ErrorCode ec = meta_searcher->CleanupLocationsByPredicate(&cleanup_ctx, storage_type, /*scan_batch_size=*/1000, std::move(should_delete), - std::move(should_abort)); + std::move(should_abort), + std::move(acquire_cleanup_lease)); const auto elapsed_ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - scan_begin).count(); KVCM_LOG_INFO("SnapshotReclaimer: scanned instance [%s] host [%s] token [%s] in [%" PRId64 "] ms, ec [%d]", @@ -3557,6 +4284,7 @@ ErrorCode CacheManager::GetCacheLocationByQueryType(MetaSearcher *meta_searcher, } ErrorCode CacheManager::DoRecoverOnce() { + ActivateEventCleanupCallbacks(); if (!registry_manager_) { KVCM_LOG_ERROR("CacheManager do recover failed, registry_manager is nullptr"); return EC_ERROR; @@ -3655,19 +4383,38 @@ void CacheManager::ClearEventCleanupCallbacks() { } auto dsm = registry_manager_->data_storage_manager(); for (const auto &name : dsm->GetAllStorageNames()) { - auto *erb = dynamic_cast(dsm->GetDataStorageBackend(name).get()); - if (erb) { - erb->SetCleanupCallback(nullptr); + auto backend = dsm->GetDataStorageBackend(name); + auto event_backend = std::dynamic_pointer_cast(std::move(backend)); + if (event_backend) { + event_backend->SetCleanupCallback(nullptr); } } } +void CacheManager::DeactivateEventCleanupCallbacks() { + if (!event_cleanup_callback_state_) { + return; + } + std::unique_lock callback_fence(event_cleanup_callback_state_->mutex); + event_cleanup_callback_state_->accepting = false; + ++event_cleanup_callback_state_->epoch; +} + +void CacheManager::ActivateEventCleanupCallbacks() { + if (!event_cleanup_callback_state_) { + return; + } + std::unique_lock callback_fence(event_cleanup_callback_state_->mutex); + event_cleanup_callback_state_->accepting = true; +} + ErrorCode CacheManager::DoCleanup() { if (cache_garbage_collector_) { cache_garbage_collector_->Stop(); } ClearEventCleanupCallbacks(); StopRecoverRetryLoop(); + DeactivateEventCleanupCallbacks(); // aborting write session need meta indexer if (write_location_manager_) { write_location_manager_->DoCleanup(); @@ -3772,7 +4519,7 @@ CheckLocDataExistFunc CacheManager::GetCheckLocDataExistFunc(const std::string & } if (IsEventReportStorageType(loc.type())) { - auto event_backend_holder = LookupEventReportBackend(registry_manager_, instance_id, loc.type()); + auto event_backend_holder = LookupEventReportBackend(registry_manager_, instance_id, loc.type(), true); auto *event_backend = dynamic_cast(event_backend_holder.get()); if (!event_backend || loc.type() != event_backend->GetStorageType()) { return false; @@ -3803,7 +4550,83 @@ CheckLocDataExistFunc CacheManager::GetCheckLocDataExistFunc(const std::string & const std::string storage_unique_name = storage_uris.front().GetHostName(); const auto result = registry_manager_->data_storage_manager()->Exist(storage_unique_name, storage_uris, true); - return std::all_of(result.cbegin(), result.cend(), [](bool value) { return value; }); + return result.size() == storage_uris.size() && + std::all_of(result.cbegin(), result.cend(), [](bool value) { return value; }); + }; +} + +MetaSearcher::CheckHostCacheLocationFunc +CacheManager::GetHostCacheStateCheckLocDataExistFunc(const std::string &instance_id) const { + auto fallback = GetCheckLocDataExistFunc(instance_id); + struct EventVisibilitySnapshot { + std::shared_ptr backend; + EventReportBackend::QueryVisibilitySnapshot reporters; + }; + struct EventVisibilitySnapshots { + std::once_flag initialize_once; + std::map by_storage_type; + }; + auto event_snapshots = std::make_shared(); + auto initialize_event_snapshots = [registry_manager = registry_manager_, instance_id, event_snapshots] { + if (!registry_manager || !registry_manager->data_storage_manager()) { + return; + } + const std::string group_name = registry_manager->GetInstanceGroupName(instance_id); + const auto instance_group = registry_manager->GetInstanceGroupConfig(group_name); + const auto storage_manager = registry_manager->data_storage_manager(); + if (!instance_group || !storage_manager) { + return; + } + for (const auto &candidate_name : instance_group->event_report_storage_candidates()) { + auto event_backend = + std::dynamic_pointer_cast(storage_manager->GetDataStorageBackend(candidate_name)); + if (!event_backend) { + continue; + } + const DataStorageType storage_type = event_backend->GetStorageType(); + if (event_snapshots->by_storage_type.find(storage_type) != event_snapshots->by_storage_type.end()) { + continue; + } + EventVisibilitySnapshot snapshot; + snapshot.backend = std::move(event_backend); + snapshot.backend->GetQueryVisibilitySnapshot(instance_id, snapshot.reporters); + event_snapshots->by_storage_type.emplace(storage_type, std::move(snapshot)); + } + }; + + return [fallback = std::move(fallback), + event_snapshots = std::move(event_snapshots), + initialize_event_snapshots = std::move(initialize_event_snapshots)]( + const CacheLocation &location, MetaSearcher::HostCacheLocationInfo &out_info) -> bool { + out_info = {}; + if (!IsEventReportStorageType(location.type())) { + return fallback ? fallback(location) : true; + } + // Initialization is intentionally lazy: MetaSearcher reads metadata + // before invoking this callback, so the request-level liveness/version + // snapshot is taken after the potentially expensive metadata I/O. + std::call_once(event_snapshots->initialize_once, initialize_event_snapshots); + const auto snapshot_it = event_snapshots->by_storage_type.find(location.type()); + if (snapshot_it == event_snapshots->by_storage_type.end() || !snapshot_it->second.backend) { + return false; + } + std::string_view reporter_medium; + std::string_view reporter_host; + if (!snapshot_it->second.backend->ParseLocationIdView(location.id(), reporter_medium, reporter_host)) { + return false; + } + const auto reporter_it = snapshot_it->second.reporters.find(reporter_host); + if (reporter_it == snapshot_it->second.reporters.end()) { + return false; + } + if (!IsEventReportLocationReadable( + location, reporter_it->second.strict, reporter_it->second.committed_version)) { + return false; + } + out_info.has_reporter_identity = true; + out_info.reporter_medium = reporter_medium; + out_info.reporter_host = reporter_host; + return true; }; } @@ -3833,7 +4656,8 @@ CacheManager::GetHostCacheState(RequestContext *request_context, const std::string &instance_id, QueryType query_type, const KeyVector &block_cache_keys, - const std::vector &medium_filter) { + const std::vector &medium_filter, + size_t p2p_host_count) { SPAN_TRACER(request_context); const std::string &trace_id = request_context->trace_id(); auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); @@ -3868,16 +4692,22 @@ CacheManager::GetHostCacheState(RequestContext *request_context, QueryTypeToString(query_type).c_str()); } - PREFIX_LOG(INFO, "GetHostCacheState query_type [%s]", QueryTypeToString(query_type).c_str()); + PREFIX_LOG(DEBUG, "GetHostCacheState query_type [%s]", QueryTypeToString(query_type).c_str()); KVCM_METRICS_COLLECTOR_SET_METRICS(service_metrics_collector, manager, request_key_count, block_cache_keys.size()); auto query_scope = KVCM_METRICS_COLLECTOR_CHRONO_SCOPE(service_metrics_collector, ManagerPrefixMatch); + const auto request_check_location = GetHostCacheStateCheckLocDataExistFunc(instance_id); std::vector host_matches; ErrorCode ec = EC_ERROR; switch (query_type) { case QueryType::QT_PREFIX_MATCH: { - ec = meta_searcher->PrefixMatchByHost( - request_context, block_cache_keys, use_eagle_pop, medium_filter, host_matches); + ec = meta_searcher->PrefixMatchByHost(request_context, + block_cache_keys, + use_eagle_pop, + medium_filter, + host_matches, + &request_check_location, + p2p_host_count); break; } case QueryType::QT_PREFIX_MATCH_WITH_MAMBA: { @@ -3886,7 +4716,9 @@ CacheManager::GetHostCacheState(RequestContext *request_context, use_eagle_pop, medium_filter, instance_info->location_spec_groups(), - host_matches); + host_matches, + &request_check_location, + p2p_host_count); break; } default: @@ -3899,7 +4731,7 @@ CacheManager::GetHostCacheState(RequestContext *request_context, std::vector result; result.reserve(host_matches.size()); for (const auto &match : host_matches) { - result.push_back(HostCacheMatch{match.host_ip_port, match.prefix_match_blocks}); + result.push_back(HostCacheMatch{match.host_ip_port, match.local, match.p2p_1_fetch, match.p2p_1_total_match}); } return {EC_OK, std::move(result)}; diff --git a/kv_cache_manager/manager/cache_manager.h b/kv_cache_manager/manager/cache_manager.h index 871c45a30..96c9ceb95 100644 --- a/kv_cache_manager/manager/cache_manager.h +++ b/kv_cache_manager/manager/cache_manager.h @@ -1,9 +1,11 @@ #pragma once #include +#include #include #include #include +#include #include #include #include @@ -40,6 +42,9 @@ struct MetricsLifecycle; class MigrationManager; constexpr unsigned int DEFAULT_SCHEDULE_PLAN_EXECUTOR_THREAD_COUNT = 2; constexpr unsigned int DEFAULT_SCHEDULE_PLAN_MIGRATION_WORKER_BUDGET = 1; +constexpr unsigned int DEFAULT_META_QUERY_WORKER_COUNT = 4; +constexpr std::size_t DEFAULT_META_QUERY_PARALLEL_THRESHOLD = 256; +constexpr std::size_t DEFAULT_META_QUERY_CHUNK_SIZE = 128; class CacheManager { // TODO should not public @@ -74,7 +79,9 @@ class CacheManager { struct HostCacheMatch { std::string host_ip_port; - int64_t prefix_match_blocks; + int64_t local; + int64_t p2p_1_fetch; + int64_t p2p_1_total_match; }; CacheManager(std::shared_ptr metrics_registry, @@ -90,6 +97,9 @@ class CacheManager { uint32_t cache_reclaimer_worker_size = 16, CacheReclaimerAsyncDeleteConfig cache_reclaimer_async_delete_config = {}, uint32_t schedule_plan_migration_worker_budget = DEFAULT_SCHEDULE_PLAN_MIGRATION_WORKER_BUDGET, + uint32_t meta_query_worker_count = DEFAULT_META_QUERY_WORKER_COUNT, + std::size_t meta_query_parallel_threshold = DEFAULT_META_QUERY_PARALLEL_THRESHOLD, + std::size_t meta_query_chunk_size = DEFAULT_META_QUERY_CHUNK_SIZE, CacheGarbageCollector::Config cache_gc_config = {}); ErrorCode DoRecover(); ErrorCode DoRecoverOnce(); @@ -206,7 +216,8 @@ class CacheManager { const std::string &instance_id, QueryType query_type, const KeyVector &block_cache_keys, - const std::vector &medium_filter = {}); + const std::vector &medium_filter = {}, + size_t p2p_host_count = 0); ErrorCode TrimCache(RequestContext *request_context, const std::string &instance_id, const proto::meta::TrimStrategy &trim_strategy, @@ -236,6 +247,15 @@ class CacheManager { void SetRevisitHistogramConfig(const std::vector &boundaries); private: + struct EventCleanupCallbackState { + std::shared_mutex mutex; + bool accepting = true; + // Advances whenever cleanup is deactivated. A task admitted before a + // leader cleanup must stay stale even if the same CacheManager is later + // activated again during recovery. + uint64_t epoch = 1; + }; + ErrorCode FilterWriteCache(RequestContext *request_context, const std::string &instance_id, MetaSearcher *meta_searcher, @@ -318,12 +338,14 @@ class CacheManager { void CleanupHostLocations(const std::string &instance_id, const std::string &host_ip_port, uint64_t cleanup_generation, - DataStorageType storage_type); + DataStorageType storage_type, + const std::shared_ptr &expected_backend); ErrorCode CleanupStaleSnapshotLocations(const ReporterSnapshotKey &reporter_key, const std::string &snapshot_version, DataStorageType storage_type, const std::shared_ptr &event_backend, - uint64_t snapshot_attempt_epoch = 0); + uint64_t snapshot_attempt_epoch = 0, + uint64_t lifecycle_generation = 0); ErrorCode GetCacheLocationByQueryType(MetaSearcher *meta_searcher, RequestContext *request_context, const std::string &instance_id, @@ -346,8 +368,12 @@ class CacheManager { std::unique_ptr genSelectLocationPolicy(RequestContext *request_context, const std::string &instance_id) const; CheckLocDataExistFunc GetCheckLocDataExistFunc(const std::string &instance_id) const; + MetaSearcher::CheckHostCacheLocationFunc + GetHostCacheStateCheckLocDataExistFunc(const std::string &instance_id) const; SubmitDelReqFunc GetSubmitDelReqFunc(const std::string &instance_id) const; void ClearEventCleanupCallbacks(); + void DeactivateEventCleanupCallbacks(); + void ActivateEventCleanupCallbacks(); // purge metrics registry entries and invoke the removal callback // for a given instance_id @@ -388,6 +414,11 @@ class CacheManager { std::shared_ptr event_manager_; // 无需清理 std::shared_ptr metrics_lifecycle_; + // EventReportBackend owns a callback that cannot retain CacheManager. + // This separate gate lets destruction drain a callback copy that was + // already taken by the liveness thread and reject copies invoked later. + std::shared_ptr event_cleanup_callback_state_ = + std::make_shared(); // 需要清理 - 避免有metrics遗留 std::shared_ptr metrics_recorder_; // 无需清理 diff --git a/kv_cache_manager/manager/cache_reclaimer.cc b/kv_cache_manager/manager/cache_reclaimer.cc index 4e0cb594c..811ed422b 100644 --- a/kv_cache_manager/manager/cache_reclaimer.cc +++ b/kv_cache_manager/manager/cache_reclaimer.cc @@ -1488,6 +1488,10 @@ bool CacheReclaimer::FilterLocID(RequestContext *request_context, if (!loc_ptr) { continue; } + // Reporter-owned locations are not reclaim candidates, but still + // keep the metadata key alive after all ordinary locations have + // been removed. Count them before filtering so key-count credit is + // only granted when the deletion can actually remove the key. ++valid_location_count; const auto &loc = *loc_ptr; if (IsEventReportStorageType(loc.type())) { diff --git a/kv_cache_manager/manager/meta_searcher.cc b/kv_cache_manager/manager/meta_searcher.cc index 7a1437c9c..a2dfd3be6 100644 --- a/kv_cache_manager/manager/meta_searcher.cc +++ b/kv_cache_manager/manager/meta_searcher.cc @@ -1,10 +1,14 @@ #include "kv_cache_manager/manager/meta_searcher.h" #include +#include #include +#include #include -#include +#include #include +#include +#include #include #include #include @@ -38,14 +42,26 @@ void LogErrorCodes(const std::string &operation_name, } } +bool TryGetLocationSpecSize(const LocationSpec &spec, std::uint64_t &size) { + size = 0; + DataStorageUri uri(spec.uri()); + if (!uri.Valid()) { + return false; + } + uri.GetParamAs("size", size); + return true; +} + +std::uint64_t GetLocationSpecSize(const LocationSpec &spec) { + std::uint64_t size = 0; + (void)TryGetLocationSpecSize(spec, size); + return size; +} + std::uint64_t GetLocationSpecsSize(const std::vector &specs) { std::uint64_t total_size = 0; for (const auto &loc_spec : specs) { - if (DataStorageUri ds_uri(loc_spec.uri()); ds_uri.Valid()) { - std::uint64_t spec_size = 0; - ds_uri.GetParamAs("size", spec_size); - total_size += spec_size; - } + total_size += GetLocationSpecSize(loc_spec); } return total_size; } @@ -67,11 +83,33 @@ struct StorageUsageChange { bool has_old = false; }; -ErrorCode ValidateConsistentSnapshotVersion(const std::vector &specs) { +template +ErrorCode +ValidateConsistentSnapshotVersion(size_t spec_count, SpecAccessor spec_at, std::uint64_t *out_total_size = nullptr) { + if (spec_count == 0) { + return EC_BADARGS; + } bool has_snapshot_version = false; std::string snapshot_version; bool has_unversioned_spec = false; - for (const auto &spec : specs) { + std::uint64_t total_size = 0; + std::unordered_set spec_names; + if (spec_count > 1) { + spec_names.reserve(spec_count); + } + for (size_t spec_index = 0; spec_index < spec_count; ++spec_index) { + const auto &spec = spec_at(spec_index); + const DataStorageUri uri(spec.uri()); + if (spec.name().empty() || (spec_count > 1 && !spec_names.insert(std::string_view(spec.name())).second) || + !uri.Valid()) { + return EC_BADARGS; + } + std::uint64_t spec_size = 0; + uri.GetParamAs("size", spec_size); + if (spec_size > std::numeric_limits::max() - total_size) { + return EC_BADARGS; + } + total_size += spec_size; const size_t version_param_count = SnapshotUriUtils::CountUriParam(spec.uri(), SnapshotUriUtils::kSnapshotVersionParam); if (version_param_count == 0) { @@ -82,8 +120,7 @@ ErrorCode ValidateConsistentSnapshotVersion(const std::vector &spe continue; } SnapshotUriInfo info; - if (has_unversioned_spec || version_param_count != 1 || - !SnapshotUriUtils::ParseSnapshotUriInfo(spec.uri(), info)) { + if (has_unversioned_spec || version_param_count != 1 || !SnapshotUriUtils::ParseSnapshotUriInfo(uri, info)) { return EC_BADARGS; } if (!has_snapshot_version) { @@ -93,35 +130,63 @@ ErrorCode ValidateConsistentSnapshotVersion(const std::vector &spe return EC_BADARGS; } } + if (out_total_size) { + *out_total_size = total_size; + } return EC_OK; } -ErrorCode MergeLocationSpecsByName(const std::vector &old_specs, - const std::vector &new_specs, - std::vector &out_specs) { - const ErrorCode parse_ec = ValidateConsistentSnapshotVersion(new_specs); - if (parse_ec != EC_OK) { - return parse_ec; - } +ErrorCode ValidateConsistentSnapshotVersion(const std::vector &specs, + std::uint64_t *out_total_size = nullptr) { + return ValidateConsistentSnapshotVersion( + specs.size(), [&specs](size_t index) -> const LocationSpec & { return specs[index]; }, out_total_size); +} - std::map merged_specs; - for (const auto &spec : old_specs) { - // Snapshot generations are reconciliation/cleanup tags, not a - // visibility fence. After a KVCM restart, a new delta may use a fresh - // generation while untouched specs still carry an older one. Preserve - // those specs and overwrite only names present in this delta. - merged_specs[spec.name()] = spec; - } - for (const auto &spec : new_specs) { - merged_specs[spec.name()] = spec; +ErrorCode ValidateConsistentSnapshotVersion(const MetaSearcher::MergeLocationSpecsTask &task, + std::uint64_t *out_total_size = nullptr) { + return ValidateConsistentSnapshotVersion( + task.SpecCount(), [&task](size_t index) -> const LocationSpec & { return task.SpecAt(index); }, out_total_size); +} + +void MergeLocationSpecsByName(std::vector &merged_specs, + const MetaSearcher::MergeLocationSpecsTask &task) { + // Snapshot generations are reconciliation/cleanup tags, not a visibility + // fence. After a KVCM restart, a new delta may use a fresh generation while + // untouched specs still carry an older one. Preserve those specs and + // overwrite only names present in this delta. Normalize any legacy + // duplicate names in place with the same last-value-wins behavior the old + // std::map implementation provided. + size_t unique_count = 0; + for (size_t i = 0; i < merged_specs.size(); ++i) { + auto duplicate = + std::find_if(merged_specs.begin(), + merged_specs.begin() + unique_count, + [&merged_specs, i](const auto &spec) { return spec.name() == merged_specs[i].name(); }); + if (duplicate != merged_specs.begin() + unique_count) { + *duplicate = std::move(merged_specs[i]); + continue; + } + if (i != unique_count) { + merged_specs[unique_count] = std::move(merged_specs[i]); + } + ++unique_count; } + merged_specs.resize(unique_count); - out_specs.clear(); - out_specs.reserve(merged_specs.size()); - for (auto &[name, spec] : merged_specs) { - out_specs.push_back(std::move(spec)); + for (size_t spec_index = 0; spec_index < task.SpecCount(); ++spec_index) { + const auto &spec = task.SpecAt(spec_index); + auto existing = std::find_if(merged_specs.begin(), merged_specs.end(), [&spec](const auto &candidate) { + return candidate.name() == spec.name(); + }); + if (existing == merged_specs.end()) { + merged_specs.push_back(spec); + } else { + *existing = spec; + } } - return EC_OK; + std::sort(merged_specs.begin(), merged_specs.end(), [](const auto &lhs, const auto &rhs) { + return lhs.name() < rhs.name(); + }); } CacheLocationConstPtr SelectAndMergeForMatch(SelectLocationPolicy *policy, @@ -191,11 +256,27 @@ CacheLocationConstPtr SelectAndMergeForMatch(SelectLocationPolicy *policy, return result; } -std::string ExtractPeerAddrFromLocation(const CacheLocation &loc) { - if (loc.location_specs().empty()) { +const LocationSpec *FindRequestedSpec(const CacheLocation &loc, std::string_view requested_spec_name) { + if (requested_spec_name.empty()) { + return loc.location_specs().empty() ? nullptr : &loc.location_specs().front(); + } + const auto it = + std::find_if(loc.location_specs().begin(), + loc.location_specs().end(), + [requested_spec_name](const LocationSpec &spec) { return spec.name() == requested_spec_name; }); + return it == loc.location_specs().end() ? nullptr : &*it; +} + +bool MatchesRequestedSpec(const CacheLocation &loc, std::string_view requested_spec_name) { + return requested_spec_name.empty() || FindRequestedSpec(loc, requested_spec_name) != nullptr; +} + +std::string ExtractPeerAddrFromLocation(const CacheLocation &loc, std::string_view requested_spec_name) { + const auto *spec = FindRequestedSpec(loc, requested_spec_name); + if (spec == nullptr) { return {}; } - StandardUri uri(loc.location_specs().front().uri()); + StandardUri uri(spec->uri()); if (!uri.Valid() || uri.GetHostName().empty()) { return {}; } @@ -232,7 +313,9 @@ V6DPeerSelection SelectV6DByPrefix(const std::vector &candidate_indices, } prefix_covered.push_back(ci); } - if (prefix_covered.size() > best.covered_indices.size()) { + if (prefix_covered.size() > best.covered_indices.size() || + (prefix_covered.size() == best.covered_indices.size() && + (best.peer_addr.empty() || addr < best.peer_addr))) { best.peer_addr = addr; best.covered_indices = std::move(prefix_covered); } @@ -258,7 +341,8 @@ SelectV6DByCoverage(const std::vector &candidate_indices, } V6DPeerSelection best; for (auto &[addr, indices] : addr_to_indices) { - if (indices.size() > best.covered_indices.size()) { + if (indices.size() > best.covered_indices.size() || + (indices.size() == best.covered_indices.size() && (best.peer_addr.empty() || addr < best.peer_addr))) { best.peer_addr = addr; best.covered_indices = indices; } @@ -287,70 +371,912 @@ CacheLocationMap FilterValidLocations(const CacheLocationMap &location_map, return valid; } -bool IsMediumMatched(const StandardUri &uri, const std::unordered_set &medium_set) { +using MediumViewSet = std::unordered_set; + +constexpr size_t WordCountForBits(size_t bit_count) noexcept { return bit_count == 0 ? 0 : 1 + (bit_count - 1) / 64; } + +// Measures the union of concurrently executing callback intervals. Summing +// per-worker durations would turn the existing wall-time metric into CPU time +// and could make it exceed the enclosing request latency. +class ConcurrentWallTimer { +public: + class Scope { + public: + explicit Scope(ConcurrentWallTimer *timer) noexcept : timer_(timer) { timer_->Begin(); } + ~Scope() { timer_->End(); } + + Scope(const Scope &) = delete; + Scope &operator=(const Scope &) = delete; + + private: + ConcurrentWallTimer *timer_; + }; + + [[nodiscard]] Scope Measure() noexcept { return Scope(this); } + [[nodiscard]] int64_t elapsed_us() const noexcept { return elapsed_us_.load(std::memory_order_relaxed); } + +private: + void Begin() noexcept { + const int64_t now = TimestampUtil::GetCurrentTimeUs(); + if (active_.fetch_add(1, std::memory_order_acq_rel) == 0) { + interval_start_us_.store(now, std::memory_order_release); + } + } + + void End() noexcept { + const int64_t now = TimestampUtil::GetCurrentTimeUs(); + const int64_t interval_start = interval_start_us_.load(std::memory_order_acquire); + if (active_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + elapsed_us_.fetch_add(std::max(now - interval_start, 0), std::memory_order_relaxed); + } + } + +private: + std::atomic active_{0}; + std::atomic interval_start_us_{0}; + std::atomic elapsed_us_{0}; +}; + +MediumViewSet BuildMediumViewSet(const std::vector &medium_filter) { + MediumViewSet mediums; + mediums.reserve(medium_filter.size()); + for (const auto &medium : medium_filter) { + mediums.emplace(medium); + } + return mediums; +} + +bool IsMediumMatched(const StandardUri &uri, const MediumViewSet &medium_set) { if (medium_set.empty()) { return true; } - std::string medium = uri.GetPath(); - if (!medium.empty() && medium[0] == '/') { - medium = medium.substr(1); + std::string_view medium(uri.GetPath()); + if (!medium.empty() && medium.front() == '/') { + medium.remove_prefix(1); } return medium_set.find(medium) != medium_set.end(); } -using HostToSpecNames = std::map>; -using KeyToHostSpecNames = std::vector; // key -> host -> spec names +bool IsReporterMediumMatched(std::string_view medium, const MediumViewSet &medium_set) { + return medium_set.empty() || medium_set.find(medium) != medium_set.end(); +} -void BuildHostSpecNamesForOneKey(const CacheLocationMap &location_map, - CheckLocDataExistFunc check_loc_data_exist, - const std::unordered_set &medium_set, - HostToSpecNames &host_specs) { - for (const auto &kv : location_map) { - const auto &loc = kv.second; - if (!loc || loc->location_specs().empty()) { +template +void VisitHostSpecsForOneKey(const LocationRange &locations, + const CheckLocDataExistFunc &check_loc_data_exist, + const MetaSearcher::CheckHostCacheLocationFunc *request_check_location, + const MediumViewSet &medium_set, + bool visit_spec_names, + Visitor &&visitor) { + for (const auto &loc : locations) { + // GetHostCacheState reports readable data, not metadata intent. A URI + // attached to WRITING/DELETING/NEW must never become a cache hit. + if (!loc || loc->status() != CacheLocationStatus::CLS_SERVING || loc->location_specs().empty()) { continue; } - if (check_loc_data_exist && !check_loc_data_exist(*loc)) { + + MetaSearcher::HostCacheLocationInfo location_info; + if (request_check_location) { + if (!(*request_check_location)(*loc, location_info)) { + continue; + } + } else if (check_loc_data_exist && !check_loc_data_exist(*loc)) { continue; } - std::string event_medium; - std::string reporter_host; - const bool has_reporter_identity = - IsEventReportStorageType(loc->type()) && - SnapshotUriUtils::ParseEventReportLocationId(kv.first, event_medium, reporter_host); - for (const auto &spec : loc->location_specs()) { - StandardUri uri(spec.uri()); - const bool medium_matches = has_reporter_identity ? medium_set.empty() || medium_set.count(event_medium) > 0 - : IsMediumMatched(uri, medium_set); - if (!uri.Valid() || !medium_matches) { + + const bool is_event_report = IsEventReportStorageType(loc->type()); + bool has_reporter_identity = is_event_report && location_info.has_reporter_identity; + if (!has_reporter_identity && is_event_report) { + std::string_view storage_type; + has_reporter_identity = SnapshotUriUtils::ParseEventReportLocationIdView( + loc->id(), storage_type, location_info.reporter_medium, location_info.reporter_host); + } + const bool is_vineyard = loc->type() == DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2; + if (has_reporter_identity) { + if (!IsReporterMediumMatched(location_info.reporter_medium, medium_set) || + location_info.reporter_host.empty()) { + continue; + } + // The request checker already scans every EventReport URI while + // applying the reporter generation fence. Reuse that result. + const bool specs_already_validated = + request_check_location != nullptr && is_event_report && location_info.has_reporter_identity; + if (!visit_spec_names) { + if (specs_already_validated) { + visitor(location_info.reporter_host, std::string_view{}, is_vineyard); + continue; + } + for (const auto &spec : loc->location_specs()) { + if (StandardUri(spec.uri()).Valid()) { + visitor(location_info.reporter_host, std::string_view{}, is_vineyard); + break; + } + } continue; } - std::string host = has_reporter_identity ? reporter_host : uri.GetHostPort(); - if (host.empty()) { + for (const auto &spec : loc->location_specs()) { + if (!specs_already_validated && !StandardUri(spec.uri()).Valid()) { + continue; + } + visitor(location_info.reporter_host, std::string_view(spec.name()), is_vineyard); + } + continue; + } + + for (const auto &spec : loc->location_specs()) { + const StandardUri uri(spec.uri()); + if (!uri.Valid() || !IsMediumMatched(uri, medium_set)) { continue; } - host_specs[std::move(host)].insert(spec.name()); + const std::string host = uri.GetHostPort(); + if (!host.empty()) { + visitor(std::string_view(host), std::string_view(spec.name()), false); + } } } } -bool IsFullLocationSpecGroup(const LocationSpecGroup &group) { - const auto &name = group.name(); - return name.rfind("full", 0) == 0 || name.rfind("FULL", 0) == 0; +template +void BuildHostsForOneKey(const LocationRange &locations, + const CheckLocDataExistFunc &check_loc_data_exist, + const MetaSearcher::CheckHostCacheLocationFunc *request_check_location, + const MediumViewSet &medium_set, + std::vector &hosts) { + VisitHostSpecsForOneKey(locations, + check_loc_data_exist, + request_check_location, + medium_set, + false, + [&hosts](std::string_view host, std::string_view, bool) { hosts.emplace_back(host); }); + std::sort(hosts.begin(), hosts.end()); + hosts.erase(std::unique(hosts.begin(), hosts.end()), hosts.end()); +} + +template +void BuildCandidatePresenceForOneKey(const LocationRange &locations, + const CheckLocDataExistFunc &check_loc_data_exist, + const MetaSearcher::CheckHostCacheLocationFunc *request_check_location, + const MediumViewSet &medium_set, + const std::vector &candidate_hosts, + std::uint64_t *presence_words) { + VisitHostSpecsForOneKey(locations, + check_loc_data_exist, + request_check_location, + medium_set, + false, + [&candidate_hosts, presence_words](std::string_view host, std::string_view, bool) { + const auto it = + std::lower_bound(candidate_hosts.begin(), + candidate_hosts.end(), + host, + [](const std::string &candidate, std::string_view value) { + return std::string_view(candidate) < value; + }); + if (it != candidate_hosts.end() && std::string_view(*it) == host) { + const size_t index = static_cast(it - candidate_hosts.begin()); + presence_words[index / 64] |= std::uint64_t{1} << (index % 64); + } + }); +} + +std::vector SelectTopHostIndicesByLocal(const std::vector &host_matches, + size_t p2p_host_count) { + if (p2p_host_count == 0) { + return {}; + } + std::vector host_indices; + host_indices.reserve(host_matches.size()); + for (size_t i = 0; i < host_matches.size(); ++i) { + if (host_matches[i].local > 0) { + host_indices.push_back(i); + } + } + const size_t selected_count = std::min(p2p_host_count, host_indices.size()); + std::partial_sort(host_indices.begin(), + host_indices.begin() + selected_count, + host_indices.end(), + [&host_matches](size_t lhs, size_t rhs) { + if (host_matches[lhs].local != host_matches[rhs].local) { + return host_matches[lhs].local > host_matches[rhs].local; + } + return host_matches[lhs].host_ip_port < host_matches[rhs].host_ip_port; + }); + host_indices.resize(selected_count); + return host_indices; } -bool HasAllLocationSpecGroups(const std::set &spec_names, - const std::vector &groups) { - for (const auto *group : groups) { - for (const auto &spec_name : group->spec_names()) { - if (spec_names.find(spec_name) == spec_names.end()) { +class DistinctFetchedKeys { +public: + void Add(size_t key_index, int64_t key) { + // Projection is ordered. Collapse adjacent additions from multiple + // groups before de-duplicating repeated block-key values. + if (last_key_index_ == key_index) { + return; + } + last_key_index_ = key_index; + keys_.push_back(key); + } + + int64_t DistinctCount() { + std::sort(keys_.begin(), keys_.end()); + keys_.erase(std::unique(keys_.begin(), keys_.end()), keys_.end()); + return static_cast(keys_.size()); + } + +private: + size_t last_key_index_ = std::numeric_limits::max(); + std::vector keys_; +}; + +class HostCatalog { +public: + size_t GetOrAdd(std::string_view host) { + auto it = host_to_id_.lower_bound(host); + if (it == host_to_id_.end() || std::string_view(it->first) != host) { + const size_t host_id = hosts_by_id_.size(); + it = host_to_id_.emplace_hint(it, std::string(host), host_id); + hosts_by_id_.push_back(&it->first); + } + return it->second; + } + + [[nodiscard]] const std::string &host(size_t host_id) const { return *hosts_by_id_[host_id]; } + [[nodiscard]] size_t size() const { return hosts_by_id_.size(); } + +private: + std::map> host_to_id_; + std::vector hosts_by_id_; +}; + +struct HostSpecEntry { + size_t host_id = 0; + size_t mask_offset = 0; + bool has_vineyard_spec = false; +}; + +// Reused for one metadata key at a time. Host strings are interned once per +// request; entries and masks are cleared after each key, so only the catalog +// grows with the number of distinct hosts rather than with key count. +class HostSpecKeyView { +public: + HostSpecKeyView(HostCatalog &host_catalog, const std::vector &required_spec_names) + : host_catalog_(host_catalog) + , required_spec_names_(required_spec_names) + , spec_word_count_(WordCountForBits(required_spec_names.size())) {} + + template + void Build(const LocationRange &locations, + const CheckLocDataExistFunc &check_loc_data_exist, + const MetaSearcher::CheckHostCacheLocationFunc *request_check_location, + const MediumViewSet &medium_set) { + entries_.clear(); + all_spec_words_.clear(); + vineyard_spec_words_.clear(); + ++generation_; + if (generation_ == 0) { + std::fill(entry_generations_.begin(), entry_generations_.end(), 0); + generation_ = 1; + } + VisitHostSpecsForOneKey( + locations, + check_loc_data_exist, + request_check_location, + medium_set, + spec_word_count_ != 0, + [this](std::string_view host, std::string_view spec_name, bool is_vineyard) { + const size_t host_id = host_catalog_.GetOrAdd(host); + if (entry_generations_.size() <= host_id) { + entry_generations_.resize(host_id + 1, 0); + entry_indices_.resize(host_id + 1, 0); + } + if (entry_generations_[host_id] != generation_) { + entry_generations_[host_id] = generation_; + entry_indices_[host_id] = entries_.size(); + const size_t mask_offset = all_spec_words_.size(); + entries_.push_back({host_id, mask_offset, false}); + all_spec_words_.insert(all_spec_words_.end(), spec_word_count_, 0); + vineyard_spec_words_.insert(vineyard_spec_words_.end(), spec_word_count_, 0); + } + auto &entry = entries_[entry_indices_[host_id]]; + if (is_vineyard) { + entry.has_vineyard_spec = true; + } + if (spec_word_count_ == 0) { + return; + } + const auto spec_it = std::lower_bound(required_spec_names_.begin(), + required_spec_names_.end(), + spec_name, + [](const std::string &candidate, std::string_view value) { + return std::string_view(candidate) < value; + }); + if (spec_it == required_spec_names_.end() || std::string_view(*spec_it) != spec_name) { + return; + } + const size_t spec_index = static_cast(spec_it - required_spec_names_.begin()); + const std::uint64_t bit = std::uint64_t{1} << (spec_index % 64); + all_spec_words_[entry.mask_offset + spec_index / 64] |= bit; + if (is_vineyard) { + vineyard_spec_words_[entry.mask_offset + spec_index / 64] |= bit; + } + }); + } + + [[nodiscard]] const HostSpecEntry *Find(size_t host_id) const { + if (host_id >= entry_generations_.size() || entry_generations_[host_id] != generation_) { + return nullptr; + } + return &entries_[entry_indices_[host_id]]; + } + + [[nodiscard]] bool HasMask(const HostSpecEntry *entry, const std::vector &required) const { + if (!entry) { + // LocationSpecGroup permits an empty spec list. Its all-of + // predicate is vacuously true even when the target host has no + // entry for the key. + return std::all_of(required.begin(), required.end(), [](std::uint64_t word) { return word == 0; }); + } + for (size_t word = 0; word < required.size(); ++word) { + if ((all_spec_words_[entry->mask_offset + word] & required[word]) != required[word]) { + return false; + } + } + return true; + } + + [[nodiscard]] bool CoversMask(const HostSpecEntry *local, + const HostSpecEntry &peer, + const std::vector &required) const { + for (size_t word = 0; word < required.size(); ++word) { + const std::uint64_t local_word = local ? all_spec_words_[local->mask_offset + word] : 0; + const std::uint64_t peer_word = vineyard_spec_words_[peer.mask_offset + word]; + if (((local_word | peer_word) & required[word]) != required[word]) { + return false; + } + } + return true; + } + + void CopyAllWords(const HostSpecEntry *entry, std::vector &out_words) const { + out_words.assign(spec_word_count_, 0); + if (entry) { + std::copy_n(all_spec_words_.begin() + static_cast(entry->mask_offset), + spec_word_count_, + out_words.begin()); + } + } + + void MergeVineyardGroup(const HostSpecEntry &peer, + const std::vector &group_mask, + std::vector &in_out_words) const { + for (size_t word = 0; word < spec_word_count_; ++word) { + in_out_words[word] |= vineyard_spec_words_[peer.mask_offset + word] & group_mask[word]; + } + } + + [[nodiscard]] bool WordsHaveMask(const std::vector &words, + const std::vector &required) const { + for (size_t word = 0; word < required.size(); ++word) { + if ((words[word] & required[word]) != required[word]) { return false; } } + return true; + } + + [[nodiscard]] const std::vector &entries() const { return entries_; } + +private: + HostCatalog &host_catalog_; + const std::vector &required_spec_names_; + size_t spec_word_count_ = 0; + std::vector entries_; + std::vector all_spec_words_; + std::vector vineyard_spec_words_; + std::vector entry_generations_; + std::vector entry_indices_; + std::uint64_t generation_ = 0; +}; + +std::vector BuildSpecMask(const std::vector &required_spec_names, + const LocationSpecGroup &group) { + std::vector mask(WordCountForBits(required_spec_names.size()), 0); + for (const auto &spec_name : group.spec_names()) { + const auto it = std::lower_bound(required_spec_names.begin(), required_spec_names.end(), spec_name); + assert(it != required_spec_names.end() && *it == spec_name); + const size_t index = static_cast(it - required_spec_names.begin()); + mask[index / 64] |= std::uint64_t{1} << (index % 64); + } + return mask; +} + +std::vector MergeSpecMasks(const std::vector> &masks, size_t word_count) { + std::vector merged(word_count, 0); + for (const auto &mask : masks) { + for (size_t word = 0; word < word_count; ++word) { + merged[word] |= mask[word]; + } + } + return merged; +} + +ErrorCode ClassifySpecGroups(RequestContext *request_context, + const std::vector &location_spec_groups, + std::vector &full_groups, + std::vector &mamba_state_groups) { + full_groups.clear(); + mamba_state_groups.clear(); + for (const auto &group : location_spec_groups) { + const auto &group_name = group.name(); + const char category = group_name.empty() ? '?' : group_name.front(); + switch (category) { + case 'F': + full_groups.push_back(&group); + break; + case 'L': + mamba_state_groups.push_back(&group); + break; + case 'W': + case 'C': + case 'E': + case 'X': { + std::string error_msg = + "unsupported location spec category for QT_PREFIX_MATCH_WITH_MAMBA, group: " + group_name + + ", category: " + std::string(1, category); + request_context->error_tracer()->AddErrorMsg(error_msg); + KVCM_LOG_WARN("%s", error_msg.c_str()); + return EC_BADARGS; + } + default: { + std::string error_msg = + "invalid location spec category for QT_PREFIX_MATCH_WITH_MAMBA, group: " + group_name + + ", category: " + std::string(1, category); + request_context->error_tracer()->AddErrorMsg(error_msg); + KVCM_LOG_WARN("%s", error_msg.c_str()); + return EC_BADARGS; + } + } + } + + if (full_groups.empty() || mamba_state_groups.empty()) { + std::string group_names = Jsonizable::ToJsonString(location_spec_groups); + std::string error_msg = + full_groups.empty() ? "no full location spec group" : "no mamba state location spec group"; + error_msg += ", location_spec_groups: " + group_names; + request_context->error_tracer()->AddErrorMsg(error_msg); + KVCM_LOG_WARN("%s", error_msg.c_str()); + return EC_BADARGS; + } + return EC_OK; +} + +size_t SelectLexicographicallySmallestPeer(const std::vector &peer_ids, const HostCatalog &host_catalog) { + if (peer_ids.empty()) { + return std::numeric_limits::max(); + } + return *std::min_element(peer_ids.begin(), peer_ids.end(), [&host_catalog](size_t lhs, size_t rhs) { + return host_catalog.host(lhs) < host_catalog.host(rhs); + }); +} + +struct PrefixPeerTracker { + bool initialized = false; + bool stopped = false; + size_t stop_index = 0; + std::vector active_peer_ids; +}; + +bool AdvancePrefixPeerTracker(const HostSpecKeyView &key_view, + size_t target_host_id, + const std::vector &required_mask, + size_t key_index, + PrefixPeerTracker &tracker) { + const auto *local = key_view.Find(target_host_id); + if (key_view.HasMask(local, required_mask)) { + return true; + } + + const auto is_candidate = [&key_view, local, target_host_id, &required_mask](size_t peer_id) { + const auto *peer = key_view.Find(peer_id); + return peer && peer->host_id != target_host_id && peer->has_vineyard_spec && + key_view.CoversMask(local, *peer, required_mask); + }; + if (!tracker.initialized) { + for (const auto &peer : key_view.entries()) { + if (peer.host_id != target_host_id && peer.has_vineyard_spec && + key_view.CoversMask(local, peer, required_mask)) { + tracker.active_peer_ids.push_back(peer.host_id); + } + } + tracker.initialized = true; + } else if (!std::any_of(tracker.active_peer_ids.begin(), tracker.active_peer_ids.end(), is_candidate)) { + // Preserve the last non-empty intersection; it selects the peer that + // covers all earlier local gaps. + tracker.stopped = true; + tracker.stop_index = key_index; + return false; + } else { + tracker.active_peer_ids.erase(std::remove_if(tracker.active_peer_ids.begin(), + tracker.active_peer_ids.end(), + [&](size_t peer_id) { return !is_candidate(peer_id); }), + tracker.active_peer_ids.end()); + } + if (tracker.active_peer_ids.empty()) { + tracker.stopped = true; + tracker.stop_index = key_index; + return false; } return true; } +ErrorCode PrefixMatchByHostWithoutP2P(MetaIndexer *meta_indexer, + const CheckLocDataExistFunc &check_loc_data_exist, + RequestContext *request_context, + const MetaSearcher::KeyVector &keys, + bool use_eagle_pop, + const std::vector &medium_filter, + std::vector &out_matches, + const MetaSearcher::CheckHostCacheLocationFunc *request_check_location) { + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + const MediumViewSet medium_set = BuildMediumViewSet(medium_filter); + std::vector candidate_hosts; + std::unique_ptr[]> prefix_stops; + std::size_t presence_word_count = 0; + ConcurrentWallTimer projection_wall_timer; + + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerGet); + const auto visitor = + [&keys, + &check_loc_data_exist, + request_check_location, + &medium_set, + &candidate_hosts, + &prefix_stops, + &presence_word_count, + &projection_wall_timer](std::size_t begin, const CompactLocationsPerKey &locations, std::size_t valid_count) { + auto projection_scope = projection_wall_timer.Measure(); + std::size_t first_local_index = 0; + if (begin == 0) { + BuildHostsForOneKey( + locations[0], check_loc_data_exist, request_check_location, medium_set, candidate_hosts); + if (candidate_hosts.empty()) { + return size_t{0}; + } + presence_word_count = WordCountForBits(candidate_hosts.size()); + prefix_stops = std::make_unique[]>(candidate_hosts.size()); + for (std::size_t i = 0; i < candidate_hosts.size(); ++i) { + prefix_stops[i].store(keys.size(), std::memory_order_relaxed); + } + first_local_index = 1; + } + + std::vector presence_words(presence_word_count, 0); + for (std::size_t local_index = first_local_index; local_index < valid_count; ++local_index) { + const std::size_t key_index = begin + local_index; + bool any_host_needs_key = false; + for (std::size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + if (key_index < prefix_stops[host_index].load(std::memory_order_relaxed)) { + any_host_needs_key = true; + break; + } + } + if (!any_host_needs_key) { + // Prefix stops are monotonic. Once every candidate has + // stopped at or before this key, no later key in this + // ordered callback range can change the result. + break; + } + + std::fill(presence_words.begin(), presence_words.end(), 0); + BuildCandidatePresenceForOneKey(locations[local_index], + check_loc_data_exist, + request_check_location, + medium_set, + candidate_hosts, + presence_words.data()); + for (std::size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + std::size_t current = prefix_stops[host_index].load(std::memory_order_relaxed); + if (key_index >= current) { + continue; + } + const std::uint64_t bit = std::uint64_t{1} << (host_index % 64); + if ((presence_words[host_index / 64] & bit) != 0) { + continue; + } + while (key_index < current && + !prefix_stops[host_index].compare_exchange_weak( + current, key_index, std::memory_order_relaxed, std::memory_order_relaxed)) {} + } + } + + bool every_host_stopped = true; + std::size_t last_required_key = 0; + for (std::size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + const std::size_t stop = prefix_stops[host_index].load(std::memory_order_relaxed); + if (stop == keys.size()) { + every_host_stopped = false; + break; + } + last_required_key = std::max(last_required_key, stop); + } + return every_host_stopped ? last_required_key : keys.size(); + }; + const auto result = meta_indexer->VisitLocationValuesForPrefix(request_context, keys, visitor); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerGet); + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_searcher, host_projection_time_us, projection_wall_timer.elapsed_us()); + const std::size_t valid_key_count = result.valid_key_count; + if (valid_key_count < keys.size() && result.terminal_ec != EC_OK) { + KVCM_LOG_DEBUG("prefix match by host end because Get keys[%lu](%lu) return %d", + valid_key_count, + keys[valid_key_count], + result.terminal_ec); + if (result.terminal_ec != ErrorCode::EC_NOENT) { + request_context->error_tracer()->AddErrorMsg("prefix match metadata read failed"); + return result.terminal_ec; + } + } + if (candidate_hosts.empty() || valid_key_count == 0) { + return EC_OK; + } + + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherHostPrefixReduce); + out_matches.reserve(candidate_hosts.size()); + for (std::size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + int64_t prefix_len = + static_cast(std::min(valid_key_count, prefix_stops[host_index].load(std::memory_order_relaxed))); + if (use_eagle_pop) { + prefix_len = std::max(prefix_len - 1, 0); + } + if (prefix_len > 0) { + out_matches.push_back( + MetaSearcher::HostCacheMatch{std::move(candidate_hosts[host_index]), prefix_len, 0, prefix_len}); + } + } + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherHostPrefixReduce); + return EC_OK; +} + +ErrorCode PrefixMatchWithMambaByHostWithoutP2P(MetaIndexer *meta_indexer, + const CheckLocDataExistFunc &check_loc_data_exist, + RequestContext *request_context, + const MetaSearcher::KeyVector &keys, + bool use_eagle_pop, + const std::vector &medium_filter, + const std::vector &full_groups, + const std::vector &mamba_state_groups, + std::vector &out_matches, + const MetaSearcher::CheckHostCacheLocationFunc *request_check_location) { + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + const MediumViewSet medium_set = BuildMediumViewSet(medium_filter); + + std::vector required_spec_names; + auto append_required_names = [&required_spec_names](const std::vector &groups) { + for (const auto *group : groups) { + required_spec_names.insert( + required_spec_names.end(), group->spec_names().begin(), group->spec_names().end()); + } + }; + append_required_names(full_groups); + append_required_names(mamba_state_groups); + std::sort(required_spec_names.begin(), required_spec_names.end()); + required_spec_names.erase(std::unique(required_spec_names.begin(), required_spec_names.end()), + required_spec_names.end()); + const size_t spec_word_count = WordCountForBits(required_spec_names.size()); + std::vector full_required(spec_word_count, 0); + std::vector state_required(spec_word_count, 0); + auto build_required_mask = [&required_spec_names](const std::vector &groups, + std::vector &mask) { + for (const auto *group : groups) { + for (const auto &spec_name : group->spec_names()) { + const auto it = std::lower_bound(required_spec_names.begin(), required_spec_names.end(), spec_name); + assert(it != required_spec_names.end() && *it == spec_name); + const size_t index = static_cast(it - required_spec_names.begin()); + mask[index / 64] |= std::uint64_t{1} << (index % 64); + } + } + }; + build_required_mask(full_groups, full_required); + build_required_mask(mamba_state_groups, state_required); + + std::vector candidate_hosts; + std::unique_ptr[]> full_prefix_stops; + std::vector state_present_words; + std::size_t state_key_word_count = 0; + std::atomic projection_invalid(false); + ConcurrentWallTimer projection_wall_timer; + + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerGet); + const auto visitor = [&keys, + &check_loc_data_exist, + request_check_location, + &medium_set, + &candidate_hosts, + &required_spec_names, + &full_required, + &state_required, + &full_prefix_stops, + &state_present_words, + &state_key_word_count, + &projection_invalid, + &projection_wall_timer, + spec_word_count]( + std::size_t begin, const CompactLocationsPerKey &locations, std::size_t valid_count) { + auto projection_scope = projection_wall_timer.Measure(); + if (begin == 0) { + BuildHostsForOneKey( + locations[0], check_loc_data_exist, request_check_location, medium_set, candidate_hosts); + if (candidate_hosts.empty()) { + return size_t{0}; + } + if (spec_word_count > std::numeric_limits::max() / candidate_hosts.size()) { + projection_invalid.store(true, std::memory_order_relaxed); + return size_t{0}; + } + full_prefix_stops = std::make_unique[]>(candidate_hosts.size()); + for (std::size_t i = 0; i < candidate_hosts.size(); ++i) { + full_prefix_stops[i].store(keys.size(), std::memory_order_relaxed); + } + state_key_word_count = WordCountForBits(keys.size()); + if (state_key_word_count > std::numeric_limits::max() / candidate_hosts.size()) { + projection_invalid.store(true, std::memory_order_relaxed); + return size_t{0}; + } + state_present_words.assign(state_key_word_count * candidate_hosts.size(), 0); + } + + std::vector seen_words(candidate_hosts.size() * spec_word_count, 0); + std::vector host_present(candidate_hosts.size(), 0); + for (std::size_t local_index = 0; local_index < valid_count; ++local_index) { + const std::size_t key_index = begin + local_index; + bool any_host_needs_key = false; + for (size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + if (key_index < full_prefix_stops[host_index].load(std::memory_order_relaxed)) { + any_host_needs_key = true; + break; + } + } + if (!any_host_needs_key) { + break; + } + std::fill(seen_words.begin(), seen_words.end(), 0); + std::fill(host_present.begin(), host_present.end(), 0); + VisitHostSpecsForOneKey( + locations[local_index], + check_loc_data_exist, + request_check_location, + medium_set, + true, + [&candidate_hosts, &required_spec_names, &seen_words, &host_present, spec_word_count]( + std::string_view host, std::string_view spec_name, bool) { + const auto host_it = std::lower_bound(candidate_hosts.begin(), + candidate_hosts.end(), + host, + [](const std::string &candidate, std::string_view value) { + return std::string_view(candidate) < value; + }); + if (host_it == candidate_hosts.end() || std::string_view(*host_it) != host) { + return; + } + const size_t host_index = static_cast(host_it - candidate_hosts.begin()); + host_present[host_index] = 1; + const auto spec_it = std::lower_bound(required_spec_names.begin(), + required_spec_names.end(), + spec_name, + [](const std::string &candidate, std::string_view value) { + return std::string_view(candidate) < value; + }); + if (spec_it == required_spec_names.end() || std::string_view(*spec_it) != spec_name) { + return; + } + const size_t spec_index = static_cast(spec_it - required_spec_names.begin()); + seen_words[host_index * spec_word_count + spec_index / 64] |= std::uint64_t{1} << (spec_index % 64); + }); + + for (size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + bool has_full = host_present[host_index] != 0; + bool has_state = has_full; + for (size_t word = 0; word < spec_word_count && (has_full || has_state); ++word) { + const auto seen = seen_words[host_index * spec_word_count + word]; + has_full = has_full && (seen & full_required[word]) == full_required[word]; + has_state = has_state && (seen & state_required[word]) == state_required[word]; + } + if (has_state) { + state_present_words[host_index * state_key_word_count + key_index / 64] |= std::uint64_t{1} + << (key_index % 64); + } + if (!has_full) { + std::size_t current = full_prefix_stops[host_index].load(std::memory_order_relaxed); + while (key_index < current && + !full_prefix_stops[host_index].compare_exchange_weak( + current, key_index, std::memory_order_relaxed, std::memory_order_relaxed)) {} + } + } + } + + bool every_host_stopped = true; + std::size_t last_required_key = 0; + for (std::size_t host_index = 0; host_index < candidate_hosts.size(); ++host_index) { + const std::size_t stop = full_prefix_stops[host_index].load(std::memory_order_relaxed); + if (stop == keys.size()) { + every_host_stopped = false; + break; + } + last_required_key = std::max(last_required_key, stop); + } + return every_host_stopped ? last_required_key : keys.size(); + }; + const auto result = meta_indexer->VisitLocationValuesForPrefix(request_context, keys, visitor); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerGet); + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_searcher, host_projection_time_us, projection_wall_timer.elapsed_us()); + if (projection_invalid.load(std::memory_order_relaxed)) { + request_context->error_tracer()->AddErrorMsg("mamba host/spec flag matrix size overflow"); + return EC_ERROR; + } + const std::size_t valid_key_count = result.valid_key_count; + if (valid_key_count < keys.size() && result.terminal_ec != EC_OK) { + KVCM_LOG_DEBUG("prefix match with mamba by host end because Get keys[%lu](%lu) return %d", + valid_key_count, + keys[valid_key_count], + result.terminal_ec); + if (result.terminal_ec != ErrorCode::EC_NOENT) { + request_context->error_tracer()->AddErrorMsg("mamba prefix match metadata read failed"); + return result.terminal_ec; + } + } + if (candidate_hosts.empty() || valid_key_count == 0) { + return EC_OK; + } + + std::vector prefix_lengths(candidate_hosts.size(), 0); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherHostPrefixReduce); + const bool reduce_ok = meta_indexer->ParallelForQuery( + candidate_hosts.size(), + [&full_prefix_stops, + &state_present_words, + state_key_word_count, + valid_key_count, + &prefix_lengths, + use_eagle_pop](std::size_t begin, std::size_t end) { + for (std::size_t host_index = begin; host_index < end; ++host_index) { + size_t full_prefix_len = + std::min(valid_key_count, full_prefix_stops[host_index].load(std::memory_order_relaxed)); + if (use_eagle_pop && full_prefix_len > 0) { + --full_prefix_len; + } + while (full_prefix_len > 0) { + const size_t last_index = full_prefix_len - 1; + const size_t word_index = last_index / 64; + const unsigned used_bits = static_cast(last_index % 64) + 1; + const std::uint64_t prefix_mask = used_bits == 64 ? std::numeric_limits::max() + : (std::uint64_t{1} << used_bits) - 1; + const std::uint64_t present = + state_present_words[host_index * state_key_word_count + word_index] & prefix_mask; + if (present != 0) { + const size_t highest_bit = 63U - static_cast(__builtin_clzll(present)); + prefix_lengths[host_index] = static_cast(word_index * 64 + highest_bit + 1); + break; + } + full_prefix_len = word_index * 64; + } + } + }); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherHostPrefixReduce); + if (!reduce_ok) { + request_context->error_tracer()->AddErrorMsg("parallel mamba host prefix reduction failed"); + return EC_ERROR; + } + out_matches.reserve(candidate_hosts.size()); + for (std::size_t i = 0; i < candidate_hosts.size(); ++i) { + if (prefix_lengths[i] > 0) { + out_matches.push_back( + MetaSearcher::HostCacheMatch{std::move(candidate_hosts[i]), prefix_lengths[i], 0, prefix_lengths[i]}); + } + } + return EC_OK; +} + } // namespace MetaSearcher::MetaSearcher(const std::shared_ptr &meta_indexer) : meta_indexer_(meta_indexer) {} @@ -533,27 +1459,61 @@ ErrorCode MetaSearcher::BatchGetBestLocationByBackend(RequestContext *request_co const KeyVector &keys, LocationsPerKey &out_locations, SelectLocationPolicy *policy, - const std::vector &selectors) const { + const std::vector &selectors, + const std::vector &requested_spec_names, + const BlockMask &input_mask) const { assert(policy != nullptr); SPAN_TRACER(request_context); out_locations.clear(); out_locations.resize(keys.size()); + if (!requested_spec_names.empty() && + (requested_spec_names.size() != keys.size() || + std::any_of(requested_spec_names.begin(), requested_spec_names.end(), [](const std::string &name) { + return name.empty(); + }))) { + request_context->error_tracer()->AddErrorMsg( + "requested_spec_names must be empty or contain one non-empty name per key"); + return EC_BADARGS; + } + const bool has_implicit_empty_mask = + std::holds_alternative(input_mask) && std::get(input_mask).empty(); + if (!has_implicit_empty_mask && !IsBlockMaskValid(input_mask, keys.size())) { + return EC_BADARGS; + } + + // A masked block is already available to the caller. Do not send it to + // the metadata backend, but preserve the request's positional response + // shape so callers can safely correlate each entry with the original key. + KeyVector query_keys; + std::vector query_to_output_index; + query_keys.reserve(keys.size()); + query_to_output_index.reserve(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + if (IsIndexInMaskRange(input_mask, i)) { + continue; + } + query_keys.push_back(keys[i]); + query_to_output_index.push_back(i); + } + if (query_keys.empty()) { + return EC_OK; + } auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerGet); CacheLocationMapVector location_maps; - auto result = meta_indexer_->GetLocations(request_context, keys, location_maps); + auto result = meta_indexer_->GetLocations(request_context, query_keys, location_maps); KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerGet); KeyVector prune_keys; std::vector> prune_loc_ids_vec; - std::vector valid_maps(keys.size()); + std::vector valid_maps(query_keys.size()); bool has_error = false; - for (size_t i = 0; i < keys.size(); ++i) { + for (size_t i = 0; i < query_keys.size(); ++i) { if (result.error_codes[i] == ErrorCode::EC_NOENT) { continue; } if (result.error_codes[i] != ErrorCode::EC_OK) { - KVCM_LOG_WARN("get key failed, key[%lu](%lu), error_code: %d", i, keys[i], result.error_codes[i]); + KVCM_LOG_WARN("get key failed, key[%lu](%lu), error_code: %d", i, query_keys[i], result.error_codes[i]); has_error = true; break; } @@ -563,7 +1523,7 @@ ErrorCode MetaSearcher::BatchGetBestLocationByBackend(RequestContext *request_co std::vector prune_loc_ids; valid_maps[i] = FilterValidLocations(location_maps[i], check_loc_data_exist_func_, prune_loc_ids); if (!prune_loc_ids.empty()) { - prune_keys.emplace_back(keys[i]); + prune_keys.emplace_back(query_keys[i]); prune_loc_ids_vec.emplace_back(std::move(prune_loc_ids)); } } @@ -584,17 +1544,21 @@ ErrorCode MetaSearcher::BatchGetBestLocationByBackend(RequestContext *request_co std::unordered_map> key_peer_to_location; bool stop_vineyard = false; - for (size_t i = 0; i < keys.size(); ++i) { + for (size_t i = 0; i < query_keys.size(); ++i) { if (stop_vineyard) break; const auto &vmap = valid_maps[i]; + const std::string_view requested_spec_name = + requested_spec_names.empty() ? std::string_view{} : requested_spec_names[query_to_output_index[i]]; std::vector vineyard_addrs; for (const auto &[id, loc] : vmap) { if (loc->type() != target_type) continue; - std::string addr = ExtractPeerAddrFromLocation(*loc); + if (!MatchesRequestedSpec(*loc, requested_spec_name)) + continue; + std::string addr = ExtractPeerAddrFromLocation(*loc, requested_spec_name); if (addr.empty()) continue; // Dedup: only add if not already present @@ -632,17 +1596,19 @@ ErrorCode MetaSearcher::BatchGetBestLocationByBackend(RequestContext *request_co auto loc_it = peer_it->second.find(selection.peer_addr); if (loc_it == peer_it->second.end()) continue; - out_locations[idx].push_back(loc_it->second); + out_locations[query_to_output_index[idx]].push_back(loc_it->second); } } } else { // --- Per-key independent selection (WEIGHTED_RANDOM or other non-event-report) --- - for (size_t i = 0; i < keys.size(); ++i) { + for (size_t i = 0; i < query_keys.size(); ++i) { + const std::string_view requested_spec_name = + requested_spec_names.empty() ? std::string_view{} : requested_spec_names[query_to_output_index[i]]; const auto &vmap = valid_maps[i]; CacheLocationMap filtered; for (const auto &[id, loc] : vmap) { - if (loc->type() == target_type) { + if (loc->type() == target_type && MatchesRequestedSpec(*loc, requested_spec_name)) { filtered.try_emplace(id, loc); } } @@ -677,7 +1643,7 @@ ErrorCode MetaSearcher::BatchGetBestLocationByBackend(RequestContext *request_co } merged->set_spec_size(specs.size()); merged->set_location_specs(std::move(specs)); - out_locations[i].push_back(std::move(merged)); + out_locations[query_to_output_index[i]].push_back(std::move(merged)); } } } @@ -767,58 +1733,242 @@ ErrorCode MetaSearcher::PrefixMatchByHost(RequestContext *request_context, const KeyVector &keys, bool use_eagle_pop, const std::vector &medium_filter, - std::vector &out_matches) const { + std::vector &out_matches, + const CheckHostCacheLocationFunc *request_check_location, + size_t p2p_host_count) const { SPAN_TRACER(request_context); out_matches.clear(); if (keys.empty()) { return EC_OK; } + if (p2p_host_count == 0) { + return PrefixMatchByHostWithoutP2P(meta_indexer_.get(), + check_loc_data_exist_func_, + request_context, + keys, + use_eagle_pop, + medium_filter, + out_matches, + request_check_location); + } - auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); - KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerGet); - CacheLocationMapVector location_maps; - auto result = meta_indexer_->GetLocations(request_context, keys, location_maps); - KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerGet); - LogErrorCodes("PrefixMatchByHost", result.error_codes, keys); - assert(keys.size() == location_maps.size()); + // Local metadata streams this reduction in bounded ordered chunks. Other + // backends preserve their existing single batched read through the same + // visitor API, avoiding a second P2P implementation. + { + struct TrackedPeerPrefix { + size_t host_index = 0; + bool peer_initialized = false; + bool stopped = false; + size_t stop_index = 0; + std::vector active_peer_ids; + DistinctFetchedKeys fetched_keys; + }; + + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + const MediumViewSet medium_set = BuildMediumViewSet(medium_filter); + const auto &check_loc_data_exist = check_loc_data_exist_func_; + HostCatalog host_catalog; + const std::vector no_required_specs; + HostSpecKeyView key_view(host_catalog, no_required_specs); + std::vector candidate_host_ids; + std::vector raw_prefix_stops; + std::vector local_active_indices; + std::vector tracked_prefixes; + std::vector newly_stopped; + size_t visited_until = 0; + bool projection_valid = true; + int64_t projection_cpu_time_us = 0; + + const auto visitor = [&keys, + &check_loc_data_exist, + request_check_location, + &medium_set, + &host_catalog, + &key_view, + &candidate_host_ids, + &raw_prefix_stops, + &local_active_indices, + &tracked_prefixes, + &newly_stopped, + &visited_until, + &projection_valid, + &projection_cpu_time_us, + p2p_host_count, + use_eagle_pop]( + size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + const int64_t projection_begin = TimestampUtil::GetCurrentTimeUs(); + if (begin != visited_until) { + projection_valid = false; + return size_t{0}; + } + for (size_t local_index = 0; local_index < valid_count; ++local_index) { + const size_t key_index = begin + local_index; + key_view.Build(locations[local_index], check_loc_data_exist, request_check_location, medium_set); + if (key_index == 0) { + for (const auto &entry : key_view.entries()) { + candidate_host_ids.push_back(entry.host_id); + } + std::sort( + candidate_host_ids.begin(), candidate_host_ids.end(), [&host_catalog](size_t lhs, size_t rhs) { + return host_catalog.host(lhs) < host_catalog.host(rhs); + }); + raw_prefix_stops.assign(candidate_host_ids.size(), keys.size()); + local_active_indices.resize(candidate_host_ids.size()); + std::iota(local_active_indices.begin(), local_active_indices.end(), size_t{0}); + if (candidate_host_ids.empty()) { + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return size_t{0}; + } + } - KeyToHostSpecNames key_to_host_spec_names; - key_to_host_spec_names.reserve(keys.size()); - const std::unordered_set medium_set(medium_filter.begin(), medium_filter.end()); - for (size_t i = 0; i < keys.size(); ++i) { - if (result.error_codes[i] != ErrorCode::EC_OK) { - KVCM_LOG_DEBUG( - "prefix match by host end because Get keys[%lu](%lu) return %d", i, keys[i], result.error_codes[i]); - break; - } - HostToSpecNames host_to_spec_names; - BuildHostSpecNamesForOneKey(location_maps[i], check_loc_data_exist_func_, medium_set, host_to_spec_names); - // 只保留可参与前缀匹配的连续 key 前缀。 - key_to_host_spec_names.push_back(std::move(host_to_spec_names)); - } + newly_stopped.clear(); + size_t active_write = 0; + for (size_t host_index : local_active_indices) { + const auto *entry = key_view.Find(candidate_host_ids[host_index]); + if (!entry) { + raw_prefix_stops[host_index] = key_index; + newly_stopped.push_back(host_index); + } else { + local_active_indices[active_write++] = host_index; + } + } + local_active_indices.resize(active_write); + + if (!newly_stopped.empty()) { + for (size_t host_index : newly_stopped) { + const size_t raw_prefix = raw_prefix_stops[host_index]; + const size_t local = use_eagle_pop && raw_prefix != 0 ? raw_prefix - 1 : raw_prefix; + if (local != 0) { + tracked_prefixes.push_back( + TrackedPeerPrefix{host_index, false, false, keys.size(), {}, {}}); + } + } + std::sort( + tracked_prefixes.begin(), + tracked_prefixes.end(), + [&raw_prefix_stops, &candidate_host_ids, &host_catalog](const auto &lhs, const auto &rhs) { + if (raw_prefix_stops[lhs.host_index] != raw_prefix_stops[rhs.host_index]) { + return raw_prefix_stops[lhs.host_index] > raw_prefix_stops[rhs.host_index]; + } + return host_catalog.host(candidate_host_ids[lhs.host_index]) < + host_catalog.host(candidate_host_ids[rhs.host_index]); + }); + // Stops only move forward. After considering every host + // that stopped at this key, a discarded lower-ranked host + // can never re-enter the final top-N. + if (tracked_prefixes.size() > p2p_host_count) { + tracked_prefixes.resize(p2p_host_count); + } + } - if (key_to_host_spec_names.empty() || key_to_host_spec_names.front().empty()) { - return EC_OK; - } + for (auto &tracked : tracked_prefixes) { + if (tracked.stopped) { + continue; + } + const size_t target_host_id = candidate_host_ids[tracked.host_index]; + const auto *local = key_view.Find(target_host_id); + if (local) { + continue; + } + if (!tracked.peer_initialized) { + for (const auto &peer : key_view.entries()) { + if (peer.host_id != target_host_id && peer.has_vineyard_spec) { + tracked.active_peer_ids.push_back(peer.host_id); + } + } + tracked.peer_initialized = true; + } else { + tracked.active_peer_ids.erase(std::remove_if(tracked.active_peer_ids.begin(), + tracked.active_peer_ids.end(), + [&key_view](size_t peer_id) { + const auto *peer = key_view.Find(peer_id); + return !peer || !peer->has_vineyard_spec; + }), + tracked.active_peer_ids.end()); + } + if (tracked.active_peer_ids.empty()) { + tracked.stopped = true; + tracked.stop_index = key_index; + continue; + } + tracked.fetched_keys.Add(key_index, keys[key_index]); + } - out_matches.reserve(key_to_host_spec_names.front().size()); - // 只有命中第一个 key 的 host 才可能有大于 0 的前缀长度。 - for (const auto &[host, _] : key_to_host_spec_names.front()) { - int64_t prefix_len = 1; - for (size_t i = 1; i < key_to_host_spec_names.size(); ++i) { - if (key_to_host_spec_names[i].find(host) == key_to_host_spec_names[i].end()) { - break; + visited_until = key_index + 1; + const bool all_tracked_stopped = std::all_of(tracked_prefixes.begin(), + tracked_prefixes.end(), + [](const auto &tracked) { return tracked.stopped; }); + if (local_active_indices.empty() && all_tracked_stopped) { + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return key_index; + } } - ++prefix_len; + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return keys.size(); + }; + + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerGet); + const auto result = meta_indexer_->VisitLocationValuesForPrefix( + request_context, keys, visitor, MetaIndexer::PrefixVisitOrder::ORDERED); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerGet); + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_searcher, host_projection_time_us, projection_cpu_time_us); + if (!projection_valid) { + request_context->error_tracer()->AddErrorMsg("ordered host projection callback mismatch"); + return result.terminal_ec == EC_OK ? EC_MISMATCH : result.terminal_ec; } - if (use_eagle_pop) { - prefix_len = std::max(prefix_len - 1, 0); + if (result.valid_key_count < keys.size() && result.terminal_ec != EC_OK) { + KVCM_LOG_DEBUG("prefix match by host end because Get keys[%lu](%lu) return %d", + result.valid_key_count, + keys[result.valid_key_count], + result.terminal_ec); + if (result.terminal_ec != EC_NOENT) { + request_context->error_tracer()->AddErrorMsg("prefix match metadata read failed"); + return result.terminal_ec; + } } - if (prefix_len > 0) { - out_matches.push_back(HostCacheMatch{host, prefix_len}); + if (candidate_host_ids.empty()) { + return EC_OK; + } + + std::vector host_matches(candidate_host_ids.size()); + for (size_t host_index = 0; host_index < candidate_host_ids.size(); ++host_index) { + raw_prefix_stops[host_index] = std::min(raw_prefix_stops[host_index], result.valid_key_count); + int64_t local = static_cast(raw_prefix_stops[host_index]); + if (use_eagle_pop) { + local = std::max(local - 1, 0); + } + host_matches[host_index] = {host_catalog.host(candidate_host_ids[host_index]), local, 0, local}; + } + + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherHostPrefixReduce); + const auto top_host_indices = SelectTopHostIndicesByLocal(host_matches, p2p_host_count); + for (size_t host_index : top_host_indices) { + const auto tracked = std::find_if(tracked_prefixes.begin(), + tracked_prefixes.end(), + [host_index](const auto &item) { return item.host_index == host_index; }); + if (tracked == tracked_prefixes.end()) { + continue; + } + const size_t raw_total = tracked->stopped ? tracked->stop_index : result.valid_key_count; + int64_t total_match = static_cast(raw_total); + if (use_eagle_pop) { + total_match = std::max(total_match - 1, 0); + } + host_matches[host_index].p2p_1_fetch = tracked->fetched_keys.DistinctCount(); + host_matches[host_index].p2p_1_total_match = total_match; } + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherHostPrefixReduce); + + out_matches.reserve(host_matches.size()); + for (auto &match : host_matches) { + if (match.local > 0) { + out_matches.push_back(std::move(match)); + } + } + return EC_OK; } - return EC_OK; } ErrorCode MetaSearcher::PrefixMatchWithMambaByHost(RequestContext *request_context, @@ -826,7 +1976,9 @@ ErrorCode MetaSearcher::PrefixMatchWithMambaByHost(RequestContext *request_conte bool use_eagle_pop, const std::vector &medium_filter, const std::vector &location_spec_groups, - std::vector &out_matches) const { + std::vector &out_matches, + const CheckHostCacheLocationFunc *request_check_location, + size_t p2p_host_count) const { SPAN_TRACER(request_context); out_matches.clear(); if (keys.empty()) { @@ -835,80 +1987,628 @@ ErrorCode MetaSearcher::PrefixMatchWithMambaByHost(RequestContext *request_conte std::vector full_groups; std::vector mamba_state_groups; - for (const auto &group : location_spec_groups) { - if (IsFullLocationSpecGroup(group)) { - full_groups.push_back(&group); - } else { - mamba_state_groups.push_back(&group); - } + auto ec = ClassifySpecGroups(request_context, location_spec_groups, full_groups, mamba_state_groups); + if (ec != EC_OK) { + return ec; } - if (full_groups.empty() || mamba_state_groups.empty()) { - std::string group_names = Jsonizable::ToJsonString(location_spec_groups); - std::string error_msg = - full_groups.empty() ? "no full location spec group" : "no mamba state location spec group"; - error_msg += ", location_spec_groups: " + group_names; - request_context->error_tracer()->AddErrorMsg(error_msg); - KVCM_LOG_WARN("%s", error_msg.c_str()); - return EC_BADARGS; + if (p2p_host_count == 0) { + return PrefixMatchWithMambaByHostWithoutP2P(meta_indexer_.get(), + check_loc_data_exist_func_, + request_context, + keys, + use_eagle_pop, + medium_filter, + full_groups, + mamba_state_groups, + out_matches, + request_check_location); } - auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); - KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerGet); - CacheLocationMapVector location_maps; - auto result = meta_indexer_->GetLocations(request_context, keys, location_maps); - KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerGet); - LogErrorCodes("PrefixMatchWithMambaByHost", result.error_codes, keys); - assert(keys.size() == location_maps.size()); + // Local metadata uses up to three bounded ordered passes (local, peer + // planning, and final projection when a peer plan is selected). Backends + // that cannot stream retain their first compact batch and replay it, + // preserving one backend read without a second P2P implementation. + { + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + const MediumViewSet medium_set = BuildMediumViewSet(medium_filter); + const auto &check_loc_data_exist = check_loc_data_exist_func_; + + std::vector required_spec_names; + auto append_required_names = [&required_spec_names](const std::vector &groups) { + for (const auto *group : groups) { + required_spec_names.insert( + required_spec_names.end(), group->spec_names().begin(), group->spec_names().end()); + } + }; + append_required_names(full_groups); + append_required_names(mamba_state_groups); + std::sort(required_spec_names.begin(), required_spec_names.end()); + required_spec_names.erase(std::unique(required_spec_names.begin(), required_spec_names.end()), + required_spec_names.end()); + + const size_t spec_word_count = WordCountForBits(required_spec_names.size()); + std::vector> full_group_masks; + std::vector> state_group_masks; + full_group_masks.reserve(full_groups.size()); + state_group_masks.reserve(mamba_state_groups.size()); + for (const auto *group : full_groups) { + // Empty groups are vacuously satisfied and need neither a peer + // tracker nor a fetch range. + if (!group->spec_names().empty()) { + full_group_masks.push_back(BuildSpecMask(required_spec_names, *group)); + } + } + for (const auto *group : mamba_state_groups) { + // Likewise, an empty state group contributes no coverage query. + if (!group->spec_names().empty()) { + state_group_masks.push_back(BuildSpecMask(required_spec_names, *group)); + } + } + const auto full_required = MergeSpecMasks(full_group_masks, spec_word_count); + const auto state_required = MergeSpecMasks(state_group_masks, spec_word_count); + + int64_t total_indexer_time_us = 0; + int64_t total_backend_read_wall_time_us = 0; + int64_t projection_cpu_time_us = 0; + const bool reuse_batched_locations = !meta_indexer_->SupportsProgressiveLocationValueReads(); + CompactLocationsPerKey batched_locations; + size_t batched_valid_count = 0; + ErrorCode batched_terminal_ec = EC_OK; + bool batched_locations_ready = false; + auto run_ordered_scan = [this, + request_context, + &keys, + &total_indexer_time_us, + &total_backend_read_wall_time_us, + &batched_locations, + &batched_valid_count, + &batched_terminal_ec, + &batched_locations_ready](const MetaIndexer::PrefixLocationVisitor &visitor) { + const int64_t begin = TimestampUtil::GetCurrentTimeUs(); + MetaIndexer::PrefixLocationResult result; + if (!batched_locations_ready) { + result = meta_indexer_->VisitLocationValuesForPrefix( + request_context, keys, visitor, MetaIndexer::PrefixVisitOrder::ORDERED); + } else { + size_t requested_stop = keys.size(); + try { + requested_stop = std::min(keys.size(), visitor(0, batched_locations, batched_valid_count)); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("replayed location prefix visitor failed: %s", e.what()); + result.terminal_ec = EC_ERROR; + } catch (...) { + KVCM_LOG_ERROR("replayed location prefix visitor failed with unknown exception"); + result.terminal_ec = EC_ERROR; + } + if (result.terminal_ec == EC_OK) { + result.valid_key_count = std::min(batched_valid_count, requested_stop); + result.stopped_by_visitor = requested_stop <= batched_valid_count && requested_stop < keys.size(); + if (batched_valid_count < requested_stop && batched_valid_count < keys.size()) { + result.terminal_ec = batched_terminal_ec; + } + } + } + total_indexer_time_us += TimestampUtil::GetCurrentTimeUs() - begin; + total_backend_read_wall_time_us += result.backend_read_wall_time_us; + return result; + }; + auto publish_metrics = [service_metrics_collector, + &total_indexer_time_us, + &total_backend_read_wall_time_us, + &projection_cpu_time_us, + reuse_batched_locations] { + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_searcher, indexer_get_time_us, total_indexer_time_us); + if (!reuse_batched_locations) { + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_indexer, get_io_time_us, total_backend_read_wall_time_us); + } + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_searcher, host_projection_time_us, projection_cpu_time_us); + }; + auto hard_read_error = [](const MetaIndexer::PrefixLocationResult &result) { + return result.terminal_ec != EC_OK && result.terminal_ec != EC_NOENT; + }; + + struct LocalMambaState { + size_t raw_full_stop = 0; + size_t previous_state_match = 0; + size_t latest_state_match = 0; + }; + + HostCatalog host_catalog; + HostSpecKeyView key_view(host_catalog, required_spec_names); + std::vector candidate_host_ids; + std::vector local_states; + std::vector full_active_indices; + size_t local_visited_until = 0; + bool local_projection_valid = true; + bool local_prefix_complete = false; + const auto local_visitor = [&keys, + &check_loc_data_exist, + request_check_location, + &medium_set, + &host_catalog, + &key_view, + &candidate_host_ids, + &local_states, + &full_active_indices, + &batched_locations, + &batched_valid_count, + &local_visited_until, + &local_projection_valid, + &local_prefix_complete, + &projection_cpu_time_us, + &full_required, + &state_required, + reuse_batched_locations]( + size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + const int64_t projection_begin = TimestampUtil::GetCurrentTimeUs(); + if (begin != local_visited_until) { + local_projection_valid = false; + return size_t{0}; + } + if (reuse_batched_locations && begin == 0) { + batched_locations = locations; + batched_valid_count = valid_count; + } + for (size_t local_index = 0; local_index < valid_count; ++local_index) { + const size_t key_index = begin + local_index; + if (key_index != 0 && full_active_indices.empty()) { + local_visited_until = begin + valid_count; + break; + } + key_view.Build(locations[local_index], check_loc_data_exist, request_check_location, medium_set); + if (key_index == 0) { + for (const auto &entry : key_view.entries()) { + candidate_host_ids.push_back(entry.host_id); + } + std::sort( + candidate_host_ids.begin(), candidate_host_ids.end(), [&host_catalog](size_t lhs, size_t rhs) { + return host_catalog.host(lhs) < host_catalog.host(rhs); + }); + local_states.assign(candidate_host_ids.size(), LocalMambaState{keys.size(), 0, 0}); + full_active_indices.resize(candidate_host_ids.size()); + std::iota(full_active_indices.begin(), full_active_indices.end(), size_t{0}); + if (candidate_host_ids.empty()) { + local_prefix_complete = true; + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return size_t{0}; + } + } - KeyToHostSpecNames key_to_host_spec_names(keys.size()); - const std::unordered_set medium_set(medium_filter.begin(), medium_filter.end()); - for (size_t i = 0; i < keys.size(); ++i) { - if (result.error_codes[i] != ErrorCode::EC_OK) { + size_t active_write = 0; + for (size_t host_index : full_active_indices) { + auto &state = local_states[host_index]; + const auto *local = key_view.Find(candidate_host_ids[host_index]); + if (!key_view.HasMask(local, full_required)) { + state.raw_full_stop = key_index; + continue; + } + full_active_indices[active_write++] = host_index; + if (key_view.HasMask(local, state_required)) { + state.previous_state_match = state.latest_state_match; + state.latest_state_match = key_index + 1; + } + } + full_active_indices.resize(active_write); + local_visited_until = key_index + 1; + if (full_active_indices.empty()) { + local_prefix_complete = true; + if (!reuse_batched_locations) { + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return key_index; + } + } + } + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return keys.size(); + }; + + const auto local_result = run_ordered_scan(local_visitor); + if (reuse_batched_locations) { + batched_terminal_ec = local_result.terminal_ec; + batched_locations_ready = true; + } + if (!local_projection_valid) { + publish_metrics(); + request_context->error_tracer()->AddErrorMsg("ordered mamba local projection callback mismatch"); + return local_result.terminal_ec == EC_OK ? EC_MISMATCH : local_result.terminal_ec; + } + const bool defer_batched_suffix_error = + hard_read_error(local_result) && reuse_batched_locations && local_prefix_complete; + if (local_result.valid_key_count < keys.size() && local_result.terminal_ec != EC_OK && + !defer_batched_suffix_error) { KVCM_LOG_DEBUG("prefix match with mamba by host end because Get keys[%lu](%lu) return %d", - i, - keys[i], - result.error_codes[i]); - break; + local_result.valid_key_count, + keys[local_result.valid_key_count], + local_result.terminal_ec); + } + // A batched backend may have already observed an error after the local + // prefix was proven complete. Defer that suffix error to peer planning, + // which either proves it irrelevant or propagates it if a selected plan + // actually needs to cross the error boundary. + if (hard_read_error(local_result) && !defer_batched_suffix_error) { + publish_metrics(); + request_context->error_tracer()->AddErrorMsg("mamba prefix match metadata read failed"); + return local_result.terminal_ec; + } + if (candidate_host_ids.empty()) { + publish_metrics(); + return EC_OK; } - BuildHostSpecNamesForOneKey( - location_maps[i], check_loc_data_exist_func_, medium_set, key_to_host_spec_names[i]); - } - out_matches.reserve(key_to_host_spec_names.front().size()); - // 只有命中第一个 key 的 host 才可能有大于 0 的前缀长度。 - for (const auto &kv : key_to_host_spec_names.front()) { - const auto &host = kv.first; - size_t full_prefix_len = 0; - for (; full_prefix_len < key_to_host_spec_names.size(); ++full_prefix_len) { - auto host_it = key_to_host_spec_names[full_prefix_len].find(host); - if (host_it == key_to_host_spec_names[full_prefix_len].end() || - !HasAllLocationSpecGroups(host_it->second, full_groups)) { - break; + std::vector host_matches(candidate_host_ids.size()); + for (size_t host_index = 0; host_index < candidate_host_ids.size(); ++host_index) { + auto &state = local_states[host_index]; + state.raw_full_stop = std::min(state.raw_full_stop, local_result.valid_key_count); + size_t adjusted_full = state.raw_full_stop; + if (use_eagle_pop && adjusted_full > 0) { + --adjusted_full; + } + size_t local = 0; + if (state.latest_state_match <= adjusted_full) { + local = state.latest_state_match; + } else if (state.previous_state_match <= adjusted_full) { + local = state.previous_state_match; } + host_matches[host_index] = {host_catalog.host(candidate_host_ids[host_index]), + static_cast(local), + 0, + static_cast(local)}; } - if (use_eagle_pop && full_prefix_len > 0) { - --full_prefix_len; + auto move_positive_matches_to_output = [&host_matches, &out_matches] { + out_matches.reserve(host_matches.size()); + for (auto &match : host_matches) { + if (match.local > 0) { + out_matches.push_back(std::move(match)); + } + } + }; + if (full_group_masks.empty() && state_group_masks.empty()) { + publish_metrics(); + move_positive_matches_to_output(); + return EC_OK; } - if (full_prefix_len == 0) { - continue; + + struct MambaP2PPlan { + size_t host_index = 0; + size_t target_host_id = 0; + std::vector full_trackers; + std::vector full_peer_ids; + std::vector full_group_stops; + bool combined_full_active = true; + size_t full_stop = 0; + size_t adjusted_full = 0; + std::vector coverage_counts; + std::vector pending_coverage_counts; + std::vector pending_coverage_peer_ids; + size_t coverage_peer_id = std::numeric_limits::max(); + bool final_projection_required = false; + DistinctFetchedKeys fetched_keys; + std::vector combined_words; + int64_t total_match = 0; + }; + + const auto top_host_indices = SelectTopHostIndicesByLocal(host_matches, p2p_host_count); + std::vector plans; + plans.reserve(top_host_indices.size()); + for (size_t host_index : top_host_indices) { + MambaP2PPlan plan; + plan.host_index = host_index; + plan.target_host_id = candidate_host_ids[host_index]; + plan.full_trackers.resize(full_group_masks.size()); + plan.full_peer_ids.assign(full_group_masks.size(), std::numeric_limits::max()); + plan.combined_words.reserve(spec_word_count); + plan.total_match = host_matches[host_index].local; + plans.push_back(std::move(plan)); + } + if (plans.empty()) { + publish_metrics(); + move_positive_matches_to_output(); + return EC_OK; + } + + auto add_state_coverage_for_key = + [&key_view, &host_catalog, &state_group_masks]( + MambaP2PPlan &plan, std::vector &counts, std::vector *touched_peer_ids = nullptr) { + if (state_group_masks.empty()) { + return; + } + if (counts.size() < host_catalog.size()) { + counts.resize(host_catalog.size(), 0); + } + const auto *local = key_view.Find(plan.target_host_id); + for (const auto &group_mask : state_group_masks) { + if (key_view.HasMask(local, group_mask)) { + continue; + } + for (const auto &peer : key_view.entries()) { + if (peer.host_id != plan.target_host_id && peer.has_vineyard_spec && + key_view.CoversMask(local, peer, group_mask)) { + if (touched_peer_ids && counts[peer.host_id] == 0) { + touched_peer_ids->push_back(peer.host_id); + } + ++counts[peer.host_id]; + } + } + } + }; + auto commit_pending_coverage = [](MambaP2PPlan &plan) { + if (plan.coverage_counts.size() < plan.pending_coverage_counts.size()) { + plan.coverage_counts.resize(plan.pending_coverage_counts.size(), 0); + } + for (size_t peer_id : plan.pending_coverage_peer_ids) { + plan.coverage_counts[peer_id] += plan.pending_coverage_counts[peer_id]; + plan.pending_coverage_counts[peer_id] = 0; + } + plan.pending_coverage_peer_ids.clear(); + }; + + size_t full_visited_until = 0; + bool full_phase_valid = true; + const auto full_phase_visitor = [&keys, + &check_loc_data_exist, + request_check_location, + &medium_set, + &key_view, + &plans, + &local_states, + &full_group_masks, + &add_state_coverage_for_key, + &commit_pending_coverage, + &full_visited_until, + &full_phase_valid, + &projection_cpu_time_us, + use_eagle_pop]( + size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + const int64_t projection_begin = TimestampUtil::GetCurrentTimeUs(); + if (begin != full_visited_until) { + full_phase_valid = false; + return size_t{0}; + } + for (size_t local_index = 0; local_index < valid_count; ++local_index) { + const size_t key_index = begin + local_index; + key_view.Build(locations[local_index], check_loc_data_exist, request_check_location, medium_set); + for (auto &plan : plans) { + if (key_index >= local_states[plan.host_index].raw_full_stop) { + for (size_t group_index = 0; group_index < full_group_masks.size(); ++group_index) { + auto &tracker = plan.full_trackers[group_index]; + if (tracker.stopped) { + continue; + } + (void)AdvancePrefixPeerTracker( + key_view, plan.target_host_id, full_group_masks[group_index], key_index, tracker); + } + } + + if (!plan.combined_full_active) { + continue; + } + const bool combined_full_present = + std::none_of(plan.full_trackers.begin(), plan.full_trackers.end(), [](const auto &tracker) { + return tracker.stopped; + }); + if (!combined_full_present) { + plan.combined_full_active = false; + plan.pending_coverage_counts.clear(); + plan.pending_coverage_peer_ids.clear(); + continue; + } + if (use_eagle_pop) { + commit_pending_coverage(plan); + add_state_coverage_for_key(plan, plan.pending_coverage_counts, &plan.pending_coverage_peer_ids); + } else { + add_state_coverage_for_key(plan, plan.coverage_counts); + } + } + full_visited_until = key_index + 1; + // The first stopped group fixes the combined match prefix, but + // every full group independently selects a prefix peer and its + // complete covered range contributes to p2p_1_fetch. Preserve + // that externally visible selection semantic while retaining + // only one tracker per plan/group. + const bool every_plan_stopped = + !full_group_masks.empty() && std::all_of(plans.begin(), plans.end(), [](const auto &plan) { + return std::all_of(plan.full_trackers.begin(), + plan.full_trackers.end(), + [](const auto &tracker) { return tracker.stopped; }); + }); + if (every_plan_stopped) { + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return key_index; + } + } + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return keys.size(); + }; + + const auto full_result = run_ordered_scan(full_phase_visitor); + if (!full_phase_valid) { + publish_metrics(); + request_context->error_tracer()->AddErrorMsg("ordered mamba full-prefix phase callback mismatch"); + return full_result.terminal_ec == EC_OK ? EC_MISMATCH : full_result.terminal_ec; + } + if (hard_read_error(full_result)) { + publish_metrics(); + request_context->error_tracer()->AddErrorMsg("mamba full-prefix metadata read failed"); + return full_result.terminal_ec; + } + for (auto &plan : plans) { + plan.full_stop = full_result.valid_key_count; + plan.full_group_stops.resize(plan.full_trackers.size()); + for (size_t group_index = 0; group_index < plan.full_trackers.size(); ++group_index) { + auto &tracker = plan.full_trackers[group_index]; + if (!tracker.stopped) { + tracker.stop_index = full_result.valid_key_count; + } + plan.full_group_stops[group_index] = tracker.stop_index; + plan.full_stop = std::min(plan.full_stop, tracker.stop_index); + plan.full_peer_ids[group_index] = + SelectLexicographicallySmallestPeer(tracker.active_peer_ids, host_catalog); + } + plan.adjusted_full = plan.full_stop; + if (use_eagle_pop && plan.adjusted_full > 0) { + --plan.adjusted_full; + } + std::vector().swap(plan.full_trackers); + std::vector().swap(plan.pending_coverage_counts); + std::vector().swap(plan.pending_coverage_peer_ids); + } + + for (auto &plan : plans) { + for (size_t peer_id = 0; peer_id < plan.coverage_counts.size(); ++peer_id) { + if (plan.coverage_counts[peer_id] == 0) { + continue; + } + if (plan.coverage_peer_id == std::numeric_limits::max() || + plan.coverage_counts[peer_id] > plan.coverage_counts[plan.coverage_peer_id] || + (plan.coverage_counts[peer_id] == plan.coverage_counts[plan.coverage_peer_id] && + host_catalog.host(peer_id) < host_catalog.host(plan.coverage_peer_id))) { + plan.coverage_peer_id = peer_id; + } + } + std::vector().swap(plan.coverage_counts); + } + + size_t final_limit = 0; + for (auto &plan : plans) { + // Validate and count every independently selected full-group peer + // range, including ranges beyond the shortest combined prefix and + // the boundary removed by Eagle pop. This is what the materialized + // implementation reported as actually selected P2P fetches. + for (size_t group_index = 0; group_index < plan.full_group_stops.size(); ++group_index) { + if (plan.full_peer_ids[group_index] != std::numeric_limits::max()) { + plan.final_projection_required = true; + final_limit = std::max(final_limit, plan.full_group_stops[group_index]); + } + } + if (plan.coverage_peer_id != std::numeric_limits::max()) { + plan.final_projection_required = true; + final_limit = std::max(final_limit, plan.adjusted_full); + } } - - int64_t prefix_len = 0; - // Mamba 模式要求 full 前缀范围内最后一个可用 block 同时具备 mamba state。 - for (size_t offset = full_prefix_len; offset > 0; --offset) { - const size_t index = offset - 1; - auto host_it = key_to_host_spec_names[index].find(host); - if (host_it != key_to_host_spec_names[index].end() && - HasAllLocationSpecGroups(host_it->second, mamba_state_groups)) { - prefix_len = static_cast(index + 1); - break; + if (final_limit > 0) { + size_t final_visited_until = 0; + bool final_phase_valid = true; + const auto final_visitor = [&keys, + &check_loc_data_exist, + request_check_location, + &medium_set, + &key_view, + &plans, + &full_group_masks, + &state_group_masks, + &final_visited_until, + &final_phase_valid, + &projection_cpu_time_us, + final_limit]( + size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + const int64_t projection_begin = TimestampUtil::GetCurrentTimeUs(); + if (begin != final_visited_until) { + final_phase_valid = false; + return size_t{0}; + } + for (size_t local_index = 0; local_index < valid_count; ++local_index) { + const size_t key_index = begin + local_index; + if (key_index >= final_limit) { + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return final_limit; + } + key_view.Build(locations[local_index], check_loc_data_exist, request_check_location, medium_set); + for (auto &plan : plans) { + if (!plan.final_projection_required) { + continue; + } + const auto *local = key_view.Find(plan.target_host_id); + const bool build_combined = key_index < plan.full_stop; + if (build_combined) { + key_view.CopyAllWords(local, plan.combined_words); + } + bool fetched_full = false; + for (size_t group_index = 0; group_index < full_group_masks.size(); ++group_index) { + if (key_index >= plan.full_group_stops[group_index]) { + continue; + } + const auto &group_mask = full_group_masks[group_index]; + if (key_view.HasMask(local, group_mask)) { + continue; + } + const size_t peer_id = plan.full_peer_ids[group_index]; + const auto *peer = + peer_id == std::numeric_limits::max() ? nullptr : key_view.Find(peer_id); + if (!peer || !peer->has_vineyard_spec || !key_view.CoversMask(local, *peer, group_mask)) { + final_phase_valid = false; + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return key_index; + } + fetched_full = true; + if (build_combined) { + key_view.MergeVineyardGroup(*peer, group_mask, plan.combined_words); + } + } + if (fetched_full) { + plan.fetched_keys.Add(key_index, keys[key_index]); + } + if (!build_combined) { + continue; + } + if (key_index >= plan.adjusted_full) { + continue; + } + bool fetched_state = false; + const auto *coverage_peer = plan.coverage_peer_id == std::numeric_limits::max() + ? nullptr + : key_view.Find(plan.coverage_peer_id); + for (const auto &group_mask : state_group_masks) { + if (key_view.HasMask(local, group_mask)) { + continue; + } + if (!coverage_peer || !coverage_peer->has_vineyard_spec || + !key_view.CoversMask(local, *coverage_peer, group_mask)) { + continue; + } + key_view.MergeVineyardGroup(*coverage_peer, group_mask, plan.combined_words); + fetched_state = true; + } + if (fetched_state) { + plan.fetched_keys.Add(key_index, keys[key_index]); + } + const bool all_state_groups_present = std::all_of( + state_group_masks.begin(), state_group_masks.end(), [&](const auto &group_mask) { + return key_view.WordsHaveMask(plan.combined_words, group_mask); + }); + if (all_state_groups_present) { + plan.total_match = std::max(plan.total_match, static_cast(key_index + 1)); + } + } + final_visited_until = key_index + 1; + if (final_visited_until >= final_limit) { + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return final_limit; + } + } + projection_cpu_time_us += TimestampUtil::GetCurrentTimeUs() - projection_begin; + return keys.size(); + }; + + const auto final_result = run_ordered_scan(final_visitor); + if (!final_phase_valid || final_result.valid_key_count < final_limit) { + publish_metrics(); + request_context->error_tracer()->AddErrorMsg("mamba final phase observed an inconsistent peer plan"); + return hard_read_error(final_result) ? final_result.terminal_ec : EC_MISMATCH; } } - if (prefix_len > 0) { - out_matches.push_back(HostCacheMatch{host, prefix_len}); + + for (auto &plan : plans) { + host_matches[plan.host_index].p2p_1_fetch = plan.fetched_keys.DistinctCount(); + host_matches[plan.host_index].p2p_1_total_match = plan.total_match; } + + publish_metrics(); + move_positive_matches_to_output(); + return EC_OK; } - return EC_OK; } ErrorCode MetaSearcher::BatchGetLocation(RequestContext *request_context, @@ -1177,35 +2877,53 @@ MetaSearcher::BatchReplaceLocationSpecs(RequestContext *request_context, LocationIdsPerKey location_ids_per_key(keys.size()); std::vector> usage_changes(keys.size()); + std::unordered_set seen_keys; for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + if (!seen_keys.insert(keys[key_index]).second) { + std::fill(out_per_key_ec.begin(), out_per_key_ec.end(), EC_BADARGS); + return EC_BADARGS; + } auto &location_ids = location_ids_per_key[key_index]; auto &key_usage_changes = usage_changes[key_index]; location_ids.reserve(tasks_per_key[key_index].size()); key_usage_changes.resize(tasks_per_key[key_index].size()); std::unordered_set seen_location_ids; - for (const auto &task : tasks_per_key[key_index]) { - if (task.location_id.empty() || !seen_location_ids.insert(task.location_id).second) { - out_per_key_ec[key_index] = EC_BADARGS; + for (size_t task_index = 0; task_index < tasks_per_key[key_index].size(); ++task_index) { + const auto &task = tasks_per_key[key_index][task_index]; + const auto &location_id = task.ResolvedLocationId(); + std::uint64_t incoming_size = 0; + if (location_id.empty() || !seen_location_ids.insert(location_id).second || + ValidateConsistentSnapshotVersion(task.specs, &incoming_size) != EC_OK) { + std::fill(out_per_key_ec.begin(), out_per_key_ec.end(), EC_BADARGS); return EC_BADARGS; } - location_ids.push_back(task.location_id); + location_ids.push_back(location_id); + key_usage_changes[task_index].new_size = incoming_size; } } const int64_t batch_create_time = TimestampUtil::GetCurrentTimeUs(); - std::vector write_leases; - auto modifier = [&keys, &tasks_per_key, &usage_changes, &acquire_write_lease, &write_leases, batch_create_time]( - const std::vector &get_ecs, - const LocationIdVector &location_ids, - size_t key_index, - CacheLocationVector &locations, - PropertyMap & /*upsert_property_map*/) -> LocationModifierResult { - if (acquire_write_lease) { - auto [lease_ec, lease] = acquire_write_lease(); - if (lease_ec != EC_OK) { - return {ModifierAction::MA_FAIL, std::vector(location_ids.size(), lease_ec)}; - } - write_leases.push_back(std::move(lease)); + bool write_lease_attempted = false; + ErrorCode write_lease_ec = EC_OK; + MetadataWriteLease write_lease; + auto modifier = [&keys, + &tasks_per_key, + &usage_changes, + &acquire_write_lease, + &write_lease_attempted, + &write_lease_ec, + &write_lease, + batch_create_time](const std::vector &get_ecs, + const LocationIdVector &location_ids, + size_t key_index, + CacheLocationVector &locations, + PropertyMap & /*upsert_property_map*/) -> LocationModifierResult { + if (acquire_write_lease && !write_lease_attempted) { + std::tie(write_lease_ec, write_lease) = acquire_write_lease(); + write_lease_attempted = true; + } + if (write_lease_ec != EC_OK) { + return {ModifierAction::MA_FAIL, std::vector(location_ids.size(), write_lease_ec)}; } const auto &tasks = tasks_per_key[key_index]; std::vector modifier_ecs(location_ids.size(), ErrorCode::EC_OK); @@ -1240,6 +2958,12 @@ MetaSearcher::BatchReplaceLocationSpecs(RequestContext *request_context, new_location = std::make_shared(*locations[location_index]); } else { new_location = std::make_shared(); + } + if (const auto *interned_location_id = task.ResolvedInternedLocationId()) { + // New locations and legacy owned-id locations converge to the + // request's canonical reporter/medium id on the next write. + new_location->set_id(*interned_location_id); + } else if (get_ec != ErrorCode::EC_OK || !locations[location_index]) { new_location->set_id(task.location_id); } @@ -1253,7 +2977,7 @@ MetaSearcher::BatchReplaceLocationSpecs(RequestContext *request_context, new_location->set_status(task.status); new_location->set_spec_size(new_location->location_specs().size()); new_location->set_create_time(batch_create_time); - usage.new_size = GetLocationSpecsSize(task.specs); + new_location->set_validated_total_size(usage.new_size); locations[location_index] = std::move(new_location); updated = true; } @@ -1269,13 +2993,22 @@ MetaSearcher::BatchReplaceLocationSpecs(RequestContext *request_context, meta_indexer_->ReadModifyWriteLocation(request_context, keys, location_ids_per_key, std::move(modifier)); KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerReadModifyWriteLocation); + bool malformed_result = result.per_location_error_codes.size() != keys.size(); + if (malformed_result) { + KVCM_LOG_ERROR("BatchReplaceLocationSpecs result size mismatch, keys[%zu], results[%zu]", + keys.size(), + result.per_location_error_codes.size()); + } for (size_t key_index = 0; key_index < keys.size(); ++key_index) { ErrorCode key_ec = ErrorCode::EC_OK; - if (key_index >= result.per_location_error_codes.size()) { - key_ec = result.ec == ErrorCode::EC_OK ? ErrorCode::EC_ERROR : result.ec; + const size_t expected_location_count = tasks_per_key[key_index].size(); + if (key_index >= result.per_location_error_codes.size() || + result.per_location_error_codes[key_index].size() != expected_location_count) { + key_ec = ErrorCode::EC_MISMATCH; + malformed_result = true; } else { const auto &location_ecs = result.per_location_error_codes[key_index]; - for (size_t location_index = 0; location_index < location_ecs.size(); ++location_index) { + for (size_t location_index = 0; location_index < expected_location_count; ++location_index) { const ErrorCode location_ec = location_ecs[location_index]; if (location_ec != ErrorCode::EC_OK) { if (key_ec == ErrorCode::EC_OK) { @@ -1299,198 +3032,369 @@ MetaSearcher::BatchReplaceLocationSpecs(RequestContext *request_context, if (result.ec != ErrorCode::EC_OK) { KVCM_LOG_WARN("meta_indexer_->ReadModifyWriteLocation failed, ec: %d", result.ec); } - return result.ec; + return malformed_result ? ErrorCode::EC_MISMATCH : result.ec; } +class MetaSearcher::MergeLocationSpecsTaskView { +public: + explicit MergeLocationSpecsTaskView(const std::vector> &nested_tasks) + : nested_tasks_(&nested_tasks) {} + + MergeLocationSpecsTaskView(const std::vector &offsets, std::vector &flat_tasks) + : offsets_(&offsets), flat_tasks_(&flat_tasks) {} + + [[nodiscard]] bool Valid(size_t key_count) const noexcept { + if (nested_tasks_) { + return nested_tasks_->size() == key_count; + } + if (!offsets_ || !flat_tasks_ || offsets_->size() != key_count + 1 || offsets_->front() != 0 || + offsets_->back() != flat_tasks_->size()) { + return false; + } + return std::is_sorted(offsets_->begin(), offsets_->end()); + } + + [[nodiscard]] size_t Size(size_t key_index) const noexcept { + if (nested_tasks_) { + return (*nested_tasks_)[key_index].size(); + } + return (*offsets_)[key_index + 1] - (*offsets_)[key_index]; + } + + [[nodiscard]] const MergeLocationSpecsTask &At(size_t key_index, size_t task_index) const noexcept { + if (nested_tasks_) { + return (*nested_tasks_)[key_index][task_index]; + } + return (*flat_tasks_)[(*offsets_)[key_index] + task_index]; + } + + // The ReportEvent-only flat representation owns its input strings and is + // dead after this call except for spec names used to map failures back to + // request items. Transfer the potentially large URI into the immutable + // CacheLocation while restoring the usually-SSO name in the task. Generic + // nested callers retain their historical copy semantics. + [[nodiscard]] LocationSpec CopyOrConsumeSpec(size_t key_index, size_t task_index, size_t spec_index) { + if (!flat_tasks_) { + return At(key_index, task_index).SpecAt(spec_index); + } + auto &task = (*flat_tasks_)[(*offsets_)[key_index] + task_index]; + if (!task.prevalidated_total_size.has_value()) { + return task.SpecAt(spec_index); + } + auto &source = task.MutableSpecAt(spec_index); + std::string retained_name = source.name(); + LocationSpec result = std::move(source); + source.set_name(std::move(retained_name)); + return result; + } + +private: + const std::vector> *nested_tasks_ = nullptr; + const std::vector *offsets_ = nullptr; + std::vector *flat_tasks_ = nullptr; +}; + ErrorCode MetaSearcher::BatchMergeLocationSpecs(RequestContext *request_context, const KeyVector &keys, const std::vector> &tasks_per_key, std::vector &out_per_key_ec, AcquireMetadataWriteLeaseFunc acquire_write_lease) { - if (keys.size() != tasks_per_key.size()) { + return BatchMergeLocationSpecsImpl(request_context, + keys, + MergeLocationSpecsTaskView(tasks_per_key), + out_per_key_ec, + std::move(acquire_write_lease)); +} + +ErrorCode MetaSearcher::BatchMergeLocationSpecsFlat(RequestContext *request_context, + const KeyVector &keys, + const std::vector &task_offsets, + std::vector &flat_tasks, + std::vector &out_per_key_ec, + AcquireMetadataWriteLeaseFunc acquire_write_lease) { + return BatchMergeLocationSpecsImpl(request_context, + keys, + MergeLocationSpecsTaskView(task_offsets, flat_tasks), + out_per_key_ec, + std::move(acquire_write_lease)); +} + +ErrorCode MetaSearcher::BatchMergeLocationSpecsImpl(RequestContext *request_context, + const KeyVector &keys, + MergeLocationSpecsTaskView tasks, + std::vector &out_per_key_ec, + AcquireMetadataWriteLeaseFunc acquire_write_lease) { + if (!tasks.Valid(keys.size())) { return EC_BADARGS; } out_per_key_ec.assign(keys.size(), ErrorCode::EC_OK); + if (keys.empty()) { + return EC_OK; + } - std::vector>> created_locs_sz(keys.size()); - const int64_t batch_create_time = TimestampUtil::GetCurrentTimeUs(); - std::vector> merge_tasks_per_key(keys.size()); - std::vector write_leases; - - auto create_modifier = [&tasks_per_key, - &merge_tasks_per_key, - &keys, - &created_locs_sz, - &acquire_write_lease, - &write_leases, - batch_create_time](const LocationIdVector &existing_ids, - ErrorCode get_ec, - size_t index, - PropertyMap & /*upsert_property_map*/, - CacheLocationMap &out_new_locations) -> ModifierResult { - if (acquire_write_lease) { - auto [lease_ec, lease] = acquire_write_lease(); - if (lease_ec != EC_OK) { - return {ModifierAction::MA_FAIL, lease_ec}; - } - write_leases.push_back(std::move(lease)); - } - if (get_ec != ErrorCode::EC_OK && get_ec != ErrorCode::EC_NOENT) { - KVCM_LOG_WARN("load location ids failed, key[%lu](%lu) return %d", index, keys[index], get_ec); - return {ModifierAction::MA_FAIL, get_ec}; + bool has_duplicate_keys = false; + if (std::is_sorted(keys.begin(), keys.end())) { + has_duplicate_keys = std::adjacent_find(keys.begin(), keys.end()) != keys.end(); + } else { + std::unordered_set seen_keys; + seen_keys.reserve(keys.size()); + for (const int64_t key : keys) { + if (!seen_keys.insert(key).second) { + has_duplicate_keys = true; + break; + } } + } + if (has_duplicate_keys) { + std::fill(out_per_key_ec.begin(), out_per_key_ec.end(), EC_BADARGS); + return EC_BADARGS; + } - const std::unordered_set existing_id_set(existing_ids.begin(), existing_ids.end()); - bool created = false; - for (const auto &entry : tasks_per_key[index]) { - const ErrorCode validation_ec = ValidateConsistentSnapshotVersion(entry.specs); - if (validation_ec != EC_OK) { - return {ModifierAction::MA_FAIL, validation_ec}; + bool use_single_location_fast_path = meta_indexer_->SupportsSingleLocationRmw(); + for (size_t key_index = 0; key_index < keys.size() && use_single_location_fast_path; ++key_index) { + use_single_location_fast_path = tasks.Size(key_index) == 1; + } + LocationIdsPerKey location_ids_per_key(use_single_location_fast_path ? 0 : keys.size()); + LocationIdRefVector single_location_ids; + if (use_single_location_fast_path) { + single_location_ids.reserve(keys.size()); + } + // Keep the incoming and final usage in one request-shaped allocation. + // The dominant pure-local ReportEvent shape has exactly one location per + // key, so its usage index is the key index and needs no offsets array. + // Multi-location callers retain the generic flattened-offset layout. + std::vector usage_offsets(use_single_location_fast_path ? 0 : keys.size() + 1, 0); + std::vector usage_changes; + usage_changes.reserve(keys.size()); + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + const size_t task_count = tasks.Size(key_index); + std::unordered_set seen_location_ids; + if (task_count > 1) { + seen_location_ids.reserve(task_count); + } + if (!use_single_location_fast_path) { + location_ids_per_key[key_index].reserve(task_count); + } + for (size_t task_index = 0; task_index < task_count; ++task_index) { + const auto &task = tasks.At(key_index, task_index); + const auto &task_location_id = task.ResolvedLocationId(); + std::uint64_t incoming_size = 0; + const ErrorCode validation_ec = task.prevalidated_total_size.has_value() + ? (task.SpecsEmpty() ? EC_BADARGS : EC_OK) + : ValidateConsistentSnapshotVersion(task, &incoming_size); + if (task.prevalidated_total_size.has_value()) { + incoming_size = task.prevalidated_total_size->value(); } - if (get_ec == ErrorCode::EC_OK && existing_id_set.count(entry.location_id) > 0) { - merge_tasks_per_key[index].push_back(entry); - continue; + if (task_location_id.empty() || + (task_count > 1 && !seen_location_ids.insert(std::string_view(task_location_id)).second) || + validation_ec != EC_OK) { + std::fill(out_per_key_ec.begin(), out_per_key_ec.end(), EC_BADARGS); + return EC_BADARGS; } - - CacheLocation loc; - loc.set_id(entry.location_id); - loc.set_type(entry.type); - loc.set_status(entry.status); - loc.set_spec_size(entry.specs.size()); - loc.set_create_time(batch_create_time); - for (const auto &ls : entry.specs) { - loc.push_location_spec(LocationSpec(ls.name(), ls.uri())); + if (use_single_location_fast_path) { + single_location_ids.push_back(&task_location_id); + } else { + location_ids_per_key[key_index].push_back(task_location_id); } - out_new_locations[entry.location_id] = std::make_shared(std::move(loc)); - created_locs_sz[index].emplace_back(entry.type, GetLocationSpecsSize(entry.specs)); - created = true; + usage_changes.push_back(StorageUsageChange{0, incoming_size, false}); } - if (!created) { - return {ModifierAction::MA_SKIP, ErrorCode::EC_OK}; - } - return {ModifierAction::MA_OK, ErrorCode::EC_OK}; - }; - - auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); - KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerReadModifyWriteBlock); - auto result = meta_indexer_->ReadModifyWriteBlock(request_context, keys, create_modifier); - KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerReadModifyWriteBlock); - ErrorCode final_ec = result.ec; - - for (size_t i = 0; i < keys.size(); ++i) { - ErrorCode key_ec = (i < result.error_codes.size()) ? result.error_codes[i] : result.ec; - out_per_key_ec[i] = key_ec; - if (key_ec == ErrorCode::EC_OK) { - for (const auto &[type, size] : created_locs_sz[i]) { - meta_indexer_->AddStorageUsageByType(type, size); - } + if (!use_single_location_fast_path) { + usage_offsets[key_index + 1] = usage_offsets[key_index] + task_count; } } - if (result.ec != ErrorCode::EC_OK) { - LogErrorCodes("meta_indexer_->ReadModifyWriteBlock", result.error_codes, keys); - } + const int64_t batch_create_time = TimestampUtil::GetCurrentTimeUs(); + bool write_lease_attempted = false; + ErrorCode write_lease_ec = EC_OK; + MetadataWriteLease write_lease; + auto ensure_write_lease = [&acquire_write_lease, &write_lease_attempted, &write_lease_ec, &write_lease]() { + if (acquire_write_lease && !write_lease_attempted) { + std::tie(write_lease_ec, write_lease) = acquire_write_lease(); + write_lease_attempted = true; + } + return write_lease_ec; + }; - KeyVector merge_keys; - std::vector merge_key_indices; - LocationIdsPerKey merge_location_ids; - for (size_t i = 0; i < keys.size(); ++i) { - if (out_per_key_ec[i] != ErrorCode::EC_OK || merge_tasks_per_key[i].empty()) { - continue; + auto merge_one_location = + [&keys, &tasks, &usage_offsets, &usage_changes, use_single_location_fast_path, batch_create_time]( + ErrorCode get_ec, + const LocationId &location_id, + size_t key_index, + size_t location_index, + const CacheLocation *existing_location, + CacheLocationConstPtr &out_location) -> ErrorCode { + if (key_index >= keys.size() || location_index >= tasks.Size(key_index)) { + return EC_MISMATCH; } - merge_keys.push_back(keys[i]); - merge_key_indices.push_back(i); - auto &ids = merge_location_ids.emplace_back(); - ids.reserve(merge_tasks_per_key[i].size()); - for (const auto &task : merge_tasks_per_key[i]) { - ids.push_back(task.location_id); + const auto &task = tasks.At(key_index, location_index); + const auto &task_location_id = task.ResolvedLocationId(); + if (location_id != task_location_id) { + return EC_MISMATCH; } - } - if (merge_keys.empty()) { - return final_ec; - } + if (get_ec != EC_OK && get_ec != EC_NOENT) { + KVCM_LOG_WARN("load target location failed, key[%lu](%lu), location_id[%s], return[%d]", + key_index, + keys[key_index], + task_location_id.c_str(), + get_ec); + return get_ec; + } - std::vector> merge_usage_changes(keys.size()); - for (size_t i = 0; i < keys.size(); ++i) { - merge_usage_changes[i].resize(merge_tasks_per_key[i].size()); - } - write_leases.clear(); - auto merge_modifier = [&keys, - &merge_tasks_per_key, - &merge_key_indices, - &merge_usage_changes, - &acquire_write_lease, - &write_leases, - batch_create_time](const std::vector &get_ecs, - const LocationIdVector &loc_ids, - size_t key_index, - CacheLocationVector &locs, - PropertyMap &upsert_property_map) -> LocationModifierResult { - (void)upsert_property_map; - if (acquire_write_lease) { - auto [lease_ec, lease] = acquire_write_lease(); - if (lease_ec != EC_OK) { - return {ModifierAction::MA_FAIL, std::vector(loc_ids.size(), lease_ec)}; + const size_t usage_index = + use_single_location_fast_path ? key_index : usage_offsets[key_index] + location_index; + auto &usage = usage_changes[usage_index]; + const std::uint64_t incoming_size = usage.new_size; + std::shared_ptr new_location; + bool final_specs_validated = true; + if (get_ec == EC_OK) { + if (!existing_location || existing_location->type() != task.type) { + return existing_location ? ErrorCode::EC_BADARGS : ErrorCode::EC_MISMATCH; } - write_leases.push_back(std::move(lease)); - } - const size_t original_key_index = merge_key_indices[key_index]; - const auto &tasks = merge_tasks_per_key[original_key_index]; - std::vector modifier_ecs(loc_ids.size(), ErrorCode::EC_OK); - bool updated = false; - for (size_t loc_index = 0; loc_index < loc_ids.size(); ++loc_index) { - const ErrorCode ec = get_ecs[loc_index]; - if (loc_index >= tasks.size()) { - modifier_ecs[loc_index] = ErrorCode::EC_ERROR; - continue; + std::uint64_t replaced_old_size = 0; + bool has_legacy_duplicate_names = false; + const auto &old_specs = existing_location->location_specs(); + std::uint64_t cached_single_spec_size = 0; + const bool has_cached_single_spec_size = + old_specs.size() == 1 && existing_location->GetValidatedTotalSize(cached_single_spec_size); + bool old_size_overflow = false; + for (size_t old_index = 0; old_index < old_specs.size(); ++old_index) { + const auto &old_spec = old_specs[old_index]; + std::uint64_t old_spec_size = cached_single_spec_size; + const bool old_spec_validated = + has_cached_single_spec_size || TryGetLocationSpecSize(old_spec, old_spec_size); + if (old_spec_size > std::numeric_limits::max() - usage.old_size) { + old_size_overflow = true; + break; + } + usage.old_size += old_spec_size; + bool replaces_old_spec = false; + for (size_t spec_index = 0; spec_index < task.SpecCount(); ++spec_index) { + if (task.SpecAt(spec_index).name() == old_spec.name()) { + replaces_old_spec = true; + break; + } + } + if (!replaces_old_spec && !old_spec_validated) { + final_specs_validated = false; + } + if (replaces_old_spec) { + if (old_spec_size > std::numeric_limits::max() - replaced_old_size) { + old_size_overflow = true; + break; + } + replaced_old_size += old_spec_size; + } + has_legacy_duplicate_names = + has_legacy_duplicate_names || + std::any_of(old_specs.begin(), old_specs.begin() + old_index, [&old_spec](const auto &prior) { + return prior.name() == old_spec.name(); + }); } - const auto &task = tasks[loc_index]; - if (ec != ErrorCode::EC_OK && ec != ErrorCode::EC_NOENT) { - modifier_ecs[loc_index] = ec; - KVCM_LOG_WARN("load location failed, key[%lu](%lu), location_id: %s, return %d", - original_key_index, - keys[original_key_index], - task.location_id.c_str(), - ec); - continue; + if (old_size_overflow) { + return EC_BADARGS; } - - auto &usage = merge_usage_changes[original_key_index][loc_index]; - std::shared_ptr new_loc; - if (ec == ErrorCode::EC_OK && locs[loc_index]) { - if (locs[loc_index]->type() != task.type) { - modifier_ecs[loc_index] = ErrorCode::EC_BADARGS; - continue; + usage.has_old = true; + const bool replaces_only_spec = + old_specs.size() == 1 && task.SpecCount() == 1 && old_specs.front().name() == task.SpecAt(0).name(); + if (replaces_only_spec) { + // The dominant ReportEvent update replaces the only stored + // spec with one same-named spec. Copying CacheLocation first + // allocates/copies the old URI only to destroy it immediately + // in MergeLocationSpecsByName. Build the final immutable value + // directly for this exact case; every CacheLocation field is + // assigned below and the incoming spec is still copied once. + new_location = std::make_shared(); + if (const auto *interned_location_id = task.ResolvedInternedLocationId()) { + new_location->set_id(*interned_location_id); + } else { + new_location->set_id(existing_location->id()); } - usage.old_size = GetLocationSpecsSize(locs[loc_index]->location_specs()); - usage.has_old = true; - new_loc = std::make_shared(*locs[loc_index]); - std::vector merged_specs; - const ErrorCode merge_ec = - MergeLocationSpecsByName(new_loc->location_specs(), task.specs, merged_specs); - if (merge_ec != EC_OK) { - modifier_ecs[loc_index] = merge_ec; - continue; + std::vector specs; + specs.reserve(1); + specs.push_back(tasks.CopyOrConsumeSpec(key_index, location_index, 0)); + new_location->set_location_specs(std::move(specs)); + } else { + new_location = std::make_shared(*existing_location); + if (const auto *interned_location_id = task.ResolvedInternedLocationId()) { + new_location->set_id(*interned_location_id); + } + MergeLocationSpecsByName(new_location->mutable_location_specs(), task); + } + if (has_legacy_duplicate_names) { + usage.new_size = 0; + final_specs_validated = true; + for (const auto &merged_spec : new_location->location_specs()) { + std::uint64_t merged_spec_size = 0; + if (!TryGetLocationSpecSize(merged_spec, merged_spec_size)) { + final_specs_validated = false; + } + if (merged_spec_size > std::numeric_limits::max() - usage.new_size) { + return EC_BADARGS; + } + usage.new_size += merged_spec_size; } - new_loc->set_location_specs(std::move(merged_specs)); } else { - new_loc = std::make_shared(); - new_loc->set_id(task.location_id); - std::vector specs; - specs.reserve(task.specs.size()); - for (const auto &spec : task.specs) { - specs.emplace_back(spec.name(), spec.uri()); + const std::uint64_t retained_size = usage.old_size - replaced_old_size; + if (incoming_size > std::numeric_limits::max() - retained_size) { + return EC_BADARGS; } - new_loc->set_location_specs(std::move(specs)); + usage.new_size = retained_size + incoming_size; } - new_loc->set_type(task.type); - new_loc->set_status(task.status); - new_loc->set_create_time(batch_create_time); - new_loc->set_spec_size(new_loc->location_specs().size()); - usage.new_size = GetLocationSpecsSize(new_loc->location_specs()); - locs[loc_index] = std::move(new_loc); - updated = true; + } else { + new_location = std::make_shared(); + if (const auto *interned_location_id = task.ResolvedInternedLocationId()) { + new_location->set_id(*interned_location_id); + } else { + new_location->set_id(task.location_id); + } + std::vector specs; + specs.reserve(task.SpecCount()); + for (size_t spec_index = 0; spec_index < task.SpecCount(); ++spec_index) { + specs.push_back(tasks.CopyOrConsumeSpec(key_index, location_index, spec_index)); + } + new_location->set_location_specs(std::move(specs)); + usage.new_size = incoming_size; + } + new_location->set_type(task.type); + new_location->set_status(task.status); + new_location->set_create_time(batch_create_time); + new_location->set_spec_size(new_location->location_specs().size()); + if (final_specs_validated) { + new_location->set_validated_total_size(usage.new_size); + } + out_location = std::move(new_location); + return EC_OK; + }; + + auto modifier = [&tasks, &keys, &ensure_write_lease, &merge_one_location]( + const std::vector &get_ecs, + const LocationIdVector &location_ids, + size_t key_index, + CacheLocationVector &locations, + PropertyMap & /*upsert_property_map*/) -> LocationModifierResult { + if (location_ids.empty()) { + return {ModifierAction::MA_SKIP, {}}; + } + const ErrorCode lease_ec = ensure_write_lease(); + if (lease_ec != EC_OK) { + return {ModifierAction::MA_FAIL, std::vector(location_ids.size(), lease_ec)}; + } + if (key_index >= keys.size() || get_ecs.size() != location_ids.size() || + locations.size() != location_ids.size() || tasks.Size(key_index) != location_ids.size()) { + return {ModifierAction::MA_FAIL, std::vector(location_ids.size(), EC_MISMATCH)}; + } + + std::vector modifier_ecs(location_ids.size(), ErrorCode::EC_OK); + bool updated = false; + for (size_t location_index = 0; location_index < location_ids.size(); ++location_index) { + modifier_ecs[location_index] = merge_one_location(get_ecs[location_index], + location_ids[location_index], + key_index, + location_index, + locations[location_index].get(), + locations[location_index]); + updated = updated || modifier_ecs[location_index] == EC_OK; } if (!updated) { return {ModifierAction::MA_SKIP, std::move(modifier_ecs)}; @@ -1498,48 +3402,106 @@ ErrorCode MetaSearcher::BatchMergeLocationSpecs(RequestContext *request_context, return {ModifierAction::MA_OK, std::move(modifier_ecs)}; }; + auto single_modifier = [&tasks, &keys, &ensure_write_lease, &merge_one_location]( + ErrorCode get_ec, + const LocationId &location_id, + size_t key_index, + const CacheLocation *existing_location, + CacheLocationConstPtr &out_location) -> ModifierResult { + const ErrorCode lease_ec = ensure_write_lease(); + if (lease_ec != EC_OK) { + return {ModifierAction::MA_FAIL, lease_ec}; + } + if (key_index >= keys.size() || tasks.Size(key_index) != 1) { + return {ModifierAction::MA_FAIL, EC_MISMATCH}; + } + const ErrorCode ec = merge_one_location(get_ec, location_id, key_index, 0, existing_location, out_location); + return {ec == EC_OK ? ModifierAction::MA_OK : ModifierAction::MA_SKIP, ec}; + }; + + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(service_metrics_collector, MetaSearcherIndexerReadModifyWriteLocation); - auto merge_result = - meta_indexer_->ReadModifyWriteLocation(request_context, merge_keys, merge_location_ids, merge_modifier); + if (use_single_location_fast_path) { + auto result = meta_indexer_->ReadModifyWriteSingleTargetLocations( + request_context, keys, single_location_ids, single_modifier); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerReadModifyWriteLocation); + + bool malformed_result = result.error_codes.size() != keys.size(); + if (malformed_result) { + KVCM_LOG_ERROR("BatchMergeLocationSpecs flat result size mismatch, keys[%zu], results[%zu]", + keys.size(), + result.error_codes.size()); + } + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + const ErrorCode key_ec = + key_index < result.error_codes.size() ? result.error_codes[key_index] : EC_MISMATCH; + out_per_key_ec[key_index] = key_ec; + if (key_ec == EC_OK) { + const auto &usage = usage_changes[key_index]; + const auto &task = tasks.At(key_index, 0); + if (usage.has_old) { + ApplyStorageUsageChange(meta_indexer_.get(), task.type, usage.old_size, usage.new_size); + } else { + meta_indexer_->AddStorageUsageByType(task.type, usage.new_size); + } + } + } + ErrorCode final_ec = result.ec; + if (result.ec != EC_OK) { + KVCM_LOG_WARN("meta_indexer_->ReadModifyWriteSingleTargetLocations failed, ec: %d", result.ec); + } + if (malformed_result) { + final_ec = final_ec == EC_OK ? EC_MISMATCH : (final_ec == EC_MISMATCH ? EC_MISMATCH : EC_PARTIAL_OK); + } + return final_ec; + } + + auto result = meta_indexer_->ReadModifyWriteTargetLocations(request_context, keys, location_ids_per_key, modifier); KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerReadModifyWriteLocation); - for (size_t i = 0; i < merge_key_indices.size(); ++i) { - const size_t original_key_index = merge_key_indices[i]; - ErrorCode key_ec = ErrorCode::EC_OK; - if (i >= merge_result.per_location_error_codes.size()) { - key_ec = merge_result.ec == ErrorCode::EC_OK ? ErrorCode::EC_ERROR : merge_result.ec; + bool malformed_result = result.per_location_error_codes.size() != keys.size(); + if (malformed_result) { + KVCM_LOG_ERROR("BatchMergeLocationSpecs fused result size mismatch, keys[%zu], results[%zu]", + keys.size(), + result.per_location_error_codes.size()); + } + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + ErrorCode key_ec = EC_OK; + const size_t expected_location_count = tasks.Size(key_index); + if (key_index >= result.per_location_error_codes.size() || + result.per_location_error_codes[key_index].size() != expected_location_count) { + key_ec = EC_MISMATCH; + malformed_result = true; } else { - for (size_t loc_index = 0; loc_index < merge_result.per_location_error_codes[i].size(); ++loc_index) { - const auto loc_ec = merge_result.per_location_error_codes[i][loc_index]; - if (loc_ec == ErrorCode::EC_OK) { - const auto &usage = merge_usage_changes[original_key_index][loc_index]; - if (usage.has_old) { - ApplyStorageUsageChange(meta_indexer_.get(), - merge_tasks_per_key[original_key_index][loc_index].type, - usage.old_size, - usage.new_size); - } else { - meta_indexer_->AddStorageUsageByType(merge_tasks_per_key[original_key_index][loc_index].type, - usage.new_size); + for (size_t location_index = 0; location_index < expected_location_count; ++location_index) { + const ErrorCode location_ec = result.per_location_error_codes[key_index][location_index]; + if (location_ec != EC_OK) { + if (key_ec == EC_OK) { + key_ec = location_ec; } continue; } - if (key_ec == ErrorCode::EC_OK) { - key_ec = loc_ec; + const auto &usage = usage_changes[usage_offsets[key_index] + location_index]; + const auto &task = tasks.At(key_index, location_index); + if (usage.has_old) { + ApplyStorageUsageChange(meta_indexer_.get(), task.type, usage.old_size, usage.new_size); + } else { + meta_indexer_->AddStorageUsageByType(task.type, usage.new_size); } } } - if (key_ec != ErrorCode::EC_OK) { - out_per_key_ec[original_key_index] = key_ec; - } + out_per_key_ec[key_index] = key_ec; } - if (merge_result.ec != ErrorCode::EC_OK) { - KVCM_LOG_WARN("meta_indexer_->ReadModifyWriteLocation failed, ec: %d", merge_result.ec); - if (final_ec == ErrorCode::EC_OK) { - final_ec = merge_result.ec; - } else if (merge_result.ec != final_ec) { - final_ec = ErrorCode::EC_PARTIAL_OK; + ErrorCode final_ec = result.ec; + if (result.ec != EC_OK) { + KVCM_LOG_WARN("meta_indexer_->ReadModifyWriteTargetLocations failed, ec: %d", result.ec); + } + if (malformed_result) { + if (final_ec == EC_OK) { + final_ec = EC_MISMATCH; + } else if (final_ec != EC_MISMATCH) { + final_ec = EC_PARTIAL_OK; } } return final_ec; @@ -1561,22 +3523,45 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context out_missing_targets->resize(keys.size()); } - // 每个 DeleteLocationSpecsTask 需要独立返回结果；同一个 key 下多个 task - // 也可能删除同一个 location 的不同 specs，因此先展开成 task 维度执行。 + std::unordered_set seen_keys; + bool invalid_input = false; + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + out_batch_results[key_index].assign(tasks_per_key[key_index].size(), ErrorCode::EC_OK); + if (out_missing_targets) { + (*out_missing_targets)[key_index].assign(tasks_per_key[key_index].size(), false); + } + if (!seen_keys.insert(keys[key_index]).second) { + invalid_input = true; + } + std::unordered_set seen_location_ids; + if (tasks_per_key[key_index].size() > 1) { + seen_location_ids.reserve(tasks_per_key[key_index].size()); + } + for (const auto &task : tasks_per_key[key_index]) { + const auto &location_id = task.ResolvedLocationId(); + if (location_id.empty() || !seen_location_ids.insert(location_id).second) { + invalid_input = true; + } + } + } + if (invalid_input) { + for (auto &per_key_results : out_batch_results) { + std::fill(per_key_results.begin(), per_key_results.end(), EC_BADARGS); + } + return EC_BADARGS; + } + + // 每个 DeleteLocationSpecsTask 需要独立返回结果。一个 key 下的 location + // 必须唯一；否则多个 RMW 槽会从同一旧值计算并发生 last-write-wins 丢更新。 KeyVector flat_keys; LocationIdsPerKey flat_location_ids; std::vector> flat_to_original_task_indices; std::vector> flat_deleted_specs_size; std::vector flat_missing_targets; - std::vector write_leases; for (size_t i = 0; i < keys.size(); ++i) { - out_batch_results[i].assign(tasks_per_key[i].size(), ErrorCode::EC_OK); - if (out_missing_targets) { - (*out_missing_targets)[i].assign(tasks_per_key[i].size(), false); - } for (size_t task_index = 0; task_index < tasks_per_key[i].size(); ++task_index) { flat_keys.push_back(keys[i]); - flat_location_ids.push_back({tasks_per_key[i][task_index].location_id}); + flat_location_ids.push_back({tasks_per_key[i][task_index].ResolvedLocationId()}); flat_to_original_task_indices.push_back({i, task_index}); flat_deleted_specs_size.emplace_back(DataStorageType::DATA_STORAGE_TYPE_UNKNOWN, 0); flat_missing_targets.push_back(0); @@ -1588,24 +3573,29 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context // 每条 flat task 独立处理：NOENT 视为幂等成功；spec_names 为空返回 BADARGS； // 删除后无剩余 specs 则删除整个 location，否则 COW 更新 location_specs。 + bool write_lease_attempted = false; + ErrorCode write_lease_ec = EC_OK; + MetadataWriteLease write_lease; auto modifier = [&keys, &tasks_per_key, &flat_to_original_task_indices, &flat_deleted_specs_size, &flat_missing_targets, &acquire_write_lease, - &write_leases](const std::vector &get_ecs, - const LocationIdVector &loc_ids, - size_t key_index, - CacheLocationVector &locs, - PropertyMap &upsert_property_map) -> LocationModifierResult { + &write_lease_attempted, + &write_lease_ec, + &write_lease](const std::vector &get_ecs, + const LocationIdVector &loc_ids, + size_t key_index, + CacheLocationVector &locs, + PropertyMap &upsert_property_map) -> LocationModifierResult { (void)upsert_property_map; - if (acquire_write_lease) { - auto [lease_ec, lease] = acquire_write_lease(); - if (lease_ec != EC_OK) { - return {ModifierAction::MA_FAIL, std::vector(loc_ids.size(), lease_ec)}; - } - write_leases.push_back(std::move(lease)); + if (acquire_write_lease && !write_lease_attempted) { + std::tie(write_lease_ec, write_lease) = acquire_write_lease(); + write_lease_attempted = true; + } + if (write_lease_ec != EC_OK) { + return {ModifierAction::MA_FAIL, std::vector(loc_ids.size(), write_lease_ec)}; } std::vector modifier_ecs(loc_ids.size(), ErrorCode::EC_OK); if (loc_ids.size() != 1 || key_index >= flat_to_original_task_indices.size()) { @@ -1615,6 +3605,12 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context const auto [original_key_index, original_task_index] = flat_to_original_task_indices[key_index]; const auto &task = tasks_per_key[original_key_index][original_task_index]; + if (task.spec_names.empty() || std::any_of(task.spec_names.begin(), + task.spec_names.end(), + [](const std::string &name) { return name.empty(); })) { + modifier_ecs[0] = ErrorCode::EC_BADARGS; + return {ModifierAction::MA_SKIP, std::move(modifier_ecs)}; + } const ErrorCode ec = get_ecs.empty() ? ErrorCode::EC_ERROR : get_ecs[0]; const std::string &loc_id = loc_ids[0]; @@ -1639,11 +3635,8 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context return {ModifierAction::MA_SKIP, std::move(modifier_ecs)}; } - if (task.spec_names.empty()) { - modifier_ecs[0] = ErrorCode::EC_BADARGS; - return {ModifierAction::MA_SKIP, std::move(modifier_ecs)}; - } - + std::uint64_t validated_total_size = 0; + const bool old_specs_validated = locs[0]->GetValidatedTotalSize(validated_total_size); std::unordered_set delete_spec_names(task.spec_names.begin(), task.spec_names.end()); std::vector kept_specs; std::vector deleted_specs; @@ -1660,14 +3653,21 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context return {ModifierAction::MA_SKIP, std::move(modifier_ecs)}; } - flat_deleted_specs_size[key_index] = std::make_pair(locs[0]->type(), GetLocationSpecsSize(deleted_specs)); + const std::uint64_t deleted_specs_size = GetLocationSpecsSize(deleted_specs); + flat_deleted_specs_size[key_index] = std::make_pair(locs[0]->type(), deleted_specs_size); if (kept_specs.empty()) { return {ModifierAction::MA_DELETE, std::move(modifier_ecs)}; } auto new_loc = std::make_shared(*locs[0]); + if (const auto *interned_location_id = task.ResolvedInternedLocationId()) { + new_loc->set_id(*interned_location_id); + } new_loc->set_location_specs(std::move(kept_specs)); new_loc->set_spec_size(new_loc->location_specs().size()); + if (old_specs_validated && deleted_specs_size <= validated_total_size) { + new_loc->set_validated_total_size(validated_total_size - deleted_specs_size); + } locs[0] = std::move(new_loc); return {ModifierAction::MA_OK, std::move(modifier_ecs)}; }; @@ -1678,11 +3678,19 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(service_metrics_collector, MetaSearcherIndexerReadModifyWriteLocation); // ReadModifyWriteLocation 返回 flat 维度结果，这里映射回原始 key/task 维度。 + bool malformed_result = result.per_location_error_codes.size() != flat_to_original_task_indices.size(); + if (malformed_result) { + KVCM_LOG_ERROR("BatchDeleteLocationSpecs result size mismatch, tasks[%zu], results[%zu]", + flat_to_original_task_indices.size(), + result.per_location_error_codes.size()); + } for (size_t i = 0; i < flat_to_original_task_indices.size(); ++i) { const auto [original_key_index, original_task_index] = flat_to_original_task_indices[i]; - ErrorCode ec = result.ec; - if (i < result.per_location_error_codes.size() && !result.per_location_error_codes[i].empty()) { + ErrorCode ec = ErrorCode::EC_MISMATCH; + if (i < result.per_location_error_codes.size() && result.per_location_error_codes[i].size() == 1) { ec = result.per_location_error_codes[i][0]; + } else { + malformed_result = true; } out_batch_results[original_key_index][original_task_index] = ec; if (out_missing_targets) { @@ -1702,7 +3710,7 @@ ErrorCode MetaSearcher::BatchDeleteLocationSpecs(RequestContext *request_context if (result.ec != ErrorCode::EC_OK) { KVCM_LOG_WARN("meta_indexer_->ReadModifyWriteLocation failed, ec: %d", result.ec); } - return result.ec; + return malformed_result ? ErrorCode::EC_MISMATCH : result.ec; } ErrorCode MetaSearcher::BatchUpdateLocationStatus(RequestContext *request_context, @@ -2098,7 +4106,8 @@ ErrorCode MetaSearcher::CleanupLocationsByPredicate(RequestContext *request_cont DataStorageType storage_type, size_t scan_batch_size, LocationCleanupPredicate should_delete, - std::function should_abort) { + std::function should_abort, + AcquireMetadataWriteLeaseFunc acquire_cleanup_lease) { if (!should_delete) { return EC_BADARGS; } @@ -2150,11 +4159,59 @@ ErrorCode MetaSearcher::CleanupLocationsByPredicate(RequestContext *request_cont } } if (has_deletes) { - if (!submit_del_req_func_) { - KVCM_LOG_WARN("CleanupLocationsByPredicate: reclaimer submit callback is unavailable"); - return EC_ERROR; + // Callers without a lifecycle lease historically cancel only at + // scan-batch boundaries. Lease-aware cleanup rechecks here so + // a lifecycle change during the scan cannot reach the delete. + if (acquire_cleanup_lease && should_abort && should_abort()) { + KVCM_LOG_INFO("CleanupLocationsByPredicate: aborted before delete"); + return EC_OK; + } + if (acquire_cleanup_lease) { + auto [lease_ec, cleanup_lease] = acquire_cleanup_lease(); + if (lease_ec == EC_MISMATCH || lease_ec == EC_NODE_NOT_REGISTERED || + lease_ec == EC_INSTANCE_NOT_EXIST) { + KVCM_LOG_INFO("CleanupLocationsByPredicate: lifecycle changed before delete, ec %d", lease_ec); + return EC_OK; + } + if (lease_ec != EC_OK) { + KVCM_LOG_WARN("CleanupLocationsByPredicate: failed to acquire cleanup lease, ec %d", lease_ec); + return lease_ec; + } + + KeyVector delete_keys; + LocationIdsPerKey compact_location_ids; + std::vector> compact_expected_values; + delete_keys.reserve(keys.size()); + compact_location_ids.reserve(keys.size()); + compact_expected_values.reserve(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + if (delete_location_ids[i].empty()) { + continue; + } + delete_keys.push_back(keys[i]); + compact_location_ids.push_back(std::move(delete_location_ids[i])); + compact_expected_values.push_back(std::move(expected_location_values[i])); + } + std::vector> per_location_ec; + const ErrorCode delete_ec = BatchDeleteLocations( + request_context, delete_keys, compact_location_ids, per_location_ec, compact_expected_values); + if (delete_ec != EC_OK) { + has_failure = true; + } + for (const auto &per_key_ec : per_location_ec) { + for (const ErrorCode location_ec : per_key_ec) { + if (location_ec != EC_OK && location_ec != EC_NOENT && location_ec != EC_MISMATCH) { + has_failure = true; + } + } + } + } else { + if (!submit_del_req_func_) { + KVCM_LOG_WARN("CleanupLocationsByPredicate: reclaimer submit callback is unavailable"); + return EC_ERROR; + } + submit_del_req_func_(keys, delete_location_ids, expected_location_values, true); } - submit_del_req_func_(keys, delete_location_ids, expected_location_values, true); } } cursor = next_cursor; @@ -2206,6 +4263,10 @@ ErrorCode MetaSearcher::CleanupLocationsByHost(RequestContext *request_context, } for (const auto &kv : location_maps[i]) { const std::string &loc_id = kv.first; + if (!kv.second) { + has_failure = true; + continue; + } const CacheLocation &loc = *kv.second; if (loc.type() == storage_type && loc_id.size() >= host_suffix.size() && loc_id.compare(loc_id.size() - host_suffix.size(), host_suffix.size(), host_suffix) == 0) { diff --git a/kv_cache_manager/manager/meta_searcher.h b/kv_cache_manager/manager/meta_searcher.h index fc1db4fe6..87d1422d3 100644 --- a/kv_cache_manager/manager/meta_searcher.h +++ b/kv_cache_manager/manager/meta_searcher.h @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include "kv_cache_manager/common/error_code.h" @@ -24,6 +26,7 @@ using SubmitDelReqFunc = std::function &blk class MetaIndexer; class LocationSpecGroup; +class CacheManager; enum class LocationSelectStrategy : int32_t { LSS_UNSPECIFIED = 0, @@ -48,9 +51,24 @@ class MetaSearcher { struct HostCacheMatch { std::string host_ip_port; - int64_t prefix_match_blocks; + int64_t local; + int64_t p2p_1_fetch; + int64_t p2p_1_total_match; }; + struct HostCacheLocationInfo { + // When true, the checker has already parsed the EventReport location + // id and validated every spec URI while applying query visibility. + bool has_reporter_identity = false; + // Views borrow the immutable CacheLocation id and are consumed while + // that location is still held by the current projection; they are + // never retained in output. + std::string_view reporter_medium; + std::string_view reporter_host; + }; + using CheckHostCacheLocationFunc = + std::function; + explicit MetaSearcher(const std::shared_ptr &meta_manager); MetaSearcher(const std::shared_ptr &meta_indexer, CheckLocDataExistFunc check_loc_data_exist, @@ -72,7 +90,9 @@ class MetaSearcher { const KeyVector &keys, LocationsPerKey &out_locations, SelectLocationPolicy *policy, - const std::vector &selectors) const; + const std::vector &selectors, + const std::vector &requested_spec_names = {}, + const BlockMask &input_mask = BlockMask{}) const; ErrorCode ReverseRollSlideWindowMatch(RequestContext *request_context, const KeyVector &keys, int32_t sw_size, @@ -82,13 +102,17 @@ class MetaSearcher { const KeyVector &keys, bool use_eagle_pop, const std::vector &medium_filter, - std::vector &out_matches) const; + std::vector &out_matches, + const CheckHostCacheLocationFunc *request_check_location = nullptr, + size_t p2p_host_count = 0) const; ErrorCode PrefixMatchWithMambaByHost(RequestContext *request_context, const KeyVector &keys, bool use_eagle_pop, const std::vector &medium_filter, const std::vector &location_spec_groups, - std::vector &out_matches) const; + std::vector &out_matches, + const CheckHostCacheLocationFunc *request_check_location = nullptr, + size_t p2p_host_count = 0) const; ErrorCode BatchGetLocation(RequestContext *request_context, const KeyVector &keys, const BlockMask &input_mask, @@ -134,32 +158,147 @@ class MetaSearcher { DataStorageType type; CacheLocationStatus status; std::vector specs; + InternedLocationId interned_location_id; + const InternedLocationId *borrowed_interned_location_id = nullptr; + + [[nodiscard]] const std::string &ResolvedLocationId() const noexcept { + const auto *interned = ResolvedInternedLocationId(); + return interned ? **interned : location_id; + } + [[nodiscard]] const InternedLocationId *ResolvedInternedLocationId() const noexcept { + if (interned_location_id) { + return &interned_location_id; + } + return borrowed_interned_location_id && *borrowed_interned_location_id ? borrowed_interned_location_id + : nullptr; + } }; // Replaces existing specs or creates the stable location in one metadata - // read-modify-write operation per batch. + // read-modify-write operation per batch. Keys and location ids within a + // key must be unique. Every task requires non-empty, uniquely named specs + // with valid URIs and cannot mix versioned/unversioned specs or multiple + // snapshot versions. When supplied, the write-lease callback is invoked + // once after the metadata read and before the first mutation; the returned + // lease is retained until the operation returns. ErrorCode BatchReplaceLocationSpecs(RequestContext *request_context, const KeyVector &keys, const std::vector> &tasks_per_key, std::vector &out_per_key_ec, AcquireMetadataWriteLeaseFunc acquire_write_lease = nullptr); + class PrevalidatedTotalSize { + public: + [[nodiscard]] std::uint64_t value() const noexcept { return value_; } + + private: + friend class CacheManager; + explicit PrevalidatedTotalSize(std::uint64_t value) noexcept : value_(value) {} + + std::uint64_t value_ = 0; + }; struct MergeLocationSpecsTask { std::string location_id; DataStorageType type; CacheLocationStatus status; std::vector specs; + // ReportEvent fully validates and parses every input URI before it + // acquires the reporter mutation fence. Supplying this value lets its + // internal merge path reuse the already computed aggregate size + // instead of parsing every versioned URI a second time. Other callers + // must leave it empty and receive the normal strict validation. + std::optional prevalidated_total_size; + InternedLocationId interned_location_id; + // ReportEvent owns one canonical id per medium for the duration of + // the synchronous Batch* call. Borrow it here so a 20K-key request + // does not perform 20K atomic shared_ptr increments/decrements merely + // to route tasks. A persisted CacheLocation still takes ownership. + const InternedLocationId *borrowed_interned_location_id = nullptr; + // ReportEvent overwhelmingly carries one spec per block. Keep that + // value inline so a 20K-block request does not allocate 20K one-item + // vectors; generic/multi-spec callers continue using specs unchanged. + std::optional inline_spec; + + [[nodiscard]] const std::string &ResolvedLocationId() const noexcept { + const auto *interned = ResolvedInternedLocationId(); + return interned ? **interned : location_id; + } + [[nodiscard]] const InternedLocationId *ResolvedInternedLocationId() const noexcept { + if (interned_location_id) { + return &interned_location_id; + } + return borrowed_interned_location_id && *borrowed_interned_location_id ? borrowed_interned_location_id + : nullptr; + } + [[nodiscard]] size_t SpecCount() const noexcept { return specs.size() + (inline_spec ? 1 : 0); } + [[nodiscard]] bool SpecsEmpty() const noexcept { return SpecCount() == 0; } + [[nodiscard]] const LocationSpec &SpecAt(size_t index) const noexcept { + return index < specs.size() ? specs[index] : *inline_spec; + } + [[nodiscard]] LocationSpec &MutableSpecAt(size_t index) noexcept { + return index < specs.size() ? specs[index] : *inline_spec; + } + void PushReportEventSpec(LocationSpec &&spec, size_t max_spec_count) { + if (specs.empty() && !inline_spec) { + inline_spec.emplace(std::move(spec)); + return; + } + if (inline_spec) { + specs.reserve(max_spec_count < 2 ? 2 : max_spec_count); + specs.push_back(std::move(*inline_spec)); + inline_spec.reset(); + } + specs.push_back(std::move(spec)); + } }; + // Keys and location ids within a key must be unique. Every task requires + // non-empty, uniquely named specs with valid URIs and cannot mix + // versioned/unversioned specs or multiple snapshot versions. The block + // existence state and every requested target location are read + // together, then merged in one targeted RMW phase. The optional write + // lease is acquired once after that read and before the first mutation; + // it is retained through the upsert so a concurrent lifecycle change can + // fence work that was admitted under an older reporter generation. ErrorCode BatchMergeLocationSpecs(RequestContext *request_context, const KeyVector &keys, const std::vector> &tasks_per_key, std::vector &out_per_key_ec, AcquireMetadataWriteLeaseFunc acquire_write_lease = nullptr); + // Allocation-light equivalent used by ReportEvent. Tasks for key i are + // stored in [task_offsets[i], task_offsets[i + 1]); offsets must start at + // zero, be nondecreasing, and end at flat_tasks.size(). URI ownership in + // CacheManager-prevalidated tasks can be consumed during the synchronous + // call; their spec names remain available for per-item failure mapping. + // Non-prevalidated flat tasks retain the generic copy semantics. + ErrorCode BatchMergeLocationSpecsFlat(RequestContext *request_context, + const KeyVector &keys, + const std::vector &task_offsets, + std::vector &flat_tasks, + std::vector &out_per_key_ec, + AcquireMetadataWriteLeaseFunc acquire_write_lease = nullptr); struct DeleteLocationSpecsTask { std::string location_id; std::vector spec_names; + InternedLocationId interned_location_id; + const InternedLocationId *borrowed_interned_location_id = nullptr; + + [[nodiscard]] const std::string &ResolvedLocationId() const noexcept { + const auto *interned = ResolvedInternedLocationId(); + return interned ? **interned : location_id; + } + [[nodiscard]] const InternedLocationId *ResolvedInternedLocationId() const noexcept { + if (interned_location_id) { + return &interned_location_id; + } + return borrowed_interned_location_id && *borrowed_interned_location_id ? borrowed_interned_location_id + : nullptr; + } }; - // Missing block/location targets are idempotent EC_OK. When requested, - // out_missing_targets mirrors tasks_per_key and marks those no-op targets; - // an existing location with only missing spec_names is not marked. + // Missing block/location targets are idempotent EC_OK. Keys and location + // ids within a key must be unique, and every task must name at least one + // non-empty spec. When requested, out_missing_targets mirrors + // tasks_per_key and marks missing block/location no-ops; an existing + // location with only missing spec_names is not marked. The optional write + // lease is acquired once after the metadata read and before the first + // mutation. ErrorCode BatchDeleteLocationSpecs(RequestContext *request_context, const KeyVector &keys, const std::vector> &tasks_per_key, @@ -214,7 +353,8 @@ class MetaSearcher { DataStorageType storage_type, size_t scan_batch_size, LocationCleanupPredicate should_delete, - std::function should_abort = nullptr); + std::function should_abort = nullptr, + AcquireMetadataWriteLeaseFunc acquire_cleanup_lease = nullptr); ErrorCode CleanupLocationsByHost(RequestContext *request_context, const std::string &host_suffix, DataStorageType storage_type, @@ -223,6 +363,14 @@ class MetaSearcher { AcquireMetadataWriteLeaseFunc acquire_cleanup_lease = nullptr); private: + class MergeLocationSpecsTaskView; + + ErrorCode BatchMergeLocationSpecsImpl(RequestContext *request_context, + const KeyVector &keys, + MergeLocationSpecsTaskView tasks, + std::vector &out_per_key_ec, + AcquireMetadataWriteLeaseFunc acquire_write_lease); + struct StorageTypeWeights { static constexpr size_t NFS = 5; // NFS存储权重较高 static constexpr size_t MOONCAKE = 3; // Mooncake存储权重中等 diff --git a/kv_cache_manager/manager/test/BUILD b/kv_cache_manager/manager/test/BUILD index b2616ae3f..c4b76c725 100644 --- a/kv_cache_manager/manager/test/BUILD +++ b/kv_cache_manager/manager/test/BUILD @@ -87,8 +87,22 @@ cc_test( data = [], deps = [ "//kv_cache_manager/common:unittest", + "//kv_cache_manager/config:instance_info", + "//kv_cache_manager/manager:meta_searcher", + "//kv_cache_manager/meta", "//kv_cache_manager/meta:cache_location", "//kv_cache_manager/meta:meta_local_backend", + ], +) + +cc_test( + name = "GetHostCacheStateBenchmark", + srcs = ["get_host_cache_state_benchmark.cc"], + copts = ["-fno-access-control"], + tags = ["manual"], + deps = [ + "//kv_cache_manager/common:unittest", + "//kv_cache_manager/config", "//kv_cache_manager/manager:meta_searcher", "//kv_cache_manager/meta", ], @@ -157,9 +171,9 @@ cc_test( ], deps = [ "//kv_cache_manager/common:unittest", - "//kv_cache_manager/meta:cache_location", "//kv_cache_manager/manager:meta_searcher", "//kv_cache_manager/meta", + "//kv_cache_manager/meta:cache_location", ], ) diff --git a/kv_cache_manager/manager/test/cache_manager_test.cc b/kv_cache_manager/manager/test/cache_manager_test.cc index 9410bb841..8b497cb23 100644 --- a/kv_cache_manager/manager/test/cache_manager_test.cc +++ b/kv_cache_manager/manager/test/cache_manager_test.cc @@ -1,8 +1,11 @@ +#include #include #include #include #include #include +#include +#include #include #include #include @@ -172,6 +175,15 @@ class ControllableMetaLocalBackend : public MetaLocalBackend { void BlockNextLocationRead() { std::lock_guard lock(control_mutex_); block_next_location_read_ = true; + blocked_location_read_thread_ = {}; + location_read_entered_ = false; + release_location_read_ = false; + } + + void BlockNextLocationReadOnCurrentThread() { + std::lock_guard lock(control_mutex_); + block_next_location_read_ = true; + blocked_location_read_thread_ = std::this_thread::get_id(); location_read_entered_ = false; release_location_read_ = false; } @@ -253,6 +265,39 @@ class ControllableMetaLocalBackend : public MetaLocalBackend { return MetaLocalBackend::GetLocations(request_context, keys, location_ids, out_locations); } + std::vector> + GetLocationsWithKeyStatus(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept override { + MaybeBlockLocationRead(); + return MetaLocalBackend::GetLocationsWithKeyStatus( + request_context, keys, location_ids, out_locations, out_key_error_codes); + } + + std::vector GetLocations(RequestContext *request_context, + const KeyTypeVec &keys, + CacheLocationMapVector &out_locations) noexcept override { + MaybeBlockLocationRead(); + return MetaLocalBackend::GetLocations(request_context, keys, out_locations); + } + + std::vector GetLocationValues(RequestContext *request_context, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept override { + MaybeBlockLocationRead(); + return MetaLocalBackend::GetLocationValues(request_context, keys, out_locations); + } + + std::vector GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept override { + MaybeBlockLocationRead(); + return MetaLocalBackend::GetLocationValuesCompact(request_context, keys, key_count, out_locations); + } + bool Sync(const KeyTypeVec &keys) noexcept override { { std::lock_guard lock(control_mutex_); @@ -264,10 +309,12 @@ class ControllableMetaLocalBackend : public MetaLocalBackend { private: void MaybeBlockLocationRead() { std::unique_lock lock(control_mutex_); - if (!block_next_location_read_) { + if (!block_next_location_read_ || (blocked_location_read_thread_ != std::thread::id{} && + blocked_location_read_thread_ != std::this_thread::get_id())) { return; } block_next_location_read_ = false; + blocked_location_read_thread_ = {}; location_read_entered_ = true; control_cv_.notify_all(); control_cv_.wait(lock, [&] { return release_location_read_; }); @@ -290,6 +337,7 @@ class ControllableMetaLocalBackend : public MetaLocalBackend { bool upsert_entered_ = false; bool release_upsert_ = false; bool block_next_location_read_ = false; + std::thread::id blocked_location_read_thread_; bool location_read_entered_ = false; bool release_location_read_ = false; std::optional fail_key_on_next_upsert_; @@ -484,8 +532,8 @@ class CacheManagerTest : public TESTBASE { } } - std::shared_ptr InstallEventReportBackend() { - const std::string storage_name = "event_report_default"; + std::shared_ptr + InstallEventReportBackend(const std::string &storage_name = "event_report_default") { const std::string group_name = registry_manager_->GetInstanceGroupName("test_instance"); auto group = registry_manager_->instance_group_configs_.at(group_name); group->set_event_report_storage_candidates({storage_name}); @@ -729,6 +777,22 @@ TEST_F(CacheManagerTest, TestRegisterInstance) { } } +TEST_F(CacheManagerTest, TestRegisterInstanceRejectsDifferentInstanceGroup) { + auto [ec, storage_configs] = cache_manager_->RegisterInstance(request_context_.get(), + "different_group", + "test_instance", + 64, + createLocationSpecInfos(), + createModelDeployment(), + {}); + EXPECT_EQ(EC_DUPLICATE_ENTITY, ec); + EXPECT_TRUE(storage_configs.empty()); + const auto existing = registry_manager_->GetInstanceInfo(request_context_.get(), "test_instance"); + ASSERT_NE(nullptr, existing); + EXPECT_EQ("default", existing->instance_group_name()); + EXPECT_NE(std::string::npos, request_context_->error_tracer()->ToJsonString().find("instance_group_name")); +} + TEST_F(CacheManagerTest, TestRegisterInstanceReturnsTieredMigrationStorageConfigs) { const std::string migration_source = "nfs_migration_source"; const std::string migration_target = "nfs_migration_target"; @@ -942,8 +1006,7 @@ TEST_F(CacheManagerTest, TestStartWriteCacheRollsBackPartialBatchAdd) { } std::vector keys{1001, 1002}; - while (GetShardIndex(keys[0], meta_indexer->mutex_shard_mask_) == - GetShardIndex(keys[1], meta_indexer->mutex_shard_mask_)) { + while (meta_indexer->GetMutexShardIndex(keys[0]) == meta_indexer->GetMutexShardIndex(keys[1])) { ++keys[1]; } auto [ec, start_write_cache_info] = @@ -2563,16 +2626,33 @@ TEST_F(CacheManagerTest, TestGetCheckLocDataExistFunc_VerifiesUriPassthrough) { } TEST_F(CacheManagerTest, TestGetCheckLocDataExistFunc_UnregisteredBackend) { - // valid URIs whose hostname does not match any registered backend; - // DataStorageManager::Exist returns an empty vector, and - // std::all_of on an empty range is true -> functor returns true + // A missing backend returns no per-URI result and must fail closed. auto func = cache_manager_->GetCheckLocDataExistFunc("test_instance"); CacheLocation loc; loc.set_status(CLS_SERVING); loc.set_type(DataStorageType::DATA_STORAGE_TYPE_NFS); loc.set_location_specs({LocationSpec("tp0", "file://nonexistent_backend/path")}); - ASSERT_EQ(func(loc), true); + EXPECT_FALSE(func(loc)); +} + +TEST_F(CacheManagerTest, TestGetCheckLocDataExistFunc_ShortBackendResultFailsClosed) { + auto mock_backend = std::make_shared(cache_manager_->metrics_registry_); + EXPECT_CALL(*mock_backend, MightExist(_)).WillOnce([](const std::vector &) { + return std::vector{true}; + }); + auto dsm = registry_manager_->data_storage_manager_; + dsm->storage_map_["short_result_store"] = mock_backend; + + const auto func = cache_manager_->GetCheckLocDataExistFunc("test_instance"); + CacheLocation loc; + loc.set_status(CLS_SERVING); + loc.set_type(DataStorageType::DATA_STORAGE_TYPE_NFS); + loc.set_location_specs({LocationSpec("tp0", "file://short_result_store/path_a"), + LocationSpec("tp1", "file://short_result_store/path_b")}); + EXPECT_FALSE(func(loc)); + + dsm->storage_map_.erase("short_result_store"); } TEST(ReportEventContractTest, SnapshotAndResponseFieldNumbersMatchContract) { @@ -2848,24 +2928,126 @@ TEST_F(CacheManagerTest, TestHostDownMakesAlreadyAdmittedDeltaInvisibleWithoutDe EXPECT_TRUE(QueryEventReportUris({key}).empty()); } +TEST_F(CacheManagerTest, TestEventCleanupTasksDrainAndRemainFencedAcrossReactivation) { + const std::string host = "192.168.10.78:8080"; + const int64_t key = 94'478; + auto event_backend = InstallEventReportBackend(); + auto *meta_backend = InstallControllableMetaBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_NE(nullptr, meta_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + ASSERT_EQ(EC_OK, CallReportEvent(MakeSnapshotRequest(host, {{key, "baseline"}}), "cleanup_gate_baseline").first); + ASSERT_EQ(1u, QueryRawEventReportUris(key).size()); + ASSERT_TRUE(event_backend->IsCleanupCallbackSet()); + + // Stop executor workers so the test can deterministically take ownership + // of the cleanup closure admitted by the backend callback. + cache_manager_->reclaimer_task_supervisor_->Stop(); + auto &executor = cache_manager_->schedule_plan_executor_; + executor->stop_.store(true); + executor->condition_.notify_all(); + for (auto &worker : executor->workers_) { + if (worker.joinable()) { + worker.join(); + } + } + executor->workers_.clear(); + executor->stop_.store(false); + { + std::lock_guard lock(executor->queue_mutex_); + for (auto &queue : executor->task_queues_) { + queue.clear(); + } + } + + EventReportBackend::CleanupCallback callback; + { + std::lock_guard lock(event_backend->cleanup_cb_mutex_); + callback = event_backend->cleanup_callback_; + } + ASSERT_TRUE(callback); + + auto take_only_queued_task = [&]() { + std::function task; + std::lock_guard lock(executor->queue_mutex_); + EXPECT_EQ(1u, executor->WaitingTaskCountLocked()); + for (auto &queue : executor->task_queues_) { + if (!queue.empty()) { + task = queue.begin()->task; + queue.clear(); + } + } + return task; + }; + + uint64_t cleanup_generation = 0; + ASSERT_EQ(EC_OK, event_backend->UnregisterNodeForHostDown("test_instance", host, cleanup_generation)); + callback("test_instance", host, cleanup_generation); + auto running_cleanup = take_only_queued_task(); + ASSERT_TRUE(running_cleanup); + + // A cleanup already executing must hold the lifetime lease until its + // metadata access completes. Deactivation therefore waits rather than + // racing manager cleanup against the raw-this task. + meta_backend->BlockNextLocationRead(); + auto cleanup_future = std::async(std::launch::async, std::move(running_cleanup)); + ASSERT_TRUE(meta_backend->WaitUntilLocationReadEntered(std::chrono::seconds(1))); + auto deactivate_future = + std::async(std::launch::async, [this] { cache_manager_->DeactivateEventCleanupCallbacks(); }); + EXPECT_EQ(std::future_status::timeout, deactivate_future.wait_for(std::chrono::milliseconds(50))); + meta_backend->ReleaseLocationRead(); + ASSERT_EQ(std::future_status::ready, cleanup_future.wait_for(std::chrono::seconds(2))); + cleanup_future.get(); + ASSERT_EQ(std::future_status::ready, deactivate_future.wait_for(std::chrono::seconds(2))); + deactivate_future.get(); + EXPECT_TRUE(QueryRawEventReportUris(key).empty()); + + cache_manager_->ActivateEventCleanupCallbacks(); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "after_reactivate"), "cleanup_gate_delta").first); + ASSERT_EQ(1u, QueryRawEventReportUris(key).size()); + + // A task queued in the old epoch must remain inert after reactivation; a + // boolean-only gate would incorrectly make it live again here. + uint64_t stale_cleanup_generation = 0; + ASSERT_EQ(EC_OK, event_backend->UnregisterNodeForHostDown("test_instance", host, stale_cleanup_generation)); + callback("test_instance", host, stale_cleanup_generation); + auto stale_queued_cleanup = take_only_queued_task(); + ASSERT_TRUE(stale_queued_cleanup); + cache_manager_->DeactivateEventCleanupCallbacks(); + cache_manager_->ActivateEventCleanupCallbacks(); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + stale_queued_cleanup(); + EXPECT_EQ(1u, QueryRawEventReportUris(key).size()); +} + TEST_F(CacheManagerTest, TestOldDeltaCannotCrossReporterLifecycleAfterReregisterAndSnapshot) { const std::string host = "192.168.10.45:8080"; const int64_t key = 94'422; - const int64_t new_key = 94'423; + int64_t new_key = 94'423; auto event_backend = InstallEventReportBackend(); auto *meta_backend = InstallControllableMetaBackend(); ASSERT_NE(nullptr, event_backend); ASSERT_NE(nullptr, meta_backend); + auto meta_indexer = cache_manager_->meta_indexer_manager_->GetMetaIndexer("test_instance"); + ASSERT_NE(nullptr, meta_indexer); + while (meta_indexer->GetMutexShardIndex(key) == meta_indexer->GetMutexShardIndex(new_key)) { + ++new_key; + } ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); ASSERT_EQ(EC_OK, CallReportEvent(MakeSnapshotRequest(host, {{key, "baseline"}}), "lifecycle_baseline").first); // Pause the old request after it entered metadata read I/O but before its // modifier can acquire the generation-pinned write lease. - meta_backend->BlockNextLocationRead(); - auto old_delta = std::async(std::launch::async, [this, host, key] { + auto old_delta = std::async(std::launch::async, [this, host, key, meta_backend] { + meta_backend->BlockNextLocationReadOnCurrentThread(); return CallReportEvent(MakeAddRequest(host, key, "stale_old_lifecycle"), "old_lifecycle_delta"); }); - ASSERT_TRUE(meta_backend->WaitUntilLocationReadEntered(std::chrono::seconds(1))); + const bool old_delta_entered = meta_backend->WaitUntilLocationReadEntered(std::chrono::seconds(1)); + if (!old_delta_entered) { + meta_backend->ReleaseLocationRead(); + } + ASSERT_TRUE(old_delta_entered); proto::meta::ReportEventRequest host_down; host_down.set_instance_id("test_instance"); @@ -3034,6 +3216,439 @@ TEST_F(CacheManagerTest, TestReportEventSameRequestDeltaOrderUsesLastOperationPe EXPECT_NE(std::string::npos, visible.front().find("s_version=" + token)); } +TEST_F(CacheManagerTest, TestReportEventFlatFoldMatchesReferenceAcrossBlocksMediaAndSpecs) { + const std::string host = "192.168.10.81:8080"; + constexpr int64_t first_key = 96'000; + constexpr size_t key_count = 48; + constexpr size_t event_count = 768; + std::vector mediums; + for (size_t i = 0; i < 17; ++i) { + mediums.push_back("medium_" + std::to_string(i)); + } + const std::array spec_names{"tp0", "tp1", "tp2", "tp3"}; + + struct ExpectedSpec { + std::string raw_uri; + std::uint64_t size = 0; + }; + std::map>> expected; + + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, mediums)); + + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + + std::uint64_t random_state = 0x9e3779b97f4a7c15ULL; + auto next_random = [&random_state] { + random_state = random_state * 6364136223846793005ULL + 1442695040888963407ULL; + return random_state; + }; + for (size_t event_index = 0; event_index < event_count; ++event_index) { + const int64_t key = first_key + static_cast(next_random() % key_count); + const std::string &medium = mediums[next_random() % mediums.size()]; + const size_t first_spec_index = next_random() % spec_names.size(); + const bool is_add = next_random() % 4 != 0; + auto *event = request.add_events(); + if (is_add) { + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *params = event->mutable_block_add(); + params->set_block_key(std::to_string(key)); + params->set_medium(medium); + auto add_spec = [&](size_t spec_index) { + const std::string &name = spec_names[spec_index]; + const std::uint64_t size = next_random() % 97 + 1; + const std::string raw_uri = "event_report://" + host + "/" + medium + "?source=model_" + + std::to_string(event_index) + "_" + name + "&size=" + std::to_string(size); + auto *spec = params->add_specs(); + spec->set_name(name); + spec->set_uri(raw_uri); + expected[key][medium][name] = ExpectedSpec{raw_uri, size}; + }; + add_spec(first_spec_index); + if (next_random() % 7 == 0) { + add_spec((first_spec_index + 1) % spec_names.size()); + } + } else { + event->set_event_type(proto::meta::EVENT_BLOCK_DELETE); + auto *params = event->mutable_block_delete(); + params->set_block_key(std::to_string(key)); + params->set_medium(medium); + params->add_spec_names(spec_names[first_spec_index]); + expected[key][medium].erase(spec_names[first_spec_index]); + if (next_random() % 7 == 0) { + const std::string &second_name = spec_names[(first_spec_index + 1) % spec_names.size()]; + params->add_spec_names(second_name); + expected[key][medium].erase(second_name); + } + } + } + + const auto [ec, response] = CallReportEvent(request, "flat_fold_reference_model"); + ASSERT_EQ(EC_OK, ec); + EXPECT_EQ(proto::meta::OK, response.header().status().code()); + EXPECT_EQ(0, response.item_results_size()); + ASSERT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken(response.committed_snapshot_version())); + + std::vector keys; + keys.reserve(key_count); + for (size_t i = 0; i < key_count; ++i) { + keys.push_back(first_key + static_cast(i)); + } + MetaSearcher *meta_searcher = cache_manager_->meta_searcher_manager_->GetMetaSearcher("test_instance"); + ASSERT_NE(nullptr, meta_searcher); + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + ASSERT_EQ(keys.size(), location_maps.size()); + + std::uint64_t expected_total_size = 0; + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + using FlattenedSpecs = std::map, std::string>; + FlattenedSpecs expected_specs; + FlattenedSpecs actual_specs; + size_t expected_location_count = 0; + for (const auto &medium : mediums) { + const auto expected_medium = expected[keys[key_index]].find(medium); + if (expected_medium != expected[keys[key_index]].end()) { + expected_location_count += static_cast(!expected_medium->second.empty()); + for (const auto &[name, expected_spec] : expected_medium->second) { + std::string versioned_uri; + ASSERT_TRUE(SnapshotUriUtils::AddSnapshotVersionToUri( + expected_spec.raw_uri, response.committed_snapshot_version(), versioned_uri)); + expected_specs[{medium, name}] = std::move(versioned_uri); + expected_total_size += expected_spec.size; + } + } + + const auto location_it = location_maps[key_index].find(event_backend->BuildLocationId(medium, host)); + if (location_it == location_maps[key_index].end()) { + continue; + } + ASSERT_TRUE(location_it->second); + EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, location_it->second->type()); + EXPECT_EQ(location_it->second->location_specs().size(), location_it->second->spec_size()); + EXPECT_TRUE(std::is_sorted(location_it->second->location_specs().begin(), + location_it->second->location_specs().end(), + [](const auto &lhs, const auto &rhs) { return lhs.name() < rhs.name(); })); + for (const auto &spec : location_it->second->location_specs()) { + actual_specs[{medium, spec.name()}] = spec.uri(); + } + } + EXPECT_EQ(expected_location_count, location_maps[key_index].size()) << "block key " << keys[key_index]; + EXPECT_EQ(expected_specs, actual_specs) << "block key " << keys[key_index]; + } + + auto meta_indexer = cache_manager_->meta_indexer_manager_->GetMetaIndexer("test_instance"); + ASSERT_NE(nullptr, meta_indexer); + EXPECT_EQ(expected_total_size, + meta_indexer->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); +} + +TEST_F(CacheManagerTest, TestReportEventCrossRequestMultiReporterStateMatchesReference) { + const std::array hosts{"192.168.10.85:8080", "192.168.10.86:8080"}; + const std::array mediums{"mem", "disk", "hbm", "ssd", "remote"}; + const std::array spec_names{"tp0", "tp1", "tp2"}; + constexpr int64_t first_key = 96'200; + constexpr size_t key_count = 32; + constexpr size_t round_count = 8; + constexpr size_t random_events_per_round = 95; + + struct ExpectedSpec { + std::string versioned_uri; + std::uint64_t size = 0; + }; + using ExpectedSpecs = std::map; + using ExpectedLocations = std::map; + std::map expected; + + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + const std::vector registered_mediums(mediums.begin(), mediums.end()); + std::array reporter_versions; + for (size_t reporter = 0; reporter < hosts.size(); ++reporter) { + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", hosts[reporter], registered_mediums)); + const auto [ec, response] = + CallReportEvent(MakeSnapshotRequest(hosts[reporter], {}), "multi_reporter_initial_snapshot"); + ASSERT_EQ(EC_OK, ec); + ASSERT_EQ(proto::meta::OK, response.header().status().code()); + ASSERT_EQ(0, response.item_results_size()); + ASSERT_FALSE(response.snapshot_required()); + ASSERT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken(response.committed_snapshot_version())); + reporter_versions[reporter] = response.committed_snapshot_version(); + } + ASSERT_NE(reporter_versions[0], reporter_versions[1]); + + std::uint64_t random_state = 0xd1b54a32d192ed03ULL; + auto next_random = [&random_state] { + random_state = random_state * 2862933555777941757ULL + 3037000493ULL; + return random_state; + }; + + for (size_t round = 0; round < round_count; ++round) { + const size_t reporter = round % hosts.size(); + const std::string &host = hosts[reporter]; + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + + auto add_spec = [&](int64_t key, + const std::string &medium, + const std::string &name, + std::uint64_t size, + const std::string &source, + proto::meta::BlockAddEventParams *params) { + const std::string raw_uri = + "event_report://" + host + "/" + medium + "?size=" + std::to_string(size) + "&source=" + source; + auto *spec = params->add_specs(); + spec->set_name(name); + spec->set_uri(raw_uri); + + std::string versioned_uri; + ASSERT_TRUE(SnapshotUriUtils::AddSnapshotVersionToUri(raw_uri, reporter_versions[reporter], versioned_uri)); + const std::string location_id = event_backend->BuildLocationId(medium, host); + ASSERT_FALSE(location_id.empty()); + expected[key][location_id][name] = ExpectedSpec{std::move(versioned_uri), size}; + }; + auto erase_spec = [&](int64_t key, const std::string &medium, const std::string &name) { + const std::string location_id = event_backend->BuildLocationId(medium, host); + auto key_it = expected.find(key); + if (key_it == expected.end()) { + return; + } + auto location_it = key_it->second.find(location_id); + if (location_it == key_it->second.end()) { + return; + } + location_it->second.erase(name); + if (location_it->second.empty()) { + key_it->second.erase(location_it); + } + if (key_it->second.empty()) { + expected.erase(key_it); + } + }; + + for (size_t event_index = 0; event_index < random_events_per_round; ++event_index) { + const int64_t key = first_key + static_cast(next_random() % key_count); + const std::string &medium = mediums[next_random() % mediums.size()]; + const size_t spec_index = next_random() % spec_names.size(); + const bool is_add = next_random() % 5 != 0; + auto *event = request.add_events(); + if (is_add) { + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *params = event->mutable_block_add(); + params->set_block_key(std::to_string(key)); + params->set_medium(medium); + const std::string source = "r" + std::to_string(round) + "_e" + std::to_string(event_index); + add_spec(key, medium, spec_names[spec_index], next_random() % 251 + 1, source + "_0", params); + if (next_random() % 5 == 0) { + add_spec(key, + medium, + spec_names[(spec_index + 1) % spec_names.size()], + next_random() % 251 + 1, + source + "_1", + params); + } + } else { + event->set_event_type(proto::meta::EVENT_BLOCK_DELETE); + auto *params = event->mutable_block_delete(); + params->set_block_key(std::to_string(key)); + params->set_medium(medium); + params->add_spec_names(spec_names[spec_index]); + erase_spec(key, medium, spec_names[spec_index]); + if (next_random() % 5 == 0) { + const std::string &second_name = spec_names[(spec_index + 1) % spec_names.size()]; + params->add_spec_names(second_name); + erase_spec(key, medium, second_name); + } + } + } + + // End every reporter request with a write to the same block/location. + // From round 2 onward this updates one reporter's existing location + // while the other reporter's location on the same key must survive. + auto *shared_event = request.add_events(); + shared_event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *shared_add = shared_event->mutable_block_add(); + shared_add->set_block_key(std::to_string(first_key)); + shared_add->set_medium(mediums[0]); + add_spec(first_key, + mediums[0], + spec_names[0], + 1000 + round, + "forced_shared_round_" + std::to_string(round), + shared_add); + + const auto [ec, response] = + CallReportEvent(request, "cross_request_multi_reporter_round_" + std::to_string(round)); + ASSERT_EQ(EC_OK, ec) << "round " << round; + EXPECT_EQ(proto::meta::OK, response.header().status().code()) << "round " << round; + EXPECT_EQ(0, response.item_results_size()) << "round " << round; + EXPECT_FALSE(response.snapshot_required()) << "round " << round; + EXPECT_EQ(reporter_versions[reporter], response.committed_snapshot_version()) << "round " << round; + EXPECT_EQ(reporter_versions[reporter], event_backend->GetSnapshotVersion({"test_instance", host})); + + std::vector keys; + keys.reserve(key_count); + for (size_t key_offset = 0; key_offset < key_count; ++key_offset) { + keys.push_back(first_key + static_cast(key_offset)); + } + MetaSearcher *meta_searcher = cache_manager_->meta_searcher_manager_->GetMetaSearcher("test_instance"); + ASSERT_NE(nullptr, meta_searcher); + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + ASSERT_EQ(keys.size(), location_maps.size()); + + std::uint64_t expected_total_size = 0; + const ExpectedLocations empty_locations; + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + const auto expected_key_it = expected.find(keys[key_index]); + const ExpectedLocations &expected_locations = + expected_key_it == expected.end() ? empty_locations : expected_key_it->second; + ASSERT_EQ(expected_locations.size(), location_maps[key_index].size()) + << "round " << round << ", key " << keys[key_index]; + + std::set allowed_visible_uris; + for (const auto &[location_id, expected_specs] : expected_locations) { + const auto actual_location_it = location_maps[key_index].find(location_id); + ASSERT_NE(actual_location_it, location_maps[key_index].end()) + << "round " << round << ", key " << keys[key_index] << ", location " << location_id; + ASSERT_TRUE(actual_location_it->second); + EXPECT_EQ(location_id, actual_location_it->second->id()); + EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, actual_location_it->second->type()); + EXPECT_EQ(expected_specs.size(), actual_location_it->second->location_specs().size()); + + std::map actual_specs; + for (const auto &spec : actual_location_it->second->location_specs()) { + actual_specs[spec.name()] = spec.uri(); + } + std::map expected_spec_uris; + for (const auto &[name, expected_spec] : expected_specs) { + expected_spec_uris[name] = expected_spec.versioned_uri; + allowed_visible_uris.insert(expected_spec.versioned_uri); + expected_total_size += expected_spec.size; + } + EXPECT_EQ(expected_spec_uris, actual_specs) + << "round " << round << ", key " << keys[key_index] << ", location " << location_id; + } + + const auto visible_uris = QueryEventReportUris({keys[key_index]}); + if (allowed_visible_uris.empty()) { + EXPECT_TRUE(visible_uris.empty()) << "round " << round << ", key " << keys[key_index]; + } else { + ASSERT_FALSE(visible_uris.empty()) << "round " << round << ", key " << keys[key_index]; + for (const auto &uri : visible_uris) { + EXPECT_TRUE(allowed_visible_uris.count(uri) != 0) + << "round " << round << ", key " << keys[key_index] << ", URI " << uri; + } + } + } + + auto meta_indexer = cache_manager_->meta_indexer_manager_->GetMetaIndexer("test_instance"); + ASSERT_NE(nullptr, meta_indexer); + EXPECT_EQ(expected_total_size, + meta_indexer->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)) + << "round " << round; + } +} + +TEST_F(CacheManagerTest, TestReportEventFoldedTotalSizeOverflowFailsWithoutMetadata) { + const std::string host = "192.168.10.82:8080"; + constexpr int64_t key = 96'100; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto add_event = [&](const std::string &name, const std::string &size) { + auto *event = request.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *params = event->mutable_block_add(); + params->set_block_key(std::to_string(key)); + params->set_medium("mem"); + auto *spec = params->add_specs(); + spec->set_name(name); + spec->set_uri("event_report://" + host + "/mem?size=" + size); + }; + add_event("tp0", "18446744073709551615"); + add_event("tp1", "1"); + + const auto [ec, response] = CallReportEvent(request, "folded_total_size_overflow"); + EXPECT_EQ(EC_PARTIAL_OK, ec); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.header().status().code()); + ASSERT_EQ(2, response.item_results_size()); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.item_results(0)); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.item_results(1)); + EXPECT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken(response.committed_snapshot_version())); + EXPECT_TRUE(response.snapshot_required()); + EXPECT_TRUE(QueryRawEventReportUris(key).empty()); + + auto meta_indexer = cache_manager_->meta_indexer_manager_->GetMetaIndexer("test_instance"); + ASSERT_NE(nullptr, meta_indexer); + EXPECT_EQ(0u, meta_indexer->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); +} + +TEST_F(CacheManagerTest, TestReportEventRejectsMergeThatOverflowsExistingLocation) { + const std::string host = "192.168.10.83:8080"; + constexpr int64_t key = 96'101; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + auto make_add = [&](const std::string &name, const std::string &size) { + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto *event = request.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *params = event->mutable_block_add(); + params->set_block_key(std::to_string(key)); + params->set_medium("mem"); + auto *spec = params->add_specs(); + spec->set_name(name); + spec->set_uri("event_report://" + host + "/mem?size=" + size); + return request; + }; + + const auto [initial_ec, initial_response] = + CallReportEvent(make_add("tp0", "18446744073709551615"), "existing_size_max"); + ASSERT_EQ(EC_OK, initial_ec); + ASSERT_EQ(proto::meta::OK, initial_response.header().status().code()); + + const auto [overflow_ec, overflow_response] = + CallReportEvent(make_add("tp1", "1"), "existing_plus_new_size_overflow"); + EXPECT_EQ(EC_PARTIAL_OK, overflow_ec); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, overflow_response.header().status().code()); + ASSERT_EQ(1, overflow_response.item_results_size()); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, overflow_response.item_results(0)); + + MetaSearcher *meta_searcher = cache_manager_->meta_searcher_manager_->GetMetaSearcher("test_instance"); + ASSERT_NE(nullptr, meta_searcher); + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher->BatchGetLocation(request_context_.get(), {key}, mask, location_maps)); + ASSERT_EQ(1u, location_maps.size()); + ASSERT_EQ(1u, location_maps[0].size()); + const auto &stored_specs = location_maps[0].begin()->second->location_specs(); + ASSERT_EQ(1u, stored_specs.size()); + EXPECT_EQ("tp0", stored_specs[0].name()); + + auto meta_indexer = cache_manager_->meta_indexer_manager_->GetMetaIndexer("test_instance"); + ASSERT_NE(nullptr, meta_indexer); + EXPECT_EQ(std::numeric_limits::max(), + meta_indexer->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); +} + TEST_F(CacheManagerTest, TestReportEventFoldedDeltaEventsShareFinalWriteFailure) { const std::string host = "192.168.10.52:8080"; const int64_t key = 94'440; @@ -3065,6 +3680,116 @@ TEST_F(CacheManagerTest, TestReportEventFoldedDeltaEventsShareFinalWriteFailure) EXPECT_TRUE(QueryEventReportUris({key}).empty()); } +TEST_F(CacheManagerTest, TestReportEventCapacityFailurePreservesExistingUpdateAndItemMapping) { + const std::string host = "192.168.10.84:8080"; + constexpr int64_t existing_key = 96'102; + constexpr int64_t rejected_key = 96'103; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + auto meta_indexer = cache_manager_->meta_indexer_manager_->GetMetaIndexer("test_instance"); + ASSERT_NE(nullptr, meta_indexer); + meta_indexer->max_key_count_ = 1; + + const auto [initial_ec, initial_response] = + CallReportEvent(MakeAddRequest(host, existing_key, "capacity_baseline"), "capacity_baseline"); + ASSERT_EQ(EC_OK, initial_ec); + ASSERT_EQ(proto::meta::OK, initial_response.header().status().code()); + ASSERT_EQ(1u, meta_indexer->GetKeyCount()); + + auto mixed_request = MakeAddRequest(host, existing_key, "capacity_existing_update"); + *mixed_request.add_events() = MakeAddRequest(host, rejected_key, "capacity_rejected_new_key").events(0); + const auto [mixed_ec, mixed_response] = CallReportEvent(mixed_request, "capacity_mixed_update_and_insert"); + + // Both tasks have already built their immutable replacement values before + // the fused writer applies max_key_count. The existing-key update must + // remain admissible, and the consumed source task must retain enough + // identity to map EC_NOSPC back to only the rejected event. + EXPECT_EQ(EC_PARTIAL_OK, mixed_ec); + EXPECT_EQ(proto::meta::INTERNAL_ERROR, mixed_response.header().status().code()); + ASSERT_EQ(2, mixed_response.item_results_size()); + EXPECT_EQ(proto::meta::OK, mixed_response.item_results(0)); + EXPECT_EQ(proto::meta::INTERNAL_ERROR, mixed_response.item_results(1)); + EXPECT_EQ(1u, meta_indexer->GetKeyCount()); + + const auto existing_uris = QueryRawEventReportUris(existing_key); + ASSERT_EQ(1u, existing_uris.size()); + EXPECT_NE(std::string::npos, existing_uris.front().find("source=capacity_existing_update")); + EXPECT_TRUE(QueryRawEventReportUris(rejected_key).empty()); +} + +TEST_F(CacheManagerTest, TestReportEventUnsortedDeltaFailureMapsBackToOnlyAffectedBlock) { + const std::string host = "192.168.10.53:8080"; + constexpr int64_t low_key = 94'441; + constexpr int64_t middle_key = 94'442; + constexpr int64_t high_key = 94'443; + auto event_backend = InstallEventReportBackend(); + auto *meta_backend = InstallControllableMetaBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_NE(nullptr, meta_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + // Force the fold to build its sorted permutation instead of retaining the + // input order. Failure ranges are now located lazily in that view. + auto request = MakeAddRequest(host, high_key, "high"); + *request.add_events() = MakeAddRequest(host, low_key, "low").events(0); + *request.add_events() = MakeAddRequest(host, middle_key, "middle").events(0); + + meta_backend->FailKeyOnNextUpsert(middle_key); + const auto [ec, response] = CallReportEvent(request, "unsorted_delta_failure_range"); + EXPECT_EQ(EC_PARTIAL_OK, ec); + EXPECT_EQ(proto::meta::INTERNAL_ERROR, response.header().status().code()); + ASSERT_EQ(3, response.item_results_size()); + EXPECT_EQ(proto::meta::OK, response.item_results(0)); + EXPECT_EQ(proto::meta::OK, response.item_results(1)); + EXPECT_EQ(proto::meta::INTERNAL_ERROR, response.item_results(2)); + EXPECT_FALSE(QueryEventReportUris({high_key}).empty()); + EXPECT_FALSE(QueryEventReportUris({low_key}).empty()); + EXPECT_TRUE(QueryEventReportUris({middle_key}).empty()); +} + +TEST_F(CacheManagerTest, TestReportEventDeltaFailureMarksSafeRetryDependencyClosure) { + const std::string host = "192.168.10.72:8080"; + const int64_t key = 94'472; + auto event_backend = InstallEventReportBackend(); + auto *meta_backend = InstallControllableMetaBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_NE(nullptr, meta_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "baseline_a"), "retry_closure_baseline").first); + + auto request = MakeAddRequest(host, key, "readd_a"); + auto *spec_b = request.mutable_events(0)->mutable_block_add()->add_specs(); + spec_b->set_name("tp1"); + spec_b->set_uri("event_report://" + host + "/mem?source=add_b"); + auto *delete_a = request.add_events(); + delete_a->set_event_type(proto::meta::EVENT_BLOCK_DELETE); + delete_a->mutable_block_delete()->set_block_key(std::to_string(key)); + delete_a->mutable_block_delete()->set_medium("mem"); + delete_a->mutable_block_delete()->add_spec_names("tp0"); + + // The final ADD phase writes only tp1; tp0's final operation is DELETE. + // Fail that ADD, then let DELETE succeed. Retrying only event 0 would + // otherwise resurrect tp0, so both dependent events must be reported as + // failed and retried in their original order. + meta_backend->FailKeyOnNextUpsert(key); + const auto [failed_ec, failed_response] = CallReportEvent(request, "retry_closure_injected_failure"); + EXPECT_EQ(EC_PARTIAL_OK, failed_ec); + ASSERT_EQ(2, failed_response.item_results_size()); + EXPECT_EQ(proto::meta::INTERNAL_ERROR, failed_response.item_results(0)); + EXPECT_EQ(proto::meta::INTERNAL_ERROR, failed_response.item_results(1)); + EXPECT_TRUE(QueryRawEventReportUris(key).empty()); + + const auto [retry_ec, retry_response] = CallReportEvent(request, "retry_closure_retry_all_failed_items"); + EXPECT_EQ(EC_OK, retry_ec); + EXPECT_EQ(0, retry_response.item_results_size()); + const auto final_uris = QueryRawEventReportUris(key); + ASSERT_EQ(1u, final_uris.size()); + EXPECT_NE(std::string::npos, final_uris.front().find("source=add_b")); +} + TEST_F(CacheManagerTest, TestReportEventLazilyRestoresReporterWithoutRegisterOrSnapshot) { const std::string host = "192.168.10.32:8080"; const std::string snapshot_host = "192.168.10.132:8080"; @@ -3129,6 +3854,33 @@ TEST_F(CacheManagerTest, TestReportEventLazilyRestoresReporterWithoutRegisterOrS EXPECT_NE(std::string::npos, visible.front().find("source=registered_but_unavailable")); } +TEST_F(CacheManagerTest, TestReportEventHeartbeatFailureDoesNotOverwriteMalformedItems) { + const std::string host = "192.168.10.79:8080"; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + ASSERT_EQ(EC_OK, event_backend->UnregisterNode("test_instance", host)); + + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + // This item is structurally invalid and must keep INVALID_ARGUMENT even + // when the valid heartbeat in the same request fails at the backend. + request.add_events()->set_event_type(proto::meta::EVENT_HEARTBEAT); + auto *valid_heartbeat = request.add_events(); + valid_heartbeat->set_event_type(proto::meta::EVENT_HEARTBEAT); + valid_heartbeat->mutable_heartbeat(); + + const auto [ec, response] = CallReportEvent(request, "mixed_invalid_and_tombstoned_heartbeat"); + EXPECT_EQ(EC_PARTIAL_OK, ec); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.header().status().code()); + ASSERT_EQ(2, response.item_results_size()); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.item_results(0)); + EXPECT_EQ(proto::meta::NODE_NOT_REGISTERED, response.item_results(1)); + EXPECT_FALSE(event_backend->IsNodeRegistered("test_instance", host)); +} + TEST_F(CacheManagerTest, TestReportEventSnapshotRequiredOnlyForGenerationCreatingDelta) { auto event_backend = InstallEventReportBackend(); ASSERT_NE(nullptr, event_backend); @@ -3221,9 +3973,70 @@ TEST_F(CacheManagerTest, TestReportEventSnapshotWhileUnavailableCommitsButStaysH EXPECT_NE(std::string::npos, visible.front().find("s_version=" + snapshot_generation)); } -TEST_F(CacheManagerTest, TestReportEventRegisterThenFirstDeltaInSameRequest) { - const std::string host = "192.168.10.48:8080"; - const int64_t key = 94'433; +TEST_F(CacheManagerTest, TestReportEventHeartbeatRecoveryCarriesSameRequestMutationsIntoNewLifecycle) { + const std::string host = "192.168.10.73:8080"; + const int64_t add_key = 94'473; + const int64_t delete_key = 94'474; + const int64_t snapshot_key = 94'475; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + ASSERT_EQ(EC_OK, + CallReportEvent(MakeAddRequest(host, delete_key, "delete_baseline"), "recovery_batch_baseline").first); + + event_backend->SetNodeUnavailable("test_instance", host); + const uint64_t add_old_generation = event_backend->GetNodeGeneration("test_instance", host); + auto heartbeat_then_add = MakeAddRequest(host, add_key, "heartbeat_then_add"); + const auto add_event = heartbeat_then_add.events(0); + heartbeat_then_add.clear_events(); + auto *heartbeat = heartbeat_then_add.add_events(); + heartbeat->set_event_type(proto::meta::EVENT_HEARTBEAT); + (*heartbeat->mutable_heartbeat()->mutable_system_status())["phase"] = "recover_add"; + *heartbeat_then_add.add_events() = add_event; + const auto [add_ec, add_response] = CallReportEvent(heartbeat_then_add, "recovery_batch_heartbeat_then_add"); + ASSERT_EQ(EC_OK, add_ec); + EXPECT_EQ(0, add_response.item_results_size()); + EXPECT_GT(event_backend->GetNodeGeneration("test_instance", host), add_old_generation); + ASSERT_EQ(1u, QueryEventReportUris({add_key}).size()); + + event_backend->SetNodeUnavailable("test_instance", host); + proto::meta::ReportEventRequest delete_then_heartbeat; + delete_then_heartbeat.set_instance_id("test_instance"); + delete_then_heartbeat.set_host_ip_port(host); + delete_then_heartbeat.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto *delete_event = delete_then_heartbeat.add_events(); + delete_event->set_event_type(proto::meta::EVENT_BLOCK_DELETE); + delete_event->mutable_block_delete()->set_block_key(std::to_string(delete_key)); + delete_event->mutable_block_delete()->set_medium("mem"); + delete_event->mutable_block_delete()->add_spec_names("tp0"); + heartbeat = delete_then_heartbeat.add_events(); + heartbeat->set_event_type(proto::meta::EVENT_HEARTBEAT); + heartbeat->mutable_heartbeat(); + const auto [delete_ec, delete_response] = + CallReportEvent(delete_then_heartbeat, "recovery_batch_delete_then_heartbeat"); + ASSERT_EQ(EC_OK, delete_ec); + EXPECT_EQ(0, delete_response.item_results_size()); + EXPECT_TRUE(QueryEventReportUris({delete_key}).empty()); + + event_backend->SetNodeUnavailable("test_instance", host); + auto heartbeat_then_snapshot = MakeSnapshotRequest(host, {{snapshot_key, "heartbeat_then_snapshot"}}); + const auto snapshot_event = heartbeat_then_snapshot.events(0); + heartbeat_then_snapshot.clear_events(); + heartbeat = heartbeat_then_snapshot.add_events(); + heartbeat->set_event_type(proto::meta::EVENT_HEARTBEAT); + heartbeat->mutable_heartbeat(); + *heartbeat_then_snapshot.add_events() = snapshot_event; + const auto [snapshot_ec, snapshot_response] = + CallReportEvent(heartbeat_then_snapshot, "recovery_batch_heartbeat_then_snapshot"); + ASSERT_EQ(EC_OK, snapshot_ec); + EXPECT_EQ(0, snapshot_response.item_results_size()); + EXPECT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken(snapshot_response.committed_snapshot_version())); + ASSERT_EQ(1u, QueryEventReportUris({snapshot_key}).size()); +} + +TEST_F(CacheManagerTest, TestReportEventRegisterThenFirstDeltaInSameRequest) { + const std::string host = "192.168.10.48:8080"; + const int64_t key = 94'433; auto event_backend = InstallEventReportBackend(); ASSERT_NE(nullptr, event_backend); @@ -3281,6 +4094,217 @@ TEST_F(CacheManagerTest, TestReportEventDeltaBeforeExplicitRegisterSucceedsInSam EXPECT_NE(std::string::npos, visible.front().find("s_version=" + response.committed_snapshot_version())); } +TEST_F(CacheManagerTest, TestReportEventAdmissionFailurePropagatesToLaterRelatedMutation) { + const std::string host = "192.168.10.81:8080"; + const int64_t key = 94'481; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + ASSERT_EQ(EC_OK, event_backend->UnregisterNode("test_instance", host)); + + auto request = MakeAddRequest(host, key, "must_be_deleted_after_retry"); + auto *register_event = request.add_events(); + register_event->set_event_type(proto::meta::EVENT_NODE_REGISTER); + register_event->mutable_node_register()->add_mediums("mem"); + auto *delete_event = request.add_events(); + delete_event->set_event_type(proto::meta::EVENT_BLOCK_DELETE); + delete_event->mutable_block_delete()->set_block_key(std::to_string(key)); + delete_event->mutable_block_delete()->set_medium("mem"); + delete_event->mutable_block_delete()->add_spec_names("tp0"); + + const auto [ec, response] = CallReportEvent(request, "admission_failure_dependency_closure"); + EXPECT_EQ(EC_PARTIAL_OK, ec); + EXPECT_EQ(proto::meta::NODE_NOT_REGISTERED, response.header().status().code()); + ASSERT_EQ(3, response.item_results_size()); + EXPECT_EQ(proto::meta::NODE_NOT_REGISTERED, response.item_results(0)); + EXPECT_EQ(proto::meta::OK, response.item_results(1)); + // The DELETE physically succeeded, but it shares the retry dependency + // group with the earlier failed ADD. Returning success here would let a + // caller retry only ADD and reverse the request's last-operation-wins + // result. + EXPECT_EQ(proto::meta::NODE_NOT_REGISTERED, response.item_results(2)); + EXPECT_TRUE(QueryEventReportUris({key}).empty()); + + proto::meta::ReportEventRequest retry = request; + retry.clear_events(); + *retry.add_events() = request.events(0); + *retry.add_events() = request.events(2); + const auto [retry_ec, retry_response] = CallReportEvent(retry, "admission_failure_dependency_retry"); + EXPECT_EQ(EC_OK, retry_ec); + EXPECT_EQ(0, retry_response.item_results_size()); + EXPECT_TRUE(QueryEventReportUris({key}).empty()); +} + +TEST_F(CacheManagerTest, TestReportEventValidatesMultipleRegisterItemsIndependently) { + const std::string host = "192.168.10.74:8080"; + const int64_t key = 94'476; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + + auto request = MakeAddRequest(host, key, "valid_register_survives_invalid_sibling"); + const auto add_event = request.events(0); + request.clear_events(); + auto *valid_register = request.add_events(); + valid_register->set_event_type(proto::meta::EVENT_NODE_REGISTER); + valid_register->mutable_node_register()->add_mediums("mem"); + auto *invalid_register = request.add_events(); + invalid_register->set_event_type(proto::meta::EVENT_NODE_REGISTER); + invalid_register->mutable_node_register()->add_mediums("bad#medium"); + *request.add_events() = add_event; + + const auto [ec, response] = CallReportEvent(request, "multiple_register_item_validation"); + EXPECT_EQ(EC_PARTIAL_OK, ec); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.header().status().code()); + ASSERT_EQ(3, response.item_results_size()); + EXPECT_EQ(proto::meta::OK, response.item_results(0)); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.item_results(1)); + EXPECT_EQ(proto::meta::OK, response.item_results(2)); + EXPECT_TRUE(event_backend->IsNodeRegistered("test_instance", host)); + ASSERT_EQ(1u, QueryEventReportUris({key}).size()); +} + +TEST_F(CacheManagerTest, TestReportEventCoalescesMultipleValidRegistersIntoOneLifecycle) { + const std::string host = "192.168.10.82:8080"; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto *mem_register = request.add_events(); + mem_register->set_event_type(proto::meta::EVENT_NODE_REGISTER); + mem_register->mutable_node_register()->add_mediums("mem"); + auto *disk_register = request.add_events(); + disk_register->set_event_type(proto::meta::EVENT_NODE_REGISTER); + disk_register->mutable_node_register()->add_mediums("disk"); + disk_register->mutable_node_register()->add_mediums("mem"); + + const auto [ec, response] = CallReportEvent(request, "multiple_valid_registers_one_lifecycle"); + ASSERT_EQ(EC_OK, ec); + EXPECT_EQ(proto::meta::OK, response.header().status().code()); + EXPECT_EQ(0, response.item_results_size()); + EXPECT_EQ(1u, event_backend->GetNodeGeneration("test_instance", host)); + + const auto [retry_ec, retry_response] = CallReportEvent(request, "multiple_valid_registers_next_request"); + ASSERT_EQ(EC_OK, retry_ec); + EXPECT_EQ(0, retry_response.item_results_size()); + EXPECT_EQ(2u, event_backend->GetNodeGeneration("test_instance", host)); +} + +TEST_F(CacheManagerTest, TestReportEventDisabledBackendRejectsReportsAndHidesExistingLocations) { + const std::string host = "192.168.10.75:8080"; + const int64_t baseline_key = 94'477; + const int64_t rejected_key = 94'478; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, + CallReportEvent(MakeAddRequest(host, baseline_key, "before_disable"), "disable_backend_baseline").first); + ASSERT_EQ(1u, QueryEventReportUris({baseline_key}).size()); + + event_backend->SetAvailable(false); + EXPECT_TRUE(QueryEventReportUris({baseline_key}).empty()); + const auto [rejected_ec, rejected_response] = + CallReportEvent(MakeAddRequest(host, rejected_key, "must_not_write"), "disable_backend_reject_report"); + EXPECT_EQ(EC_INSTANCE_NOT_EXIST, rejected_ec); + EXPECT_EQ(proto::meta::INSTANCE_NOT_EXIST, rejected_response.header().status().code()); + EXPECT_TRUE(QueryRawEventReportUris(rejected_key).empty()); + + event_backend->SetAvailable(true); + ASSERT_EQ(1u, QueryEventReportUris({baseline_key}).size()); +} + +TEST_F(CacheManagerTest, TestHostCleanupCannotCrossEventBackendIncarnations) { + const std::string host = "192.168.10.76:8080"; + const int64_t key = 94'479; + auto old_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, old_backend); + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "old_incarnation"), "old_incarnation_add").first); + const uint64_t old_generation = old_backend->GetNodeGeneration("test_instance", host); + ASSERT_NE(0u, old_generation); + ASSERT_EQ(EC_OK, old_backend->Close()); + + auto new_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, new_backend); + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "new_incarnation"), "new_incarnation_add").first); + ASSERT_EQ(old_generation, new_backend->GetNodeGeneration("test_instance", host)); + + cache_manager_->CleanupHostLocations( + "test_instance", host, old_generation, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, old_backend); + const auto uris = QueryRawEventReportUris(key); + ASSERT_EQ(1u, uris.size()); + EXPECT_NE(std::string::npos, uris.front().find("source=new_incarnation")); +} + +TEST_F(CacheManagerTest, TestHostCleanupUsesTheBackendThatCurrentlyWinsCandidateRouting) { + const std::string host = "192.168.10.77:8080"; + const int64_t key = 94'480; + auto old_backend = InstallEventReportBackend("event_report_old_candidate"); + ASSERT_NE(nullptr, old_backend); + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "old_candidate"), "old_candidate_add").first); + const uint64_t old_generation = old_backend->GetNodeGeneration("test_instance", host); + ASSERT_NE(0u, old_generation); + + auto current_backend = InstallEventReportBackend("event_report_current_candidate"); + ASSERT_NE(nullptr, current_backend); + const std::string group_name = registry_manager_->GetInstanceGroupName("test_instance"); + registry_manager_->instance_group_configs_.at(group_name) + ->set_event_report_storage_candidates({"event_report_current_candidate", "event_report_old_candidate"}); + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "current_candidate"), "current_candidate_add").first); + ASSERT_EQ(old_generation, current_backend->GetNodeGeneration("test_instance", host)); + + cache_manager_->CleanupHostLocations( + "test_instance", host, old_generation, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, old_backend); + const auto uris = QueryRawEventReportUris(key); + ASSERT_EQ(1u, uris.size()); + EXPECT_NE(std::string::npos, uris.front().find("source=current_candidate")); +} + +TEST_F(CacheManagerTest, TestSnapshotCleanupCannotCrossBackendReplacementDuringScan) { + const std::string host = "192.168.10.78:8080"; + const int64_t key = 94'482; + auto old_backend = InstallEventReportBackend(); + auto *meta_backend = InstallControllableMetaBackend(); + ASSERT_NE(nullptr, old_backend); + ASSERT_NE(nullptr, meta_backend); + + const auto [baseline_ec, baseline] = + CallReportEvent(MakeAddRequest(host, key, "old_snapshot"), "old_snapshot_for_cleanup"); + ASSERT_EQ(EC_OK, baseline_ec); + const ReporterSnapshotKey reporter_key{"test_instance", host}; + const uint64_t cleanup_generation = old_backend->GetNodeGeneration("test_instance", host); + + std::string cleanup_version; + uint64_t retry_after_ms = 0; + ASSERT_EQ(EC_OK, old_backend->BeginSnapshot(reporter_key, cleanup_version, retry_after_ms)); + ASSERT_NE(baseline.committed_snapshot_version(), cleanup_version); + ASSERT_TRUE(old_backend->CommitSnapshotVersion(reporter_key, cleanup_version)); + const uint64_t cleanup_epoch = old_backend->GetSnapshotAttemptEpoch(reporter_key); + + meta_backend->BlockNextLocationRead(); + auto cleanup = std::async(std::launch::async, [&] { + return cache_manager_->CleanupStaleSnapshotLocations(reporter_key, + cleanup_version, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + old_backend, + cleanup_epoch, + cleanup_generation); + }); + ASSERT_TRUE(meta_backend->WaitUntilLocationReadEntered(std::chrono::seconds(2))); + + ASSERT_EQ(EC_OK, old_backend->Close()); + auto new_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, new_backend); + ASSERT_EQ(EC_OK, CallReportEvent(MakeAddRequest(host, key, "new_backend_value"), "new_backend_value_add").first); + + meta_backend->ReleaseLocationRead(); + ASSERT_EQ(std::future_status::ready, cleanup.wait_for(std::chrono::seconds(2))); + EXPECT_EQ(EC_OK, cleanup.get()); + const auto uris = QueryRawEventReportUris(key); + ASSERT_EQ(1u, uris.size()); + EXPECT_NE(std::string::npos, uris.front().find("source=new_backend_value")); +} + TEST_F(CacheManagerTest, TestReportEventInvalidFirstDeltaDoesNotCreateVersionButPartialBatchDoes) { const std::string host = "192.168.10.49:8080"; const int64_t invalid_key = 94'434; @@ -3381,6 +4405,46 @@ TEST_F(CacheManagerTest, TestReportEventMissingBlockDeletesRemainSuccessfulAsOne EXPECT_TRUE(QueryEventReportUris({94'437, 94'438, 94'439}).empty()); } +TEST_F(CacheManagerTest, TestReportEventLargeDeltaBatchAcrossRepeatedMediums) { + const std::string host = "192.168.10.56:8080"; + constexpr int64_t first_key = 95'000; + constexpr size_t event_count = 512; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + + auto request = MakeAddRequest(host, first_key, "batch_0"); + for (size_t i = 1; i < event_count; ++i) { + auto event_request = MakeAddRequest(host, first_key + static_cast(i), "batch_" + std::to_string(i)); + if (i % 2 != 0) { + event_request.mutable_events(0)->mutable_block_add()->set_medium("disk"); + event_request.mutable_events(0)->mutable_block_add()->mutable_specs(0)->set_uri( + "event_report://" + host + "/disk?source=batch_" + std::to_string(i)); + } + *request.add_events() = event_request.events(0); + } + + const auto [ec, response] = CallReportEvent(request, "large_repeated_medium_delta_batch"); + ASSERT_EQ(EC_OK, ec); + EXPECT_EQ(proto::meta::OK, response.header().status().code()); + EXPECT_EQ(0, response.item_results_size()); + ASSERT_TRUE(SnapshotUriUtils::IsValidSnapshotVersionToken(response.committed_snapshot_version())); + EXPECT_TRUE(response.snapshot_required()); + + const auto visible = QueryEventReportUris({first_key, + first_key + static_cast(event_count / 2), + first_key + static_cast(event_count - 1)}); + ASSERT_EQ(3u, visible.size()); + EXPECT_TRUE(std::any_of(visible.begin(), visible.end(), [](const auto &uri) { + return uri.find("source=batch_0") != std::string::npos; + })); + EXPECT_TRUE(std::any_of(visible.begin(), visible.end(), [](const auto &uri) { + return uri.find("source=batch_256") != std::string::npos; + })); + EXPECT_TRUE(std::any_of(visible.begin(), visible.end(), [](const auto &uri) { + return uri.find("source=batch_511") != std::string::npos; + })); +} + TEST_F(CacheManagerTest, TestReportEventRestartKeepsHistoricalCacheAndAcceptsDeltaWithoutSnapshot) { const std::string host = "192.168.10.47:8080"; const int64_t historical_key = 94'431; @@ -3398,8 +4462,13 @@ TEST_F(CacheManagerTest, TestReportEventRestartKeepsHistoricalCacheAndAcceptsDel const StorageConfig storage_config = event_backend->GetStorageConfig(); ASSERT_EQ(EC_OK, event_backend->Close()); + // A process/configuration restart creates a fresh backend incarnation. A + // closed object deliberately cannot be reopened because queued callbacks + // and lifecycle fences are scoped to exactly one incarnation. + event_backend = std::make_shared(metrics_registry_); ASSERT_EQ(EC_OK, event_backend->Open(storage_config, "delta_only_restart")); event_backend->SetSnapshotMinIntervalMsForTest(0); + registry_manager_->data_storage_manager_->storage_map_[storage_config.global_unique_name()] = event_backend; EXPECT_TRUE(event_backend->GetSnapshotVersion({"test_instance", host}).empty()); EXPECT_TRUE(QueryEventReportUris({historical_key}).empty()); @@ -3864,17 +4933,25 @@ TEST_F(CacheManagerTest, TestSnapshotCleanupPreservesCurrentDeltaBesideLegacySpe MetaSearcher *meta_searcher = cache_manager_->meta_searcher_manager_->GetMetaSearcher("test_instance"); ASSERT_NE(nullptr, meta_searcher); - std::vector replace_results; + std::vector merge_results; ASSERT_EQ(EC_OK, - meta_searcher->BatchReplaceLocationSpecs( - request_context_.get(), - {key}, - {{{event_backend->BuildLocationId("mem", host), - DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, - CacheLocationStatus::CLS_SERVING, - {LocationSpec("tp0", current_uri), LocationSpec("tp1", legacy_uri)}}}}, - replace_results)); - ASSERT_EQ((std::vector{EC_OK}), replace_results); + meta_searcher->BatchMergeLocationSpecs(request_context_.get(), + {key}, + {{{event_backend->BuildLocationId("mem", host), + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp1", legacy_uri)}}}}, + merge_results)); + ASSERT_EQ((std::vector{EC_OK}), merge_results); + ASSERT_EQ(EC_OK, + meta_searcher->BatchMergeLocationSpecs(request_context_.get(), + {key}, + {{{event_backend->BuildLocationId("mem", host), + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", current_uri)}}}}, + merge_results)); + ASSERT_EQ((std::vector{EC_OK}), merge_results); ASSERT_EQ( EC_OK, @@ -4333,6 +5410,11 @@ TEST_F(CacheManagerTest, TestGetCheckLocDataExistFuncEventReportUriValidationMat {"event_report://physical-cache:9600/mem?s_version=" + token + "&s_version=" + upper_token}, false}, {"invalid_uri", {"not-a-uri"}, false}, + {"invalid_empty_port", {"event_report://physical-cache:/mem"}, false}, + {"invalid_negative_port", {"event_report://physical-cache:-1/mem"}, false}, + {"invalid_text_port", {"event_report://physical-cache:not-a-port/mem"}, false}, + {"invalid_overflow_port", {"event_report://physical-cache:9223372036854775808/mem"}, false}, + {"invalid_userinfo_port", {"event_report://user:secret@physical-cache:not-a-port/mem"}, false}, {"mixed_versioned_and_legacy", {"event_report://physical-cache:9600/mem?s_version=" + token, "event_report://physical-cache:9600/mem?source=legacy"}, @@ -4724,6 +5806,11 @@ TEST_F(CacheManagerTest, TestFilterWriteCache_StaleBreaksPrefix) { createModelDeployment(), std::vector())); + // This test removes queued executor tasks for direct inspection. Stop the + // supervisor first so it cannot wait on a packaged task that the test then + // destroys, which would surface as a nondeterministic broken promise. + cache_manager_->reclaimer_task_supervisor_->Stop(); + // write keys {1,2,3} and finish as CLS_SERVING std::vector write_keys{1, 2, 3}; auto [ec1, swci1] = @@ -4897,6 +5984,11 @@ TEST_F(CacheManagerTest, TestFilterWriteCache_StaleSuffix) { createModelDeployment(), std::vector())); + // This test removes queued executor tasks for direct inspection. Stop the + // supervisor first so it cannot wait on a packaged task that the test then + // destroys, which would surface as a nondeterministic broken promise. + cache_manager_->reclaimer_task_supervisor_->Stop(); + // write keys {1,2,3} and finish as CLS_SERVING std::vector write_keys{1, 2, 3}; auto [ec1, swci1] = @@ -5421,6 +6513,30 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects [=](auto *event, int64_t key, const std::string &host) { configure_valid_add(event, key, host)->set_block_key("not-a-number"); }}, + {"add_positive_overflow_key", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->set_block_key("18446744073709551616"); + }}, + {"add_negative_overflow_key", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->set_block_key("-9223372036854775809"); + }}, + {"add_negative_uint64_alias_key", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->set_block_key("-18446744073709551615"); + }}, + {"add_leading_plus_key", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->set_block_key("+1"); + }}, + {"add_leading_space_key", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->set_block_key(" 1"); + }}, + {"add_trailing_space_key", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->set_block_key("1 "); + }}, {"add_empty_medium", [=](auto *event, int64_t key, const std::string &host) { configure_valid_add(event, key, host)->clear_medium(); @@ -5441,6 +6557,10 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects [=](auto *event, int64_t key, const std::string &host) { configure_valid_add(event, key, host)->mutable_specs(0)->clear_name(); }}, + {"add_unregistered_spec_name", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host)->mutable_specs(0)->set_name("unregistered"); + }}, {"add_duplicate_spec_name", [=](auto *event, int64_t key, const std::string &host) { auto *params = configure_valid_add(event, key, host); @@ -5450,6 +6570,20 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects [=](auto *event, int64_t key, const std::string &host) { configure_valid_add(event, key, host)->mutable_specs(0)->set_uri("not-a-uri"); }}, + {"add_invalid_uri_port", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_add(event, key, host) + ->mutable_specs(0) + ->set_uri("event_report://cache-node:not-a-port/mem"); + }}, + {"add_total_size_overflow", + [=](auto *event, int64_t key, const std::string &host) { + auto *params = configure_valid_add(event, key, host); + params->mutable_specs(0)->set_uri("event_report://" + host + "/mem?size=18446744073709551615"); + auto *second = params->add_specs(); + second->set_name("tp1"); + second->set_uri("event_report://" + host + "/mem?size=1"); + }}, {"add_client_snapshot_version", [=](auto *event, int64_t key, const std::string &host) { configure_valid_add(event, key, host) @@ -5474,6 +6608,10 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects [=](auto *event, int64_t key, const std::string &) { configure_valid_delete(event, key)->set_spec_names(0, ""); }}, + {"delete_unregistered_spec_name", + [=](auto *event, int64_t key, const std::string &) { + configure_valid_delete(event, key)->set_spec_names(0, "unregistered"); + }}, {"delete_duplicate_spec_name", [=](auto *event, int64_t key, const std::string &) { configure_valid_delete(event, key)->add_spec_names("tp0"); @@ -5498,6 +6636,10 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects [=](auto *event, int64_t key, const std::string &host) { configure_valid_snapshot(event, key, host)->mutable_specs(0)->clear_name(); }}, + {"snapshot_unregistered_spec_name", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_snapshot(event, key, host)->mutable_specs(0)->set_name("unregistered"); + }}, {"snapshot_duplicate_spec_name", [=](auto *event, int64_t key, const std::string &host) { auto *block = configure_valid_snapshot(event, key, host); @@ -5507,6 +6649,20 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects [=](auto *event, int64_t key, const std::string &host) { configure_valid_snapshot(event, key, host)->mutable_specs(0)->set_uri("not-a-uri"); }}, + {"snapshot_invalid_uri_port", + [=](auto *event, int64_t key, const std::string &host) { + configure_valid_snapshot(event, key, host) + ->mutable_specs(0) + ->set_uri("event_report://cache-node:not-a-port/mem"); + }}, + {"snapshot_total_size_overflow", + [=](auto *event, int64_t key, const std::string &host) { + auto *block = configure_valid_snapshot(event, key, host); + block->mutable_specs(0)->set_uri("event_report://" + host + "/mem?size=18446744073709551615"); + auto *second = block->add_specs(); + second->set_name("tp1"); + second->set_uri("event_report://" + host + "/mem?size=1"); + }}, {"snapshot_client_snapshot_version", [=](auto *event, int64_t key, const std::string &host) { configure_valid_snapshot(event, key, host) @@ -5546,6 +6702,58 @@ TEST_F(CacheManagerTest, TestReportEventMutationValidationMatrixHasNoSideEffects } } +TEST_F(CacheManagerTest, TestReportEventUnregisteredSpecIsIsolatedFromValidItem) { + const std::string host = "192.168.12.200:8080"; + const int64_t invalid_key = 95'900; + const int64_t valid_key = 95'901; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + auto request = MakeAddRequest(host, invalid_key, "unregistered_spec"); + request.mutable_events(0)->mutable_block_add()->mutable_specs(0)->set_name("unregistered"); + *request.add_events() = MakeAddRequest(host, valid_key, "registered_spec").events(0); + + const auto [ec, response] = CallReportEvent(request, "unregistered_spec_isolated"); + EXPECT_EQ(EC_PARTIAL_OK, ec); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.header().status().code()); + ASSERT_EQ(2, response.item_results_size()); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.item_results(0)); + EXPECT_EQ(proto::meta::OK, response.item_results(1)); + EXPECT_TRUE(QueryRawEventReportUris(invalid_key).empty()); + EXPECT_FALSE(QueryRawEventReportUris(valid_key).empty()); +} + +TEST_F(CacheManagerTest, TestReportEventAcceptsSignedAndUnsignedBlockKeySpellingsWithoutChangingBitPattern) { + const std::string host = "192.168.12.100:8080"; + auto event_backend = InstallEventReportBackend(); + ASSERT_NE(nullptr, event_backend); + ASSERT_EQ(EC_OK, event_backend->RegisterNode("test_instance", host, {"mem"})); + + struct TestCase { + const char *text; + int64_t expected_key; + }; + const std::vector cases = { + {"0", 0}, + {"9223372036854775807", std::numeric_limits::max()}, + {"-9223372036854775808", std::numeric_limits::min()}, + {"9223372036854775808", std::numeric_limits::min()}, + {"-1", -1}, + {"18446744073709551615", -1}, + }; + + for (size_t index = 0; index < cases.size(); ++index) { + SCOPED_TRACE(cases[index].text); + auto request = MakeAddRequest(host, 0, "block_key_boundary_" + std::to_string(index)); + request.mutable_events(0)->mutable_block_add()->set_block_key(cases[index].text); + const auto [ec, response] = CallReportEvent(request, "block_key_boundary"); + ASSERT_EQ(EC_OK, ec); + ASSERT_EQ(proto::meta::OK, response.header().status().code()); + EXPECT_FALSE(QueryRawEventReportUris(cases[index].expected_key).empty()); + } +} + TEST_F(CacheManagerTest, TestReportEventRejectsMismatchedPayloadsWithoutSideEffects) { auto event_backend = InstallEventReportBackend(); ASSERT_NE(nullptr, event_backend); @@ -5750,6 +6958,22 @@ TEST_F(CacheManagerTest, TestReportEventRejectsInvalidRequestsAndMapsItemErrors) EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.header().status().code()); } + { + proto::meta::ReportEventRequest request; + request.set_instance_id("test_instance"); + request.set_host_ip_port("10.0.0.30:8080"); + // 263 has the same low byte as ST_EVENT_REPORT_L1P5 (7). It must remain + // an unsupported open-enum value rather than being truncated and + // routed to a real backend. + request.set_storage_type(static_cast(263)); + add_register_event(request); + + proto::meta::ReportEventResponse response; + EXPECT_EQ(EC_BADARGS, cache_manager_->ReportEvent(request_context_.get(), &request, &response)); + EXPECT_EQ(proto::meta::INVALID_ARGUMENT, response.header().status().code()); + EXPECT_EQ("unsupported event-report storage_type: 263", response.header().status().message()); + } + { proto::meta::ReportEventRequest request; request.set_instance_id("missing_instance"); @@ -6027,6 +7251,37 @@ TEST_F(CacheManagerTest, TestReportEventBlockAddMergesLocationSpecs) { EXPECT_EQ(mem_uri, mem_specs["linear_0"]); EXPECT_EQ(disk_uri, disk_specs["linear_1"]); } + + // Case 5: all blocks for one reporter/medium in a request retain the same + // immutable location-id string instead of allocating one copy per block. + const int64_t interned_key_a = 9004; + const int64_t interned_key_b = 9005; + { + proto::meta::ReportEventRequest req; + req.set_instance_id(instance_id); + req.set_host_ip_port(host); + req.set_storage_type(proto::meta::ST_EVENT_REPORT_L1P5); + for (const int64_t key : {interned_key_a, interned_key_b}) { + auto *event = req.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *add = event->mutable_block_add(); + add->set_block_key(std::to_string(key)); + add->set_medium("mem"); + auto *spec = add->add_specs(); + spec->set_name("linear_0"); + spec->set_uri("event_report://10.0.0.9:8080/mem"); + } + proto::meta::ReportEventResponse resp; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); + } + const auto interned_a = get_location_map(interned_key_a); + const auto interned_b = get_location_map(interned_key_b); + const std::string mem_location_id = event_backend->BuildLocationId("mem", host); + ASSERT_EQ(1u, interned_a.size()); + ASSERT_EQ(1u, interned_b.size()); + ASSERT_TRUE(interned_a.at(mem_location_id)); + ASSERT_TRUE(interned_b.at(mem_location_id)); + EXPECT_EQ(&interned_a.at(mem_location_id)->id(), &interned_b.at(mem_location_id)->id()); } TEST_F(CacheManagerTest, TestReportEventL1P5L2BlockAddAreIsolated) { @@ -6251,14 +7506,20 @@ TEST_F(CacheManagerTest, TestReportEventBlockDeleteRemovesLocationSpecs) { } } -TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { +TEST_F(CacheManagerTest, TestGetCacheLocationsByBackend) { auto expected_reg = std::pair(EC_OK, default_storage_configs); + const std::string instance_id = "test_backend_selectors"; + auto location_spec_infos = createLocationSpecInfos(); + location_spec_infos.emplace_back("full_0", 512); + location_spec_infos.emplace_back("linear_1", 512); + location_spec_infos.emplace_back("linear_2", 512); + location_spec_infos.emplace_back("linear_3", 512); ASSERT_EQ(expected_reg, cache_manager_->RegisterInstance(request_context_.get(), "default", - "test_instance", + instance_id, 64, - createLocationSpecInfos(), + location_spec_infos, createModelDeployment(), std::vector())); @@ -6268,12 +7529,11 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { std::vector nfs_keys{300, 500, 700}; { auto [ec, swci] = - cache_manager_->StartWriteCache(request_context_.get(), "test_instance", nfs_keys, {}, {}, 100000000); + cache_manager_->StartWriteCache(request_context_.get(), instance_id, nfs_keys, {}, {}, 100000000); ASSERT_EQ(EC_OK, ec); BlockMask bm = static_cast(nfs_keys.size()); - ASSERT_EQ( - EC_OK, - cache_manager_->FinishWriteCache(request_context_.get(), "test_instance", swci.write_session_id(), bm)); + ASSERT_EQ(EC_OK, + cache_manager_->FinishWriteCache(request_context_.get(), instance_id, swci.write_session_id(), bm)); } // Set up EventReportBackend @@ -6304,9 +7564,9 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { {"192.168.1.3:8080", {300}}, }; for (const auto &pd : peer_data) { - InitializeEventReporter("test_instance", pd.host, proto::meta::ST_EVENT_REPORT_L2); + InitializeEventReporter(instance_id, pd.host, proto::meta::ST_EVENT_REPORT_L2); proto::meta::ReportEventRequest req; - req.set_instance_id("test_instance"); + req.set_instance_id(instance_id); req.set_host_ip_port(pd.host); req.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); @@ -6328,16 +7588,92 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { // --- Test 1: empty backend_selectors → returns EC_BADARGS --- { BlockMask bm = static_cast(0); + auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend( + request_context_.get(), instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, bm, 0, {}, {}); + ASSERT_EQ(EC_BADARGS, ec); + } + + // Invalid, incompatible, and duplicate selectors fail closed instead of + // silently behaving like weighted-random selection or duplicating output. + for (const auto &selectors : std::vector>{ + {{DataStorageType::DATA_STORAGE_TYPE_UNKNOWN, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}}, + {{DataStorageType::DATA_STORAGE_TYPE_NFS, LocationSelectStrategy::LSS_UNSPECIFIED}}, + {{DataStorageType::DATA_STORAGE_TYPE_NFS, LocationSelectStrategy::LSS_V6D_PREFIX}}, + {{DataStorageType::DATA_STORAGE_TYPE_NFS, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}, + {DataStorageType::DATA_STORAGE_TYPE_NFS, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}}, + }) { + BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, bm, 0, {}, - {}); - ASSERT_EQ(EC_BADARGS, ec); + selectors); + EXPECT_EQ(EC_BADARGS, ec); + EXPECT_TRUE(locs.empty()); + } + + // Invalid masks must fail closed. An omitted protobuf mask is represented + // as an empty bool vector and remains backward-compatible with no mask. + const std::vector nfs_selectors = { + {DataStorageType::DATA_STORAGE_TYPE_NFS, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}, + }; + for (const BlockMask &invalid_mask : std::vector{ + BlockMaskOffset{all_keys.size() + 1}, + BlockMaskVector{true, false}, + }) { + auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_BATCH_GET, + all_keys, + {}, + invalid_mask, + 0, + {}, + nfs_selectors); + EXPECT_EQ(EC_BADARGS, ec); + EXPECT_TRUE(locs.empty()); + } + { + const BlockMask implicit_empty_mask = BlockMaskVector{}; + auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_BATCH_GET, + all_keys, + {}, + implicit_empty_mask, + 0, + {}, + nfs_selectors); + EXPECT_EQ(EC_OK, ec); + EXPECT_EQ(all_keys.size(), locs.size()); + } + + // Masked entries stay positionally aligned but do not expose a remote + // location. Both vector and prefix-offset forms are supported. + for (const BlockMask &mask : std::vector{ + BlockMaskVector{true, false, true, false, true}, + BlockMaskOffset{2}, + }) { + auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_BATCH_GET, + all_keys, + {}, + mask, + 0, + {}, + nfs_selectors); + ASSERT_EQ(EC_OK, ec); + ASSERT_EQ(all_keys.size(), locs.size()); + for (size_t i = 0; i < all_keys.size(); ++i) { + if (IsIndexInMaskRange(mask, i)) { + EXPECT_TRUE(locs[i].cache_locations_view().empty()); + } + } } // --- Test 2: EVENT_REPORT PREFIX + NFS (NFS on 300,500,700 should not affect event report peer selection) --- @@ -6348,7 +7684,7 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { }; BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, @@ -6402,7 +7738,7 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { }; BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, @@ -6437,7 +7773,7 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { }; BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, @@ -6472,7 +7808,7 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { }; BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, keys_no_er_first, {}, @@ -6503,7 +7839,7 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { }; BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, keys_gap, {}, @@ -6532,7 +7868,7 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { }; BlockMask bm = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, bad_keys, {}, @@ -6546,39 +7882,169 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { EXPECT_TRUE(locs[1].cache_locations_view().empty()); } - // --- Test 8: location_spec_names filter still works with backend_selectors --- + // --- Test 8: non-empty location_spec_names must align one-to-one with query keys --- { std::vector selectors = { - {DataStorageType::DATA_STORAGE_TYPE_NFS, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}, + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_COVERAGE}, }; BlockMask bm = static_cast(0); + auto [size_ec, size_locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_BATCH_GET, + {300, 400, 300}, + {}, + bm, + 0, + {"full_0", "linear_1"}, + selectors); + EXPECT_EQ(EC_BADARGS, size_ec); + EXPECT_TRUE(size_locs.empty()); + + auto [empty_ec, empty_locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_BATCH_GET, + {300, 400, 300}, + {}, + bm, + 0, + {"full_0", "", "linear_1"}, + selectors); + EXPECT_EQ(EC_BADARGS, empty_ec); + EXPECT_TRUE(empty_locs.empty()); + } + + // --- Test 9: non-hybrid attention sends one full spec per key and uses prefix selection --- + // Current group-aware Vineyard represents each FullAttention object as + // (block key, full_0), preserving the original query order. + { + auto report_full_keys = [&](const std::string &host, const std::vector &keys) { + proto::meta::ReportEventRequest req; + req.set_instance_id(instance_id); + req.set_host_ip_port(host); + req.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + for (int64_t key : keys) { + auto *ev = req.add_events(); + ev->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *ba = ev->mutable_block_add(); + ba->set_block_key(std::to_string(key)); + ba->set_medium("mem"); + auto *spec = ba->add_specs(); + spec->set_name("full_0"); + spec->set_uri("event_report://" + host + "/mem"); + } + proto::meta::ReportEventResponse resp; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); + }; + + const std::string full_prefix_host = "192.168.2.3:8080"; + const std::string short_full_host = "192.168.2.4:8080"; + InitializeEventReporter(instance_id, full_prefix_host, proto::meta::ST_EVENT_REPORT_L2); + InitializeEventReporter(instance_id, short_full_host, proto::meta::ST_EVENT_REPORT_L2); + report_full_keys(full_prefix_host, {900, 901}); + report_full_keys(short_full_host, {900}); + + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, + }; + BlockMask block_mask = static_cast(0); auto [ec, locs] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, - {300}, + {900, 901, 902}, {}, - bm, + block_mask, 0, - {"tp0", "tp2"}, + {"full_0", "full_0", "full_0"}, selectors); ASSERT_EQ(EC_OK, ec); - ASSERT_EQ(1u, locs.size()); - const auto &kl = locs[0].cache_locations_view(); - ASSERT_EQ(1u, kl.size()); - EXPECT_EQ(2u, kl[0].location_specs().size()); + ASSERT_EQ(3u, locs.size()); + for (size_t i = 0; i < 2; ++i) { + const auto &key_locations = locs[i].cache_locations_view(); + ASSERT_EQ(1u, key_locations.size()) << "query index=" << i; + ASSERT_EQ(1u, key_locations[0].location_specs().size()) << "query index=" << i; + EXPECT_EQ(1u, key_locations[0].spec_size()) << "query index=" << i; + EXPECT_EQ("full_0", key_locations[0].location_specs()[0].name()) << "query index=" << i; + EXPECT_NE(std::string::npos, key_locations[0].location_specs()[0].uri().find(full_prefix_host)) + << "query index=" << i; + } + EXPECT_TRUE(locs[2].cache_locations_view().empty()); } - // --- Test 9: backend-selected queries enforce reporter liveness --- + // --- Test 10: mixed-attention Mamba groups share one ordered best-effort query --- + // Current Vineyard sends all group-aware objects from one lookup in their + // original order. Different Mamba groups can therefore repeat the same block + // key and are distinguished only by per-position location_spec_names. + { + auto report_specs = [&](const std::string &host, int64_t key, const std::vector &spec_names) { + proto::meta::ReportEventRequest req; + req.set_instance_id(instance_id); + req.set_host_ip_port(host); + req.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto *ev = req.add_events(); + ev->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *ba = ev->mutable_block_add(); + ba->set_block_key(std::to_string(key)); + ba->set_medium("mem"); + for (const auto &spec_name : spec_names) { + auto *spec = ba->add_specs(); + spec->set_name(spec_name); + spec->set_uri("event_report://" + host + "/mem"); + } + proto::meta::ReportEventResponse resp; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); + }; + + const std::string linear_1_host = "192.168.2.1:8080"; + const std::string remaining_groups_host = "192.168.2.2:8080"; + InitializeEventReporter(instance_id, linear_1_host, proto::meta::ST_EVENT_REPORT_L2); + InitializeEventReporter(instance_id, remaining_groups_host, proto::meta::ST_EVENT_REPORT_L2); + report_specs(linear_1_host, 800, {"linear_1"}); + report_specs(linear_1_host, 801, {"linear_1"}); + report_specs(remaining_groups_host, 800, {"linear_2", "linear_3"}); + report_specs(remaining_groups_host, 801, {"linear_3"}); + + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_COVERAGE}, + }; + BlockMask block_mask = static_cast(0); + auto [ec, locs] = + cache_manager_->GetCacheLocationsByBackend(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_BATCH_GET, + {800, 801, 800, 800, 801}, + {}, + block_mask, + 0, + {"linear_1", "linear_1", "linear_2", "linear_3", "linear_3"}, + selectors); + ASSERT_EQ(EC_OK, ec); + ASSERT_EQ(5u, locs.size()); + EXPECT_TRUE(locs[0].cache_locations_view().empty()); + EXPECT_TRUE(locs[1].cache_locations_view().empty()); + + const std::vector expected_specs = {"linear_2", "linear_3", "linear_3"}; + for (size_t i = 0; i < expected_specs.size(); ++i) { + const auto &key_locations = locs[i + 2].cache_locations_view(); + ASSERT_EQ(1u, key_locations.size()) << "query index=" << i + 2; + ASSERT_EQ(1u, key_locations[0].location_specs().size()) << "query index=" << i + 2; + EXPECT_EQ(1u, key_locations[0].spec_size()) << "query index=" << i + 2; + EXPECT_EQ(expected_specs[i], key_locations[0].location_specs()[0].name()) << "query index=" << i + 2; + EXPECT_NE(std::string::npos, key_locations[0].location_specs()[0].uri().find(remaining_groups_host)) + << "query index=" << i + 2; + } + } + + // --- Test 11: backend-selected queries enforce reporter liveness --- { for (const auto &peer : peer_data) { - event_report_backend->SetNodeUnavailable("test_instance", peer.host); + event_report_backend->SetNodeUnavailable(instance_id, peer.host); } const std::vector selectors = { {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, }; BlockMask block_mask = static_cast(0); auto [hidden_ec, hidden] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, @@ -6593,10 +8059,10 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { } for (const auto &peer : peer_data) { - ASSERT_EQ(EC_OK, event_report_backend->OnHeartbeat("test_instance", peer.host, {})); + ASSERT_EQ(EC_OK, event_report_backend->OnHeartbeat(instance_id, peer.host, {})); } auto [restored_ec, restored] = cache_manager_->GetCacheLocationsByBackend(request_context_.get(), - "test_instance", + instance_id, CacheManager::QueryType::QT_BATCH_GET, all_keys, {}, @@ -6633,6 +8099,165 @@ TEST_F(CacheManagerTest, TestGetCacheLocationsByBackendWithBackendSelectors) { // host_B: 100→200→300→400→(miss 500) → prefix=4 // host_C: 100→(miss 200) → prefix=1 // +TEST_F(CacheManagerTest, TestGetHostCacheStateDoesNotExposeStartWriteBeforeFinish) { + const std::string instance_id = "test_host_cache_state_writing"; + ASSERT_EQ(std::make_pair(EC_OK, default_storage_configs), + cache_manager_->RegisterInstance(request_context_.get(), + "default", + instance_id, + 64, + createLocationSpecInfos(), + createModelDeployment(), + std::vector(), + CacheManager::QueryType::QT_PREFIX_MATCH)); + + const CacheManager::KeyVector keys = {8999}; + auto [start_ec, write_info] = + cache_manager_->StartWriteCache(request_context_.get(), instance_id, keys, {}, {}, 100000000); + ASSERT_EQ(EC_OK, start_ec); + + auto [query_ec, hosts] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, keys); + EXPECT_EQ(EC_OK, query_ec); + EXPECT_TRUE(hosts.empty()); + + const BlockMask success_mask = static_cast(keys.size()); + EXPECT_EQ(EC_OK, + cache_manager_->FinishWriteCache( + request_context_.get(), instance_id, write_info.write_session_id(), success_mask)); +} + +TEST_F(CacheManagerTest, TestGetHostCacheStateSnapshotsHostLivenessAfterMetadataRead) { + auto event_backend = InstallEventReportBackend(); + ASSERT_TRUE(event_backend); + auto *meta_backend = InstallControllableMetaBackend(); + ASSERT_TRUE(meta_backend); + + const std::string instance_id = "test_instance"; + const std::string host = "10.0.9.1:8080"; + InitializeEventReporter(instance_id, host, proto::meta::ST_EVENT_REPORT_L2); + proto::meta::ReportEventRequest report; + report.set_instance_id(instance_id); + report.set_host_ip_port(host); + report.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto *event = report.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + event->mutable_block_add()->set_block_key("9001"); + event->mutable_block_add()->set_medium("mem"); + auto *spec = event->mutable_block_add()->add_specs(); + spec->set_name("tp0"); + spec->set_uri("event_report://" + host + "/mem"); + proto::meta::ReportEventResponse report_response; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &report, &report_response)); + + meta_backend->BlockNextLocationRead(); + auto query = std::async(std::launch::async, [&] { + RequestContext context("host_liveness_after_meta_read"); + return cache_manager_->GetHostCacheState( + &context, instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, {9001}); + }); + const bool read_entered = meta_backend->WaitUntilLocationReadEntered(std::chrono::seconds(2)); + if (!read_entered) { + meta_backend->ReleaseLocationRead(); + (void)query.get(); + FAIL() << "GetHostCacheState did not enter the controlled metadata read"; + } + + event_backend->SetNodeUnavailable(instance_id, host); + meta_backend->ReleaseLocationRead(); + auto [ec, hosts] = query.get(); + EXPECT_EQ(EC_OK, ec); + EXPECT_TRUE(hosts.empty()); +} + +TEST_F(CacheManagerTest, TestGetHostCacheStateConcurrentWithReportEventAndHostDown) { + auto event_backend = InstallEventReportBackend(); + ASSERT_TRUE(event_backend); + const std::string instance_id = "test_instance"; + const std::string host = "10.0.9.2:8080"; + constexpr std::size_t kBlockCount = 384; + InitializeEventReporter(instance_id, host, proto::meta::ST_EVENT_REPORT_L2); + + auto make_add_request = [&](std::size_t round) { + proto::meta::ReportEventRequest request; + request.set_instance_id(instance_id); + request.set_host_ip_port(host); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + for (std::size_t i = 0; i < kBlockCount; ++i) { + auto *event = request.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *add = event->mutable_block_add(); + add->set_block_key(std::to_string(10000 + i)); + add->set_medium("mem"); + auto *spec = add->add_specs(); + spec->set_name("tp0"); + spec->set_uri("event_report://" + host + "/mem?round=" + std::to_string(round)); + } + return request; + }; + + auto setup_request = make_add_request(0); + proto::meta::ReportEventResponse setup_response; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &setup_request, &setup_response)); + CacheManager::KeyVector keys; + keys.reserve(kBlockCount); + for (std::size_t i = 0; i < kBlockCount; ++i) { + keys.push_back(10000 + i); + } + + std::atomic start{false}; + std::atomic writer_done{false}; + std::atomic failures{0}; + std::thread writer([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (std::size_t round = 1; round <= 12; ++round) { + auto request = make_add_request(round); + proto::meta::ReportEventResponse response; + RequestContext context("concurrent_report_" + std::to_string(round)); + if (cache_manager_->ReportEvent(&context, &request, &response) != EC_OK) { + failures.fetch_add(1, std::memory_order_relaxed); + } + } + writer_done.store(true, std::memory_order_release); + }); + std::thread reader([&] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + std::size_t query_count = 0; + while (!writer_done.load(std::memory_order_acquire) || query_count < 24) { + RequestContext context("concurrent_host_query_" + std::to_string(query_count)); + auto [ec, hosts] = cache_manager_->GetHostCacheState( + &context, instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, keys, {"mem"}); + if (ec != EC_OK || hosts.size() != 1 || hosts.front().host_ip_port != host || + hosts.front().local != static_cast(kBlockCount)) { + failures.fetch_add(1, std::memory_order_relaxed); + } + ++query_count; + } + }); + start.store(true, std::memory_order_release); + writer.join(); + reader.join(); + EXPECT_EQ(0u, failures.load(std::memory_order_relaxed)); + + proto::meta::ReportEventRequest host_down; + host_down.set_instance_id(instance_id); + host_down.set_host_ip_port(host); + host_down.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + auto *host_down_event = host_down.add_events(); + host_down_event->set_event_type(proto::meta::EVENT_HOST_DOWN); + host_down_event->mutable_host_down(); + proto::meta::ReportEventResponse host_down_response; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &host_down, &host_down_response)); + auto [ec, hosts] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, keys); + EXPECT_EQ(EC_OK, ec); + EXPECT_TRUE(hosts.empty()); +} + TEST_F(CacheManagerTest, TestGetHostCacheState) { auto expected_reg = std::pair(EC_OK, default_storage_configs); const std::string instance_id = "test_host_cache_state_prefix"; @@ -6694,11 +8319,11 @@ TEST_F(CacheManagerTest, TestGetHostCacheState) { ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); } - // Helper: find a host's prefix_match_blocks in the result + // Helper: find a host's local in the result auto find_prefix = [](const std::vector &hosts, const std::string &host) -> int64_t { for (const auto &h : hosts) { if (h.host_ip_port == host) { - return h.prefix_match_blocks; + return h.local; } } return -1; // not found @@ -6824,7 +8449,28 @@ TEST_F(CacheManagerTest, TestGetHostCacheState) { EXPECT_EQ(-1, find_prefix(hosts, "10.0.0.4:8080")); } - // --- Test 9: unavailable host is filtered even before metadata cleanup --- + // --- Test 9: requests above the parallel threshold preserve ordering and + // prefix semantics. Repeated keys also stress concurrent reads of the same + // local-cache item rather than only independent LRU shards. --- + { + CacheManager::KeyVector keys; + keys.reserve(384); + for (std::size_t i = 0; i < 96; ++i) { + keys.insert(keys.end(), {100, 200, 300, 400}); + } + auto [ec, hosts] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, keys); + ASSERT_EQ(EC_OK, ec); + ASSERT_EQ(3u, hosts.size()); + EXPECT_EQ("10.0.0.1:8080", hosts[0].host_ip_port); + EXPECT_EQ("10.0.0.2:8080", hosts[1].host_ip_port); + EXPECT_EQ("10.0.0.3:8080", hosts[2].host_ip_port); + EXPECT_EQ(2, find_prefix(hosts, "10.0.0.1:8080")); + EXPECT_EQ(384, find_prefix(hosts, "10.0.0.2:8080")); + EXPECT_EQ(1, find_prefix(hosts, "10.0.0.3:8080")); + } + + // --- Test 10: unavailable host is filtered even before metadata cleanup --- { event_backend->SetNodeUnavailable(instance_id, "10.0.0.2:8080"); CacheManager::KeyVector keys = {100, 200, 300, 400}; @@ -6841,6 +8487,416 @@ TEST_F(CacheManagerTest, TestGetHostCacheState) { dsm->storage_map_.erase("event_backend_default"); } +TEST_F(CacheManagerTest, TestGetHostCacheStateP2P) { + auto expected_reg = std::pair(EC_OK, default_storage_configs); + const std::string instance_id = "test_host_cache_state_single_p2p"; + ASSERT_EQ(expected_reg, + cache_manager_->RegisterInstance(request_context_.get(), + "default", + instance_id, + 64, + createLocationSpecInfos(), + createModelDeployment(), + std::vector(), + CacheManager::QueryType::QT_PREFIX_MATCH)); + + auto make_backend = [&](const std::string &name, DataStorageType type) { + auto backend = std::make_shared(cache_manager_->metrics_registry_); + StorageConfig cfg; + cfg.set_global_unique_name(name); + cfg.set_type(type); + cfg.set_storage_spec(std::make_shared()); + return std::make_pair(backend, backend->Open(cfg, "test_trace")); + }; + auto subscriber_backend_result = + make_backend("host_state_subscriber", DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5); + auto vineyard_backend_result = + make_backend("host_state_vineyard", DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2); + ASSERT_EQ(EC_OK, subscriber_backend_result.second); + ASSERT_EQ(EC_OK, vineyard_backend_result.second); + + auto dsm = registry_manager_->data_storage_manager_; + dsm->storage_map_["host_state_subscriber"] = subscriber_backend_result.first; + dsm->storage_map_["host_state_vineyard"] = vineyard_backend_result.first; + registry_manager_->instance_group_configs_["default"]->set_event_report_storage_candidates( + {"host_state_subscriber", "host_state_vineyard"}); + + auto report_keys = [&](proto::meta::StorageType type, const std::string &host, const std::vector &keys) { + InitializeEventReporter(instance_id, host, type); + proto::meta::ReportEventRequest req; + req.set_instance_id(instance_id); + req.set_host_ip_port(host); + req.set_storage_type(type); + for (int64_t key : keys) { + auto *event = req.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *block_add = event->mutable_block_add(); + block_add->set_block_key(std::to_string(key)); + block_add->set_medium("mem"); + auto *spec = block_add->add_specs(); + spec->set_name("tp0"); + spec->set_uri("event_report://" + host + "/mem"); + } + proto::meta::ReportEventResponse resp; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); + }; + + const std::string host_a = "10.0.2.1:8080"; + const std::string host_b = "10.0.2.2:8080"; + const std::string host_c = "10.0.2.3:8080"; + report_keys(proto::meta::ST_EVENT_REPORT_L1P5, host_a, {100, 300}); + report_keys(proto::meta::ST_EVENT_REPORT_L2, host_b, {100, 200, 400}); + report_keys(proto::meta::ST_EVENT_REPORT_L2, host_c, {100, 200, 300}); + + auto [ec, hosts] = cache_manager_->GetHostCacheState(request_context_.get(), + instance_id, + CacheManager::QueryType::QT_PREFIX_MATCH, + {100, 200, 300, 400, 500}, + {}, + 5); + ASSERT_EQ(EC_OK, ec); + ASSERT_EQ(3u, hosts.size()); + + // host A selects B for local-miss keys {200, 400}; host B selects C for {300}; + // host C selects B for {400}. All three stop at the uncached key 500. + auto expect_match = [&](const std::string &host, int64_t local, int64_t p2p_1_fetch, int64_t p2p_1_total_match) { + auto it = + std::find_if(hosts.begin(), hosts.end(), [&](const auto &match) { return match.host_ip_port == host; }); + ASSERT_NE(hosts.end(), it); + EXPECT_EQ(local, it->local); + EXPECT_EQ(p2p_1_fetch, it->p2p_1_fetch); + EXPECT_EQ(p2p_1_total_match, it->p2p_1_total_match); + }; + expect_match(host_a, 1, 2, 4); + expect_match(host_b, 2, 1, 4); + expect_match(host_c, 3, 1, 4); + + // Only the five hosts with the largest local prefix compute P2P. Hosts + // after the cutoff are still returned in host order with local-only totals. + const std::vector top5_keys = {1000, 1001, 1002, 1003, 1004, 1005, 1006}; + const std::vector> top5_hosts = { + {"10.0.5.1:8080", 7}, + {"10.0.5.2:8080", 6}, + {"10.0.5.3:8080", 5}, + {"10.0.5.4:8080", 4}, + {"10.0.5.5:8080", 3}, + {"10.0.5.6:8080", 3}, + {"10.0.5.7:8080", 1}, + }; + for (size_t i = 0; i < top5_hosts.size(); ++i) { + const auto &[host, prefix_len] = top5_hosts[i]; + report_keys(i == 0 ? proto::meta::ST_EVENT_REPORT_L2 : proto::meta::ST_EVENT_REPORT_L1P5, + host, + std::vector(top5_keys.begin(), top5_keys.begin() + prefix_len)); + } + const std::string zero_local_host = "10.0.5.8:8080"; + report_keys(proto::meta::ST_EVENT_REPORT_L1P5, zero_local_host, {top5_keys[1]}); + + auto [top5_ec, top5_matches] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, top5_keys, {}, 5); + ASSERT_EQ(EC_OK, top5_ec); + ASSERT_EQ(top5_hosts.size(), top5_matches.size()); + const std::vector> expected_top5_matches = { + {7, 0, 7}, + {6, 1, 7}, + {5, 2, 7}, + {4, 3, 7}, + {3, 4, 7}, + {3, 0, 3}, + {1, 0, 1}, + }; + for (size_t i = 0; i < top5_hosts.size(); ++i) { + EXPECT_EQ(top5_hosts[i].first, top5_matches[i].host_ip_port); + EXPECT_EQ(std::get<0>(expected_top5_matches[i]), top5_matches[i].local); + EXPECT_EQ(std::get<1>(expected_top5_matches[i]), top5_matches[i].p2p_1_fetch); + EXPECT_EQ(std::get<2>(expected_top5_matches[i]), top5_matches[i].p2p_1_total_match); + } + EXPECT_EQ(top5_matches.end(), std::find_if(top5_matches.begin(), top5_matches.end(), [&](const auto &match) { + return match.host_ip_port == zero_local_host; + })); + + auto [top2_ec, top2_matches] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, top5_keys, {}, 2); + ASSERT_EQ(EC_OK, top2_ec); + ASSERT_EQ(top5_hosts.size(), top2_matches.size()); + const std::vector> expected_top2_matches = { + {7, 0, 7}, + {6, 1, 7}, + {5, 0, 5}, + {4, 0, 4}, + {3, 0, 3}, + {3, 0, 3}, + {1, 0, 1}, + }; + for (size_t i = 0; i < top5_hosts.size(); ++i) { + EXPECT_EQ(top5_hosts[i].first, top2_matches[i].host_ip_port); + EXPECT_EQ(std::get<0>(expected_top2_matches[i]), top2_matches[i].local); + EXPECT_EQ(std::get<1>(expected_top2_matches[i]), top2_matches[i].p2p_1_fetch); + EXPECT_EQ(std::get<2>(expected_top2_matches[i]), top2_matches[i].p2p_1_total_match); + } + + auto [top0_ec, top0_matches] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, top5_keys, {}, 0); + ASSERT_EQ(EC_OK, top0_ec); + ASSERT_EQ(top5_hosts.size(), top0_matches.size()); + for (size_t i = 0; i < top5_hosts.size(); ++i) { + EXPECT_EQ(top5_hosts[i].first, top0_matches[i].host_ip_port); + EXPECT_EQ(static_cast(top5_hosts[i].second), top0_matches[i].local); + EXPECT_EQ(0, top0_matches[i].p2p_1_fetch); + EXPECT_EQ(top0_matches[i].local, top0_matches[i].p2p_1_total_match); + } + + auto [default_ec, default_matches] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH, top5_keys); + ASSERT_EQ(EC_OK, default_ec); + ASSERT_EQ(top0_matches.size(), default_matches.size()); + for (size_t i = 0; i < top0_matches.size(); ++i) { + EXPECT_EQ(top0_matches[i].host_ip_port, default_matches[i].host_ip_port); + EXPECT_EQ(top0_matches[i].local, default_matches[i].local); + EXPECT_EQ(top0_matches[i].p2p_1_fetch, default_matches[i].p2p_1_fetch); + EXPECT_EQ(top0_matches[i].p2p_1_total_match, default_matches[i].p2p_1_total_match); + } + + // Prefix match with Mamba: local and P2P specs jointly complete the blocks, + // then the merged prefix is evaluated with the Mamba state and Eagle POP rules. + const std::string mamba_instance_id = "test_host_cache_state_mamba_p2p"; + std::vector mamba_location_spec_infos = { + LocationSpecInfo("F0", 512), + LocationSpecInfo("L0", 512), + LocationSpecInfo("L1", 512), + }; + std::vector mamba_location_spec_groups = { + LocationSpecGroup("F0", {"F0"}), + LocationSpecGroup("L0", {"L0"}), + LocationSpecGroup("L1", {"L1"}), + }; + ASSERT_EQ(expected_reg, + cache_manager_->RegisterInstance(request_context_.get(), + "default", + mamba_instance_id, + 64, + mamba_location_spec_infos, + createModelDeploymentWithEaglePop(), + mamba_location_spec_groups, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA)); + + const std::string mamba_host = "10.0.3.1:8080"; + const std::string mamba_p2p_host = "10.0.3.2:8080"; + InitializeEventReporter(mamba_instance_id, mamba_host, proto::meta::ST_EVENT_REPORT_L1P5); + InitializeEventReporter(mamba_instance_id, mamba_p2p_host, proto::meta::ST_EVENT_REPORT_L2); + + auto report_mamba_specs = [&](proto::meta::StorageType type, + const std::string &host, + int64_t key, + const std::vector &spec_names) { + proto::meta::ReportEventRequest req; + req.set_instance_id(mamba_instance_id); + req.set_host_ip_port(host); + req.set_storage_type(type); + auto *event = req.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *block_add = event->mutable_block_add(); + block_add->set_block_key(std::to_string(key)); + block_add->set_medium("mem"); + for (const auto &spec_name : spec_names) { + auto *spec = block_add->add_specs(); + spec->set_name(spec_name); + spec->set_uri("event_report://" + host + "/mem"); + } + proto::meta::ReportEventResponse resp; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); + }; + + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L1P5, mamba_host, 100, {"F0", "L0", "L1"}); + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L1P5, mamba_host, 200, {"F0"}); + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L1P5, mamba_host, 300, {"F0", "L0", "L1"}); + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L1P5, mamba_host, 400, {"F0", "L0"}); + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L2, mamba_p2p_host, 200, {"L0", "L1"}); + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L2, mamba_p2p_host, 400, {"L1"}); + report_mamba_specs(proto::meta::ST_EVENT_REPORT_L2, mamba_p2p_host, 500, {"F0", "L0", "L1"}); + + auto [mamba_ec, mamba_hosts] = + cache_manager_->GetHostCacheState(request_context_.get(), + mamba_instance_id, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA, + {100, 200, 300, 400, 500}, + {}, + 5); + ASSERT_EQ(EC_OK, mamba_ec); + ASSERT_EQ(1u, mamba_hosts.size()); + EXPECT_EQ(mamba_host, mamba_hosts[0].host_ip_port); + EXPECT_EQ(3, mamba_hosts[0].local); + EXPECT_EQ(3, mamba_hosts[0].p2p_1_fetch); + EXPECT_EQ(4, mamba_hosts[0].p2p_1_total_match); + + // Hybrid P2P uses coverage across missing spec positions. Peer A owns the + // first two L1 positions, while peer B owns four later positions on + // only three distinct block keys. Coverage must select B and fetch=3. + const std::string coverage_instance_id = "test_host_cache_state_mamba_coverage"; + std::vector coverage_spec_infos = { + LocationSpecInfo("F0", 512), + LocationSpecInfo("L1", 512), + LocationSpecInfo("L2", 512), + LocationSpecInfo("L3", 512), + }; + std::vector coverage_spec_groups = { + LocationSpecGroup("F0", {"F0"}), + LocationSpecGroup("L1", {"L1"}), + LocationSpecGroup("L2", {"L2"}), + LocationSpecGroup("L3", {"L3"}), + }; + ASSERT_EQ(expected_reg, + cache_manager_->RegisterInstance(request_context_.get(), + "default", + coverage_instance_id, + 64, + coverage_spec_infos, + createModelDeployment(), + coverage_spec_groups, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA)); + + const std::string coverage_host = "10.0.4.1:8080"; + const std::string prefix_peer = "10.0.4.2:8080"; + const std::string coverage_peer = "10.0.4.3:8080"; + InitializeEventReporter(coverage_instance_id, coverage_host, proto::meta::ST_EVENT_REPORT_L1P5); + InitializeEventReporter(coverage_instance_id, prefix_peer, proto::meta::ST_EVENT_REPORT_L2); + InitializeEventReporter(coverage_instance_id, coverage_peer, proto::meta::ST_EVENT_REPORT_L2); + + auto report_coverage_specs = [&](proto::meta::StorageType type, + const std::string &host, + int64_t key, + const std::vector &spec_names) { + proto::meta::ReportEventRequest req; + req.set_instance_id(coverage_instance_id); + req.set_host_ip_port(host); + req.set_storage_type(type); + auto *event = req.add_events(); + event->set_event_type(proto::meta::EVENT_BLOCK_ADD); + auto *block_add = event->mutable_block_add(); + block_add->set_block_key(std::to_string(key)); + block_add->set_medium("mem"); + for (const auto &spec_name : spec_names) { + auto *spec = block_add->add_specs(); + spec->set_name(spec_name); + spec->set_uri("event_report://" + host + "/mem"); + } + proto::meta::ReportEventResponse resp; + ASSERT_EQ(EC_OK, cache_manager_->ReportEvent(request_context_.get(), &req, &resp)); + }; + + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L1P5, coverage_host, 100, {"F0", "L1", "L2", "L3"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L1P5, coverage_host, 200, {"F0"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L1P5, coverage_host, 300, {"F0"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L1P5, coverage_host, 400, {"F0", "L1"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L2, prefix_peer, 200, {"L1"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L2, prefix_peer, 300, {"L1"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L2, coverage_peer, 200, {"L2"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L2, coverage_peer, 300, {"L2"}); + report_coverage_specs(proto::meta::ST_EVENT_REPORT_L2, coverage_peer, 400, {"L2", "L3"}); + + auto [coverage_ec, coverage_hosts] = + cache_manager_->GetHostCacheState(request_context_.get(), + coverage_instance_id, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA, + {100, 200, 300, 400}, + {}, + 5); + ASSERT_EQ(EC_OK, coverage_ec); + ASSERT_EQ(1u, coverage_hosts.size()); + EXPECT_EQ(coverage_host, coverage_hosts[0].host_ip_port); + EXPECT_EQ(1, coverage_hosts[0].local); + EXPECT_EQ(3, coverage_hosts[0].p2p_1_fetch); + EXPECT_EQ(4, coverage_hosts[0].p2p_1_total_match); + + const std::vector mamba_top5_keys = {2000, 2001, 2002, 2003, 2004, 2005, 2006}; + const std::vector> mamba_top5_hosts = { + {"10.0.6.1:8080", 7}, + {"10.0.6.2:8080", 6}, + {"10.0.6.3:8080", 5}, + {"10.0.6.4:8080", 4}, + {"10.0.6.5:8080", 3}, + {"10.0.6.6:8080", 3}, + {"10.0.6.7:8080", 1}, + }; + const std::vector all_coverage_specs = {"F0", "L1", "L2", "L3"}; + for (size_t i = 0; i < mamba_top5_hosts.size(); ++i) { + const auto &[host, prefix_len] = mamba_top5_hosts[i]; + const auto storage_type = i == 0 ? proto::meta::ST_EVENT_REPORT_L2 : proto::meta::ST_EVENT_REPORT_L1P5; + InitializeEventReporter(coverage_instance_id, host, storage_type); + for (size_t key_index = 0; key_index < prefix_len; ++key_index) { + report_coverage_specs(storage_type, host, mamba_top5_keys[key_index], all_coverage_specs); + } + } + const std::string mamba_zero_local_host = "10.0.6.8:8080"; + InitializeEventReporter(coverage_instance_id, mamba_zero_local_host, proto::meta::ST_EVENT_REPORT_L1P5); + report_coverage_specs( + proto::meta::ST_EVENT_REPORT_L1P5, mamba_zero_local_host, mamba_top5_keys[1], all_coverage_specs); + + auto [mamba_top5_ec, mamba_top5_matches] = + cache_manager_->GetHostCacheState(request_context_.get(), + coverage_instance_id, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA, + mamba_top5_keys, + {}, + 5); + ASSERT_EQ(EC_OK, mamba_top5_ec); + ASSERT_EQ(mamba_top5_hosts.size(), mamba_top5_matches.size()); + const std::vector> expected_mamba_top5_matches = { + {7, 0, 7}, + {6, 1, 7}, + {5, 2, 7}, + {4, 3, 7}, + {3, 4, 7}, + {3, 0, 3}, + {1, 0, 1}, + }; + for (size_t i = 0; i < mamba_top5_hosts.size(); ++i) { + EXPECT_EQ(mamba_top5_hosts[i].first, mamba_top5_matches[i].host_ip_port); + EXPECT_EQ(std::get<0>(expected_mamba_top5_matches[i]), mamba_top5_matches[i].local); + EXPECT_EQ(std::get<1>(expected_mamba_top5_matches[i]), mamba_top5_matches[i].p2p_1_fetch); + EXPECT_EQ(std::get<2>(expected_mamba_top5_matches[i]), mamba_top5_matches[i].p2p_1_total_match); + } + EXPECT_EQ(mamba_top5_matches.end(), + std::find_if(mamba_top5_matches.begin(), mamba_top5_matches.end(), [&](const auto &match) { + return match.host_ip_port == mamba_zero_local_host; + })); + + auto [mamba_top2_ec, mamba_top2_matches] = + cache_manager_->GetHostCacheState(request_context_.get(), + coverage_instance_id, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA, + mamba_top5_keys, + {}, + 2); + ASSERT_EQ(EC_OK, mamba_top2_ec); + ASSERT_EQ(mamba_top5_hosts.size(), mamba_top2_matches.size()); + for (size_t i = 0; i < mamba_top5_hosts.size(); ++i) { + EXPECT_EQ(mamba_top5_hosts[i].first, mamba_top2_matches[i].host_ip_port); + EXPECT_EQ(std::get<0>(expected_top2_matches[i]), mamba_top2_matches[i].local); + EXPECT_EQ(std::get<1>(expected_top2_matches[i]), mamba_top2_matches[i].p2p_1_fetch); + EXPECT_EQ(std::get<2>(expected_top2_matches[i]), mamba_top2_matches[i].p2p_1_total_match); + } + + auto [mamba_top0_ec, mamba_top0_matches] = + cache_manager_->GetHostCacheState(request_context_.get(), + coverage_instance_id, + CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA, + mamba_top5_keys, + {}, + 0); + ASSERT_EQ(EC_OK, mamba_top0_ec); + ASSERT_EQ(mamba_top5_hosts.size(), mamba_top0_matches.size()); + for (size_t i = 0; i < mamba_top5_hosts.size(); ++i) { + EXPECT_EQ(mamba_top5_hosts[i].first, mamba_top0_matches[i].host_ip_port); + EXPECT_EQ(static_cast(mamba_top5_hosts[i].second), mamba_top0_matches[i].local); + EXPECT_EQ(0, mamba_top0_matches[i].p2p_1_fetch); + EXPECT_EQ(mamba_top0_matches[i].local, mamba_top0_matches[i].p2p_1_total_match); + } + + dsm->storage_map_.erase("host_state_subscriber"); + dsm->storage_map_.erase("host_state_vineyard"); +} + TEST_F(CacheManagerTest, TestGetHostCacheStateUnspecifiedWithoutRegisteredQueryType) { auto expected_reg = std::pair(EC_OK, default_storage_configs); const std::string instance_id = "test_host_cache_state_no_query_type"; @@ -6864,14 +8920,14 @@ TEST_F(CacheManagerTest, TestGetHostCacheStatePrefixMatchWithMamba) { auto expected_reg = std::pair(EC_OK, default_storage_configs); const std::string instance_id = "test_host_cache_state_mamba"; std::vector location_spec_infos = { - LocationSpecInfo("full_0", 512), - LocationSpecInfo("linear_0", 512), - LocationSpecInfo("linear_1", 512), + LocationSpecInfo("F0", 512), + LocationSpecInfo("L0", 512), + LocationSpecInfo("L1", 512), }; std::vector location_spec_groups = { - LocationSpecGroup("full_0", {"full_0"}), - LocationSpecGroup("linear_0", {"linear_0"}), - LocationSpecGroup("linear_1", {"linear_1"}), + LocationSpecGroup("F0", {"F0"}), + LocationSpecGroup("L0", {"L0"}), + LocationSpecGroup("L1", {"L1"}), }; ASSERT_EQ(expected_reg, cache_manager_->RegisterInstance(request_context_.get(), @@ -6927,33 +8983,33 @@ TEST_F(CacheManagerTest, TestGetHostCacheStatePrefixMatchWithMamba) { InitializeEventReporter(instance_id, host_c, proto::meta::ST_EVENT_REPORT_L1P5); InitializeEventReporter(instance_id, host_e, proto::meta::ST_EVENT_REPORT_L1P5); - report_specs(host_a, 100, {"full_0", "linear_0", "linear_1"}); - report_specs(host_a, 200, {"full_0"}); - report_specs(host_a, 300, {"full_0"}); - report_specs(host_a, 300, {"linear_0", "linear_1"}); - report_specs(host_a, 400, {"full_0", "linear_0"}); + report_specs(host_a, 100, {"F0", "L0", "L1"}); + report_specs(host_a, 200, {"F0"}); + report_specs(host_a, 300, {"F0"}); + report_specs(host_a, 300, {"L0", "L1"}); + report_specs(host_a, 400, {"F0", "L0"}); - report_specs(host_b, 100, {"full_0"}); - report_specs(host_b, 200, {"full_0"}); - report_specs(host_b, 300, {"full_0"}); - report_specs(host_b, 400, {"full_0", "linear_0", "linear_1"}); + report_specs(host_b, 100, {"F0"}); + report_specs(host_b, 200, {"F0"}); + report_specs(host_b, 300, {"F0"}); + report_specs(host_b, 400, {"F0", "L0", "L1"}); - report_specs(host_c, 100, {"full_0"}); - report_specs(host_c, 200, {"full_0"}); + report_specs(host_c, 100, {"F0"}); + report_specs(host_c, 200, {"F0"}); - report_specs(host_e, 100, {"full_0", "linear_0", "linear_1"}); - report_specs(host_e, 200, {"full_0", "linear_0", "linear_1"}); - report_specs(host_e, 300, {"full_0", "linear_0", "linear_1"}); + report_specs(host_e, 100, {"F0", "L0", "L1"}); + report_specs(host_e, 200, {"F0", "L0", "L1"}); + report_specs(host_e, 300, {"F0", "L0", "L1"}); const std::string host_d = "10.0.1.4:8080"; InitializeEventReporter(instance_id, host_d, proto::meta::ST_EVENT_REPORT_L1P5); - report_specs(host_d, 200, {"full_0", "linear_0", "linear_1"}); - report_specs(host_d, 300, {"full_0", "linear_0", "linear_1"}); + report_specs(host_d, 200, {"F0", "L0", "L1"}); + report_specs(host_d, 300, {"F0", "L0", "L1"}); auto find_prefix = [](const std::vector &hosts, const std::string &host) -> int64_t { for (const auto &h : hosts) { if (h.host_ip_port == host) { - return h.prefix_match_blocks; + return h.local; } } return -1; @@ -7015,6 +9071,22 @@ TEST_F(CacheManagerTest, TestGetHostCacheStatePrefixMatchWithMamba) { EXPECT_EQ(2, find_prefix(absent_hosts, host_e)); } + { + CacheManager::KeyVector large_keys; + large_keys.reserve(384); + for (std::size_t i = 0; i < 128; ++i) { + large_keys.insert(large_keys.end(), {100, 200, 300}); + } + auto [large_ec, large_hosts] = cache_manager_->GetHostCacheState( + request_context_.get(), instance_id, CacheManager::QueryType::QT_PREFIX_MATCH_WITH_MAMBA, large_keys); + ASSERT_EQ(EC_OK, large_ec); + EXPECT_EQ(382, find_prefix(large_hosts, host_a)); + EXPECT_EQ(-1, find_prefix(large_hosts, host_b)); + EXPECT_EQ(-1, find_prefix(large_hosts, host_c)); + EXPECT_EQ(-1, find_prefix(large_hosts, host_d)); + EXPECT_EQ(383, find_prefix(large_hosts, host_e)); + } + dsm->storage_map_.erase("event_backend_mamba"); } // ===== 多层存储 Mark 消费（写路径）===== @@ -7245,6 +9317,7 @@ TEST_F(CacheManagerTest, TestMigrationTargetsRespectGroupQuota) { TEST_F(CacheManagerTest, TestFilterWriteCacheWithMinReplicaFallsBackOnMarkReadError) { EnableTieredMigrationStrategy(); + ASSERT_TRUE(RegisterDummyStorage("hot_02")); cache_manager_->RegisterInstance(request_context_.get(), "default", "min_replica_mark_read_error", @@ -7294,6 +9367,7 @@ TEST_F(CacheManagerTest, TestFilterWriteCacheWithMinReplicaFallsBackOnMarkReadEr TEST_F(CacheManagerTest, TestFilterWriteCacheWithMinReplicaInvalidTieredTargetUsesOrdinaryPolicy) { EnableTieredMigrationStrategy(); + ASSERT_TRUE(RegisterDummyStorage("hot_02")); cache_manager_->RegisterInstance(request_context_.get(), "default", "min_replica_invalid_target", @@ -7415,7 +9489,8 @@ TEST_F(CacheManagerTest, TestFilterWriteCacheTieredMarkSkipsExistingTarget) { LocationSpec("tp3", "dummy://cold_01/blk2/tp3?size=1"), }); std::vector ids; - ASSERT_EQ(EC_OK, BatchAddLocationForTest(meta_searcher, request_context_.get(), {1, 2}, {writing_loc, serving_loc}, ids)); + ASSERT_EQ(EC_OK, + BatchAddLocationForTest(meta_searcher, request_context_.get(), {1, 2}, {writing_loc, serving_loc}, ids)); ASSERT_EQ(2u, ids.size()); std::vector> cas_tasks{ {MetaSearcher::LocationCASTask{ids[1], CLS_WRITING, CLS_SERVING}}}; @@ -7551,6 +9626,7 @@ TEST_F(CacheManagerTest, TestFilterWriteCacheWithMinReplicaUsesTieredMarkTarget) TEST_F(CacheManagerTest, TestFilterWriteCacheWithMinReplicaHonorsTieredMarkWhenReplicaSatisfied) { EnableTieredMigrationStrategy(); + ASSERT_TRUE(RegisterDummyStorage("hot_02")); cache_manager_->RegisterInstance(request_context_.get(), "default", "min_replica_satisfied_tiered", @@ -7779,10 +9855,10 @@ TEST_F(CacheManagerTest, TestFinishWriteCacheClearsTieredMark) { ASSERT_TRUE(meta_searcher); std::vector source_ids; { - auto loc = std::make_shared( - DataStorageType::DATA_STORAGE_TYPE_DUMMY, - 1, - std::vector{LocationSpec("tp0", "dummy://hot/blk1?size=1")}); + auto loc = + std::make_shared(DataStorageType::DATA_STORAGE_TYPE_DUMMY, + 1, + std::vector{LocationSpec("tp0", "dummy://hot/blk1?size=1")}); ASSERT_EQ(EC_OK, BatchAddLocationForTest(meta_searcher, request_context_.get(), {1}, {loc}, source_ids)); } ASSERT_EQ(1u, source_ids.size()); @@ -7959,7 +10035,8 @@ TEST_F(CacheManagerTest, TestFinishWriteCacheFullBlockPolicyKeepsPartialMark) { 1, std::vector{LocationSpec("tp0", "dummy://cold_01/blk1/tp0?size=1")}); std::vector partial_ids; - ASSERT_EQ(EC_OK, BatchAddLocationForTest(meta_searcher, request_context_.get(), {1}, {partial_cold_loc}, partial_ids)); + ASSERT_EQ(EC_OK, + BatchAddLocationForTest(meta_searcher, request_context_.get(), {1}, {partial_cold_loc}, partial_ids)); auto partial_info = std::make_unique(); partial_info->keys = {1}; partial_info->location_ids = {partial_ids[0]}; @@ -7980,7 +10057,8 @@ TEST_F(CacheManagerTest, TestFinishWriteCacheFullBlockPolicyKeepsPartialMark) { LocationSpec("tp3", "dummy://cold_01/blk1/tp3?size=1"), }); std::vector remaining_ids; - ASSERT_EQ(EC_OK, BatchAddLocationForTest(meta_searcher, request_context_.get(), {1}, {remaining_cold_loc}, remaining_ids)); + ASSERT_EQ(EC_OK, + BatchAddLocationForTest(meta_searcher, request_context_.get(), {1}, {remaining_cold_loc}, remaining_ids)); auto remaining_info = std::make_unique(); remaining_info->keys = {1}; remaining_info->location_ids = {remaining_ids[0]}; diff --git a/kv_cache_manager/manager/test/cache_reclaimer_test.cc b/kv_cache_manager/manager/test/cache_reclaimer_test.cc index fa2cec520..180801fbb 100644 --- a/kv_cache_manager/manager/test/cache_reclaimer_test.cc +++ b/kv_cache_manager/manager/test/cache_reclaimer_test.cc @@ -4499,10 +4499,20 @@ TEST_F(CacheReclaimerTest, TestFilterLocIDDoesNotEvictHotWhenColdSpecsIncomplete DataStorageType::DATA_STORAGE_TYPE_DUMMY, 1, std::vector{LocationSpec("TP0", "dummy://cold_01/cold_partial/tp0")}); + // A reporter may use the migration target name as its URI host and carry + // the otherwise missing spec. It is not an ordinary cold-tier replica and + // must not make the hot location eligible for physical reclamation. + auto event_report_loc = std::make_shared( + "event_report#mem#cold_01:9600", + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + 1, + std::vector{LocationSpec("TP1", "event_report://cold_01:9600/mem?size=1")}); CacheLocationMap loc_map; loc_map.emplace("hot_full", hot_loc); loc_map.emplace("cold_partial", partial_cold_loc); + loc_map.emplace("event_report", event_report_loc); batch_get_loc_out_maps = {std::move(loc_map)}; batch_get_loc_result = ErrorCode::EC_OK; diff --git a/kv_cache_manager/manager/test/get_host_cache_state_benchmark.cc b/kv_cache_manager/manager/test/get_host_cache_state_benchmark.cc new file mode 100644 index 000000000..b1310f1de --- /dev/null +++ b/kv_cache_manager/manager/test/get_host_cache_state_benchmark.cc @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kv_cache_manager/common/request_context.h" +#include "kv_cache_manager/common/unittest.h" +#include "kv_cache_manager/config/instance_info.h" +#include "kv_cache_manager/config/meta_cache_policy_config.h" +#include "kv_cache_manager/config/meta_indexer_config.h" +#include "kv_cache_manager/config/meta_storage_backend_config.h" +#include "kv_cache_manager/data_storage/snapshot_uri_utils.h" +#include "kv_cache_manager/manager/meta_searcher.h" +#include "kv_cache_manager/meta/meta_indexer.h" +#include "kv_cache_manager/meta/query_executor.h" + +namespace kv_cache_manager { +namespace { + +class GetHostCacheStateBenchmark : public TESTBASE {}; + +// Manual, pure-memory benchmark for the manager-internal query chain. It +// deliberately excludes HTTP JSON parsing so a regression can be attributed +// to local metadata lookup/projection rather than request encoding. Run with: +// +// bazelisk test -c opt //kv_cache_manager/manager/test:GetHostCacheStateBenchmark +// --test_output=streamed --test_arg=--gtest_also_run_disabled_tests +TEST_F(GetHostCacheStateBenchmark, DISABLED_MillionKeyPureLocalPrefixScenarios) { + constexpr size_t kMaxKeyCount = 1'000'000; + constexpr size_t kSetupBatchSize = 4096; + constexpr size_t kEarlyStopIndex = 1024; + constexpr int64_t kBaseKey = 900'000'000; + constexpr std::string_view kHost = "benchmark-host:8080"; + constexpr std::string_view kOtherHost = "other-host:8080"; + + auto backend_config = std::make_shared("local"); + auto cache_config = std::make_shared(); + cache_config->SetCapacity(0); + auto indexer_config = std::make_shared(); + indexer_config->SetMaxKeyCount(kMaxKeyCount + 16); + indexer_config->SetMutexShardNum(256); + indexer_config->SetBatchKeySize(kSetupBatchSize); + indexer_config->SetMetaStorageBackendConfig(backend_config); + indexer_config->SetMetaCachePolicyConfig(cache_config); + + auto indexer = std::make_shared(); + indexer->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 256, /*chunk_size*/ 128, /*queue_capacity*/ 64)); + ASSERT_EQ(EC_OK, indexer->Init("get_host_million_key_benchmark", indexer_config)); + auto request_context = std::make_shared("get_host_million_key_benchmark"); + + auto make_location = [](std::string_view host) { + const std::string host_text(host); + auto location = std::make_shared( + "kvs#event_report_l2#mem#" + host_text, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + 1, + std::vector{LocationSpec("tp0", "event_report://" + host_text + "/mem")}); + // Model the in-memory ReportEvent write path, which has already + // validated every URI and records the zero aggregate size. + location->set_validated_total_size(0); + return location; + }; + const auto location = make_location(kHost); + const auto other_location = make_location(kOtherHost); + + KeyVector all_hit_keys(kMaxKeyCount); + std::iota(all_hit_keys.begin(), all_hit_keys.end(), kBaseKey); + for (size_t begin = 0; begin < all_hit_keys.size(); begin += kSetupBatchSize) { + const size_t end = std::min(all_hit_keys.size(), begin + kSetupBatchSize); + KeyVector batch_keys(all_hit_keys.begin() + static_cast(begin), + all_hit_keys.begin() + static_cast(end)); + CacheLocationMapVector locations(batch_keys.size()); + for (auto &location_map : locations) { + location_map.emplace(location->id(), location); + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, indexer->Put(request_context.get(), batch_keys, locations, properties).ec) + << "setup begin=" << begin; + } + + // One existing key with another host supports a host-prefix early-stop + // scenario without changing the all-hit data set. + const KeyType other_host_key = kBaseKey + static_cast(kMaxKeyCount); + KeyVector extra_keys{other_host_key}; + CacheLocationMapVector extra_locations(1); + extra_locations[0].emplace(other_location->id(), other_location); + PropertyMapVector extra_properties; + ASSERT_EQ(EC_OK, indexer->Put(request_context.get(), extra_keys, extra_locations, extra_properties).ec); + + MetaSearcher searcher(indexer, [](const CacheLocation &) { return true; }, {}); + size_t active_worker_count = 4; + const MetaSearcher::CheckHostCacheLocationFunc visibility_check = [](const CacheLocation &candidate, + MetaSearcher::HostCacheLocationInfo &out) { + out = {}; + std::string_view storage_type; + std::string_view reporter_medium; + std::string_view reporter_host; + if (!SnapshotUriUtils::ParseEventReportLocationIdView( + candidate.id(), storage_type, reporter_medium, reporter_host)) { + return false; + } + const bool uri_structure_prevalidated = candidate.HasValidatedLocationSpecs(); + for (const auto &spec : candidate.location_specs()) { + std::string_view version; + if (!SnapshotUriUtils::InspectSnapshotUriForVisibility(spec.uri(), version, uri_structure_prevalidated)) { + return false; + } + } + out.has_reporter_identity = true; + out.reporter_medium = reporter_medium; + out.reporter_host = reporter_host; + return true; + }; + + auto run_case = [&](const char *name, const KeyVector &query_keys, int64_t expected_prefix, int iterations) { + std::vector elapsed_ms; + elapsed_ms.reserve(iterations); + for (int iteration = -1; iteration < iterations; ++iteration) { + std::vector matches; + const auto begin = std::chrono::steady_clock::now(); + ASSERT_EQ(EC_OK, + searcher.PrefixMatchByHost( + request_context.get(), query_keys, false, {"mem"}, matches, &visibility_check)); + const auto elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - begin).count(); + const auto match = std::find_if( + matches.begin(), matches.end(), [kHost](const auto &item) { return item.host_ip_port == kHost; }); + ASSERT_NE(matches.end(), match) << name; + ASSERT_EQ(expected_prefix, match->local) << name; + if (iteration >= 0) { + elapsed_ms.push_back(elapsed); + } + } + std::sort(elapsed_ms.begin(), elapsed_ms.end()); + const double average = + std::accumulate(elapsed_ms.begin(), elapsed_ms.end(), 0.0) / static_cast(elapsed_ms.size()); + std::cout << "[GET_HOST_BENCH] case=" << name << " keys=" << query_keys.size() + << " workers=" << active_worker_count << " p50_ms=" << elapsed_ms[elapsed_ms.size() / 2] + << " avg_ms=" << average << std::endl; + }; + + auto run_metadata_only = [&](const KeyVector &query_keys, int iterations) { + std::vector elapsed_ms; + elapsed_ms.reserve(iterations); + for (int iteration = -1; iteration < iterations; ++iteration) { + const auto begin = std::chrono::steady_clock::now(); + const auto result = indexer->VisitLocationValuesForPrefix( + request_context.get(), query_keys, [&query_keys](size_t, const CompactLocationsPerKey &, size_t) { + return query_keys.size(); + }); + const auto elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - begin).count(); + ASSERT_EQ(EC_OK, result.terminal_ec); + ASSERT_EQ(query_keys.size(), result.valid_key_count); + if (iteration >= 0) { + elapsed_ms.push_back(elapsed); + } + } + std::sort(elapsed_ms.begin(), elapsed_ms.end()); + const double average = + std::accumulate(elapsed_ms.begin(), elapsed_ms.end(), 0.0) / static_cast(elapsed_ms.size()); + std::cout << "[GET_HOST_BENCH] case=metadata_only keys=" << query_keys.size() + << " workers=" << active_worker_count << " p50_ms=" << elapsed_ms[elapsed_ms.size() / 2] + << " avg_ms=" << average << std::endl; + }; + + auto run_p2p_case = [&](const char *name, const KeyVector &query_keys, bool mamba) { + std::vector elapsed_ms; + for (int iteration = -1; iteration < 3; ++iteration) { + std::vector matches; + const auto begin = std::chrono::steady_clock::now(); + ErrorCode ec = EC_OK; + if (mamba) { + const std::vector groups = { + LocationSpecGroup("F0", {"tp0"}), + LocationSpecGroup("L0", {"tp0"}), + }; + ec = searcher.PrefixMatchWithMambaByHost( + request_context.get(), query_keys, false, {"mem"}, groups, matches, &visibility_check, 1); + } else { + ec = searcher.PrefixMatchByHost( + request_context.get(), query_keys, false, {"mem"}, matches, &visibility_check, 1); + } + const auto elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - begin).count(); + ASSERT_EQ(EC_OK, ec); + const auto match = std::find_if( + matches.begin(), matches.end(), [kHost](const auto &item) { return item.host_ip_port == kHost; }); + ASSERT_NE(matches.end(), match) << name; + ASSERT_EQ(static_cast(kEarlyStopIndex), match->local) << name; + ASSERT_EQ(1, match->p2p_1_fetch) << name; + ASSERT_EQ(static_cast(query_keys.size()), match->p2p_1_total_match) << name; + if (iteration >= 0) { + elapsed_ms.push_back(elapsed); + } + } + std::sort(elapsed_ms.begin(), elapsed_ms.end()); + std::cout << "[GET_HOST_BENCH] case=" << name << " keys=" << query_keys.size() + << " workers=" << active_worker_count << " p50_ms=" << elapsed_ms[elapsed_ms.size() / 2] << std::endl; + }; + + for (const size_t key_count : {size_t{100'000}, size_t{500'000}, kMaxKeyCount}) { + KeyVector query_keys(all_hit_keys.begin(), all_hit_keys.begin() + static_cast(key_count)); + run_metadata_only(query_keys, 3); + run_case("all_hit", query_keys, static_cast(key_count), 3); + } + + KeyVector early_host_stop = all_hit_keys; + early_host_stop[kEarlyStopIndex] = other_host_key; + run_case("early_host_stop", early_host_stop, kEarlyStopIndex, 5); + run_p2p_case("p2p_prefix_single_gap", early_host_stop, false); + run_p2p_case("p2p_mamba_single_gap", early_host_stop, true); + + KeyVector early_metadata_miss = all_hit_keys; + early_metadata_miss[kEarlyStopIndex] = kBaseKey + static_cast(kMaxKeyCount + 10); + run_case("early_metadata_miss", early_metadata_miss, kEarlyStopIndex, 5); + + for (const size_t worker_count : {size_t{1}, size_t{2}, size_t{8}, size_t{16}}) { + active_worker_count = worker_count; + indexer->SetQueryExecutor( + std::make_shared(worker_count, /*parallel_threshold*/ 256, /*chunk_size*/ 128, 64)); + run_metadata_only(all_hit_keys, 3); + run_case("all_hit_worker_scaling", all_hit_keys, static_cast(kMaxKeyCount), 3); + } + + // Skip million-key persistence in a manual latency benchmark; all tested + // data lives solely in the local in-memory backend. + ASSERT_EQ(EC_OK, indexer->backend_manager_->Close()); + indexer->backend_manager_.reset(); +} + +} // namespace +} // namespace kv_cache_manager diff --git a/kv_cache_manager/manager/test/meta_searcher_test.cc b/kv_cache_manager/manager/test/meta_searcher_test.cc index 6782f163f..a9d2feb40 100644 --- a/kv_cache_manager/manager/test/meta_searcher_test.cc +++ b/kv_cache_manager/manager/test/meta_searcher_test.cc @@ -1,24 +1,93 @@ #include +#include +#include +#include #include #include +#include #include #include +#include +#include #include +#include #include #include #include +#include #include "kv_cache_manager/common/request_context.h" #include "kv_cache_manager/common/unittest.h" +#include "kv_cache_manager/config/instance_info.h" #include "kv_cache_manager/config/meta_indexer_config.h" #include "kv_cache_manager/config/meta_storage_backend_config.h" #include "kv_cache_manager/manager/meta_searcher.h" #include "kv_cache_manager/meta/meta_indexer.h" #include "kv_cache_manager/meta/meta_local_backend.h" #include "kv_cache_manager/meta/utils.h" +#include "kv_cache_manager/metrics/metrics_collector.h" +#include "kv_cache_manager/metrics/metrics_registry.h" using namespace kv_cache_manager; +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); + +TEST(CacheLocationMoveTest, SetAndMoveLocationSpecsTransferVectorStorage) { + std::vector specs; + specs.reserve(2); + specs.emplace_back("tp0", "event_report://move-test:8080/mem?payload=" + std::string(128, 'a')); + specs.emplace_back("tp1", "event_report://move-test:8080/mem?payload=" + std::string(128, 'b')); + const LocationSpec *const original_storage = specs.data(); + + CacheLocation location; + location.set_location_specs(std::move(specs)); + ASSERT_EQ(original_storage, location.location_specs().data()); + ASSERT_EQ(2u, location.location_specs().size()); + + CacheLocation moved(std::move(location)); + EXPECT_EQ(original_storage, moved.location_specs().data()); + EXPECT_EQ("tp0", moved.location_specs()[0].name()); + EXPECT_EQ("tp1", moved.location_specs()[1].name()); +} + +TEST(CacheLocationMoveTest, ValidatedTotalSizeHintFollowsLocationSpecs) { + CacheLocation location; + std::uint64_t size = 0; + EXPECT_FALSE(location.GetValidatedTotalSize(size)); + EXPECT_FALSE(location.HasValidatedLocationSpecs()); + + location.set_location_specs({LocationSpec("tp0", "event_report://size-hint:8080/mem?size=17")}); + location.set_validated_total_size(17); + ASSERT_TRUE(location.GetValidatedTotalSize(size)); + EXPECT_TRUE(location.HasValidatedLocationSpecs()); + EXPECT_EQ(17u, size); + + CacheLocation copied = location; + size = 0; + ASSERT_TRUE(copied.GetValidatedTotalSize(size)); + EXPECT_EQ(17u, size); + + copied.mutable_location_specs().front().set_uri("event_report://size-hint:8080/mem?size=23"); + EXPECT_FALSE(copied.GetValidatedTotalSize(size)); + EXPECT_FALSE(copied.HasValidatedLocationSpecs()); + copied.set_validated_total_size(23); + CacheLocation moved = std::move(copied); + ASSERT_TRUE(moved.GetValidatedTotalSize(size)); + EXPECT_EQ(23u, size); + + CacheLocation restored; + ASSERT_TRUE(restored.FromJsonString(moved.ToJsonString())); + EXPECT_FALSE(restored.GetValidatedTotalSize(size)); + EXPECT_FALSE(restored.HasValidatedLocationSpecs()); + + moved.push_location_spec(LocationSpec("tp1", "event_report://size-hint:8080/mem?size=29")); + EXPECT_FALSE(moved.GetValidatedTotalSize(size)); + EXPECT_FALSE(moved.HasValidatedLocationSpecs()); +} + namespace { // Helper class to create test data class MetaSearcherTestHelper { @@ -46,28 +115,472 @@ SubmitDelReqFunc dummy_submit_del_req = [](const std::vector &, const std::vector> &, bool) -> void {}; -class FaultyGetLocationIdsBackend : public MetaLocalBackend { +MetaSearcher::MergeLocationSpecsTask MakeEventReportTask(const std::string &host, + DataStorageType type, + const std::vector &spec_names = {"tp0"}) { + const std::string storage_type = + type == DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2 ? "event_report_l2" : "event_report_l1p5"; + std::vector specs; + specs.reserve(spec_names.size()); + for (const auto &name : spec_names) { + specs.emplace_back(name, "event_report://" + host + "/mem"); + } + return {"kvs#" + storage_type + "#mem#" + host, type, CacheLocationStatus::CLS_SERVING, std::move(specs)}; +} + +// Deliberately simple, materialized model of the intended P2P semantics. +// It is kept independent from the production dense-id/bitmask reducers so the +// randomized differential test below can catch errors in peer selection, +// group merging, Eagle pop, and fetched-key de-duplication. +using ReferenceSpecNames = std::set; +using ReferenceHostSpecs = std::map; + +struct ReferenceP2PMatrix { + std::vector all; + std::vector vineyard; +}; + +const ReferenceSpecNames &ReferenceSpecsFor(const ReferenceHostSpecs &host_specs, const std::string &host) { + static const ReferenceSpecNames empty; + const auto it = host_specs.find(host); + return it == host_specs.end() ? empty : it->second; +} + +bool ReferenceHasGroup(const ReferenceSpecNames &specs, const LocationSpecGroup &group) { + return std::all_of(group.spec_names().begin(), group.spec_names().end(), [&](const std::string &name) { + return specs.find(name) != specs.end(); + }); +} + +bool ReferenceHasGroups(const ReferenceSpecNames &specs, const std::vector &groups) { + return std::all_of( + groups.begin(), groups.end(), [&](const LocationSpecGroup *group) { return ReferenceHasGroup(specs, *group); }); +} + +bool ReferenceLocalAndPeerCoverGroup(const ReferenceSpecNames &local, + const ReferenceSpecNames &peer, + const LocationSpecGroup &group) { + return std::all_of(group.spec_names().begin(), group.spec_names().end(), [&](const std::string &name) { + return local.find(name) != local.end() || peer.find(name) != peer.end(); + }); +} + +struct ReferencePrefixPeerSelection { + std::string peer; + std::vector covered_key_indices; +}; + +ReferencePrefixPeerSelection ReferenceSelectPrefixPeer(const std::string &target_host, + const ReferenceP2PMatrix &matrix, + const LocationSpecGroup *required_group = nullptr) { + struct GapCandidates { + size_t key_index = 0; + std::vector peers; + }; + std::vector gaps; + for (size_t key_index = 0; key_index < matrix.all.size(); ++key_index) { + const auto local_it = matrix.all[key_index].find(target_host); + const auto &local = ReferenceSpecsFor(matrix.all[key_index], target_host); + const bool local_hit = + required_group ? ReferenceHasGroup(local, *required_group) : local_it != matrix.all[key_index].end(); + if (local_hit) { + continue; + } + + GapCandidates gap; + gap.key_index = key_index; + for (const auto &[peer, peer_specs] : matrix.vineyard[key_index]) { + if (peer == target_host) { + continue; + } + const bool peer_hit = required_group ? ReferenceLocalAndPeerCoverGroup(local, peer_specs, *required_group) + : !peer_specs.empty(); + if (peer_hit) { + gap.peers.push_back(peer); + } + } + if (gap.peers.empty()) { + break; + } + gaps.push_back(std::move(gap)); + } + if (gaps.empty()) { + return {}; + } + + ReferencePrefixPeerSelection best; + for (const auto &peer : gaps.front().peers) { + std::vector covered; + for (const auto &gap : gaps) { + if (std::find(gap.peers.begin(), gap.peers.end(), peer) == gap.peers.end()) { + break; + } + covered.push_back(gap.key_index); + } + if (covered.size() > best.covered_key_indices.size() || + (covered.size() == best.covered_key_indices.size() && (best.peer.empty() || peer < best.peer))) { + best.peer = peer; + best.covered_key_indices = std::move(covered); + } + } + return best; +} + +struct ReferenceCoverageSelection { + std::string peer; + std::vector> covered_queries; +}; + +ReferenceCoverageSelection ReferenceSelectCoveragePeer(const std::string &target_host, + const ReferenceP2PMatrix &matrix, + size_t block_count, + const std::vector &required_groups) { + struct CoverageQuery { + size_t key_index = 0; + const LocationSpecGroup *group = nullptr; + std::vector peers; + }; + std::vector queries; + std::map peer_counts; + for (const auto *group : required_groups) { + for (size_t key_index = 0; key_index < block_count; ++key_index) { + const auto &local = ReferenceSpecsFor(matrix.all[key_index], target_host); + if (ReferenceHasGroup(local, *group)) { + continue; + } + CoverageQuery query{key_index, group, {}}; + for (const auto &[peer, peer_specs] : matrix.vineyard[key_index]) { + if (peer != target_host && ReferenceLocalAndPeerCoverGroup(local, peer_specs, *group)) { + query.peers.push_back(peer); + ++peer_counts[peer]; + } + } + if (!query.peers.empty()) { + queries.push_back(std::move(query)); + } + } + } + if (peer_counts.empty()) { + return {}; + } + + ReferenceCoverageSelection selection; + size_t best_count = 0; + for (const auto &[peer, count] : peer_counts) { + if (count > best_count || (count == best_count && (selection.peer.empty() || peer < selection.peer))) { + selection.peer = peer; + best_count = count; + } + } + for (const auto &query : queries) { + if (std::find(query.peers.begin(), query.peers.end(), selection.peer) != query.peers.end()) { + selection.covered_queries.emplace_back(query.key_index, query.group); + } + } + return selection; +} + +void ReferenceMergeGroup(ReferenceSpecNames &target, const ReferenceSpecNames &source, const LocationSpecGroup &group) { + for (const auto &name : group.spec_names()) { + if (source.find(name) != source.end()) { + target.insert(name); + } + } +} + +int64_t ReferenceOrdinaryPrefix(const std::vector &specs_by_key, bool use_eagle_pop) { + size_t prefix = 0; + while (prefix < specs_by_key.size() && !specs_by_key[prefix].empty()) { + ++prefix; + } + return static_cast(prefix - (use_eagle_pop && prefix != 0 ? 1 : 0)); +} + +int64_t ReferenceMambaPrefix(const std::vector &specs_by_key, + bool use_eagle_pop, + const std::vector &full_groups, + const std::vector &state_groups) { + size_t full_prefix = 0; + while (full_prefix < specs_by_key.size() && ReferenceHasGroups(specs_by_key[full_prefix], full_groups)) { + ++full_prefix; + } + if (use_eagle_pop && full_prefix != 0) { + --full_prefix; + } + for (size_t offset = full_prefix; offset != 0; --offset) { + if (ReferenceHasGroups(specs_by_key[offset - 1], state_groups)) { + return static_cast(offset); + } + } + return 0; +} + +std::vector ReferenceTopHosts(const std::vector &matches, size_t p2p_host_count) { + std::vector indices; + for (size_t i = 0; i < matches.size(); ++i) { + if (matches[i].local > 0) { + indices.push_back(i); + } + } + std::sort(indices.begin(), indices.end(), [&matches](size_t lhs, size_t rhs) { + if (matches[lhs].local != matches[rhs].local) { + return matches[lhs].local > matches[rhs].local; + } + return matches[lhs].host_ip_port < matches[rhs].host_ip_port; + }); + indices.resize(std::min(indices.size(), p2p_host_count)); + return indices; +} + +std::vector ReferenceOrdinaryMatches(const MetaSearcher::KeyVector &keys, + const ReferenceP2PMatrix &matrix, + bool use_eagle_pop, + size_t p2p_host_count) { + std::vector matches; + if (matrix.all.empty()) { + return matches; + } + for (const auto &[host, ignored] : matrix.all.front()) { + (void)ignored; + std::vector local_specs(matrix.all.size()); + for (size_t i = 0; i < matrix.all.size(); ++i) { + local_specs[i] = ReferenceSpecsFor(matrix.all[i], host); + } + const int64_t local = ReferenceOrdinaryPrefix(local_specs, use_eagle_pop); + matches.push_back({host, local, 0, local}); + } + + for (size_t host_index : ReferenceTopHosts(matches, p2p_host_count)) { + auto &match = matches[host_index]; + std::vector merged(matrix.all.size()); + for (size_t i = 0; i < matrix.all.size(); ++i) { + merged[i] = ReferenceSpecsFor(matrix.all[i], match.host_ip_port); + } + const auto selection = ReferenceSelectPrefixPeer(match.host_ip_port, matrix); + std::set fetched_keys; + for (size_t key_index : selection.covered_key_indices) { + const auto &peer_specs = ReferenceSpecsFor(matrix.vineyard[key_index], selection.peer); + merged[key_index].insert(peer_specs.begin(), peer_specs.end()); + fetched_keys.insert(keys[key_index]); + } + match.p2p_1_fetch = static_cast(fetched_keys.size()); + match.p2p_1_total_match = ReferenceOrdinaryPrefix(merged, use_eagle_pop); + } + matches.erase(std::remove_if(matches.begin(), matches.end(), [](const auto &match) { return match.local == 0; }), + matches.end()); + return matches; +} + +std::vector ReferenceMambaMatches(const MetaSearcher::KeyVector &keys, + const ReferenceP2PMatrix &matrix, + bool use_eagle_pop, + const std::vector &groups, + size_t p2p_host_count) { + std::vector full_groups; + std::vector state_groups; + for (const auto &group : groups) { + (group.name().front() == 'F' ? full_groups : state_groups).push_back(&group); + } + + std::vector matches; + if (matrix.all.empty()) { + return matches; + } + for (const auto &[host, ignored] : matrix.all.front()) { + (void)ignored; + std::vector local_specs(matrix.all.size()); + for (size_t i = 0; i < matrix.all.size(); ++i) { + local_specs[i] = ReferenceSpecsFor(matrix.all[i], host); + } + const int64_t local = ReferenceMambaPrefix(local_specs, use_eagle_pop, full_groups, state_groups); + matches.push_back({host, local, 0, local}); + } + + for (size_t host_index : ReferenceTopHosts(matches, p2p_host_count)) { + auto &match = matches[host_index]; + std::vector merged(matrix.all.size()); + for (size_t i = 0; i < matrix.all.size(); ++i) { + merged[i] = ReferenceSpecsFor(matrix.all[i], match.host_ip_port); + } + std::set fetched_keys; + for (const auto *group : full_groups) { + const auto selection = ReferenceSelectPrefixPeer(match.host_ip_port, matrix, group); + for (size_t key_index : selection.covered_key_indices) { + ReferenceMergeGroup( + merged[key_index], ReferenceSpecsFor(matrix.vineyard[key_index], selection.peer), *group); + fetched_keys.insert(keys[key_index]); + } + } + + size_t full_prefix = 0; + while (full_prefix < merged.size() && ReferenceHasGroups(merged[full_prefix], full_groups)) { + ++full_prefix; + } + if (use_eagle_pop && full_prefix != 0) { + --full_prefix; + } + const auto coverage = ReferenceSelectCoveragePeer(match.host_ip_port, matrix, full_prefix, state_groups); + for (const auto &[key_index, group] : coverage.covered_queries) { + ReferenceMergeGroup( + merged[key_index], ReferenceSpecsFor(matrix.vineyard[key_index], coverage.peer), *group); + fetched_keys.insert(keys[key_index]); + } + match.p2p_1_fetch = static_cast(fetched_keys.size()); + match.p2p_1_total_match = ReferenceMambaPrefix(merged, use_eagle_pop, full_groups, state_groups); + } + matches.erase(std::remove_if(matches.begin(), matches.end(), [](const auto &match) { return match.local == 0; }), + matches.end()); + return matches; +} + +using HostCacheMatchTuple = std::tuple; + +std::vector ToMatchTuples(const std::vector &matches) { + std::vector tuples; + tuples.reserve(matches.size()); + for (const auto &match : matches) { + tuples.emplace_back(match.host_ip_port, match.local, match.p2p_1_fetch, match.p2p_1_total_match); + } + return tuples; +} + +class FaultyTargetedLocationBackend : public MetaLocalBackend { public: void SetFailedKey(KeyType key) { failed_key_ = key; } - std::vector GetLocationIds(RequestContext *request_context, - const KeyTypeVec &keys, - LocationIdsPerKey &out_location_ids) noexcept override { - auto results = MetaLocalBackend::GetLocationIds(request_context, keys, out_location_ids); + std::vector> + GetLocationsWithKeyStatus(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept override { + auto results = MetaLocalBackend::GetLocationsWithKeyStatus( + request_context, keys, location_ids, out_locations, out_key_error_codes); if (!failed_key_.has_value()) { return results; } for (size_t i = 0; i < keys.size(); ++i) { if (keys[i] == failed_key_.value()) { - results[i] = EC_ERROR; - out_location_ids[i].clear(); + out_key_error_codes[i] = EC_ERROR; + results[i].assign(location_ids[i].size(), EC_ERROR); + out_locations[i].assign(location_ids[i].size(), CacheLocationConstPtr{}); + } + } + return results; + } + +private: + std::optional failed_key_; +}; + +// A MetaLocalBackend subclass deliberately reports a non-local storage type +// so it falls back to the generic batched read path. It verifies that +// multi-pass Mamba P2P replays one retained batch instead of issuing repeated +// backend reads. +class CountingBatchedLocationBackend : public MetaLocalBackend { +public: + std::string GetStorageType() noexcept override { return "counting_batched"; } + + void SetFailedKey(std::optional key) { failed_key_ = key; } + + std::vector GetLocationValues(RequestContext *request_context, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept override { + ++location_value_read_count_; + auto results = MetaLocalBackend::GetLocationValues(request_context, keys, out_locations); + if (failed_key_) { + for (size_t key_index = 0; key_index < keys.size(); ++key_index) { + if (keys[key_index] == *failed_key_) { + results[key_index] = EC_TIMEOUT; + } } } return results; } + void ResetLocationValueReadCount() { location_value_read_count_ = 0; } + [[nodiscard]] size_t LocationValueReadCount() const { return location_value_read_count_; } + private: std::optional failed_key_; + size_t location_value_read_count_ = 0; +}; + +// Replaces one location or injects one error only in a selected compact-read +// result. The stored metadata stays unchanged, allowing tests to model a +// concurrent update or read failure between Mamba's local, planning, and final +// ordered passes deterministically. +class MutatingCompactLocationBackend : public MetaLocalBackend { +public: + void SetReadDelay(std::chrono::microseconds delay) { read_delay_ = delay; } + + void ReplaceOnRead(size_t read_number, KeyType key, std::string location_id, CacheLocationConstPtr replacement) { + mutation_read_number_ = read_number; + mutation_key_ = key; + mutation_location_id_ = std::move(location_id); + replacement_ = std::move(replacement); + failure_read_number_ = 0; + failure_ec_ = EC_OK; + compact_read_count_.store(0, std::memory_order_relaxed); + } + + void FailOnRead(size_t read_number, KeyType key, ErrorCode ec) { + mutation_read_number_ = 0; + replacement_.reset(); + failure_read_number_ = read_number; + failure_key_ = key; + failure_ec_ = ec; + compact_read_count_.store(0, std::memory_order_relaxed); + } + + [[nodiscard]] size_t CompactReadCount() const { return compact_read_count_.load(std::memory_order_relaxed); } + + std::vector GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept override { + if (read_delay_.count() != 0) { + std::this_thread::sleep_for(read_delay_); + } + auto results = MetaLocalBackend::GetLocationValuesCompact(request_context, keys, key_count, out_locations); + const size_t read_number = compact_read_count_.fetch_add(1, std::memory_order_relaxed) + 1; + if (read_number == failure_read_number_) { + for (size_t key_index = 0; key_index < key_count; ++key_index) { + if (keys[key_index] == failure_key_) { + results[key_index] = failure_ec_; + } + } + return results; + } + if (read_number != mutation_read_number_ || !replacement_) { + return results; + } + for (size_t key_index = 0; key_index < key_count; ++key_index) { + if (keys[key_index] != mutation_key_) { + continue; + } + for (size_t value_index = out_locations.offsets[key_index]; + value_index < out_locations.offsets[key_index + 1]; + ++value_index) { + const auto &location = out_locations.values[value_index]; + if (location && location->id() == mutation_location_id_) { + out_locations.values[value_index] = replacement_; + } + } + } + return results; + } + +private: + size_t mutation_read_number_ = 0; + KeyType mutation_key_ = 0; + std::string mutation_location_id_; + CacheLocationConstPtr replacement_; + size_t failure_read_number_ = 0; + KeyType failure_key_ = 0; + ErrorCode failure_ec_ = EC_OK; + std::atomic compact_read_count_{0}; + std::chrono::microseconds read_delay_{0}; }; class CommitThenFailUpsertBackend : public MetaLocalBackend { @@ -137,6 +650,81 @@ class RollbackFaultBackend : public MetaLocalBackend { std::optional get_locations_failed_key_; }; +class RecordingGetLocationsBackend : public MetaLocalBackend { +public: + std::vector GetLocations(RequestContext *request_context, + const KeyTypeVec &keys, + CacheLocationMapVector &out_locations) noexcept override { + { + std::lock_guard lock(mutex_); + requested_key_batches_.push_back(keys); + } + return MetaLocalBackend::GetLocations(request_context, keys, out_locations); + } + + void ResetReadLog() { + std::lock_guard lock(mutex_); + requested_key_batches_.clear(); + } + + std::vector RequestedKeyBatches() const { + std::lock_guard lock(mutex_); + return requested_key_batches_; + } + +private: + mutable std::mutex mutex_; + std::vector requested_key_batches_; +}; + +class PrefixReadBackend : public MetaLocalBackend { +public: + void SetFailedKey(std::optional key) { failed_key_ = key; } + + void ResetReadCounts() { + location_value_read_count_.store(0, std::memory_order_relaxed); + compact_read_count_.store(0, std::memory_order_relaxed); + } + + size_t LocationValueReadCount() const { return location_value_read_count_.load(std::memory_order_relaxed); } + size_t CompactReadCount() const { return compact_read_count_.load(std::memory_order_relaxed); } + + std::vector GetLocationValues(RequestContext *request_context, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept override { + location_value_read_count_.fetch_add(keys.size(), std::memory_order_relaxed); + auto results = MetaLocalBackend::GetLocationValues(request_context, keys, out_locations); + InjectFailure(keys.data(), keys.size(), results); + return results; + } + + std::vector GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept override { + compact_read_count_.fetch_add(key_count, std::memory_order_relaxed); + auto results = MetaLocalBackend::GetLocationValuesCompact(request_context, keys, key_count, out_locations); + InjectFailure(keys, key_count, results); + return results; + } + +private: + void InjectFailure(const KeyType *keys, size_t key_count, std::vector &results) const { + if (!failed_key_.has_value()) { + return; + } + for (size_t i = 0; i < key_count && i < results.size(); ++i) { + if (keys[i] == *failed_key_) { + results[i] = EC_TIMEOUT; + } + } + } + + std::optional failed_key_; + std::atomic location_value_read_count_{0}; + std::atomic compact_read_count_{0}; +}; + ErrorCode BatchAddLocationForTest(MetaSearcher *meta_searcher, RequestContext *request_context, const KeyVector &keys, @@ -177,12 +765,12 @@ class MetaSearcherTest : public TESTBASE { return meta_storage_backend_config; } - std::shared_ptr CreateMetaIndexer() { + std::shared_ptr CreateMetaIndexer(size_t max_key_count = 10000) { auto meta_indexer_config = std::make_shared(); auto backend_config = ConstructMetaStorageBackendConfig(); meta_indexer_config->SetMetaStorageBackendConfig(backend_config); meta_indexer_config->SetMutexShardNum(32); - meta_indexer_config->SetMaxKeyCount(10000); + meta_indexer_config->SetMaxKeyCount(max_key_count); auto indexer = std::make_shared(); auto metaCachePolicyConfig = std::make_shared(); metaCachePolicyConfig->SetCapacity(0); @@ -195,9 +783,9 @@ class MetaSearcherTest : public TESTBASE { return indexer; } - FaultyGetLocationIdsBackend *ReplaceWithFaultyBackend() { + FaultyTargetedLocationBackend *ReplaceWithFaultyBackend() { auto backend_config = ConstructMetaStorageBackendConfig(); - auto faulty_backend = std::make_unique(); + auto faulty_backend = std::make_unique(); EXPECT_EQ(EC_OK, faulty_backend->Init("test", backend_config)); EXPECT_EQ(EC_OK, faulty_backend->Open()); auto backend_raw = faulty_backend.get(); @@ -207,6 +795,30 @@ class MetaSearcherTest : public TESTBASE { return backend_raw; } + CountingBatchedLocationBackend *ReplaceWithCountingBatchedLocationBackend() { + auto backend_config = ConstructMetaStorageBackendConfig(); + auto backend = std::make_unique(); + EXPECT_EQ(EC_OK, backend->Init("test", backend_config)); + EXPECT_EQ(EC_OK, backend->Open()); + auto backend_raw = backend.get(); + meta_indexer_->backend_manager_->persistent_backend_->Close(); + meta_indexer_->backend_manager_->persistent_backend_ = std::move(backend); + meta_indexer_->backend_manager_->cache_backend_.reset(); + return backend_raw; + } + + MutatingCompactLocationBackend *ReplaceWithMutatingCompactLocationBackend() { + auto backend_config = ConstructMetaStorageBackendConfig(); + auto backend = std::make_unique(); + EXPECT_EQ(EC_OK, backend->Init("test", backend_config)); + EXPECT_EQ(EC_OK, backend->Open()); + auto backend_raw = backend.get(); + meta_indexer_->backend_manager_->persistent_backend_->Close(); + meta_indexer_->backend_manager_->persistent_backend_ = std::move(backend); + meta_indexer_->backend_manager_->cache_backend_.reset(); + return backend_raw; + } + CommitThenFailUpsertBackend *ReplaceWithCommitThenFailUpsertBackend() { auto backend_config = ConstructMetaStorageBackendConfig(); auto backend = std::make_unique(); @@ -231,6 +843,30 @@ class MetaSearcherTest : public TESTBASE { return backend_raw; } + RecordingGetLocationsBackend *ReplaceWithRecordingGetLocationsBackend() { + auto backend_config = ConstructMetaStorageBackendConfig(); + auto backend = std::make_unique(); + EXPECT_EQ(EC_OK, backend->Init("test", backend_config)); + EXPECT_EQ(EC_OK, backend->Open()); + auto backend_raw = backend.get(); + meta_indexer_->backend_manager_->persistent_backend_->Close(); + meta_indexer_->backend_manager_->persistent_backend_ = std::move(backend); + meta_indexer_->backend_manager_->cache_backend_.reset(); + return backend_raw; + } + + PrefixReadBackend *ReplaceWithPrefixReadBackend() { + auto backend_config = ConstructMetaStorageBackendConfig(); + auto backend = std::make_unique(); + EXPECT_EQ(EC_OK, backend->Init("test", backend_config)); + EXPECT_EQ(EC_OK, backend->Open()); + auto backend_raw = backend.get(); + meta_indexer_->backend_manager_->persistent_backend_->Close(); + meta_indexer_->backend_manager_->persistent_backend_ = std::move(backend); + meta_indexer_->backend_manager_->cache_backend_.reset(); + return backend_raw; + } + std::shared_ptr meta_indexer_; std::shared_ptr meta_searcher_; std::shared_ptr request_context_; @@ -286,8 +922,7 @@ TEST_F(MetaSearcherTest, TestBatchAddLocationReturnsAlignedPartialResults) { meta_indexer_->max_key_count_ = 1; MetaSearcher::KeyVector keys = {1001, 1002}; - while (GetShardIndex(keys[0], meta_indexer_->mutex_shard_mask_) == - GetShardIndex(keys[1], meta_indexer_->mutex_shard_mask_)) { + while (meta_indexer_->GetMutexShardIndex(keys[0]) == meta_indexer_->GetMutexShardIndex(keys[1])) { ++keys[1]; } auto location = MetaSearcherTestHelper::CreateCacheLocation( @@ -429,54 +1064,1442 @@ TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsAppendsAndOverwrites) { EXPECT_EQ("event_report://127.0.0.1:8080/mem", spec_uris["full_3"]); } -TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsPreservesUntouchedSpecsAcrossGenerationChange) { - const MetaSearcher::KeyVector keys = {10007}; - const std::string location_id = "kvs#event_report_l2#mem#127.0.0.1:8080"; - const std::string version_a = "00112233445566778899aabbccddeeff"; - const std::string version_b = "ffeeddccbbaa99887766554433221100"; - auto uri = [](const std::string &source, const std::string &version) { - return "event_report://127.0.0.1:8080/mem?source=" + source + "&s_version=" + version; +TEST_F(MetaSearcherTest, TestBatchMergeSingleSpecReplacementCachesValidatedTotalSize) { + const KeyType key = 10009; + const std::string location_id = "kvs#event_report_l2#mem#size-hint:8080"; + auto make_tasks = [&location_id](std::uint64_t size) { + return std::vector>{{ + {location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://size-hint:8080/mem?size=" + std::to_string(size))}}, + }}; }; std::vector per_key_ec; - std::vector> seed_tasks = {{ - {location_id, - DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, - CacheLocationStatus::CLS_SERVING, - {LocationSpec("linear_0", uri("old_linear", version_a)), - LocationSpec("mamba_0", uri("old_mamba", version_a)), - LocationSpec("legacy", "event_report://127.0.0.1:8080/mem?source=legacy")}}, - }}; - ASSERT_EQ(EC_OK, meta_searcher_->BatchReplaceLocationSpecs(request_context_.get(), keys, seed_tasks, per_key_ec)); + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {key}, make_tasks(17), per_key_ec)); + ASSERT_EQ((std::vector{EC_OK}), per_key_ec); + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {key}, make_tasks(23), per_key_ec)); + ASSERT_EQ((std::vector{EC_OK}), per_key_ec); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), {key}, mask, location_maps)); + ASSERT_EQ(1u, location_maps.size()); + const auto location_it = location_maps[0].find(location_id); + ASSERT_NE(location_maps[0].end(), location_it); + ASSERT_TRUE(location_it->second); + ASSERT_EQ(1u, location_it->second->location_specs().size()); + EXPECT_EQ("event_report://size-hint:8080/mem?size=23", location_it->second->location_specs()[0].uri()); + std::uint64_t total_size = 0; + ASSERT_TRUE(location_it->second->GetValidatedTotalSize(total_size)); + EXPECT_EQ(23u, total_size); + EXPECT_EQ(23u, meta_indexer_->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); +} + +TEST_F(MetaSearcherTest, TestBatchMergeValidationHintNeverTrustsRetainedMalformedRecoveredSpec) { + const KeyType key = 10019; + const std::string location_id = "kvs#event_report_l2#mem#recovered-hint:8080"; + auto recovered = std::make_shared( + location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + 1, + std::vector{LocationSpec("broken", "event_report://recovered-hint:not-a-port/mem?size=9")}); + CacheLocationMapVector recovered_maps(1); + recovered_maps[0].emplace(location_id, std::move(recovered)); + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), {key}, recovered_maps, properties).ec); std::vector> tasks = {{ {location_id, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, CacheLocationStatus::CLS_SERVING, - {LocationSpec("linear_0", uri("new_linear", version_b))}}, + {LocationSpec("fresh", "event_report://recovered-hint:8080/mem?size=3")}}, }}; - ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {key}, tasks, per_key_ec)); std::vector location_maps; BlockMask mask; - ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); - ASSERT_EQ(1u, location_maps.size()); - ASSERT_EQ(1u, location_maps[0].size()); - const auto &after_version_change = location_maps[0].at(location_id)->location_specs(); - std::map after_version_change_specs; - for (const auto &spec : after_version_change) { - after_version_change_specs[spec.name()] = spec.uri(); + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), {key}, mask, location_maps)); + const auto &partially_repaired = location_maps[0].at(location_id); + ASSERT_TRUE(partially_repaired); + EXPECT_FALSE(partially_repaired->HasValidatedLocationSpecs()); + std::string_view version; + EXPECT_FALSE(SnapshotUriUtils::InspectSnapshotUriForVisibility( + partially_repaired->location_specs()[0].uri(), version, partially_repaired->HasValidatedLocationSpecs())); + + // Replacing the only malformed retained spec leaves a fully validated + // final vector. The proof may then be restored for subsequent queries. + tasks[0][0].specs = {LocationSpec("broken", "event_report://recovered-hint:8080/mem?size=5")}; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {key}, tasks, per_key_ec)); + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), {key}, mask, location_maps)); + const auto &repaired = location_maps[0].at(location_id); + ASSERT_TRUE(repaired->HasValidatedLocationSpecs()); + std::uint64_t total_size = 0; + ASSERT_TRUE(repaired->GetValidatedTotalSize(total_size)); + EXPECT_EQ(8u, total_size); + for (const auto &spec : repaired->location_specs()) { + EXPECT_TRUE(SnapshotUriUtils::InspectSnapshotUriForVisibility(spec.uri(), version, true)); } - ASSERT_EQ(3u, after_version_change_specs.size()); - EXPECT_EQ(uri("new_linear", version_b), after_version_change_specs["linear_0"]); - EXPECT_EQ(uri("old_mamba", version_a), after_version_change_specs["mamba_0"]); - EXPECT_EQ("event_report://127.0.0.1:8080/mem?source=legacy", after_version_change_specs["legacy"]); +} - tasks[0][0].specs = { - LocationSpec("linear_0", uri("newer_linear", version_b)), - LocationSpec("mamba_1", uri("new_mamba", version_b)), - }; - ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); +TEST_F(MetaSearcherTest, TestBatchMergeFlatTasksPreservesPerKeyRangesAndValidatesOffsets) { + const MetaSearcher::KeyVector keys{10010, 10011}; + auto make_task = [](std::string location_id, std::string spec_name) { + return MetaSearcher::MergeLocationSpecsTask{ + std::move(location_id), + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec(std::move(spec_name), "event_report://flat-task:8080/mem?size=1")}, + }; + }; + std::vector flat_tasks; + const InternedLocationId borrowed_location_id = + std::make_shared("kvs#event_report_l2#mem#flat-a:8080"); + MetaSearcher::MergeLocationSpecsTask inline_task{ + {}, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {}, + }; + inline_task.borrowed_interned_location_id = &borrowed_location_id; + inline_task.PushReportEventSpec(LocationSpec("tp0", "event_report://flat-task:8080/mem?size=1"), 1); + inline_task.prevalidated_total_size = MetaSearcher::PrevalidatedTotalSize(1); + ASSERT_TRUE(inline_task.specs.empty()); + ASSERT_TRUE(inline_task.inline_spec.has_value()); + EXPECT_EQ(1, borrowed_location_id.use_count()); + flat_tasks.push_back(std::move(inline_task)); + flat_tasks.push_back(make_task("kvs#event_report_l2#mem#flat-b:8080", "tp0")); + flat_tasks.push_back(make_task("kvs#event_report_l2#disk#flat-b:8080", "tp1")); + + std::vector per_key_ec; + ASSERT_EQ( + EC_OK, + meta_searcher_->BatchMergeLocationSpecsFlat(request_context_.get(), keys, {0, 1, 3}, flat_tasks, per_key_ec)); + EXPECT_EQ((std::vector{EC_OK, EC_OK}), per_key_ec); + EXPECT_EQ(2, borrowed_location_id.use_count()); + EXPECT_EQ("tp0", flat_tasks[0].SpecAt(0).name()); + EXPECT_TRUE(flat_tasks[0].SpecAt(0).uri().empty()); + EXPECT_EQ("tp0", flat_tasks[1].SpecAt(0).name()); + EXPECT_FALSE(flat_tasks[1].SpecAt(0).uri().empty()); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + ASSERT_EQ(2u, location_maps.size()); + EXPECT_EQ(1u, location_maps[0].size()); + EXPECT_EQ(2u, location_maps[1].size()); + EXPECT_TRUE(location_maps[0].count(*borrowed_location_id)); + + EXPECT_EQ( + EC_BADARGS, + meta_searcher_->BatchMergeLocationSpecsFlat(request_context_.get(), keys, {0, 2, 1}, flat_tasks, per_key_ec)); + EXPECT_EQ( + EC_BADARGS, + meta_searcher_->BatchMergeLocationSpecsFlat(request_context_.get(), keys, {0, 1, 2}, flat_tasks, per_key_ec)); +} + +TEST_F(MetaSearcherTest, TestBatchMergeFusedRmwTracksNewKeysAndCapacity) { + meta_indexer_->max_key_count_ = 1; + const KeyType existing_key = 10022; + const KeyType rejected_key = 10023; + const std::string location_a = "kvs#event_report_l2#mem#capacity-a:8080"; + const std::string location_b = "kvs#event_report_l2#disk#capacity-a:8080"; + auto make_task = [](const std::string &location_id, const std::string &name, const std::string &uri) { + return MetaSearcher::MergeLocationSpecsTask{location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec(name, uri)}}; + }; + + std::vector per_key_ec; + auto task_a = make_task(location_a, "tp0", "event_report://capacity-a:8080/mem?size=3"); + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {existing_key}, {{task_a}}, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK}), per_key_ec); + ASSERT_EQ(1u, meta_indexer_->GetKeyCount()); + + // A new target location under an existing key must not consume another + // key-count slot, even though the targeted read returns EC_NOENT for that + // location. + auto task_b = make_task(location_b, "tp1", "event_report://capacity-a:8080/disk?size=5"); + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {existing_key}, {{task_b}}, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK}), per_key_ec); + EXPECT_EQ(1u, meta_indexer_->GetKeyCount()); + + // Updating an existing target also remains admissible at capacity. + task_a.specs = {LocationSpec("tp0", "event_report://capacity-a:8080/mem?size=7")}; + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {existing_key}, {{task_a}}, per_key_ec)); + EXPECT_EQ((std::vector{EC_OK}), per_key_ec); + EXPECT_EQ(1u, meta_indexer_->GetKeyCount()); + + EXPECT_NE(EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {rejected_key}, {{task_a}}, per_key_ec)); + EXPECT_EQ((std::vector{EC_NOSPC}), per_key_ec); + EXPECT_EQ(1u, meta_indexer_->GetKeyCount()); + + // A capacity failure for a new key must not reject an existing-key + // update that happens to share the same internal upsert batch. The old + // two-phase path admitted the existing update in its merge phase. + task_a.specs = {LocationSpec("tp0", "event_report://capacity-a:8080/mem?size=9")}; + const auto rejected_task = make_task(location_a, "tp0", "event_report://capacity-rejected:8080/mem?size=4"); + EXPECT_EQ(EC_PARTIAL_OK, + meta_searcher_->BatchMergeLocationSpecs( + request_context_.get(), {existing_key, rejected_key}, {{task_a}, {rejected_task}}, per_key_ec)); + EXPECT_EQ((std::vector{EC_OK, EC_NOSPC}), per_key_ec); + EXPECT_EQ(1u, meta_indexer_->GetKeyCount()); + + std::vector locations; + BlockMask mask; + ASSERT_EQ(EC_OK, + meta_searcher_->BatchGetLocation(request_context_.get(), {existing_key, rejected_key}, mask, locations)); + ASSERT_EQ(2u, locations.size()); + EXPECT_EQ(2u, locations[0].size()); + ASSERT_TRUE(locations[0].at(location_a)); + EXPECT_NE(std::string::npos, locations[0].at(location_a)->location_specs()[0].uri().find("size=9")); + EXPECT_TRUE(locations[1].empty()); + EXPECT_EQ(14u, meta_indexer_->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); +} + +TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsNormalizesLegacyDuplicateNamesInPlace) { + const MetaSearcher::KeyVector keys = {10015}; + const auto seed = + std::make_shared(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + 3, + std::vector{ + LocationSpec("z", "event_report://legacy:8080/mem?source=first"), + LocationSpec("a", "event_report://legacy:8080/mem?source=untouched"), + LocationSpec("z", "event_report://legacy:8080/mem?source=last"), + }); + std::vector add_results; + ASSERT_EQ(EC_OK, + meta_searcher_->BatchAddLocation(request_context_.get(), keys, CacheLocationVector{seed}, add_results)); + ASSERT_EQ(1u, add_results.size()); + ASSERT_EQ(EC_OK, add_results[0].ec); + + std::vector per_key_ec; + std::vector> tasks = {{ + {add_results[0].location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("b", "event_report://legacy:8080/mem?source=new")}}, + }}; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK}), per_key_ec); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + const auto &specs = location_maps[0].at(add_results[0].location_id)->location_specs(); + ASSERT_EQ(3u, specs.size()); + EXPECT_EQ("a", specs[0].name()); + EXPECT_EQ("b", specs[1].name()); + EXPECT_EQ("z", specs[2].name()); + EXPECT_EQ("event_report://legacy:8080/mem?source=last", specs[2].uri()); +} + +TEST_F(MetaSearcherTest, TestPrefixMatchWithMambaByHostSupportsMultiwordSpecsAndLocationUnion) { + const MetaSearcher::KeyVector keys = {10030, 10031, 10032, 10033}; + const std::string host = "mamba-host:8080"; + std::vector full_names; + std::vector state_names; + for (std::size_t i = 0; i < 70; ++i) { + full_names.push_back("full_spec_" + std::to_string(i)); + } + for (std::size_t i = 0; i < 5; ++i) { + state_names.push_back("state_spec_" + std::to_string(i)); + } + const std::vector groups = { + LocationSpecGroup("F0", full_names), + LocationSpecGroup("L1", state_names), + }; + + std::vector> tasks(keys.size()); + for (std::size_t key_index = 0; key_index < keys.size(); ++key_index) { + std::vector l1_specs; + std::vector l2_specs; + auto append_spec = [&](const std::string &name, std::size_t ordinal) { + LocationSpec spec(name, "event_report://" + host + "/mem?name=" + name); + (ordinal % 2 == 0 ? l1_specs : l2_specs).push_back(std::move(spec)); + }; + for (std::size_t i = 0; i < full_names.size(); ++i) { + // The fourth key terminates the full-prefix by omitting one bit in + // the second uint64_t word. + if (key_index == 3 && i == 69) { + continue; + } + append_spec(full_names[i], i); + } + if (key_index != 1) { + for (std::size_t i = 0; i < state_names.size(); ++i) { + append_spec(state_names[i], full_names.size() + i); + } + } + tasks[key_index].push_back({"kvs#event_report_l1p5#mem#" + host, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + CacheLocationStatus::CLS_SERVING, + std::move(l1_specs)}); + tasks[key_index].push_back({"kvs#event_report_l2#mem#" + host, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + std::move(l2_specs)}); + } + + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK, EC_OK, EC_OK, EC_OK}), per_key_ec); + + std::vector matches; + ASSERT_EQ( + EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost(request_context_.get(), keys, false, {"mem"}, groups, matches)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(host, matches[0].host_ip_port); + EXPECT_EQ(3, matches[0].local); + + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost(request_context_.get(), keys, true, {"mem"}, groups, matches)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(host, matches[0].host_ip_port); + EXPECT_EQ(1, matches[0].local); +} + +TEST_F(MetaSearcherTest, TestMambaProgressiveBitsetMatchesReferenceAcrossParallelReadChunks) { + constexpr size_t kKeyCount = 40000; + constexpr size_t kFullPrefix = 37001; + constexpr size_t kLastStateBeforeEaglePop = 36960; + const std::set state_indices = {0, 63, 64, 4095, 4096, 20479, 20480, kLastStateBeforeEaglePop, 37000}; + const std::string host = "parallel-mamba-host:8080"; + const std::string location_id = "kvs#event_report_l1p5#mem#" + host; + + // The fixture's normal capacity is intentionally small. Recreate only + // this test's pure-local index with enough entries to cross the 4096-key + // probe and multiple 16384-key parallel read ranges. + meta_searcher_.reset(); + meta_indexer_.reset(); + meta_indexer_ = CreateMetaIndexer(kKeyCount + 16); + ASSERT_TRUE(meta_indexer_); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 8)); + meta_searcher_ = std::make_shared(meta_indexer_, dummy_check_loc_data_exist, dummy_submit_del_req); + + auto make_location = [&](bool has_full, bool has_state) { + std::vector specs; + if (has_full) { + specs.emplace_back("full_0", "event_report://" + host + "/mem"); + } + if (has_state) { + specs.emplace_back("state_0", "event_report://" + host + "/mem"); + } + return std::make_shared(location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + specs.size(), + std::move(specs)); + }; + const auto full_only = make_location(true, false); + const auto full_and_state = make_location(true, true); + const auto state_only = make_location(false, true); + const auto other_only = std::make_shared( + location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + 1, + std::vector{LocationSpec("other", "event_report://" + host + "/mem")}); + + MetaSearcher::KeyVector keys(kKeyCount); + std::iota(keys.begin(), keys.end(), 20000); + CacheLocationMapVector location_maps(kKeyCount); + for (size_t key_index = 0; key_index < kKeyCount; ++key_index) { + const bool has_full = key_index < kFullPrefix; + const bool has_state = state_indices.find(key_index) != state_indices.end(); + CacheLocationConstPtr location; + if (has_full && has_state) { + location = full_and_state; + } else if (has_full) { + location = full_only; + } else if (has_state) { + location = state_only; + } else { + // Keep the reporter present beyond the full prefix without + // accidentally satisfying either required group. + location = other_only; + } + location_maps[key_index].emplace(location_id, std::move(location)); + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + auto run = [&](bool use_eagle_pop, size_t p2p_host_count) { + std::vector matches; + EXPECT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, use_eagle_pop, {"mem"}, groups, matches, nullptr, p2p_host_count)); + EXPECT_EQ(1u, matches.size()); + return matches.empty() ? int64_t{-1} : matches.front().local; + }; + + // Enabling P2P must not change the independently computed local result, + // including the state checkpoint immediately before Eagle pop. + EXPECT_EQ(static_cast(kFullPrefix), run(false, 0)); + EXPECT_EQ(run(false, 1), run(false, 0)); + EXPECT_EQ(static_cast(kLastStateBeforeEaglePop + 1), run(true, 0)); + EXPECT_EQ(run(true, 1), run(true, 0)); +} + +TEST_F(MetaSearcherTest, TestStreamingHostP2PMatchesMaterializedReference) { + constexpr size_t kKeyCount = 6000; + constexpr size_t kHostCount = 7; + constexpr size_t kLongMambaFullPrefix = 5501; + constexpr size_t kOrdinaryP2PGap = 5000; + + meta_searcher_.reset(); + meta_indexer_.reset(); + meta_indexer_ = CreateMetaIndexer(kKeyCount + 16); + ASSERT_TRUE(meta_indexer_); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 8)); + meta_searcher_ = std::make_shared(meta_indexer_, dummy_check_loc_data_exist, dummy_submit_del_req); + + std::vector hosts; + hosts.reserve(kHostCount); + for (size_t host_index = 0; host_index < kHostCount; ++host_index) { + hosts.push_back("random-reference-host-" + std::to_string(host_index) + ":8080"); + } + + MetaSearcher::KeyVector keys(kKeyCount); + std::iota(keys.begin(), keys.end(), 300000); + CacheLocationMapVector location_maps(kKeyCount); + std::array reference_matrices; + for (auto &matrix : reference_matrices) { + matrix.all.resize(kKeyCount); + matrix.vineyard.resize(kKeyCount); + } + std::mt19937 random(0x51A7E5u); + auto add_location = [&](size_t key_index, + size_t host_index, + std::string_view medium, + DataStorageType type, + CacheLocationStatus status, + std::vector spec_names, + bool valid_uri) { + const std::string type_name = + type == DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2 ? "event_report_l2" : "event_report_l1p5"; + const std::string location_id = "kvs#" + type_name + "#" + std::string(medium) + "#" + hosts[host_index]; + if (status == CacheLocationStatus::CLS_SERVING && valid_uri) { + for (size_t filter_index = 0; filter_index < reference_matrices.size(); ++filter_index) { + const bool medium_matches = filter_index == 0 || (filter_index == 1 && medium == "mem") || + (filter_index == 2 && medium == "ssd"); + if (!medium_matches) { + continue; + } + auto &matrix = reference_matrices[filter_index]; + matrix.all[key_index][hosts[host_index]].insert(spec_names.begin(), spec_names.end()); + if (type == DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2) { + matrix.vineyard[key_index][hosts[host_index]].insert(spec_names.begin(), spec_names.end()); + } + } + } + std::vector specs; + specs.reserve(spec_names.size()); + for (auto &name : spec_names) { + specs.emplace_back(std::move(name), + valid_uri ? "event_report://" + hosts[host_index] + "/" + std::string(medium) + : "missing-uri-scheme"); + } + location_maps[key_index].emplace( + location_id, std::make_shared(location_id, status, type, specs.size(), std::move(specs))); + }; + + for (size_t key_index = 0; key_index < kKeyCount; ++key_index) { + // Host zero guarantees that both algorithms must cross the synchronous + // 4096-key probe. Its ordinary prefix spans the request, while its + // Mamba full prefix stops inside the parallel suffix. + std::vector primary_specs = key_index < kLongMambaFullPrefix + ? std::vector{"full_a", "full_b"} + : std::vector{"other"}; + if (key_index == 0 || key_index % 97 == 0 || key_index == 4095 || key_index == 4096 || + key_index == kLongMambaFullPrefix - 1) { + primary_specs.push_back("state_a"); + primary_specs.push_back("state_b"); + } + add_location(key_index, + 0, + "mem", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + CacheLocationStatus::CLS_SERVING, + std::move(primary_specs), + true); + if (key_index % 3 != 1) { + add_location(key_index, + 0, + "ssd", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {"ssd_aux"}, + true); + } + + // The last two host ids are reserved for a deterministic ordinary-P2P + // extension below; the other hosts retain the mixed random model. + for (size_t host_index = 1; host_index + 2 < kHostCount; ++host_index) { + const uint32_t sample = random(); + if (key_index != 0 && sample % 5 == 0) { + continue; + } + const auto status = key_index == 0 || sample % 7 > 2 ? CacheLocationStatus::CLS_SERVING + : static_cast(sample % 3 + 1); + std::vector names; + if (key_index == 0 || (sample & 1U) != 0) { + names.push_back("full_a"); + } + if (key_index == 0 || (sample & 2U) != 0) { + names.push_back("full_b"); + } + if (key_index == 0 || sample % 11 == 0) { + names.push_back("state_a"); + names.push_back("state_b"); + } + if (names.empty()) { + names.push_back("other"); + } + add_location(key_index, + host_index, + sample % 2 == 0 ? "mem" : "ssd", + sample % 3 == 0 ? DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2 + : DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + status, + std::move(names), + key_index == 0 || sample % 13 != 0); + } + + const size_t ordinary_target = kHostCount - 2; + const size_t ordinary_peer = kHostCount - 1; + if (key_index != kOrdinaryP2PGap) { + add_location(key_index, + ordinary_target, + "mem", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + CacheLocationStatus::CLS_SERVING, + {"ordinary_only"}, + true); + } else { + add_location(key_index, + ordinary_peer, + "mem", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {"ordinary_peer"}, + true); + } + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + + const std::vector> medium_filters = {{}, {"mem"}, {"ssd"}}; + const std::vector groups = { + LocationSpecGroup("F0", {"full_a", "full_b"}), + LocationSpecGroup("L0", {"state_a", "state_b"}), + }; + bool saw_ordinary_fetch = false; + bool saw_ordinary_extension = false; + bool saw_mamba_fetch = false; + bool saw_mamba_extension = false; + for (size_t filter_index = 0; filter_index < medium_filters.size(); ++filter_index) { + const auto &medium_filter = medium_filters[filter_index]; + const auto &reference_matrix = reference_matrices[filter_index]; + for (const bool use_eagle_pop : {false, true}) { + for (const size_t p2p_host_count : {size_t{0}, size_t{1}, size_t{3}, kHostCount}) { + SCOPED_TRACE("filter=" + std::to_string(filter_index) + " eagle=" + std::to_string(use_eagle_pop) + + " p2p=" + std::to_string(p2p_host_count)); + std::vector actual; + ASSERT_EQ( + EC_OK, + meta_searcher_->PrefixMatchByHost( + request_context_.get(), keys, use_eagle_pop, medium_filter, actual, nullptr, p2p_host_count)); + const auto expected_ordinary = + ReferenceOrdinaryMatches(keys, reference_matrix, use_eagle_pop, p2p_host_count); + EXPECT_EQ(ToMatchTuples(expected_ordinary), ToMatchTuples(actual)); + for (const auto &match : expected_ordinary) { + saw_ordinary_fetch = saw_ordinary_fetch || match.p2p_1_fetch > 0; + saw_ordinary_extension = saw_ordinary_extension || match.p2p_1_total_match > match.local; + } + + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost(request_context_.get(), + keys, + use_eagle_pop, + medium_filter, + groups, + actual, + nullptr, + p2p_host_count)); + for (const auto &match : actual) { + EXPECT_GE(match.p2p_1_total_match, match.local); + } + const auto expected_mamba = + ReferenceMambaMatches(keys, reference_matrix, use_eagle_pop, groups, p2p_host_count); + EXPECT_EQ(ToMatchTuples(expected_mamba), ToMatchTuples(actual)); + for (const auto &match : expected_mamba) { + saw_mamba_fetch = saw_mamba_fetch || match.p2p_1_fetch > 0; + saw_mamba_extension = saw_mamba_extension || match.p2p_1_total_match > match.local; + } + } + } + } + EXPECT_TRUE(saw_ordinary_fetch); + EXPECT_TRUE(saw_ordinary_extension); + EXPECT_TRUE(saw_mamba_fetch); + EXPECT_TRUE(saw_mamba_extension); +} + +TEST_F(MetaSearcherTest, TestStreamingPrefixP2PRetainsOnlyTheFinalTopHostPlan) { + const MetaSearcher::KeyVector keys = {10050, 10051, 10052, 10053, 10054}; + const std::string host_a = "stream-host-a:8080"; + const std::string host_b = "stream-host-b:8080"; + const std::string host_c = "stream-host-c:8080"; + const std::string peer = "stream-peer:8080"; + const auto host_a_task = MakeEventReportTask(host_a, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5); + const auto host_b_task = MakeEventReportTask(host_b, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5); + const auto host_c_task = MakeEventReportTask(host_c, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5); + const auto peer_task = MakeEventReportTask(peer, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2); + std::vector> tasks = { + {host_a_task, host_b_task, host_c_task}, + {host_b_task, host_c_task, peer_task}, + {host_c_task, peer_task}, + {peer_task}, + {peer_task}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_TRUE(std::all_of(per_key_ec.begin(), per_key_ec.end(), [](ErrorCode ec) { return ec == EC_OK; })); + + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches, nullptr, 1)); + auto find_match = [&matches](const std::string &host) { + return std::find_if( + matches.begin(), matches.end(), [&host](const auto &match) { return match.host_ip_port == host; }); + }; + ASSERT_EQ(3u, matches.size()); + const auto match_a = find_match(host_a); + const auto match_b = find_match(host_b); + const auto match_c = find_match(host_c); + ASSERT_NE(matches.end(), match_a); + ASSERT_NE(matches.end(), match_b); + ASSERT_NE(matches.end(), match_c); + EXPECT_EQ((std::tuple{1, 0, 1}), + std::make_tuple(match_a->local, match_a->p2p_1_fetch, match_a->p2p_1_total_match)); + EXPECT_EQ((std::tuple{2, 0, 2}), + std::make_tuple(match_b->local, match_b->p2p_1_fetch, match_b->p2p_1_total_match)); + EXPECT_EQ((std::tuple{3, 2, 5}), + std::make_tuple(match_c->local, match_c->p2p_1_fetch, match_c->p2p_1_total_match)); +} + +TEST_F(MetaSearcherTest, TestStreamingMambaP2PReusesBatchAndDeduplicatesFetchedKeys) { + auto *backend = ReplaceWithCountingBatchedLocationBackend(); + const MetaSearcher::KeyVector stored_keys = {10060, 10061, 10062, 10063}; + const MetaSearcher::KeyVector query_keys = {10060, 10061, 10061, 10062, 10063}; + const std::string target = "mamba-stream-target:8080"; + const std::string peer_a = "mamba-stream-peer-a:8080"; + const std::string peer_b = "mamba-stream-peer-b:8080"; + const auto target_full = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"full_0", "state_0"}); + const auto target_state = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"state_0"}); + const auto a_full = MakeEventReportTask(peer_a, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_0"}); + const auto b_full = MakeEventReportTask(peer_b, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_0"}); + std::vector> tasks = { + {target_full}, + {target_state, a_full, b_full}, + {target_state, b_full}, + {target_state, a_full}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), stored_keys, tasks, per_key_ec)); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + backend->ResetLocationValueReadCount(); + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), query_keys, false, {"mem"}, groups, matches, nullptr, 1)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(target, matches[0].host_ip_port); + EXPECT_EQ((std::tuple{1, 2, 4}), + std::make_tuple(matches[0].local, matches[0].p2p_1_fetch, matches[0].p2p_1_total_match)); + EXPECT_EQ(1u, backend->LocationValueReadCount()); +} + +TEST_F(MetaSearcherTest, TestBatchedMambaP2PDefersSuffixErrorUntilNeeded) { + auto *backend = ReplaceWithCountingBatchedLocationBackend(); + ASSERT_TRUE(backend); + const MetaSearcher::KeyVector keys = {10064, 10065, 10066}; + const std::string target = "mamba-batched-stop:8080"; + const auto target_full = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"full_0"}); + const auto target_other = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"other"}); + std::vector> tasks = { + {target_full}, + {target_other}, + {target_other}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_TRUE(std::all_of(per_key_ec.begin(), per_key_ec.end(), [](ErrorCode ec) { return ec == EC_OK; })); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + backend->SetFailedKey(keys.back()); + backend->ResetLocationValueReadCount(); + std::vector matches; + + // The first two successful keys prove that the only candidate has no + // usable local checkpoint. No P2P plan can be selected, so the batched + // backend's later error is outside the required prefix just as it is for + // the progressive backend. + EXPECT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + EXPECT_TRUE(matches.empty()); + EXPECT_EQ(1u, backend->LocationValueReadCount()); + + // Give the target one valid local checkpoint and a peer that crosses the + // local full-prefix stop. Planning must now reach the failed suffix and + // propagate the original error without issuing another backend read. + const auto target_state = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"state_0"}); + const auto peer_full = + MakeEventReportTask("mamba-batched-peer:8080", DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_0"}); + tasks = { + {target_state}, + {peer_full}, + {peer_full}, + }; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_TRUE(std::all_of(per_key_ec.begin(), per_key_ec.end(), [](ErrorCode ec) { return ec == EC_OK; })); + + backend->ResetLocationValueReadCount(); + EXPECT_EQ(EC_TIMEOUT, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + EXPECT_TRUE(matches.empty()); + EXPECT_EQ(1u, backend->LocationValueReadCount()); +} + +TEST_F(MetaSearcherTest, TestStreamingMambaP2PMergesIndependentFullGroupPeers) { + const MetaSearcher::KeyVector keys = {10070, 10071, 10072, 10073}; + const std::string target = "mamba-cross-target:8080"; + const std::string peer_a = "mamba-cross-peer-a:8080"; + const std::string peer_b = "mamba-cross-peer-b:8080"; + std::vector> tasks = { + {MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"full_a", "full_b"})}, + {MakeEventReportTask(peer_a, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_a"}), + MakeEventReportTask(peer_b, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_b"})}, + {MakeEventReportTask(peer_a, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_a"})}, + {MakeEventReportTask(peer_a, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_a"})}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_a"}), + LocationSpecGroup("F1", {"full_b"}), + LocationSpecGroup("L0", {"full_a", "full_b"}), + }; + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(target, matches[0].host_ip_port); + // full_b fixes the combined prefix at key 2, while full_a's independently + // selected peer covers keys 1..3. All three selected fetch keys remain part + // of the public counter even though only key 1 extends the final match. + EXPECT_EQ((std::tuple{1, 3, 2}), + std::make_tuple(matches[0].local, matches[0].p2p_1_fetch, matches[0].p2p_1_total_match)); +} + +TEST_F(MetaSearcherTest, TestStreamingMambaP2PPreservesEmptySpecGroupSemantics) { + auto *backend = ReplaceWithPrefixReadBackend(); + ASSERT_TRUE(backend); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 1)); + const MetaSearcher::KeyVector keys = {10075, 10076, 10077}; + const std::string target = "mamba-empty-group-target:8080"; + const std::string peer = "mamba-empty-group-peer:8080"; + const auto target_task = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"full_0", "state_0"}); + const auto peer_task = + MakeEventReportTask(peer, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_0", "state_0"}); + std::vector> tasks = { + {target_task}, + {peer_task}, + {peer_task}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + + // Empty groups are valid today and their all-of predicate is true even + // when the target host is absent. The non-empty state group makes the + // empty full-group mask span allocated words and still requires a final + // state-peer projection. + const std::vector groups = { + LocationSpecGroup("F_empty", {}), + LocationSpecGroup("L_empty", {}), + LocationSpecGroup("L0", {"state_0"}), + }; + backend->ResetReadCounts(); + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(target, matches[0].host_ip_port); + EXPECT_EQ((std::tuple{1, 2, 3}), + std::make_tuple(matches[0].local, matches[0].p2p_1_fetch, matches[0].p2p_1_total_match)); + // Three Mamba phases over three keys. The empty full group is a vacuous + // predicate and must not enlarge the final peer-validation range. + EXPECT_EQ(9u, backend->CompactReadCount()); + + backend->ResetReadCounts(); + const std::vector all_empty_groups = { + LocationSpecGroup("F_empty", {}), + LocationSpecGroup("L_empty", {}), + }; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, all_empty_groups, matches, nullptr, 1)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ((std::tuple{3, 0, 3}), + std::make_tuple(matches[0].local, matches[0].p2p_1_fetch, matches[0].p2p_1_total_match)); + EXPECT_EQ(keys.size(), backend->CompactReadCount()); +} + +TEST_F(MetaSearcherTest, TestStreamingMambaP2PHandlesErrorsAndMetadataChangesBetweenPasses) { + auto *backend = ReplaceWithMutatingCompactLocationBackend(); + ASSERT_TRUE(backend); + const MetaSearcher::KeyVector keys = {10080, 10081, 10082}; + const std::string target = "mamba-mutating-target:8080"; + const std::string peer = "mamba-mutating-peer:8080"; + const auto target_task = + MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"full_0", "state_0"}); + const auto peer_task = MakeEventReportTask(peer, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_0"}); + std::vector> tasks = { + {target_task}, + {target_task}, + {peer_task}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_TRUE(std::all_of(per_key_ec.begin(), per_key_ec.end(), [](ErrorCode ec) { return ec == EC_OK; })); + + auto replacement = [&target_task, &target](std::vector spec_names) { + std::vector specs; + specs.reserve(spec_names.size()); + for (const auto &name : spec_names) { + specs.emplace_back(name, "event_report://" + target + "/mem"); + } + return std::make_shared(target_task.location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + specs.size(), + std::move(specs)); + }; + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + std::vector matches; + + // A hard metadata error that appears only during peer planning must stop + // before the final pass and must not expose a result derived from pass one. + backend->FailOnRead(/*read_number*/ 2, keys[1], EC_TIMEOUT); + EXPECT_EQ(EC_TIMEOUT, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + EXPECT_TRUE(matches.empty()); + EXPECT_EQ(2u, backend->CompactReadCount()); + + // The same error in the final pass takes precedence over the peer-plan + // mismatch caused by the truncated view; partial matches remain hidden. + backend->FailOnRead(/*read_number*/ 3, keys[1], EC_TIMEOUT); + EXPECT_EQ(EC_TIMEOUT, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + EXPECT_TRUE(matches.empty()); + EXPECT_EQ(3u, backend->CompactReadCount()); + + // The first two passes observe a two-key local result. Removing only the + // second key's state in the final pass must not make P2P total regress + // below the already returned local count. + backend->ReplaceOnRead(/*read_number*/ 3, keys[1], target_task.location_id, replacement({"full_0"})); + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(target, matches[0].host_ip_port); + EXPECT_EQ((std::tuple{2, 1, 2}), + std::make_tuple(matches[0].local, matches[0].p2p_1_fetch, matches[0].p2p_1_total_match)); + EXPECT_EQ(3u, backend->CompactReadCount()); + + // Removing a required full spec invalidates the peer plan selected by the + // second pass. Returning a partial result would be unsafe, so the final + // pass must reject the inconsistent view. + backend->ReplaceOnRead(/*read_number*/ 3, keys[1], target_task.location_id, replacement({"state_0"})); + EXPECT_EQ(EC_MISMATCH, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + EXPECT_TRUE(matches.empty()); + EXPECT_EQ(3u, backend->CompactReadCount()); + + // If the final view gains a local full spec, no remote data is actually + // needed at that key. Fetched count must come from the validated final + // plan rather than the earlier planning pass. + backend->ReplaceOnRead( + /*read_number*/ 3, keys[2], peer_task.location_id, replacement({"full_0", "state_0"})); + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, 1)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ((std::tuple{2, 0, 3}), + std::make_tuple(matches[0].local, matches[0].p2p_1_fetch, matches[0].p2p_1_total_match)); + EXPECT_EQ(3u, backend->CompactReadCount()); +} + +TEST_F(MetaSearcherTest, TestStreamingMambaP2PAggregatesBackendTimeAcrossPasses) { + auto *backend = ReplaceWithMutatingCompactLocationBackend(); + ASSERT_TRUE(backend); + backend->SetReadDelay(std::chrono::milliseconds(3)); + + const MetaSearcher::KeyVector keys = {10090, 10091}; + const std::string target = "mamba-metric-target:8080"; + const std::string peer = "mamba-metric-peer:8080"; + std::vector> tasks = { + {MakeEventReportTask(target, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, {"full_0", "state_0"})}, + {MakeEventReportTask(peer, DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, {"full_0", "state_0"})}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + + auto registry = std::make_shared(); + auto collector = std::make_shared(registry); + ASSERT_TRUE(collector->Init()); + RequestContext metrics_context("mamba_metric", collector); + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + &metrics_context, keys, false, {"mem"}, groups, matches, nullptr, 1)); + ASSERT_EQ(3u, backend->CompactReadCount()); + EXPECT_GE(collector->get_meta_indexer_get_io_time_us_metrics(), 8000.0); +} + +TEST_F(MetaSearcherTest, TestPrefixMatchByHostExcludesEveryNonServingStatusWithAndWithoutP2P) { + auto *backend = ReplaceWithPrefixReadBackend(); + ASSERT_TRUE(backend); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 1)); + const MetaSearcher::KeyVector keys = {10040, 10041}; + const std::vector> hosts = { + {"serving-host:8080", CacheLocationStatus::CLS_SERVING}, + {"new-host:8080", CacheLocationStatus::CLS_NEW}, + {"writing-host:8080", CacheLocationStatus::CLS_WRITING}, + {"deleting-host:8080", CacheLocationStatus::CLS_DELETING}, + {"not-found-host:8080", CacheLocationStatus::CLS_NOT_FOUND}, + }; + CacheLocationMapVector location_maps(keys.size()); + for (auto &location_map : location_maps) { + for (const auto &[host, status] : hosts) { + const std::string location_id = "kvs#event_report_l1p5#mem#" + host; + location_map.emplace( + location_id, + std::make_shared( + location_id, + status, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + 2, + std::vector{LocationSpec("full_0", "event_report://" + host + "/mem"), + LocationSpec("state_0", "event_report://" + host + "/mem")})); + } + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + + auto verify_prefix = [&](size_t p2p_host_count) { + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchByHost( + request_context_.get(), keys, false, {"mem"}, matches, nullptr, p2p_host_count)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ("serving-host:8080", matches[0].host_ip_port); + EXPECT_EQ(2, matches[0].local); + EXPECT_EQ(0, matches[0].p2p_1_fetch); + EXPECT_EQ(2, matches[0].p2p_1_total_match); + }; + verify_prefix(0); + verify_prefix(5); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + auto verify_mamba = [&](size_t p2p_host_count) { + std::vector matches; + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, p2p_host_count)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ("serving-host:8080", matches[0].host_ip_port); + EXPECT_EQ(2, matches[0].local); + EXPECT_EQ(0, matches[0].p2p_1_fetch); + EXPECT_EQ(2, matches[0].p2p_1_total_match); + }; + verify_mamba(0); + backend->ResetReadCounts(); + verify_mamba(5); + // Local scoring plus peer planning are sufficient when no peer is + // selected; the final validation pass is reserved for an actual plan. + EXPECT_EQ(2 * keys.size(), backend->CompactReadCount()); +} + +TEST_F(MetaSearcherTest, TestHostPrefixQueriesPropagateMetadataErrorsWithAndWithoutP2P) { + auto *backend = ReplaceWithPrefixReadBackend(); + ASSERT_TRUE(backend); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 1)); + + const MetaSearcher::KeyVector keys = {10100, 10101, 10102}; + const std::string host = "error-host:8080"; + const std::string location_id = "kvs#event_report_l1p5#mem#" + host; + CacheLocationMapVector location_maps(keys.size()); + for (auto &location_map : location_maps) { + location_map.emplace( + location_id, + std::make_shared( + location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + 2, + std::vector{LocationSpec("full_0", "event_report://" + host + "/mem"), + LocationSpec("state_0", "event_report://" + host + "/mem")})); + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + backend->SetFailedKey(keys[1]); + + for (size_t p2p_host_count : {size_t{0}, size_t{1}}) { + std::vector matches; + EXPECT_EQ(EC_TIMEOUT, + meta_searcher_->PrefixMatchByHost( + request_context_.get(), keys, false, {"mem"}, matches, nullptr, p2p_host_count)); + EXPECT_TRUE(matches.empty()); + } + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + for (size_t p2p_host_count : {size_t{0}, size_t{1}}) { + std::vector matches; + EXPECT_EQ(EC_TIMEOUT, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), keys, false, {"mem"}, groups, matches, nullptr, p2p_host_count)); + EXPECT_TRUE(matches.empty()); + } +} + +TEST_F(MetaSearcherTest, TestProgressiveHostPrefixOrdersParallelErrorsAndVisitorCancellation) { + constexpr size_t kKeyCount = 40000; + constexpr size_t kFailureIndex = 20000; + constexpr size_t kHostStopIndex = 97; + constexpr size_t kProbeKeyCount = 4096; + const std::string host = "progressive-error-host:8080"; + const std::string other_host = "progressive-other-host:8080"; + + meta_searcher_.reset(); + meta_indexer_.reset(); + meta_indexer_ = CreateMetaIndexer(kKeyCount + 16); + ASSERT_TRUE(meta_indexer_); + auto *backend = ReplaceWithPrefixReadBackend(); + ASSERT_TRUE(backend); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 8)); + meta_searcher_ = std::make_shared(meta_indexer_, dummy_check_loc_data_exist, dummy_submit_del_req); + + auto make_location = [](const std::string &target_host) { + const std::string location_id = "kvs#event_report_l1p5#mem#" + target_host; + return std::make_shared( + location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + 2, + std::vector{LocationSpec("full_0", "event_report://" + target_host + "/mem"), + LocationSpec("state_0", "event_report://" + target_host + "/mem")}); + }; + const auto location = make_location(host); + const auto other_location = make_location(other_host); + + MetaSearcher::KeyVector keys(kKeyCount); + std::iota(keys.begin(), keys.end(), 70000); + CacheLocationMapVector location_maps(kKeyCount); + for (auto &location_map : location_maps) { + location_map.emplace(location->id(), location); + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + + const KeyType other_host_key = keys.back() + 1; + CacheLocationMapVector other_location_map(1); + other_location_map[0].emplace(other_location->id(), other_location); + ASSERT_EQ(EC_OK, + meta_indexer_ + ->Put(request_context_.get(), MetaSearcher::KeyVector{other_host_key}, other_location_map, properties) + .ec); + backend->SetFailedKey(keys[kFailureIndex]); + + std::vector matches; + EXPECT_EQ(EC_TIMEOUT, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches)); + EXPECT_TRUE(matches.empty()); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + EXPECT_EQ( + EC_TIMEOUT, + meta_searcher_->PrefixMatchWithMambaByHost(request_context_.get(), keys, false, {"mem"}, groups, matches)); + EXPECT_TRUE(matches.empty()); + + // The same later backend failure is irrelevant once every candidate host + // has already stopped inside the first probe. It must neither leak as an + // RPC error nor cause the million-key-style suffix to be read. + MetaSearcher::KeyVector early_stop_keys = keys; + early_stop_keys[kHostStopIndex] = other_host_key; + backend->ResetReadCounts(); + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchByHost(request_context_.get(), early_stop_keys, false, {"mem"}, matches)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(host, matches[0].host_ip_port); + EXPECT_EQ(static_cast(kHostStopIndex), matches[0].local); + EXPECT_EQ(kProbeKeyCount, backend->CompactReadCount()); + + backend->ResetReadCounts(); + ASSERT_EQ(EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost( + request_context_.get(), early_stop_keys, false, {"mem"}, groups, matches)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ(host, matches[0].host_ip_port); + EXPECT_EQ(static_cast(kHostStopIndex), matches[0].local); + EXPECT_EQ(kProbeKeyCount, backend->CompactReadCount()); +} + +TEST_F(MetaSearcherTest, TestProgressiveHostPrefixIgnoresOnlyErrorsBeyondEveryCandidateStop) { + constexpr size_t kKeyCount = 33000; + constexpr size_t kFirstHostStop = 5000; + constexpr size_t kLastHostStop = 25000; + constexpr size_t kRequiredFailureIndex = 20000; + constexpr size_t kIrrelevantFailureIndex = 30000; + + meta_searcher_.reset(); + meta_indexer_.reset(); + meta_indexer_ = CreateMetaIndexer(kKeyCount + 16); + ASSERT_TRUE(meta_indexer_); + auto *backend = ReplaceWithPrefixReadBackend(); + ASSERT_TRUE(backend); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 8)); + meta_searcher_ = std::make_shared(meta_indexer_, dummy_check_loc_data_exist, dummy_submit_del_req); + + const std::array hosts = {"first-stop-host:8080", "last-stop-host:8080"}; + std::array locations; + for (size_t host_index = 0; host_index < hosts.size(); ++host_index) { + const std::string location_id = "kvs#event_report_l1p5#mem#" + hosts[host_index]; + locations[host_index] = std::make_shared( + location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + 2, + std::vector{LocationSpec("full_0", "event_report://" + hosts[host_index] + "/mem"), + LocationSpec("state_0", "event_report://" + hosts[host_index] + "/mem")}); + } + + MetaSearcher::KeyVector keys(kKeyCount); + std::iota(keys.begin(), keys.end(), 500000); + CacheLocationMapVector location_maps(kKeyCount); + for (size_t key_index = 0; key_index < kKeyCount; ++key_index) { + if (key_index < kFirstHostStop) { + location_maps[key_index].emplace(locations[0]->id(), locations[0]); + } + if (key_index < kLastHostStop) { + location_maps[key_index].emplace(locations[1]->id(), locations[1]); + } + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + + const std::vector groups = { + LocationSpecGroup("F0", {"full_0"}), + LocationSpecGroup("L0", {"state_0"}), + }; + auto expect_success = [&] { + std::vector matches; + ASSERT_EQ(EC_OK, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches)); + ASSERT_EQ(2u, matches.size()); + EXPECT_EQ(kFirstHostStop, matches[0].local); + EXPECT_EQ(kLastHostStop, matches[1].local); + + ASSERT_EQ( + EC_OK, + meta_searcher_->PrefixMatchWithMambaByHost(request_context_.get(), keys, false, {"mem"}, groups, matches)); + ASSERT_EQ(2u, matches.size()); + EXPECT_EQ(kFirstHostStop, matches[0].local); + EXPECT_EQ(kLastHostStop, matches[1].local); + }; + auto expect_timeout = [&] { + std::vector matches; + EXPECT_EQ(EC_TIMEOUT, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches)); + EXPECT_TRUE(matches.empty()); + EXPECT_EQ( + EC_TIMEOUT, + meta_searcher_->PrefixMatchWithMambaByHost(request_context_.get(), keys, false, {"mem"}, groups, matches)); + EXPECT_TRUE(matches.empty()); + }; + + // This error is in a range that may already be in flight, but both hosts + // have a proven shorter prefix. The speculative read must not fail the + // request. + backend->SetFailedKey(keys[kIrrelevantFailureIndex]); + expect_success(); + + // The second host still needs this key, so the same backend error class is + // now part of the observable prefix and must be propagated. + backend->SetFailedKey(keys[kRequiredFailureIndex]); + expect_timeout(); +} + +TEST_F(MetaSearcherTest, TestHostPrefixQueryStopsPureLocalReadAfterFirstBoundedWindow) { + auto *backend = ReplaceWithPrefixReadBackend(); + ASSERT_TRUE(backend); + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 1)); + + constexpr size_t kKeyCount = 8192; + constexpr size_t kFirstHostMiss = 97; + constexpr size_t kExpectedReadWindow = 4096; + MetaSearcher::KeyVector keys(kKeyCount); + std::iota(keys.begin(), keys.end(), 10200); + CacheLocationMapVector location_maps(kKeyCount); + for (size_t i = 0; i < kKeyCount; ++i) { + const std::string host = i == kFirstHostMiss ? "other-host:8080" : "prefix-host:8080"; + const std::string location_id = "kvs#event_report_l1p5#mem#" + host; + location_maps[i].emplace(location_id, + std::make_shared(location_id, + CacheLocationStatus::CLS_SERVING, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + 1, + std::vector{LocationSpec( + "tp0", "event_report://" + host + "/mem")})); + } + PropertyMapVector properties; + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), keys, location_maps, properties).ec); + + backend->ResetReadCounts(); + std::vector matches; + ASSERT_EQ(EC_OK, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches)); + ASSERT_EQ(1u, matches.size()); + EXPECT_EQ("prefix-host:8080", matches[0].host_ip_port); + EXPECT_EQ(static_cast(kFirstHostMiss), matches[0].local); + EXPECT_EQ(0u, backend->LocationValueReadCount()); + EXPECT_EQ(kExpectedReadWindow, backend->CompactReadCount()); +} + +TEST_F(MetaSearcherTest, TestPrefixMatchByHostSupportsMoreThanOnePresenceWord) { + const MetaSearcher::KeyVector keys = {10020, 10021, 10022}; + std::vector> tasks(keys.size()); + auto host_name = [](std::size_t index) { + return std::string("host-") + (index < 10 ? "0" : "") + std::to_string(index) + ":8080"; + }; + for (std::size_t host_index = 0; host_index < 70; ++host_index) { + const std::string host = host_name(host_index); + const MetaSearcher::MergeLocationSpecsTask task{ + "kvs#event_report_l2#mem#" + host, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://" + host + "/mem")}, + }; + tasks[0].push_back(task); + if (host_index != 64) { + tasks[1].push_back(task); + } + if (host_index != 63 && host_index != 64) { + tasks[2].push_back(task); + } + } + + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK, EC_OK, EC_OK}), per_key_ec); + + std::vector matches; + ASSERT_EQ(EC_OK, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches)); + ASSERT_EQ(70u, matches.size()); + auto prefix_for = [&matches](const std::string &host) { + const auto it = std::find_if( + matches.begin(), matches.end(), [&host](const auto &match) { return match.host_ip_port == host; }); + return it == matches.end() ? int64_t{-1} : it->local; + }; + EXPECT_EQ(3, prefix_for(host_name(0))); + EXPECT_EQ(2, prefix_for(host_name(63))); + EXPECT_EQ(1, prefix_for(host_name(64))); + EXPECT_EQ(3, prefix_for(host_name(69))); +} + +TEST_F(MetaSearcherTest, TestPrefixMatchByHostParallelPresenceMatrixMatchesReference) { + constexpr std::size_t kHostCount = 70; + constexpr std::size_t kKeyCount = 384; + MetaSearcher::KeyVector keys; + keys.reserve(kKeyCount); + for (std::size_t key_index = 0; key_index < kKeyCount; ++key_index) { + keys.push_back(11000 + key_index); + } + + auto host_name = [](std::size_t index) { + return std::string("parallel-host-") + (index < 10 ? "0" : "") + std::to_string(index) + ":8080"; + }; + std::vector expected_prefixes(kHostCount); + std::vector> tasks(kKeyCount); + for (std::size_t host_index = 0; host_index < kHostCount; ++host_index) { + std::size_t prefix = 1 + (host_index * 83) % kKeyCount; + if (host_index == 63) { + prefix = 257; + } else if (host_index == 64) { + prefix = 1; + } else if (host_index == 69) { + prefix = kKeyCount; + } + expected_prefixes[host_index] = prefix; + + const std::string host = host_name(host_index); + const MetaSearcher::MergeLocationSpecsTask task{ + "kvs#event_report_l2#mem#" + host, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://" + host + "/mem")}, + }; + for (std::size_t key_index = 0; key_index < prefix; ++key_index) { + tasks[key_index].push_back(task); + } + } + + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_EQ(kKeyCount, per_key_ec.size()); + EXPECT_TRUE(std::all_of(per_key_ec.begin(), per_key_ec.end(), [](ErrorCode ec) { return ec == EC_OK; })); + + auto verify_matches = [&](const std::vector &matches, bool use_eagle_pop) { + std::map actual; + for (const auto &match : matches) { + ASSERT_TRUE(actual.emplace(match.host_ip_port, match.local).second); + } + for (std::size_t host_index = 0; host_index < kHostCount; ++host_index) { + const int64_t expected = static_cast(expected_prefixes[host_index]) - (use_eagle_pop ? 1 : 0); + const auto it = actual.find(host_name(host_index)); + if (expected == 0) { + EXPECT_EQ(actual.end(), it) << "host_index=" << host_index; + } else { + ASSERT_NE(actual.end(), it) << "host_index=" << host_index; + EXPECT_EQ(expected, it->second) << "host_index=" << host_index; + } + } + }; + + std::vector matches; + ASSERT_EQ(EC_OK, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"mem"}, matches)); + ASSERT_EQ(kHostCount, matches.size()); + verify_matches(matches, false); + + ASSERT_EQ(EC_OK, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, true, {"mem"}, matches)); + verify_matches(matches, true); + + ASSERT_EQ(EC_OK, meta_searcher_->PrefixMatchByHost(request_context_.get(), keys, false, {"disk"}, matches)); + EXPECT_TRUE(matches.empty()); +} + +TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsPreservesUntouchedSpecsAcrossGenerationChange) { + const MetaSearcher::KeyVector keys = {10007}; + const std::string location_id = "kvs#event_report_l2#mem#127.0.0.1:8080"; + const std::string version_a = "00112233445566778899aabbccddeeff"; + const std::string version_b = "ffeeddccbbaa99887766554433221100"; + auto uri = [](const std::string &source, const std::string &version) { + return "event_report://127.0.0.1:8080/mem?source=" + source + "&s_version=" + version; + }; + + std::vector per_key_ec; + std::vector> seed_tasks = {{ + {location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("linear_0", uri("old_linear", version_a)), + LocationSpec("mamba_0", uri("old_mamba", version_a))}}, + }}; + ASSERT_EQ(EC_OK, meta_searcher_->BatchReplaceLocationSpecs(request_context_.get(), keys, seed_tasks, per_key_ec)); + + std::vector> tasks = {{ + {location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("legacy", "event_report://127.0.0.1:8080/mem?source=legacy")}}, + }}; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + + tasks = {{ + {location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("linear_0", uri("new_linear", version_b))}}, + }}; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + ASSERT_EQ(1u, location_maps.size()); + ASSERT_EQ(1u, location_maps[0].size()); + const auto &after_version_change = location_maps[0].at(location_id)->location_specs(); + std::map after_version_change_specs; + for (const auto &spec : after_version_change) { + after_version_change_specs[spec.name()] = spec.uri(); + } + ASSERT_EQ(3u, after_version_change_specs.size()); + EXPECT_EQ(uri("new_linear", version_b), after_version_change_specs["linear_0"]); + EXPECT_EQ(uri("old_mamba", version_a), after_version_change_specs["mamba_0"]); + EXPECT_EQ("event_report://127.0.0.1:8080/mem?source=legacy", after_version_change_specs["legacy"]); + + tasks[0][0].specs = { + LocationSpec("linear_0", uri("newer_linear", version_b)), + LocationSpec("mamba_1", uri("new_mamba", version_b)), + }; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); std::map specs; @@ -510,6 +2533,17 @@ TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsRejectsMixedOrMalformedSnaps ASSERT_EQ((std::vector{EC_OK}), per_key_ec); const std::vector> invalid_specs = { + {}, + { + LocationSpec("", "event_report://127.0.0.1:8080/mem"), + }, + { + LocationSpec("duplicate_name", "event_report://127.0.0.1:8080/mem?source=first"), + LocationSpec("duplicate_name", "event_report://127.0.0.1:8080/mem?source=second"), + }, + { + LocationSpec("invalid_uri", "not a valid uri"), + }, { LocationSpec("valid", uri("valid", version_b)), LocationSpec("missing", "event_report://127.0.0.1:8080/mem?source=missing"), @@ -527,6 +2561,11 @@ TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsRejectsMixedOrMalformedSnaps "event_report://127.0.0.1:8080/mem?source=duplicate&s_version=" + version_b + "&s_version=" + version_b), }, + { + LocationSpec("max_size", + "event_report://127.0.0.1:8080/mem?size=18446744073709551615&s_version=" + version_b), + LocationSpec("overflow_size", "event_report://127.0.0.1:8080/mem?size=1&s_version=" + version_b), + }, }; for (const auto &specs : invalid_specs) { @@ -546,16 +2585,77 @@ TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsRejectsMixedOrMalformedSnaps EXPECT_EQ(uri("baseline", version_a), stored_specs[0].uri()); } - const MetaSearcher::KeyVector new_keys = {10013}; - tasks[0][0].specs = invalid_specs.front(); - per_key_ec.clear(); - meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), new_keys, tasks, per_key_ec); + const MetaSearcher::KeyVector new_keys = {10013}; + tasks[0][0].specs = invalid_specs.front(); + per_key_ec.clear(); + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), new_keys, tasks, per_key_ec); + ASSERT_EQ((std::vector{EC_BADARGS}), per_key_ec); + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), new_keys, mask, location_maps)); + ASSERT_EQ(1u, location_maps.size()); + EXPECT_TRUE(location_maps.front().empty()); + + // Replace performs the same validation as merge, before any key in the + // batch is mutated. A valid sibling must therefore remain unwritten when + // another task carries inconsistent snapshot metadata. + const KeyVector replace_keys = {10014, 10015}; + std::vector> replace_tasks = { + {{location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("valid", uri("valid", version_a))}}}, + {{location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + invalid_specs.back()}}, + }; + EXPECT_EQ( + EC_BADARGS, + meta_searcher_->BatchReplaceLocationSpecs(request_context_.get(), replace_keys, replace_tasks, per_key_ec)); + EXPECT_EQ((std::vector{EC_BADARGS, EC_BADARGS}), per_key_ec); + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), replace_keys, mask, location_maps)); + ASSERT_EQ(2u, location_maps.size()); + EXPECT_TRUE(location_maps[0].empty()); + EXPECT_TRUE(location_maps[1].empty()); +} + +TEST_F(MetaSearcherTest, TestBatchMergeLocationSpecsRejectsOverflowAgainstExistingSpecs) { + const MetaSearcher::KeyVector keys = {10016}; + const std::string location_id = "kvs#event_report_l2#mem#127.0.0.1:8080"; + const std::string version = "00112233445566778899aabbccddeeff"; + auto uri = [&version](const std::string &source, const std::string &size) { + return "event_report://127.0.0.1:8080/mem?size=" + size + "&source=" + source + "&s_version=" + version; + }; + std::vector> tasks = {{ + {location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("baseline", uri("baseline", "18446744073709551615"))}}, + }}; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK}), per_key_ec); + EXPECT_EQ(std::numeric_limits::max(), + meta_indexer_->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); + + // Both requests are independently valid. The second one must still be + // rejected because the final merged location would overflow uint64_t. + tasks[0][0].specs = {LocationSpec("new_spec", uri("new_spec", "1"))}; + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec); ASSERT_EQ((std::vector{EC_BADARGS}), per_key_ec); + std::vector location_maps; BlockMask mask; - ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), new_keys, mask, location_maps)); + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); ASSERT_EQ(1u, location_maps.size()); - EXPECT_TRUE(location_maps.front().empty()); + ASSERT_EQ(1u, location_maps[0].size()); + const auto &stored_specs = location_maps[0].at(location_id)->location_specs(); + ASSERT_EQ(1u, stored_specs.size()); + EXPECT_EQ("baseline", stored_specs[0].name()); + EXPECT_EQ(uri("baseline", "18446744073709551615"), stored_specs[0].uri()); + EXPECT_EQ(std::numeric_limits::max(), + meta_indexer_->GetStorageUsageByType(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2)); } TEST_F(MetaSearcherTest, TestConcurrentSnapshotReplaceIsAtomicAndSameTokenDeltasDoNotLoseUpdates) { @@ -731,7 +2831,11 @@ TEST_F(MetaSearcherTest, TestMergeAndReplaceLocationSpecsKeepStorageUsageExact) ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); ASSERT_EQ(1u, location_maps.size()); ASSERT_EQ(1u, location_maps.front().size()); - EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, location_maps.front().at(location_id)->type()); + const auto &replaced_location = location_maps.front().at(location_id); + EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, replaced_location->type()); + std::uint64_t validated_size = 0; + ASSERT_TRUE(replaced_location->GetValidatedTotalSize(validated_size)); + EXPECT_EQ(7u, validated_size); } TEST_F(MetaSearcherTest, TestConditionalDeleteDoesNotRemoveRefreshedStableLocation) { @@ -925,6 +3029,7 @@ TEST_F(MetaSearcherTest, TestBatchDeleteLocationSpecsPartialDelete) { } ASSERT_EQ((std::set{"linear_1", "full_3"}), spec_names); EXPECT_EQ(2u, location_maps[0].at(location_id)->spec_size()); + EXPECT_TRUE(location_maps[0].at(location_id)->HasValidatedLocationSpecs()); delete_tasks = {{ {location_id, {"linear_1", "full_3"}}, @@ -950,6 +3055,55 @@ TEST_F(MetaSearcherTest, TestBatchDeleteLocationSpecsPartialDelete) { EXPECT_FALSE(location_maps[0].empty()); } +TEST_F(MetaSearcherTest, TestBatchDeleteFinalLocationsReclaimsDuplicateKeyOnce) { + constexpr KeyType reclaimed_key = 10014; + constexpr KeyType retained_key = 10015; + const std::string location_a = "event_report#mem#127.0.0.14:8080"; + const std::string location_b = "event_report#disk#127.0.0.14:8080"; + const std::string retained_location = "event_report#mem#127.0.0.15:8080"; + + std::vector> merge_tasks = { + {{location_a, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://127.0.0.14:8080/mem")}}, + {location_b, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp1", "event_report://127.0.0.14:8080/disk")}}}, + {{retained_location, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L1P5, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://127.0.0.15:8080/mem")}}}, + }; + std::vector per_key_ec; + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs( + request_context_.get(), {reclaimed_key, retained_key}, merge_tasks, per_key_ec)); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), per_key_ec); + ASSERT_EQ(2u, meta_indexer_->GetKeyCount()); + + std::vector> delete_tasks = {{ + {location_a, {"tp0"}}, + {location_b, {"tp1"}}, + }}; + std::vector> delete_results; + ASSERT_EQ(EC_OK, + meta_searcher_->BatchDeleteLocationSpecs( + request_context_.get(), {reclaimed_key}, delete_tasks, delete_results)); + ASSERT_EQ((std::vector>{{EC_OK, EC_OK}}), delete_results); + EXPECT_EQ(1u, meta_indexer_->GetKeyCount()); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ( + EC_OK, + meta_searcher_->BatchGetLocation(request_context_.get(), {reclaimed_key, retained_key}, mask, location_maps)); + ASSERT_EQ(2u, location_maps.size()); + EXPECT_TRUE(location_maps[0].empty()); + EXPECT_EQ(1u, location_maps[1].count(retained_location)); +} + TEST_F(MetaSearcherTest, TestBatchDeleteLocationSpecsValidatesShapeAndMissingLocationIsIdempotent) { MetaSearcher::KeyVector keys = {10004}; const std::string existing_location_id = "event_report#mem#127.0.0.1:8080"; @@ -990,6 +3144,71 @@ TEST_F(MetaSearcherTest, TestBatchDeleteLocationSpecsValidatesShapeAndMissingLoc EXPECT_EQ("linear_0", existing->second->location_specs()[0].name()); } +TEST_F(MetaSearcherTest, TestBatchSpecMutationsRejectDuplicateRmwTargetsBeforeWriting) { + const int64_t key = 10013; + const std::string location_id = "kvs#event_report_l2#mem#duplicate-target:8080"; + std::vector per_key_ec; + std::vector> seed = {{ + {location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://duplicate-target:8080/mem?source=seed_tp0"), + LocationSpec("tp1", "event_report://duplicate-target:8080/mem?source=seed_tp1")}}, + }}; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {key}, seed, per_key_ec)); + + auto duplicate_merge = seed; + duplicate_merge[0].push_back({location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp2", "event_report://duplicate-target:8080/mem?source=lost")}}); + EXPECT_EQ(EC_BADARGS, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {key}, duplicate_merge, per_key_ec)); + EXPECT_EQ((std::vector{EC_BADARGS}), per_key_ec); + + std::vector> duplicate_delete = {{ + {location_id, {"tp0"}}, + {location_id, {"tp1"}}, + }}; + std::vector> delete_results; + EXPECT_EQ( + EC_BADARGS, + meta_searcher_->BatchDeleteLocationSpecs(request_context_.get(), {key}, duplicate_delete, delete_results)); + ASSERT_EQ(1u, delete_results.size()); + EXPECT_EQ((std::vector{EC_BADARGS, EC_BADARGS}), delete_results[0]); + + std::vector> empty_delete = {{ + {"kvs#event_report_l2#mem#missing:8080", {}}, + }}; + EXPECT_EQ(EC_OK, + meta_searcher_->BatchDeleteLocationSpecs(request_context_.get(), {key}, empty_delete, delete_results)); + EXPECT_EQ((std::vector{EC_BADARGS}), delete_results[0]); + + std::vector> duplicate_key_replace = { + {{location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://duplicate-target:8080/mem?source=replace_a")}}}, + {{location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://duplicate-target:8080/mem?source=replace_b")}}}, + }; + EXPECT_EQ(EC_BADARGS, + meta_searcher_->BatchReplaceLocationSpecs( + request_context_.get(), {key, key}, duplicate_key_replace, per_key_ec)); + EXPECT_EQ((std::vector{EC_BADARGS, EC_BADARGS}), per_key_ec); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), {key}, mask, location_maps)); + ASSERT_EQ(1u, location_maps.size()); + const auto &specs = location_maps[0].at(location_id)->location_specs(); + ASSERT_EQ(2u, specs.size()); + EXPECT_EQ(std::string::npos, specs[0].uri().find("source=lost")); + EXPECT_EQ(std::string::npos, specs[1].uri().find("source=lost")); +} + TEST_F(MetaSearcherTest, TestBatchDeleteLocationSpecsIsIdempotentForMissingData) { const MetaSearcher::KeyVector keys = {10008}; const std::string location_id = "kvs#event_report_l2#mem#127.0.0.1:8080"; @@ -1039,6 +3258,183 @@ TEST_F(MetaSearcherTest, TestBatchDeleteLocationSpecsIsIdempotentForMissingData) EXPECT_EQ("linear_0", specs[0].name()); } +TEST_F(MetaSearcherTest, TestBatchMutationWriteLeaseIsAcquiredOncePerRmwPhase) { + const MetaSearcher::KeyVector keys = {10081, 10082, 10083}; + const std::string location_id = "kvs#event_report_l2#mem#lease-host:8080"; + std::vector> merge_tasks; + for (size_t i = 0; i < keys.size(); ++i) { + merge_tasks.push_back({{ + location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://lease-host:8080/mem?size=1")}, + }}); + } + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, merge_tasks, per_key_ec)); + + size_t acquire_count = 0; + MetaSearcher::AcquireMetadataWriteLeaseFunc acquire_write_lease = [&] { + ++acquire_count; + return std::make_pair(EC_OK, std::static_pointer_cast(std::make_shared(acquire_count))); + }; + + // Every key already has this location. The fused targeted RMW holds one + // lease from the post-read fence check through the single upsert phase. + for (auto &tasks : merge_tasks) { + tasks[0].specs = {LocationSpec("tp1", "event_report://lease-host:8080/mem?size=2")}; + } + ASSERT_EQ(EC_OK, + meta_searcher_->BatchMergeLocationSpecs( + request_context_.get(), keys, merge_tasks, per_key_ec, acquire_write_lease)); + EXPECT_EQ(1u, acquire_count); + EXPECT_EQ(std::vector(keys.size(), EC_OK), per_key_ec); + + acquire_count = 0; + std::vector> replace_tasks; + for (size_t i = 0; i < keys.size(); ++i) { + replace_tasks.push_back({{ + location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp2", "event_report://lease-host:8080/mem?size=3")}, + }}); + } + ASSERT_EQ(EC_OK, + meta_searcher_->BatchReplaceLocationSpecs( + request_context_.get(), keys, replace_tasks, per_key_ec, acquire_write_lease)); + EXPECT_EQ(1u, acquire_count); + EXPECT_EQ(std::vector(keys.size(), EC_OK), per_key_ec); + + acquire_count = 0; + std::vector> delete_tasks; + for (size_t i = 0; i < keys.size(); ++i) { + delete_tasks.push_back({{location_id, {"tp2"}}}); + } + std::vector> delete_results; + ASSERT_EQ(EC_OK, + meta_searcher_->BatchDeleteLocationSpecs( + request_context_.get(), keys, delete_tasks, delete_results, nullptr, acquire_write_lease)); + EXPECT_EQ(1u, acquire_count); + ASSERT_EQ(keys.size(), delete_results.size()); + for (const auto &results : delete_results) { + EXPECT_EQ((std::vector{EC_OK}), results); + } +} + +TEST_F(MetaSearcherTest, TestBatchMutationWriteLeaseFailurePreventsAllWrites) { + const MetaSearcher::KeyVector keys = {10084, 10085}; + const std::string location_id = "kvs#event_report_l2#mem#fenced-host:8080"; + size_t acquire_count = 0; + MetaSearcher::AcquireMetadataWriteLeaseFunc reject_write = [&] { + ++acquire_count; + return std::make_pair(EC_NODE_NOT_REGISTERED, MetaSearcher::MetadataWriteLease{}); + }; + + std::vector> merge_tasks; + for (size_t i = 0; i < keys.size(); ++i) { + merge_tasks.push_back({{ + location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://fenced-host:8080/mem")}, + }}); + } + std::vector per_key_ec; + EXPECT_EQ( + EC_ERROR, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, merge_tasks, per_key_ec, reject_write)); + EXPECT_EQ(1u, acquire_count); + EXPECT_EQ(std::vector(keys.size(), EC_NODE_NOT_REGISTERED), per_key_ec); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + ASSERT_EQ(keys.size(), location_maps.size()); + EXPECT_TRUE(std::all_of( + location_maps.begin(), location_maps.end(), [](const auto &locations) { return locations.empty(); })); + + acquire_count = 0; + std::vector> replace_tasks; + for (size_t i = 0; i < keys.size(); ++i) { + replace_tasks.push_back({{ + location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp1", "event_report://fenced-host:8080/mem")}, + }}); + } + EXPECT_NE(EC_OK, + meta_searcher_->BatchReplaceLocationSpecs( + request_context_.get(), keys, replace_tasks, per_key_ec, reject_write)); + EXPECT_EQ(1u, acquire_count); + EXPECT_EQ(std::vector(keys.size(), EC_NODE_NOT_REGISTERED), per_key_ec); + + acquire_count = 0; + std::vector> delete_tasks; + for (size_t i = 0; i < keys.size(); ++i) { + delete_tasks.push_back({{location_id, {"tp0"}}}); + } + std::vector> delete_results; + std::vector> missing_targets; + EXPECT_NE(EC_OK, + meta_searcher_->BatchDeleteLocationSpecs( + request_context_.get(), keys, delete_tasks, delete_results, &missing_targets, reject_write)); + EXPECT_EQ(1u, acquire_count); + ASSERT_EQ(keys.size(), delete_results.size()); + ASSERT_EQ(keys.size(), missing_targets.size()); + for (size_t i = 0; i < keys.size(); ++i) { + EXPECT_EQ((std::vector{EC_NODE_NOT_REGISTERED}), delete_results[i]); + EXPECT_EQ((std::vector{false}), missing_targets[i]); + } +} + +TEST_F(MetaSearcherTest, TestBatchMergeDoesNotReacquireLeaseInsideFusedRmw) { + const MetaSearcher::KeyVector keys = {10086, 10087}; + const std::string location_id = "kvs#event_report_l2#mem#lease-race:8080"; + std::vector> tasks; + for (size_t i = 0; i < keys.size(); ++i) { + tasks.push_back({{ + location_id, + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CacheLocationStatus::CLS_SERVING, + {LocationSpec("tp0", "event_report://lease-race:8080/mem?phase=seed")}, + }}); + } + std::vector per_key_ec; + ASSERT_EQ(EC_OK, meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec)); + + for (auto &per_key_tasks : tasks) { + per_key_tasks[0].specs = {LocationSpec("tp1", "event_report://lease-race:8080/mem?phase=stale")}; + } + size_t acquire_count = 0; + MetaSearcher::AcquireMetadataWriteLeaseFunc fail_if_reacquired = [&] { + ++acquire_count; + if (acquire_count == 1) { + return std::make_pair(EC_OK, std::static_pointer_cast(std::make_shared(acquire_count))); + } + return std::make_pair(EC_NODE_NOT_REGISTERED, MetaSearcher::MetadataWriteLease{}); + }; + EXPECT_EQ( + EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), keys, tasks, per_key_ec, fail_if_reacquired)); + EXPECT_EQ(1u, acquire_count); + EXPECT_EQ(std::vector(keys.size(), EC_OK), per_key_ec); + + std::vector location_maps; + BlockMask mask; + ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), keys, mask, location_maps)); + ASSERT_EQ(keys.size(), location_maps.size()); + for (const auto &locations : location_maps) { + ASSERT_EQ(1u, locations.size()); + const auto &specs = locations.at(location_id)->location_specs(); + ASSERT_EQ(2u, specs.size()); + EXPECT_EQ("tp0", specs[0].name()); + EXPECT_EQ("tp1", specs[1].name()); + EXPECT_NE(std::string::npos, specs[1].uri().find("phase=stale")); + } +} + TEST_F(MetaSearcherTest, TestCleanupLocationsByPredicateSubmitsExactObservedValue) { const MetaSearcher::KeyVector keys = {10009, 10010}; const std::string stale_id = "kvs#event_report_l2#mem#127.0.0.1:8080"; @@ -1704,10 +4100,8 @@ TEST_F(MetaSearcherTest, TestReconcileAddLocationRollbackClassifiesStates) { // batch: confirmed success / uncertain with id / failed without id / failed with a ghost id const KeyVector keys = {success_key, uncertain_key, no_id_key, ghost_key}; - std::vector add_results = {success_results[0], - uncertain_results[0], - {EC_ERROR, ""}, - {EC_ERROR, "ghost_location_id"}}; + std::vector add_results = { + success_results[0], uncertain_results[0], {EC_ERROR, ""}, {EC_ERROR, "ghost_location_id"}}; MetaSearcher::AddLocationRollbackPlan plan; ASSERT_EQ(EC_OK, meta_searcher_->ReconcileAddLocationRollback(request_context_.get(), keys, add_results, plan)); @@ -1721,7 +4115,9 @@ TEST_F(MetaSearcherTest, TestReconcileAddLocationRollbackClassifiesStates) { // uncertain metadata is deleted; confirmed-success metadata is left for the delete pipeline. std::vector location_maps; BlockMask mask; - ASSERT_EQ(EC_OK, meta_searcher_->BatchGetLocation(request_context_.get(), {success_key, uncertain_key}, mask, location_maps)); + ASSERT_EQ( + EC_OK, + meta_searcher_->BatchGetLocation(request_context_.get(), {success_key, uncertain_key}, mask, location_maps)); ASSERT_EQ(2u, location_maps.size()); EXPECT_EQ(1u, location_maps[0].count(success_results[0].location_id)); EXPECT_TRUE(location_maps[1].empty()); @@ -1734,8 +4130,7 @@ TEST_F(MetaSearcherTest, TestReconcileAddLocationRollbackRejectsShapeMismatch) { plan.direct_delete_indices = {0}; EXPECT_EQ(EC_BADARGS, - meta_searcher_->ReconcileAddLocationRollback( - request_context_.get(), {1, 2}, {{EC_OK, "some_id"}}, plan)); + meta_searcher_->ReconcileAddLocationRollback(request_context_.get(), {1, 2}, {{EC_OK, "some_id"}}, plan)); EXPECT_TRUE(plan.pipeline_keys.empty()); EXPECT_TRUE(plan.pipeline_location_ids.empty()); EXPECT_TRUE(plan.direct_delete_indices.empty()); @@ -1777,9 +4172,9 @@ TEST_F(MetaSearcherTest, TestReconcileAddLocationRollbackRetainsUrisOnDeleteErro backend->SetGetLocationsFailedKey(uncertain_key); MetaSearcher::AddLocationRollbackPlan plan; - ASSERT_EQ(EC_OK, - meta_searcher_->ReconcileAddLocationRollback( - request_context_.get(), {uncertain_key}, uncertain_results, plan)); + ASSERT_EQ( + EC_OK, + meta_searcher_->ReconcileAddLocationRollback(request_context_.get(), {uncertain_key}, uncertain_results, plan)); EXPECT_TRUE(plan.pipeline_keys.empty()); EXPECT_TRUE(plan.direct_delete_indices.empty()); @@ -1809,9 +4204,9 @@ TEST_F(MetaSearcherTest, TestReconcileAddLocationRollbackRetainsUrisWhenSyncFail backend->SetFailSync(true); MetaSearcher::AddLocationRollbackPlan plan; - ASSERT_EQ(EC_OK, - meta_searcher_->ReconcileAddLocationRollback( - request_context_.get(), {uncertain_key}, uncertain_results, plan)); + ASSERT_EQ( + EC_OK, + meta_searcher_->ReconcileAddLocationRollback(request_context_.get(), {uncertain_key}, uncertain_results, plan)); EXPECT_TRUE(plan.pipeline_keys.empty()); // metadata was deleted in memory but the delete could not be synced: retain the URI. EXPECT_TRUE(plan.direct_delete_indices.empty()); @@ -2539,8 +4934,77 @@ TEST_F(MetaSearcherTest, TestBatchGetMergesSpecsByStorageType) { class BatchGetBestLocationByBackendTest : public MetaSearcherTest { protected: + void AddRequestedSpecMatrixEventReportPeer() { + // The requested spec is deliberately the second spec in the first + // and third locations. The middle key has the same reporter but only + // a different spec, forming a requested-spec gap. + std::vector> upserts = { + { + {"kvs#event_report_l2#mem#matrix_peer:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + { + LocationSpec("full_0", "event_report://other_peer:8080/mem"), + LocationSpec("linear_1", "event_report://matrix_peer:8080/mem"), + }}, + }, + { + {"kvs#event_report_l2#mem#matrix_peer:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + {LocationSpec("full_0", "event_report://matrix_peer:8080/mem")}}, + }, + { + {"kvs#event_report_l2#mem#matrix_peer:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + { + LocationSpec("full_0", "event_report://other_peer:8080/mem"), + LocationSpec("linear_1", "event_report://matrix_peer:8080/mem"), + }}, + }, + }; + std::vector per_key_ec; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchMergeLocationSpecs( + request_context_.get(), {82000, 82001, 82002}, upserts, per_key_ec)); + ASSERT_EQ(3u, per_key_ec.size()); + EXPECT_TRUE(std::all_of(per_key_ec.begin(), per_key_ec.end(), [](ErrorCode ec) { return ec == EC_OK; })); + } + + void AddSpecFilteredEventReportPeers() { + // full_peer has better raw coverage, but none of its locations match + // linear_1. linear_peer must therefore win after requested-spec + // filtering for both cross-key selection strategies. + std::vector> upserts = { + { + {"kvs#event_report_l2#mem#full_peer:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + {LocationSpec("full_0", "event_report://full_peer:8080/mem")}}, + {"kvs#event_report_l2#mem#linear_peer:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + {LocationSpec("linear_1", "event_report://linear_peer:8080/mem")}}, + }, + { + {"kvs#event_report_l2#mem#full_peer:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + {LocationSpec("full_0", "event_report://full_peer:8080/mem")}}, + }, + }; + std::vector per_key_ec; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {81000, 81001}, upserts, per_key_ec)); + ASSERT_EQ(2u, per_key_ec.size()); + EXPECT_EQ(ErrorCode::EC_OK, per_key_ec[0]); + EXPECT_EQ(ErrorCode::EC_OK, per_key_ec[1]); + } + void SetUp() override { MetaSearcherTest::SetUp(); + recording_backend_ = ReplaceWithRecordingGetLocationsBackend(); // event report locations std::vector> er_upserts = { @@ -2604,7 +5068,10 @@ class BatchGetBestLocationByBackendTest : public MetaSearcherTest { std::vector> results; meta_searcher_->BatchUpdateLocationStatus(request_context_.get(), {key}, tasks, results); } + recording_backend_->ResetReadLog(); } + + RecordingGetLocationsBackend *recording_backend_ = nullptr; }; TEST_F(BatchGetBestLocationByBackendTest, EventReportPrefixStrategy) { @@ -2652,6 +5119,252 @@ TEST_F(BatchGetBestLocationByBackendTest, EventReportCoverageStrategy) { EXPECT_TRUE(out[4].empty()); } +TEST_F(BatchGetBestLocationByBackendTest, BlockMaskSkipsMetadataReadsAndPreservesOutputPositions) { + const MetaSearcher::KeyVector keys = {80000, 80001, 80002, 80003, 80004}; + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, + {DataStorageType::DATA_STORAGE_TYPE_TAIR_MEMPOOL, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}, + }; + const BlockMask mask = BlockMaskVector{true, false, true, false, true}; + + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), keys, out, &policy_, selectors, {}, mask)); + + ASSERT_EQ(keys.size(), out.size()); + EXPECT_TRUE(out[0].empty()); + ASSERT_EQ(2u, out[1].size()); + EXPECT_TRUE(out[2].empty()); + ASSERT_EQ(2u, out[3].size()); + EXPECT_TRUE(out[4].empty()); + + const auto batches = recording_backend_->RequestedKeyBatches(); + ASSERT_EQ(1u, batches.size()); + EXPECT_EQ((MetaSearcher::KeyVector{80001, 80003}), batches[0]); +} + +TEST_F(BatchGetBestLocationByBackendTest, FullyMaskedRequestAvoidsMetadataBackend) { + const MetaSearcher::KeyVector keys = {80000, 80001, 80002, 80003, 80004}; + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, + }; + const BlockMask mask = BlockMaskOffset{keys.size()}; + + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), keys, out, &policy_, selectors, {}, mask)); + + ASSERT_EQ(keys.size(), out.size()); + for (const auto &locations : out) { + EXPECT_TRUE(locations.empty()); + } + EXPECT_TRUE(recording_backend_->RequestedKeyBatches().empty()); +} + +TEST_F(BatchGetBestLocationByBackendTest, InvalidBlockMaskFailsBeforeMetadataRead) { + const MetaSearcher::KeyVector keys = {80000, 80001, 80002}; + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_TAIR_MEMPOOL, LocationSelectStrategy::LSS_WEIGHTED_RANDOM}, + }; + + for (const BlockMask &mask : std::vector{ + BlockMaskOffset{keys.size() + 1}, + BlockMaskVector{true, false}, + }) { + LocationsPerKey out; + EXPECT_EQ(ErrorCode::EC_BADARGS, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), keys, out, &policy_, selectors, {}, mask)); + ASSERT_EQ(keys.size(), out.size()); + for (const auto &locations : out) { + EXPECT_TRUE(locations.empty()); + } + } + EXPECT_TRUE(recording_backend_->RequestedKeyBatches().empty()); +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportPrefixTieBreaksByPeerAddress) { + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, + }; + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), {80000, 80001}, out, &policy_, selectors)); + + ASSERT_EQ(2u, out.size()); + for (const auto &locations : out) { + ASSERT_EQ(1u, locations.size()); + EXPECT_NE(locations[0]->location_specs()[0].uri().find("peer_a"), std::string::npos); + } +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportCoverageTieBreaksByPeerAddress) { + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_COVERAGE}, + }; + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), {80000, 80001}, out, &policy_, selectors)); + + ASSERT_EQ(2u, out.size()); + for (const auto &locations : out) { + ASSERT_EQ(1u, locations.size()); + EXPECT_NE(locations[0]->location_specs()[0].uri().find("peer_a"), std::string::npos); + } +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportPrefixFiltersRequestedSpecBeforePeerSelection) { + AddSpecFilteredEventReportPeers(); + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, + }; + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), {81000, 81001}, out, &policy_, selectors, {"linear_1", "linear_1"})); + + ASSERT_EQ(2u, out.size()); + ASSERT_EQ(1u, out[0].size()); + ASSERT_EQ(1u, out[0][0]->location_specs().size()); + EXPECT_EQ("linear_1", out[0][0]->location_specs()[0].name()); + EXPECT_NE(std::string::npos, out[0][0]->location_specs()[0].uri().find("linear_peer")); + EXPECT_TRUE(out[1].empty()); +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportCoverageFiltersRequestedSpecBeforePeerSelection) { + AddSpecFilteredEventReportPeers(); + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_COVERAGE}, + }; + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), {81000, 81001}, out, &policy_, selectors, {"linear_1", "linear_1"})); + + ASSERT_EQ(2u, out.size()); + ASSERT_EQ(1u, out[0].size()); + ASSERT_EQ(1u, out[0][0]->location_specs().size()); + EXPECT_EQ("linear_1", out[0][0]->location_specs()[0].name()); + EXPECT_NE(std::string::npos, out[0][0]->location_specs()[0].uri().find("linear_peer")); + EXPECT_TRUE(out[1].empty()); +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportUnknownRequestedSpecReturnsNoCandidate) { + AddSpecFilteredEventReportPeers(); + for (const auto strategy : {LocationSelectStrategy::LSS_V6D_PREFIX, LocationSelectStrategy::LSS_V6D_COVERAGE}) { + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, strategy}, + }; + LocationsPerKey out; + ASSERT_EQ( + ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), {81000, 81001}, out, &policy_, selectors, {"missing_spec", "missing_spec"})); + ASSERT_EQ(2u, out.size()); + EXPECT_TRUE(out[0].empty()); + EXPECT_TRUE(out[1].empty()); + } +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportRequestedSpecUsesMatchingNonFirstSpecUri) { + AddRequestedSpecMatrixEventReportPeer(); + for (const auto strategy : {LocationSelectStrategy::LSS_V6D_PREFIX, LocationSelectStrategy::LSS_V6D_COVERAGE}) { + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, strategy}, + }; + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend( + request_context_.get(), {82000}, out, &policy_, selectors, {"linear_1"})); + ASSERT_EQ(1u, out.size()); + ASSERT_EQ(1u, out[0].size()); + ASSERT_EQ(2u, out[0][0]->location_specs().size()); + const auto linear_it = std::find_if(out[0][0]->location_specs().begin(), + out[0][0]->location_specs().end(), + [](const LocationSpec &spec) { return spec.name() == "linear_1"; }); + ASSERT_NE(linear_it, out[0][0]->location_specs().end()); + EXPECT_NE(std::string::npos, linear_it->uri().find("matrix_peer")); + } +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportRequestedSpecGapStopsPrefixButNotCoverage) { + AddRequestedSpecMatrixEventReportPeer(); + const MetaSearcher::KeyVector keys = {82000, 82001, 82002}; + + LocationsPerKey prefix_out; + const std::vector prefix_selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_PREFIX}, + }; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend(request_context_.get(), + keys, + prefix_out, + &policy_, + prefix_selectors, + {"linear_1", "linear_1", "linear_1"})); + ASSERT_EQ(3u, prefix_out.size()); + ASSERT_EQ(1u, prefix_out[0].size()); + EXPECT_TRUE(prefix_out[1].empty()); + EXPECT_TRUE(prefix_out[2].empty()); + + LocationsPerKey coverage_out; + const std::vector coverage_selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_COVERAGE}, + }; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend(request_context_.get(), + keys, + coverage_out, + &policy_, + coverage_selectors, + {"linear_1", "linear_1", "linear_1"})); + ASSERT_EQ(3u, coverage_out.size()); + ASSERT_EQ(1u, coverage_out[0].size()); + EXPECT_TRUE(coverage_out[1].empty()); + ASSERT_EQ(1u, coverage_out[2].size()); +} + +TEST_F(BatchGetBestLocationByBackendTest, EventReportCoverageCountsRepeatedKeysByPerKeySpec) { + std::vector> upserts = {{ + {"kvs#event_report_l2#mem#peer_a:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + { + LocationSpec("linear_1", "event_report://peer_a:8080/mem"), + LocationSpec("linear_3", "event_report://peer_a:8080/mem"), + }}, + {"kvs#event_report_l2#mem#peer_b:8080", + DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, + CLS_SERVING, + {LocationSpec("linear_2", "event_report://peer_b:8080/mem")}}, + }}; + std::vector per_key_ec; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchMergeLocationSpecs(request_context_.get(), {83000}, upserts, per_key_ec)); + + const std::vector selectors = { + {DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2, LocationSelectStrategy::LSS_V6D_COVERAGE}, + }; + LocationsPerKey out; + ASSERT_EQ(ErrorCode::EC_OK, + meta_searcher_->BatchGetBestLocationByBackend(request_context_.get(), + {83000, 83000, 83000}, + out, + &policy_, + selectors, + {"linear_1", "linear_2", "linear_3"})); + ASSERT_EQ(3u, out.size()); + ASSERT_EQ(1u, out[0].size()); + EXPECT_TRUE(out[1].empty()); + ASSERT_EQ(1u, out[2].size()); + EXPECT_NE(std::string::npos, out[0][0]->location_specs()[0].uri().find("peer_a")); + EXPECT_NE(std::string::npos, out[2][0]->location_specs()[0].uri().find("peer_a")); +} + TEST_F(BatchGetBestLocationByBackendTest, PrefixStopsAtGap) { // key 80002 has only peer_b, not peer_a. // If we query keys in order [80000, 80002, 80003]: diff --git a/kv_cache_manager/manager/test/migration_manager_test.cc b/kv_cache_manager/manager/test/migration_manager_test.cc index b06a7b266..53cb818d2 100644 --- a/kv_cache_manager/manager/test/migration_manager_test.cc +++ b/kv_cache_manager/manager/test/migration_manager_test.cc @@ -1877,7 +1877,14 @@ TEST_F(MigrationManagerTest, TestBatchAddLocationPartialFailureKeepsSuccessfulCo indexer->batch_key_size_ = 1; indexer->max_key_count_ = 1; - const std::vector block_keys{814, 815}; + std::vector block_keys{814, 815}; + // batch_key_size is a soft limit: every key in one mutex shard remains + // in the same batch. Select two shards using this indexer's runtime hash + // seed so max_key_count=1 deterministically exercises one successful + // batch followed by one capacity failure. + while (indexer->GetMutexShardIndex(block_keys[0]) == indexer->GetMutexShardIndex(block_keys[1])) { + ++block_keys[1]; + } auto make_request = [&](int64_t block_key) { MigrationManager::MigrationRequest req; req.instance_group_name = "group_a"; diff --git a/kv_cache_manager/meta/BUILD b/kv_cache_manager/meta/BUILD index 2b4d0b376..7362168cd 100644 --- a/kv_cache_manager/meta/BUILD +++ b/kv_cache_manager/meta/BUILD @@ -7,6 +7,15 @@ cc_library( ], ) +cc_library( + name = "query_executor", + srcs = ["query_executor.cc"], + hdrs = ["query_executor.h"], + deps = [ + "//kv_cache_manager/common:logger", + ], +) + cc_library( name = "meta_indexer_manager", srcs = [ @@ -17,6 +26,7 @@ cc_library( ], deps = [ ":meta_indexer", + ":query_executor", "//kv_cache_manager/metrics:revisit_interval_histogram", ], ) @@ -53,6 +63,7 @@ cc_library( ":common", ":meta_search_cache", ":meta_storage_backend_manager", + ":query_executor", ":storage_usage_data", ":types", ":utils", @@ -274,4 +285,3 @@ cc_library( "//kv_cache_manager/config", ], ) - diff --git a/kv_cache_manager/meta/cache_location.cc b/kv_cache_manager/meta/cache_location.cc index 8b62cae62..9d8c0137e 100644 --- a/kv_cache_manager/meta/cache_location.cc +++ b/kv_cache_manager/meta/cache_location.cc @@ -33,6 +33,20 @@ bool IsBlockMaskValid(const BlockMask &mask, size_t size) { CacheLocation::CacheLocation() = default; +CacheLocation::CacheLocation(const CacheLocation &other) + : status_(other.status_) + , type_(other.type_) + , spec_size_(other.spec_size_) + , create_time_(other.create_time_) + , location_specs_(other.location_specs_) + , validated_total_size_(other.validated_total_size_) { + if (const auto *owned = std::get_if(&other.id_)) { + id_.emplace(*owned); + } else { + id_.emplace(std::get(other.id_)); + } +} + CacheLocation::CacheLocation(DataStorageType type, size_t spec_size, const std::vector &location_specs) : type_(type), spec_size_(spec_size), location_specs_(location_specs) {} diff --git a/kv_cache_manager/meta/cache_location.h b/kv_cache_manager/meta/cache_location.h index 34f3cd696..ddf04121d 100644 --- a/kv_cache_manager/meta/cache_location.h +++ b/kv_cache_manager/meta/cache_location.h @@ -1,8 +1,13 @@ #pragma once +#include +#include +#include #include #include +#include #include +#include #include #include @@ -12,12 +17,22 @@ namespace kv_cache_manager { +// A request may report the same stable location id for tens of thousands of +// block keys. CacheLocation can borrow shared ownership of one canonical +// string so immutable location copies do not allocate/copy that id again. +using InternedLocationId = std::shared_ptr; + class LocationSpec : public Jsonizable { public: LocationSpec(); LocationSpec(const std::string &name, const std::string &uri) : name_(name), uri_(uri) {} + LocationSpec(const LocationSpec &) = default; + LocationSpec &operator=(const LocationSpec &) = default; + LocationSpec(LocationSpec &&) noexcept = default; + LocationSpec &operator=(LocationSpec &&) noexcept = default; + ~LocationSpec() override; void ToRapidWriter(rapidjson::Writer &writer) const noexcept override { @@ -32,7 +47,10 @@ class LocationSpec : public Jsonizable { } void set_name(const std::string &name) { name_ = name; } + void set_name(std::string &&name) noexcept { name_ = std::move(name); } + void set_name_view(std::string_view name) { name_.assign(name.data(), name.size()); } void set_uri(const std::string &uri) { uri_ = uri; } + void set_uri(std::string &&uri) noexcept { uri_ = std::move(uri); } inline const std::string &name() const { return name_; } inline const std::string &uri() const { return uri_; } @@ -75,6 +93,10 @@ PutBlockMask(rapidjson::Writer &writer, const std::stri class CacheLocation : public Jsonizable { public: CacheLocation(); + CacheLocation(const CacheLocation &other); + CacheLocation &operator=(const CacheLocation &) = default; + CacheLocation(CacheLocation &&) noexcept = default; + CacheLocation &operator=(CacheLocation &&) noexcept = default; CacheLocation(DataStorageType type, size_t spec_size, const std::vector &location_specs); CacheLocation(const std::string &id, CacheLocationStatus status, @@ -101,7 +123,7 @@ class CacheLocation : public Jsonizable { } void ToRapidWriter(rapidjson::Writer &writer) const noexcept override { - Put(writer, "id", id_); + Put(writer, "id", id()); Put(writer, "status", status_); Put(writer, "type", type_); Put(writer, "spec_size", spec_size_); @@ -110,7 +132,10 @@ class CacheLocation : public Jsonizable { } bool FromRapidValue(const rapidjson::Value &rapid_value) override { - KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "id", id_, std::string("")); + std::string id; + validated_total_size_ = kUnknownValidatedTotalSize; + KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "id", id, std::string("")); + id_ = std::move(id); KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "status", status_, CacheLocationStatus::CLS_NOT_FOUND); KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "type", type_, DataStorageType::DATA_STORAGE_TYPE_UNKNOWN); KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "spec_size", spec_size_, size_t{0}); @@ -122,19 +147,58 @@ class CacheLocation : public Jsonizable { void set_status(CacheLocationStatus status) { status_ = status; } void set_type(DataStorageType type) { type_ = type; } void set_id(const std::string &id) { id_ = id; } + void set_id(std::string &&id) noexcept { id_ = std::move(id); } + void set_id(InternedLocationId id) noexcept { + if (id) { + id_ = std::move(id); + } else { + id_ = std::string{}; + } + } void set_spec_size(size_t spec_size) { spec_size_ = spec_size; } void set_create_time(int64_t create_time) { create_time_ = create_time; } - void push_location_spec(LocationSpec &&location_spec) { location_specs_.push_back(std::move(location_spec)); } - void set_location_specs(std::vector &&location_specs) { location_specs_ = location_specs; } + void push_location_spec(LocationSpec &&location_spec) { + validated_total_size_ = kUnknownValidatedTotalSize; + location_specs_.push_back(std::move(location_spec)); + } + void set_location_specs(std::vector &&location_specs) { + validated_total_size_ = kUnknownValidatedTotalSize; + location_specs_ = std::move(location_specs); + } + void set_validated_total_size(std::uint64_t size) noexcept { validated_total_size_ = size; } + [[nodiscard]] bool GetValidatedTotalSize(std::uint64_t &size) const noexcept { + if (validated_total_size_ == kUnknownValidatedTotalSize) { + return false; + } + size = validated_total_size_; + return true; + } + [[nodiscard]] bool HasValidatedLocationSpecs() const noexcept { + return validated_total_size_ != kUnknownValidatedTotalSize; + } [[nodiscard]] const std::vector &location_specs() const { return location_specs_; } - [[nodiscard]] const std::string &id() const { return id_; } + [[nodiscard]] std::vector &mutable_location_specs() { + validated_total_size_ = kUnknownValidatedTotalSize; + return location_specs_; + } + [[nodiscard]] const std::string &id() const { + if (const auto *owned = std::get_if(&id_)) { + return *owned; + } + const auto &interned = std::get(id_); + if (interned) { + return *interned; + } + static const std::string empty; + return empty; + } [[nodiscard]] CacheLocationStatus status() const { return status_; } [[nodiscard]] DataStorageType type() const { return type_; } [[nodiscard]] size_t spec_size() const { return spec_size_; } [[nodiscard]] int64_t create_time() const { return create_time_; } [[nodiscard]] size_t EstimateMemUsage() const { - size_t usage = sizeof(CacheLocation) + id_.size(); + size_t usage = sizeof(CacheLocation) + id().size(); for (const auto &spec : location_specs_) { usage += sizeof(LocationSpec) + spec.name().size() + spec.uri().size(); } @@ -142,12 +206,21 @@ class CacheLocation : public Jsonizable { } private: - std::string id_; + static constexpr std::uint64_t kUnknownValidatedTotalSize = std::numeric_limits::max(); + + std::variant id_; CacheLocationStatus status_ = CacheLocationStatus::CLS_NEW; DataStorageType type_ = DataStorageType::DATA_STORAGE_TYPE_UNKNOWN; size_t spec_size_ = 0; int64_t create_time_ = 0; std::vector location_specs_; + // Pure-local ReportEvent has already validated every URI and summed its + // sizes. Keep that proof and aggregate beside the immutable specs so the + // writer and query paths can skip repeated structural URI parsing. Every + // specs mutator clears it; merge/delete code may restore it only when all + // retained specs are also proven valid. It is intentionally not + // serialized, so recovered values fail closed and validate again. + std::uint64_t validated_total_size_ = kUnknownValidatedTotalSize; }; using CacheLocationConstPtr = std::shared_ptr; @@ -155,4 +228,62 @@ using CacheLocationVector = std::vector; using CacheLocationMap = std::unordered_map; using CacheLocationMapVector = std::vector; +// Compact positional representation for large all-location reads. Keeping one +// vector object per key is disproportionately expensive for GetHostCacheState: +// a million-key request otherwise constructs a million small vectors and, for +// the common one-location case, performs a million heap allocations. Offsets +// keep the same positional contract while all shared_ptr values live in one +// contiguous allocation per backend chunk. +class CacheLocationValueView { +public: + using const_iterator = CacheLocationVector::const_iterator; + + CacheLocationValueView() = default; + CacheLocationValueView(const_iterator begin, const_iterator end) : begin_(begin), end_(end) {} + + [[nodiscard]] const_iterator begin() const { return begin_; } + [[nodiscard]] const_iterator end() const { return end_; } + [[nodiscard]] bool empty() const { return begin_ == end_; } + [[nodiscard]] size_t size() const { return static_cast(end_ - begin_); } + +private: + const_iterator begin_{}; + const_iterator end_{}; +}; + +struct CompactLocationsPerKey { + std::vector offsets{0}; + CacheLocationVector values; + + void Clear(size_t key_capacity = 0, size_t location_capacity = 0) { + offsets.clear(); + offsets.reserve(key_capacity + 1); + offsets.push_back(0); + values.clear(); + values.reserve(location_capacity); + } + + [[nodiscard]] size_t size() const { return offsets.empty() ? 0 : offsets.size() - 1; } + [[nodiscard]] bool empty() const { return size() == 0; } + [[nodiscard]] bool IsValid(size_t expected_key_count) const { + if (offsets.size() != expected_key_count + 1 || offsets.empty() || offsets.front() != 0 || + offsets.back() != values.size()) { + return false; + } + for (size_t i = 1; i < offsets.size(); ++i) { + if (offsets[i] < offsets[i - 1]) { + return false; + } + } + return true; + } + + [[nodiscard]] CacheLocationValueView operator[](size_t index) const { + return CacheLocationValueView(values.begin() + static_cast(offsets[index]), + values.begin() + static_cast(offsets[index + 1])); + } + + void FinishKey() { offsets.push_back(values.size()); } +}; + } // namespace kv_cache_manager diff --git a/kv_cache_manager/meta/meta_indexer.cc b/kv_cache_manager/meta/meta_indexer.cc index 11f21dec0..858fd897a 100644 --- a/kv_cache_manager/meta/meta_indexer.cc +++ b/kv_cache_manager/meta/meta_indexer.cc @@ -4,9 +4,12 @@ #include #include #include +#include #include +#include #include #include +#include #include #include "kv_cache_manager/common/common.h" @@ -17,6 +20,7 @@ #include "kv_cache_manager/common/timestamp_util.h" #include "kv_cache_manager/config/meta_indexer_config.h" #include "kv_cache_manager/data_storage/storage_config.h" +#include "kv_cache_manager/meta/meta_local_backend.h" #include "kv_cache_manager/meta/utils.h" #include "kv_cache_manager/metrics/metrics_collector.h" #include "kv_cache_manager/metrics/metrics_registry.h" @@ -33,6 +37,53 @@ static constexpr const char *kRmwDeleteMetaOperation = "read_modify_write_delete static constexpr const char *kDeleteMetaOperation = "delete"; static constexpr const char *kExistMetaOperation = "exist"; static constexpr const char *kGetMetaOperation = "get"; +// Pure-local prefix scans first inspect a bounded window so a short prefix can +// cancel a million-key suffix promptly. Once that probe succeeds, larger +// ranges amortize the 1024-shard LRU's lookup/release locks while preserving +// enough independent work for the query executor. +static constexpr size_t kLocalPrefixProbeKeyCount = 4096; +static constexpr size_t kLocalPrefixParallelReadChunkSize = 16384; +static constexpr size_t kPrefixStateWordBits = 64; +static_assert(kLocalPrefixProbeKeyCount % kPrefixStateWordBits == 0); +static_assert(kLocalPrefixParallelReadChunkSize % kPrefixStateWordBits == 0); + +struct PrefixLocationScratch { + bool in_use = false; + CompactLocationsPerKey locations; +}; + +thread_local PrefixLocationScratch tls_prefix_location_scratch; + +class PrefixLocationScratchLease { +public: + explicit PrefixLocationScratchLease(size_t key_count) { + if (key_count <= kLocalPrefixParallelReadChunkSize && !tls_prefix_location_scratch.in_use) { + scratch_ = &tls_prefix_location_scratch; + scratch_->in_use = true; + uses_thread_local_ = true; + } else { + local_.emplace(); + scratch_ = &*local_; + } + } + + ~PrefixLocationScratchLease() { + if (uses_thread_local_) { + // Retain only allocation capacity. Holding shared_ptr values until + // this worker's next query would pin replaced CacheLocations and + // make the scratch cache an accidental object cache. + scratch_->locations.Clear(); + scratch_->in_use = false; + } + } + + CompactLocationsPerKey &locations() noexcept { return scratch_->locations; } + +private: + std::optional local_; + PrefixLocationScratch *scratch_ = nullptr; + bool uses_thread_local_ = false; +}; } // namespace class MetaIndexer::ScopedBatchLock { @@ -62,7 +113,7 @@ class MetaIndexer::ScopedBatchLock { private: MetaIndexer &indexer_; - std::vector shard_indexs_; + const std::vector &shard_indexs_; }; MetaIndexer::~MetaIndexer() { @@ -104,6 +155,15 @@ ErrorCode MetaIndexer::Init(const std::string &instance_id, const std::shared_pt backend_manager_.reset(); return ec; } + mutex_shard_hash_seed_ = kDefaultMetaShardHashSeed; + uint32_t local_cache_hash_seed = 0; + if (backend_manager_->GetPureLocalCacheHashSeed(local_cache_hash_seed)) { + // Pure-memory RMW batches already hold one metadata mutex shard at a + // time. Reusing the LRU's host-specific hash seed makes each such + // batch touch only the corresponding subset of LRU shards, avoiding + // thousands of redundant LRU lock/unlock cycles for large reports. + mutex_shard_hash_seed_ = local_cache_hash_seed; + } ec = backend_manager_->Open(); if (ec != EC_OK) { KVCM_LOG_ERROR("instance[%s] meta storage backend manager open failed, ec[%d]", instance_id_.c_str(), ec); @@ -117,10 +177,12 @@ ErrorCode MetaIndexer::Init(const std::string &instance_id, const std::shared_pt KVCM_LOG_ERROR("instance[%s] recover metadata failed, ec[%d]", instance_id_.c_str(), ec); return ec; } - KVCM_LOG_INFO("instance[%s] meta indexer init success, mutex shard num[%lu], max key count[%lu], " + KVCM_LOG_INFO("instance[%s] meta indexer init success, mutex shard num[%lu], mutex hash seed[%" PRIu64 + "], max key count[%lu], " "batch key size[%lu], key_count[%lu], persist_metadata_interval_time_ms[%zu], storage usage data[%s]", instance_id_.c_str(), mutex_shard_num, + mutex_shard_hash_seed_, max_key_count_, batch_key_size_, key_count_.load(), @@ -135,6 +197,10 @@ void MetaIndexer::SetRevisitHistogram(std::shared_ptr } } +int32_t MetaIndexer::GetMutexShardIndex(KeyType key) const noexcept { + return GetShardIndex(key, mutex_shard_mask_, mutex_shard_hash_seed_); +} + MetaIndexer::Result MetaIndexer::Put(RequestContext *request_context, const KeyVector &keys, CacheLocationMapVector &location_maps, @@ -230,26 +296,82 @@ std::pair MetaIndexer::ExecuteRmwUpsert(const std::string &tra const std::vector &put_global_indexs, const KeyVector &all_keys, RmwStats &stats, - Result &result) noexcept { + Result &result, + bool preserve_existing_updates_when_full) noexcept { if (upsert_batch.batch_keys.empty()) { return {0, 0}; } - stats.put_key_count += static_cast(put_global_indexs.size()); - stats.update_key_count += static_cast(upsert_batch.batch_keys.size() - put_global_indexs.size()); + std::unordered_set unique_put_keys; + unique_put_keys.reserve(put_global_indexs.size()); + for (const int32_t global_index : put_global_indexs) { + if (global_index >= 0 && static_cast(global_index) < all_keys.size()) { + unique_put_keys.insert(all_keys[global_index]); + } + } + const size_t unique_put_key_count = unique_put_keys.size(); + stats.put_key_count += static_cast(unique_put_key_count); + stats.update_key_count += static_cast(upsert_batch.batch_keys.size() - unique_put_key_count); std::vector upsert_ecs; - if (put_global_indexs.size() + GetKeyCount() > max_key_count_) { + BatchMetaData existing_update_batch; + std::vector existing_update_positions; + BatchMetaData *backend_batch = &upsert_batch; + const bool capacity_exceeded = unique_put_key_count + GetKeyCount() > max_key_count_; + if (capacity_exceeded) { PREFIX_INDEXER_LOG(ERROR, "ReadModifyWrite put keys count[%lu] + current key count[%lu] > max key count[%lu]", - put_global_indexs.size(), + unique_put_key_count, GetKeyCount(), max_key_count_); - upsert_ecs.assign(upsert_batch.batch_keys.size(), EC_NOSPC); - } else { + if (!preserve_existing_updates_when_full) { + upsert_ecs.assign(upsert_batch.batch_keys.size(), EC_NOSPC); + } else { + // Fused targeted RMW can mix brand-new keys and updates to keys + // that already count toward capacity. Reject only the former; + // the old two-phase merge still admitted the latter at capacity. + std::vector is_new_key(all_keys.size(), false); + for (const int32_t global_index : put_global_indexs) { + if (global_index >= 0 && static_cast(global_index) < is_new_key.size()) { + is_new_key[global_index] = true; + } + } + upsert_ecs.assign(upsert_batch.batch_keys.size(), EC_NOSPC); + existing_update_positions.reserve(upsert_batch.batch_keys.size()); + for (size_t i = 0; i < upsert_batch.batch_keys.size(); ++i) { + const int32_t global_index = upsert_batch.batch_indexs[i]; + if (global_index < 0 || static_cast(global_index) >= is_new_key.size() || + is_new_key[global_index]) { + continue; + } + existing_update_positions.push_back(i); + existing_update_batch.batch_keys.push_back(upsert_batch.batch_keys[i]); + existing_update_batch.batch_indexs.push_back(global_index); + existing_update_batch.batch_locations.push_back(upsert_batch.batch_locations[i]); + existing_update_batch.batch_properties.push_back(upsert_batch.batch_properties[i]); + } + backend_batch = &existing_update_batch; + } + } + if (upsert_ecs.empty() || !existing_update_positions.empty()) { const int64_t begin = TimestampUtil::GetCurrentTimeUs(); - upsert_ecs = backend_manager_->Upsert(request_context, upsert_batch); + std::vector backend_ecs = backend_manager_->Upsert(request_context, *backend_batch); stats.upsert_io_time_us += TimestampUtil::GetCurrentTimeUs() - begin; + if (existing_update_positions.empty()) { + upsert_ecs = std::move(backend_ecs); + } else if (backend_ecs.size() != existing_update_positions.size()) { + PREFIX_INDEXER_LOG(ERROR, + "ReadModifyWrite existing update results[%lu] mismatch keys[%lu]", + backend_ecs.size(), + existing_update_positions.size()); + for (const size_t original_position : existing_update_positions) { + upsert_ecs[original_position] = EC_MISMATCH; + } + } else { + for (size_t i = 0; i < backend_ecs.size(); ++i) { + upsert_ecs[existing_update_positions[i]] = backend_ecs[i]; + } + } int64_t v = 0; auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); KVCM_METRICS_COLLECTOR_GET_METRICS(service_metrics_collector, meta_searcher, index_serialize_time_us, v); @@ -267,15 +389,18 @@ std::pair MetaIndexer::ExecuteRmwUpsert(const std::string &tra const int32_t error_count = ProcessErrorCodes(trace_id, upsert_ecs, upsert_batch.batch_indexs, all_keys, kRmwUpsertMetaOperation, result); - int32_t put_success_count = 0; - if (error_count == 0) { - put_success_count = static_cast(put_global_indexs.size()); - } else { - for (const int32_t idx : put_global_indexs) { - if (result.error_codes[idx] == EC_OK) { - ++put_success_count; + int32_t put_success_count = static_cast(unique_put_key_count); + if (error_count != 0) { + // Reuse the existing buckets to avoid another allocation on the + // partial-failure path. The all-success path needs no second scan. + unique_put_keys.clear(); + for (const int32_t global_index : put_global_indexs) { + if (global_index >= 0 && static_cast(global_index) < all_keys.size() && + result.error_codes[global_index] == EC_OK) { + unique_put_keys.insert(all_keys[global_index]); } } + put_success_count = static_cast(unique_put_keys.size()); } return {error_count, put_success_count}; } @@ -483,6 +608,29 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext const LocationModifierFunc &modifier, bool adjust_reclaimed_key_count, bool refresh_cache_from_persistent) noexcept { + return ReadModifyWriteLocationImpl(request_context, + keys, + location_ids, + modifier, + adjust_reclaimed_key_count, + false, + refresh_cache_from_persistent); +} + +MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteTargetLocations(RequestContext *request_context, + const KeyVector &keys, + const LocationIdsPerKey &location_ids, + const LocationModifierFunc &modifier) noexcept { + return ReadModifyWriteLocationImpl(request_context, keys, location_ids, modifier, false, true, false); +} + +MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocationImpl(RequestContext *request_context, + const KeyVector &keys, + const LocationIdsPerKey &location_ids, + const LocationModifierFunc &modifier, + bool adjust_reclaimed_key_count, + bool track_created_key_count, + bool refresh_cache_from_persistent) noexcept { const auto &trace_id = request_context->trace_id(); if (keys.empty()) { return LocationResult(EC_OK); @@ -501,8 +649,9 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext std::shared_ptr ephemeral_metrics_collector = std::make_shared(ephemeral_metrics_registry); ephemeral_metrics_collector->Init(); - auto ephemeral_request_context = - std::make_shared("read_modify_write_location", ephemeral_metrics_collector); + auto ephemeral_request_context = std::make_shared( + track_created_key_count ? "read_modify_write_target_locations" : "read_modify_write_location", + ephemeral_metrics_collector); static CacheLocationMapVector empty_locations; static PropertyMapVector empty_properties; @@ -511,7 +660,13 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext LocationResult location_result(location_ids); Result rmw_result(keys.size()); - int32_t error_count = 0; + // The aggregate result reports whether the RMW machinery completed for + // each key. Per-location semantic outcomes (for example, a CAS mismatch + // or a rejected task in an otherwise valid batch) remain in + // per_location_error_codes and do not fail the whole operation. Track + // structural/read/modifier/write failures separately so malformed backend + // responses still fail closed without changing that established contract. + std::vector key_level_failures(keys.size(), false); RmwStats stats; for (auto &batch : batches) { ScopedBatchLock lock(*this, batch.batch_shard_indexs, &stats.lock_wait_time_us); @@ -519,6 +674,7 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext // 1. One batched read for every (key, location_id) return deserialised CacheLocation const auto &batch_keys = batch.batch_keys; LocationsPerKey batch_locations_per_key; + std::vector batch_key_get_ecs; const int64_t begin_get = TimestampUtil::GetCurrentTimeUs(); std::vector refresh_results; if (refresh_cache_from_persistent) { @@ -527,61 +683,130 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext // cache entry with the complete source-of-truth key first. refresh_results = backend_manager_->RefreshCacheFromPersistent(ephemeral_request_context.get(), batch_keys); } - std::vector> get_ecs_per_key = backend_manager_->GetLocations( - ephemeral_request_context.get(), batch_keys, batch.batch_location_ids, batch_locations_per_key); + std::vector> get_ecs_per_key; + if (track_created_key_count) { + get_ecs_per_key = backend_manager_->GetLocationsWithKeyStatus(ephemeral_request_context.get(), + batch_keys, + batch.batch_location_ids, + batch_locations_per_key, + batch_key_get_ecs); + } else { + get_ecs_per_key = backend_manager_->GetLocations( + ephemeral_request_context.get(), batch_keys, batch.batch_location_ids, batch_locations_per_key); + } stats.get_io_time_us += TimestampUtil::GetCurrentTimeUs() - begin_get; + int64_t v = 0; + auto *ephemeral_service_metrics_collector = + dynamic_cast(ephemeral_request_context->metrics_collector()); + KVCM_METRICS_COLLECTOR_GET_METRICS( + ephemeral_service_metrics_collector, meta_searcher, index_deserialize_time_us, v); + stats.index_deserialize_time_us += v; + stats.has_index_deserialize = true; + if (get_ecs_per_key.size() != batch_keys.size() || batch_locations_per_key.size() != batch_keys.size() || + (track_created_key_count && batch_key_get_ecs.size() != batch_keys.size()) || (refresh_cache_from_persistent && refresh_results.size() != batch_keys.size())) { PREFIX_INDEXER_LOG(ERROR, - "ReadModifyWriteLocation read shape mismatch, keys[%lu] ecs[%lu] locations[%lu] " - "refresh[%lu]", + "ReadModifyWriteLocation result size mismatch, keys[%lu], ecs[%lu], locations[%lu], " + "key_ecs[%lu]", batch_keys.size(), get_ecs_per_key.size(), batch_locations_per_key.size(), - refresh_results.size()); + batch_key_get_ecs.size()); for (const int32_t global_idx : batch.batch_indexs) { - location_result.per_location_error_codes[global_idx].assign(location_ids[global_idx].size(), EC_ERROR); + location_result.per_location_error_codes[global_idx].assign(location_ids[global_idx].size(), + EC_MISMATCH); + key_level_failures[global_idx] = true; } - error_count += static_cast(batch.batch_indexs.size()); continue; } - for (size_t i = 0; i < batch_keys.size(); ++i) { - const size_t location_count = batch.batch_location_ids[i].size(); - if (get_ecs_per_key[i].size() != location_count || batch_locations_per_key[i].size() != location_count) { - get_ecs_per_key[i].assign(location_count, EC_ERROR); - batch_locations_per_key[i].assign(location_count, nullptr); - } - if (refresh_cache_from_persistent && refresh_results[i] != EC_OK) { - get_ecs_per_key[i].assign(location_count, refresh_results[i]); - batch_locations_per_key[i].assign(location_count, nullptr); + + if (refresh_cache_from_persistent) { + for (size_t i = 0; i < batch_keys.size(); ++i) { + if (refresh_results[i] == EC_OK) { + continue; + } + get_ecs_per_key[i].assign(batch.batch_location_ids[i].size(), refresh_results[i]); + batch_locations_per_key[i].assign(batch.batch_location_ids[i].size(), nullptr); } } - int64_t v = 0; - auto *ephemeral_service_metrics_collector = - dynamic_cast(ephemeral_request_context->metrics_collector()); - KVCM_METRICS_COLLECTOR_GET_METRICS( - ephemeral_service_metrics_collector, meta_searcher, index_deserialize_time_us, v); - stats.index_deserialize_time_us += v; - stats.has_index_deserialize = true; // 2. Per-key modifier dispatch -> bucket each key into the upsert sub-batch or the delete sub-batch. BatchMetaData upsert_batch; BatchMetaData delete_batch; - std::vector> upsert_location_indexs(keys.size()); - std::vector> delete_location_indexs(keys.size()); + std::vector put_global_indexs; for (size_t i = 0; i < batch_keys.size(); ++i) { const int32_t global_idx = batch.batch_indexs[i]; const KeyType key = batch_keys[i]; - const std::vector &get_ecs = get_ecs_per_key[i]; + std::vector &get_ecs = get_ecs_per_key[i]; const LocationIdVector &loc_ids = batch.batch_location_ids[i]; CacheLocationVector &loc_values = batch_locations_per_key[i]; + if (get_ecs.size() != loc_ids.size() || loc_values.size() != loc_ids.size()) { + PREFIX_INDEXER_LOG(ERROR, + "ReadModifyWriteLocation per-key result size mismatch, key[%ld], ids[%lu], " + "ecs[%lu], locations[%lu]", + key, + loc_ids.size(), + get_ecs.size(), + loc_values.size()); + location_result.per_location_error_codes[global_idx].assign(loc_ids.size(), EC_MISMATCH); + key_level_failures[global_idx] = true; + continue; + } + const ErrorCode key_get_ec = track_created_key_count ? batch_key_get_ecs[i] : EC_OK; + if (key_get_ec != EC_OK && key_get_ec != EC_NOENT) { + location_result.per_location_error_codes[global_idx].assign(loc_ids.size(), key_get_ec); + key_level_failures[global_idx] = true; + continue; + } + // EC_OK promises a usable value for the requested id. Treat a + // null or mis-keyed value as corruption and never let a modifier + // turn it into a write based on fabricated state. + for (size_t loc_index = 0; loc_index < loc_ids.size(); ++loc_index) { + if (get_ecs[loc_index] == EC_OK && + (!loc_values[loc_index] || loc_values[loc_index]->id() != loc_ids[loc_index])) { + PREFIX_INDEXER_LOG(ERROR, + "ReadModifyWriteLocation invalid EC_OK value, key[%ld], requested id[%s]", + key, + loc_ids[loc_index].c_str()); + get_ecs[loc_index] = EC_MISMATCH; + loc_values[loc_index].reset(); + } + if (get_ecs[loc_index] != EC_OK && get_ecs[loc_index] != EC_NOENT) { + key_level_failures[global_idx] = true; + } + } + if (track_created_key_count && key_get_ec == EC_NOENT && + std::any_of(get_ecs.begin(), get_ecs.end(), [](ErrorCode ec) { return ec == EC_OK; })) { + PREFIX_INDEXER_LOG(ERROR, + "ReadModifyWriteTargetLocations key[%ld] reported missing with an existing " + "target location", + key); + location_result.per_location_error_codes[global_idx].assign(loc_ids.size(), EC_MISMATCH); + key_level_failures[global_idx] = true; + continue; + } PropertyMap upsert_property_map; auto [action, modifier_ecs] = modifier(get_ecs, loc_ids, static_cast(global_idx), loc_values, upsert_property_map); if (modifier_ecs.size() != loc_ids.size()) { modifier_ecs.assign(loc_ids.size(), EC_ERROR); action = MA_FAIL; + key_level_failures[global_idx] = true; + } + // A read error other than NOENT is not a valid basis for an RMW. + // Force it back into the corresponding result slot even if a + // buggy modifier accidentally returned EC_OK. + for (size_t loc_index = 0; loc_index < loc_ids.size(); ++loc_index) { + if (get_ecs[loc_index] != EC_OK && get_ecs[loc_index] != EC_NOENT) { + modifier_ecs[loc_index] = get_ecs[loc_index]; + } + } + if (action == MA_FAIL && + std::all_of(modifier_ecs.begin(), modifier_ecs.end(), [](ErrorCode ec) { return ec == EC_OK; })) { + modifier_ecs.assign(loc_ids.size(), EC_ERROR); + key_level_failures[global_idx] = true; } if (action == MA_OK) { CacheLocationMap upsert_loc_map; @@ -592,15 +817,21 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext } const LocationId &loc_id = loc_ids[loc_index]; const CacheLocationConstPtr &working_loc = loc_values[loc_index]; - assert(working_loc && loc_id == working_loc->id()); + if (!working_loc || loc_id != working_loc->id()) { + location_result.per_location_error_codes[global_idx][loc_index] = EC_MISMATCH; + key_level_failures[global_idx] = true; + continue; + } upsert_loc_map.emplace(loc_id, working_loc); - upsert_location_indexs[global_idx].emplace_back(loc_index); } if (!upsert_loc_map.empty() || !upsert_property_map.empty()) { upsert_batch.batch_keys.emplace_back(key); upsert_batch.batch_indexs.emplace_back(global_idx); upsert_batch.batch_locations.emplace_back(std::move(upsert_loc_map)); upsert_batch.batch_properties.emplace_back(std::move(upsert_property_map)); + if (track_created_key_count && key_get_ec == EC_NOENT) { + put_global_indexs.emplace_back(global_idx); + } } } else if (action == MA_DELETE) { LocationIdVector alive_ids; @@ -610,7 +841,6 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext continue; } alive_ids.emplace_back(loc_ids[loc_index]); - delete_location_indexs[global_idx].emplace_back(loc_index); } if (!alive_ids.empty()) { delete_batch.batch_keys.emplace_back(key); @@ -620,46 +850,436 @@ MetaIndexer::LocationResult MetaIndexer::ReadModifyWriteLocation(RequestContext } else { // MA_FAIL / MA_SKIP / unknown: surface modifier_ec if any. if (action == MA_FAIL) { - ++error_count; + key_level_failures[global_idx] = true; + } else if (action != MA_SKIP) { + modifier_ecs.assign(loc_ids.size(), EC_ERROR); + key_level_failures[global_idx] = true; } location_result.per_location_error_codes[global_idx] = std::move(modifier_ecs); } } // 3. Dispatch upsert and delete sub-batches. - static std::vector empty_put_global_indexs; - const auto [upsert_errs, put_success_count] = ExecuteRmwUpsert( - trace_id, ephemeral_request_context.get(), upsert_batch, empty_put_global_indexs, keys, stats, rmw_result); + const auto [upsert_errs, put_success_count] = ExecuteRmwUpsert(trace_id, + ephemeral_request_context.get(), + upsert_batch, + put_global_indexs, + keys, + stats, + rmw_result, + track_created_key_count); + (void)upsert_errs; for (const auto &global_index : upsert_batch.batch_indexs) { - for (const auto &location_index : upsert_location_indexs[global_index]) { - location_result.per_location_error_codes[global_index][location_index] = - rmw_result.error_codes[global_index]; + if (rmw_result.error_codes[global_index] != EC_OK) { + key_level_failures[global_index] = true; + } + for (auto &location_ec : location_result.per_location_error_codes[global_index]) { + if (location_ec == EC_OK) { + location_ec = rmw_result.error_codes[global_index]; + } } } const auto [delete_errs, delete_success_count] = ExecuteRmwDelete(trace_id, ephemeral_request_context.get(), delete_batch, keys, stats, rmw_result); + (void)delete_errs; for (const auto &global_index : delete_batch.batch_indexs) { - for (const auto &location_index : delete_location_indexs[global_index]) { - location_result.per_location_error_codes[global_index][location_index] = - rmw_result.error_codes[global_index]; + if (rmw_result.error_codes[global_index] != EC_OK) { + key_level_failures[global_index] = true; + } + for (auto &location_ec : location_result.per_location_error_codes[global_index]) { + if (location_ec == EC_OK) { + location_ec = rmw_result.error_codes[global_index]; + } } } - error_count += upsert_errs + delete_errs; AdjustKeyCountMeta(put_success_count - (adjust_reclaimed_key_count ? delete_success_count : 0)); } EmitRmwMetrics(request_context->metrics_collector(), stats, keys.size()); - if (error_count == keys.size()) { + const size_t failed_key_count = + static_cast(std::count(key_level_failures.begin(), key_level_failures.end(), true)); + if (failed_key_count == keys.size()) { location_result.ec = EC_ERROR; - PREFIX_INDEXER_LOG(DEBUG, "all locations rmw failed, error count[%d]", error_count); - } else if (error_count > 0) { + PREFIX_INDEXER_LOG(DEBUG, "all locations rmw failed, error count[%lu]", failed_key_count); + } else if (failed_key_count > 0) { location_result.ec = EC_PARTIAL_OK; PREFIX_INDEXER_LOG( - DEBUG, "partial locations rmw failed, keys count[%lu] failed count[%d]", keys.size(), error_count); + DEBUG, "partial locations rmw failed, keys count[%lu] failed count[%lu]", keys.size(), failed_key_count); } return location_result; } +bool MetaIndexer::SupportsSingleLocationRmw() const noexcept { + return backend_manager_ && backend_manager_->SupportsSingleLocationRmw(); +} + +MetaIndexer::SingleLocationResult +MetaIndexer::ReadModifyWriteSingleTargetLocations(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + const SingleLocationModifierFunc &modifier) noexcept { + const auto &trace_id = request_context->trace_id(); + if (keys.empty()) { + return SingleLocationResult(EC_OK); + } + if (keys.size() != location_ids.size() || !modifier || + std::any_of(location_ids.begin(), location_ids.end(), [](const LocationId *id) { + return id == nullptr || id->empty(); + })) { + PREFIX_INDEXER_LOG( + ERROR, "single target RMW invalid inputs, keys[%lu], location ids[%lu]", keys.size(), location_ids.size()); + return SingleLocationResult(EC_BADARGS); + } + bool has_duplicate_keys = false; + if (std::is_sorted(keys.begin(), keys.end())) { + has_duplicate_keys = std::adjacent_find(keys.begin(), keys.end()) != keys.end(); + } else { + std::unordered_set seen_keys; + seen_keys.reserve(keys.size()); + for (const KeyType key : keys) { + if (!seen_keys.insert(key).second) { + has_duplicate_keys = true; + break; + } + } + } + if (has_duplicate_keys) { + PREFIX_INDEXER_LOG(ERROR, "single target RMW requires unique keys"); + return SingleLocationResult(EC_BADARGS); + } + if (!SupportsSingleLocationRmw()) { + PREFIX_INDEXER_LOG(ERROR, "single target RMW requires a pure local metadata backend"); + return SingleLocationResult(EC_UNIMPLEMENTED); + } + + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + KVCM_METRICS_COLLECTOR_SET_METRICS(service_metrics_collector, meta_indexer, query_key_count, keys.size()); + + // Preserve MakeBatches' observable ordering and capacity behavior: shards + // are visited in ascending order, indices within one shard retain request + // order, and a whole shard is appended before the soft limit is checked. + struct SingleLocationBatch { + std::vector shard_indices; + std::vector global_indices; + }; + std::vector> indices_by_shard(mutex_shards_.size()); + for (int32_t i = 0; i < static_cast(keys.size()); ++i) { + indices_by_shard[GetMutexShardIndex(keys[i])].push_back(i); + } + std::vector batches; + batches.reserve(indices_by_shard.size()); + SingleLocationBatch current_batch; + size_t current_batch_size = 0; + size_t nonempty_shards_remaining = static_cast(std::count_if( + indices_by_shard.begin(), indices_by_shard.end(), [](const auto &indices) { return !indices.empty(); })); + for (size_t shard_index = 0; shard_index < indices_by_shard.size(); ++shard_index) { + const auto &indices = indices_by_shard[shard_index]; + if (indices.empty()) { + continue; + } + current_batch.shard_indices.push_back(static_cast(shard_index)); + current_batch.global_indices.reserve(current_batch.global_indices.size() + indices.size()); + current_batch.global_indices.insert(current_batch.global_indices.end(), indices.begin(), indices.end()); + current_batch_size += indices.size(); + --nonempty_shards_remaining; + if (current_batch_size >= batch_key_size_ || nonempty_shards_remaining == 0) { + batches.push_back(std::move(current_batch)); + current_batch = SingleLocationBatch{}; + current_batch_size = 0; + } + } + KVCM_METRICS_COLLECTOR_SET_METRICS(service_metrics_collector, meta_indexer, query_batch_num, batches.size()); + + // Prepare backend-owned lookup/release workspace once, before acquiring + // any metadata shard mutex. The same storage is reused by the read and + // upsert halves of every batch in this synchronous request. + const size_t max_batch_size = + std::max_element(batches.begin(), batches.end(), [](const auto &lhs, const auto &rhs) { + return lhs.global_indices.size() < rhs.global_indices.size(); + })->global_indices.size(); + SingleLocationRmwScratch backend_scratch; + backend_manager_->PrepareSingleLocationRmwScratch(max_batch_size, backend_scratch); + + SingleLocationResult result(keys.size()); + std::vector key_level_failures(keys.size(), false); + RmwStats stats; + stats.has_index_deserialize = true; + + KeyVector batch_keys; + LocationIdRefVector batch_location_ids; + CacheLocationViewVector batch_existing_locations; + std::vector batch_get_ecs; + std::vector batch_key_get_ecs; + KeyVector upsert_keys; + LocationIdRefVector upsert_location_ids; + CacheLocationVector upsert_locations; + std::vector upsert_global_indices; + std::vector upsert_is_new_key; + std::vector upsert_read_indices; + batch_keys.reserve(max_batch_size); + batch_location_ids.reserve(max_batch_size); + batch_existing_locations.reserve(max_batch_size); + batch_get_ecs.reserve(max_batch_size); + batch_key_get_ecs.reserve(max_batch_size); + upsert_keys.reserve(max_batch_size); + upsert_location_ids.reserve(max_batch_size); + upsert_locations.reserve(max_batch_size); + upsert_global_indices.reserve(max_batch_size); + upsert_is_new_key.reserve(max_batch_size); + upsert_read_indices.reserve(max_batch_size); + + struct DeferredLocationRelease { + CacheLocationVector &locations; + ~DeferredLocationRelease() { locations.clear(); } + }; + + for (const auto &batch : batches) { + // Reuse request-shaped buffers across internal batches. Clear happens + // before the next ScopedBatchLock, so releasing replacement locations + // and growing these vectors never extends metadata lock hold time. + batch_keys.clear(); + batch_location_ids.clear(); + batch_existing_locations.clear(); + batch_get_ecs.clear(); + batch_key_get_ecs.clear(); + upsert_keys.clear(); + upsert_location_ids.clear(); + upsert_locations.clear(); + upsert_global_indices.clear(); + upsert_is_new_key.clear(); + upsert_read_indices.clear(); + for (const int32_t global_index : batch.global_indices) { + batch_keys.push_back(keys[global_index]); + batch_location_ids.push_back(location_ids[global_index]); + } + // Allocate all request-shaped scratch vectors before taking metadata + // shard locks. Large ReportEvent requests otherwise extend every lock + // hold with allocator work and can amplify allocator futex contention. + backend_scratch.retired_locations.clear(); + // Construct this guard before the shard lock. Reverse destruction + // releases the lock first, then drops replaced CacheLocations and + // their URI strings outside the metadata critical section. + DeferredLocationRelease deferred_location_release{backend_scratch.retired_locations}; + ScopedBatchLock lock(*this, batch.shard_indices, &stats.lock_wait_time_us); + + const int64_t begin_get = TimestampUtil::GetCurrentTimeUs(); + backend_manager_->GetSingleLocationViewsWithKeyStatusInto(nullptr, + batch_keys, + batch_location_ids, + batch_existing_locations, + batch_key_get_ecs, + batch_get_ecs, + backend_scratch); + stats.get_io_time_us += TimestampUtil::GetCurrentTimeUs() - begin_get; + if (batch_get_ecs.size() != batch_keys.size() || batch_key_get_ecs.size() != batch_keys.size() || + batch_existing_locations.size() != batch_keys.size()) { + PREFIX_INDEXER_LOG(ERROR, + "single target RMW result size mismatch, keys[%lu], ecs[%lu], locations[%lu], " + "key ecs[%lu]", + batch_keys.size(), + batch_get_ecs.size(), + batch_existing_locations.size(), + batch_key_get_ecs.size()); + for (const int32_t global_index : batch.global_indices) { + result.error_codes[global_index] = EC_MISMATCH; + key_level_failures[global_index] = true; + } + backend_scratch.ReleaseRetainedHandles(); + continue; + } + + for (size_t i = 0; i < batch_keys.size(); ++i) { + const int32_t global_index = batch.global_indices[i]; + ErrorCode get_ec = batch_get_ecs[i]; + const ErrorCode key_get_ec = batch_key_get_ecs[i]; + if (key_get_ec != EC_OK && key_get_ec != EC_NOENT) { + result.error_codes[global_index] = key_get_ec; + key_level_failures[global_index] = true; + continue; + } + if (get_ec == EC_OK && + (!batch_existing_locations[i] || batch_existing_locations[i]->id() != *batch_location_ids[i])) { + PREFIX_INDEXER_LOG(ERROR, + "single target RMW invalid EC_OK value, key[%ld], requested id[%s]", + batch_keys[i], + batch_location_ids[i]->c_str()); + batch_existing_locations[i] = nullptr; + get_ec = EC_MISMATCH; + } + if (key_get_ec == EC_NOENT && get_ec == EC_OK) { + PREFIX_INDEXER_LOG(ERROR, + "single target RMW key[%ld] reported missing with an existing target location", + batch_keys[i]); + result.error_codes[global_index] = EC_MISMATCH; + key_level_failures[global_index] = true; + continue; + } + if (get_ec != EC_OK && get_ec != EC_NOENT) { + key_level_failures[global_index] = true; + } + + CacheLocationConstPtr new_location; + auto [action, modifier_ec] = modifier(get_ec, + *batch_location_ids[i], + static_cast(global_index), + batch_existing_locations[i], + new_location); + if (get_ec != EC_OK && get_ec != EC_NOENT) { + modifier_ec = get_ec; + } + if (action == MA_OK && modifier_ec == EC_OK) { + if (!new_location || new_location->id() != *batch_location_ids[i]) { + result.error_codes[global_index] = EC_MISMATCH; + key_level_failures[global_index] = true; + continue; + } + upsert_keys.push_back(batch_keys[i]); + upsert_location_ids.push_back(batch_location_ids[i]); + upsert_locations.push_back(std::move(new_location)); + upsert_global_indices.push_back(global_index); + upsert_is_new_key.push_back(key_get_ec == EC_NOENT); + upsert_read_indices.push_back(i); + } else { + if (action == MA_OK && modifier_ec != EC_OK) { + action = MA_SKIP; + } + if (action == MA_FAIL || action == MA_DELETE || (action != MA_SKIP && action != MA_OK)) { + key_level_failures[global_index] = true; + } + result.error_codes[global_index] = + action == MA_SKIP ? modifier_ec : (modifier_ec == EC_OK ? EC_ERROR : modifier_ec); + } + } + + const size_t new_key_count = + static_cast(std::count(upsert_is_new_key.begin(), upsert_is_new_key.end(), true)); + stats.put_key_count += static_cast(new_key_count); + stats.update_key_count += static_cast(upsert_keys.size() - new_key_count); + if (upsert_keys.empty()) { + backend_scratch.ReleaseRetainedHandles(); + continue; + } + + const bool capacity_exceeded = new_key_count + GetKeyCount() > max_key_count_; + if (!capacity_exceeded) { + const int64_t begin_upsert = TimestampUtil::GetCurrentTimeUs(); + // The read result is dead after modifier evaluation. Reuse its + // capacity for writer status instead of allocating another + // request-sized vector under the metadata lock. + backend_manager_->UpsertSingleLocationsUsingRetainedHandlesInto(nullptr, + upsert_keys, + upsert_location_ids, + upsert_locations, + upsert_read_indices, + batch_get_ecs, + backend_scratch); + stats.upsert_io_time_us += TimestampUtil::GetCurrentTimeUs() - begin_upsert; + } else { + PREFIX_INDEXER_LOG(ERROR, + "single target RMW put keys count[%lu] + current key count[%lu] > max key count[%lu]", + new_key_count, + GetKeyCount(), + max_key_count_); + + // Compact existing-key updates in place. The old implementation + // allocated four subset vectors while all metadata shard locks + // were held. New-key positions are already final EC_NOSPC, while + // the compacted global indices retain the mapping needed below. + for (size_t i = 0; i < upsert_is_new_key.size(); ++i) { + if (upsert_is_new_key[i]) { + const int32_t global_index = upsert_global_indices[i]; + result.error_codes[global_index] = EC_NOSPC; + key_level_failures[global_index] = true; + } + } + size_t existing_count = 0; + for (size_t i = 0; i < upsert_keys.size(); ++i) { + if (upsert_is_new_key[i]) { + continue; + } + if (existing_count != i) { + upsert_keys[existing_count] = upsert_keys[i]; + upsert_location_ids[existing_count] = upsert_location_ids[i]; + upsert_locations[existing_count] = std::move(upsert_locations[i]); + upsert_global_indices[existing_count] = upsert_global_indices[i]; + upsert_read_indices[existing_count] = upsert_read_indices[i]; + } + ++existing_count; + } + upsert_keys.resize(existing_count); + upsert_location_ids.resize(existing_count); + upsert_locations.resize(existing_count); + upsert_global_indices.resize(existing_count); + upsert_read_indices.resize(existing_count); + + if (existing_count > 0) { + const int64_t begin_upsert = TimestampUtil::GetCurrentTimeUs(); + // The get result vector is dead after modifier evaluation and + // already has sufficient capacity, so reuse it for writes. + backend_manager_->UpsertSingleLocationsUsingRetainedHandlesInto(nullptr, + upsert_keys, + upsert_location_ids, + upsert_locations, + upsert_read_indices, + batch_get_ecs, + backend_scratch); + stats.upsert_io_time_us += TimestampUtil::GetCurrentTimeUs() - begin_upsert; + if (batch_get_ecs.size() != existing_count) { + batch_get_ecs.assign(existing_count, EC_MISMATCH); + } + } else { + backend_scratch.ReleaseRetainedHandles(); + } + + // New keys cannot be admitted, but existing locations retain + // ordinary update semantics. Finish this exceptional batch here + // because its arrays were compacted in place. + for (size_t i = 0; i < existing_count; ++i) { + const int32_t global_index = upsert_global_indices[i]; + const ErrorCode ec = batch_get_ecs[i]; + result.error_codes[global_index] = ec; + if (ec != EC_OK) { + key_level_failures[global_index] = true; + PREFIX_INDEXER_LOG( + ERROR, "single target RMW upsert failed, key[%ld], ec[%d]", keys[global_index], ec); + } + } + continue; + } + + int32_t successful_new_keys = 0; + if (batch_get_ecs.size() != upsert_keys.size()) { + PREFIX_INDEXER_LOG(ERROR, + "single target RMW upsert result size[%lu] mismatch keys[%lu]", + batch_get_ecs.size(), + upsert_keys.size()); + batch_get_ecs.assign(upsert_keys.size(), EC_MISMATCH); + } + for (size_t i = 0; i < batch_get_ecs.size(); ++i) { + const int32_t global_index = upsert_global_indices[i]; + result.error_codes[global_index] = batch_get_ecs[i]; + if (batch_get_ecs[i] != EC_OK) { + key_level_failures[global_index] = true; + PREFIX_INDEXER_LOG( + ERROR, "single target RMW upsert failed, key[%ld], ec[%d]", keys[global_index], batch_get_ecs[i]); + } else if (upsert_is_new_key[i]) { + ++successful_new_keys; + } + } + AdjustKeyCountMeta(successful_new_keys); + } + + EmitRmwMetrics(request_context->metrics_collector(), stats, keys.size()); + const size_t failed_key_count = + static_cast(std::count(key_level_failures.begin(), key_level_failures.end(), true)); + if (failed_key_count == keys.size()) { + result.ec = EC_ERROR; + } else if (failed_key_count > 0) { + result.ec = EC_PARTIAL_OK; + } + return result; +} + MetaIndexer::Result MetaIndexer::Exist(RequestContext *request_context, const KeyVector &keys, std::vector &out_exists) noexcept { const auto &trace_id = request_context->trace_id(); @@ -709,6 +1329,15 @@ MetaIndexer::Result MetaIndexer::GetLocations(RequestContext *request_context, int64_t begin_get_io_time = TimestampUtil::GetCurrentTimeUs(); auto error_codes = backend_manager_->GetLocations(request_context, keys, out_location_maps); + if (error_codes.size() != keys.size() || out_location_maps.size() != keys.size()) { + PREFIX_INDEXER_LOG(ERROR, + "GetLocations result size mismatch, keys[%lu], ecs[%lu], locations[%lu]", + keys.size(), + error_codes.size(), + out_location_maps.size()); + error_codes.assign(keys.size(), EC_MISMATCH); + out_location_maps.assign(keys.size(), CacheLocationMap{}); + } KVCM_METRICS_COLLECTOR_SET_METRICS( service_metrics_collector, meta_indexer, get_io_time_us, TimestampUtil::GetCurrentTimeUs() - begin_get_io_time); @@ -740,11 +1369,501 @@ MetaIndexer::Result MetaIndexer::GetLocationsFromPersistent(RequestContext *requ return result; } +MetaIndexer::Result MetaIndexer::GetLocationValues(RequestContext *request_context, + const KeyVector &keys, + LocationsPerKey &out_locations) noexcept { + if (keys.empty()) { + out_locations.clear(); + return Result(EC_OK); + } + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + KVCM_METRICS_COLLECTOR_SET_METRICS(service_metrics_collector, meta_indexer, query_key_count, keys.size()); + const auto &trace_id = request_context->trace_id(); + + const int64_t begin_get_io_time = TimestampUtil::GetCurrentTimeUs(); + std::vector error_codes; + const bool use_parallel_local_read = query_executor_ && query_executor_->worker_count() > 1 && + keys.size() >= query_executor_->parallel_threshold() && + backend_manager_->SupportsConcurrentLocationValueReads(); + if (use_parallel_local_read) { + error_codes.assign(keys.size(), EC_ERROR); + out_locations.clear(); + out_locations.resize(keys.size()); + const bool completed = query_executor_->ParallelFor( + keys.size(), [this, &keys, &error_codes, &out_locations](std::size_t begin, std::size_t end) { + KeyVector chunk_keys(keys.begin() + begin, keys.begin() + end); + LocationsPerKey chunk_locations; + auto chunk_errors = backend_manager_->GetLocationValues(nullptr, chunk_keys, chunk_locations); + const std::size_t expected = end - begin; + if (chunk_errors.size() != expected || chunk_locations.size() != expected) { + KVCM_LOG_ERROR( + "parallel local location read size mismatch: errors[%zu] locations[%zu] expected[%zu]", + chunk_errors.size(), + chunk_locations.size(), + expected); + return; + } + for (std::size_t i = 0; i < expected; ++i) { + error_codes[begin + i] = chunk_errors[i]; + out_locations[begin + i] = std::move(chunk_locations[i]); + } + }); + if (!completed) { + KVCM_LOG_ERROR("trace_id[%s] instance[%s] | parallel local location read callback failed", + trace_id.c_str(), + instance_id_.c_str()); + } + } else { + error_codes = backend_manager_->GetLocationValues(request_context, keys, out_locations); + } + if (error_codes.size() != keys.size() || out_locations.size() != keys.size()) { + KVCM_LOG_ERROR("trace_id[%s] instance[%s] | location value result size mismatch: errors[%zu] " + "locations[%zu] keys[%zu]", + trace_id.c_str(), + instance_id_.c_str(), + error_codes.size(), + out_locations.size(), + keys.size()); + // Errors and values are one positional response. If either outer + // shape is malformed, no apparent in-range EC_OK/value pair can be + // trusted to refer to the requested key. + error_codes.assign(keys.size(), EC_MISMATCH); + out_locations.assign(keys.size(), CacheLocationVector{}); + } + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_indexer, get_io_time_us, TimestampUtil::GetCurrentTimeUs() - begin_get_io_time); + + Result result(keys.size()); + int32_t error_count = ProcessErrorCodes(trace_id, error_codes, {}, keys, kGetMetaOperation, result); + ProcessErrorResult(trace_id, kGetMetaOperation, error_count, keys.size(), result); + return result; +} + +bool MetaIndexer::SupportsProgressiveLocationValueReads() const noexcept { + return backend_manager_ && backend_manager_->SupportsConcurrentLocationValueReads(); +} + +MetaIndexer::PrefixLocationResult MetaIndexer::VisitLocationValuesForPrefix(RequestContext *request_context, + const KeyVector &keys, + const PrefixLocationVisitor &visitor, + PrefixVisitOrder visit_order) noexcept { + PrefixLocationResult prefix_result; + if (keys.empty()) { + return prefix_result; + } + + // Preserve every non-local backend's existing batching/recovery behavior. + // Only the pure-local path below performs progressive concurrent reads. + if (!backend_manager_->SupportsConcurrentLocationValueReads()) { + LocationsPerKey locations; + auto result = GetLocationValues(request_context, keys, locations); + prefix_result.read_key_count = keys.size(); + if (result.error_codes.size() != keys.size() || locations.size() != keys.size()) { + prefix_result.terminal_ec = EC_MISMATCH; + return prefix_result; + } + + size_t metadata_stop = 0; + while (metadata_stop < keys.size() && result.error_codes[metadata_stop] == EC_OK) { + ++metadata_stop; + } + size_t location_count = 0; + for (size_t i = 0; i < metadata_stop; ++i) { + location_count += locations[i].size(); + } + CompactLocationsPerKey compact; + compact.Clear(metadata_stop, location_count); + for (size_t i = 0; i < metadata_stop; ++i) { + compact.values.insert(compact.values.end(), locations[i].begin(), locations[i].end()); + compact.FinishKey(); + } + + size_t requested_stop = keys.size(); + if (metadata_stop != 0 && visitor) { + try { + requested_stop = std::min(keys.size(), visitor(0, compact, metadata_stop)); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("location prefix visitor failed: %s", e.what()); + prefix_result.terminal_ec = EC_ERROR; + return prefix_result; + } catch (...) { + KVCM_LOG_ERROR("location prefix visitor failed with unknown exception"); + prefix_result.terminal_ec = EC_ERROR; + return prefix_result; + } + } + prefix_result.valid_key_count = std::min(metadata_stop, requested_stop); + prefix_result.stopped_by_visitor = requested_stop <= metadata_stop && requested_stop < keys.size(); + if (metadata_stop < requested_stop && metadata_stop < keys.size()) { + prefix_result.terminal_ec = result.error_codes[metadata_stop]; + } + return prefix_result; + } + + auto *service_metrics_collector = dynamic_cast(request_context->metrics_collector()); + KVCM_METRICS_COLLECTOR_SET_METRICS(service_metrics_collector, meta_indexer, query_key_count, keys.size()); + const auto &trace_id = request_context->trace_id(); + // Backend reads and projection are pipelined. Track the union of backend + // call intervals so get_io_time_us does not accidentally include visitor + // CPU time or double-count overlapping worker reads. + std::atomic active_backend_reads(0); + std::atomic backend_read_interval_start_us(0); + std::atomic backend_read_wall_time_us(0); + const size_t configured_chunk_size = + std::max(kLocalPrefixParallelReadChunkSize, query_executor_ ? query_executor_->chunk_size() : size_t{256}); + // Keep callback boundaries on 64-key words. The Mamba projection stores + // per-host state as disjoint key bit ranges, so concurrent callbacks never + // update the same word. + const size_t aligned_chunk_size = + configured_chunk_size > std::numeric_limits::max() - (kPrefixStateWordBits - 1) + ? configured_chunk_size + : (configured_chunk_size + kPrefixStateWordBits - 1) & ~(kPrefixStateWordBits - 1); + // A configuration larger than the request still means one suffix range; + // clamp it so absolute range arithmetic cannot wrap around size_t. + const size_t parallel_chunk_size = std::min(aligned_chunk_size, keys.size()); + const size_t first_chunk_size = std::min(kLocalPrefixProbeKeyCount, keys.size()); + if (visit_order == PrefixVisitOrder::ORDERED) { + struct OrderedChunk { + size_t begin = 0; + size_t successful_count = 0; + ErrorCode terminal_ec = EC_OK; + CompactLocationsPerKey locations; + }; + + std::atomic read_key_count(0); + auto read_chunk = [this, + &keys, + &read_key_count, + &active_backend_reads, + &backend_read_interval_start_us, + &backend_read_wall_time_us](size_t begin, size_t count, OrderedChunk &chunk) { + chunk.begin = begin; + chunk.successful_count = 0; + chunk.terminal_ec = EC_OK; + const int64_t backend_read_begin_us = TimestampUtil::GetCurrentTimeUs(); + if (active_backend_reads.fetch_add(1, std::memory_order_acq_rel) == 0) { + backend_read_interval_start_us.store(backend_read_begin_us, std::memory_order_release); + } + auto errors = + backend_manager_->GetLocationValuesCompact(nullptr, keys.data() + begin, count, chunk.locations); + const int64_t backend_read_end_us = TimestampUtil::GetCurrentTimeUs(); + const int64_t interval_start_us = backend_read_interval_start_us.load(std::memory_order_acquire); + if (active_backend_reads.fetch_sub(1, std::memory_order_acq_rel) == 1) { + backend_read_wall_time_us.fetch_add(std::max(backend_read_end_us - interval_start_us, 0), + std::memory_order_relaxed); + } + read_key_count.fetch_add(count, std::memory_order_relaxed); + if (errors.size() != count || !chunk.locations.IsValid(count)) { + KVCM_LOG_ERROR("ordered compact local location read size mismatch: errors[%zu] locations[%zu] " + "expected[%zu]", + errors.size(), + chunk.locations.size(), + count); + chunk.terminal_ec = EC_MISMATCH; + return; + } + while (chunk.successful_count < count && errors[chunk.successful_count] == EC_OK) { + ++chunk.successful_count; + } + if (chunk.successful_count < count) { + chunk.terminal_ec = errors[chunk.successful_count]; + } + }; + + size_t metadata_stop = keys.size(); + size_t visitor_stop = keys.size(); + ErrorCode metadata_terminal_ec = EC_OK; + bool visitor_failed = false; + auto consume_chunk = [&keys, &visitor, &metadata_stop, &visitor_stop, &metadata_terminal_ec, &visitor_failed]( + const OrderedChunk &chunk) { + if (chunk.begin >= std::min(metadata_stop, visitor_stop)) { + return false; + } + const size_t visible_stop = std::min(metadata_stop, visitor_stop); + const size_t visitable_count = + std::min(chunk.successful_count, visible_stop > chunk.begin ? visible_stop - chunk.begin : size_t{0}); + if (visitable_count != 0 && visitor) { + try { + visitor_stop = std::min( + visitor_stop, std::min(keys.size(), visitor(chunk.begin, chunk.locations, visitable_count))); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("ordered location prefix visitor failed: %s", e.what()); + visitor_failed = true; + return false; + } catch (...) { + KVCM_LOG_ERROR("ordered location prefix visitor failed with unknown exception"); + visitor_failed = true; + return false; + } + } + + const size_t terminal_index = chunk.begin + chunk.successful_count; + // A visitor stop at the same position wins: the reducer already + // proved that this key and the suffix cannot affect the answer. + if (chunk.terminal_ec != EC_OK && terminal_index < visitor_stop) { + metadata_stop = terminal_index; + metadata_terminal_ec = chunk.terminal_ec; + return false; + } + return visitor_stop > terminal_index; + }; + + bool completed = true; + bool should_continue = true; + OrderedChunk first_chunk; + try { + read_chunk(0, first_chunk_size, first_chunk); + should_continue = consume_chunk(first_chunk); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("first ordered compact local location read failed: %s", e.what()); + completed = false; + } catch (...) { + KVCM_LOG_ERROR("first ordered compact local location read failed with unknown exception"); + completed = false; + } + + size_t window_begin = first_chunk_size; + const size_t max_window_chunks = + std::max(1, query_executor_ ? query_executor_->worker_count() : size_t{1}); + std::vector chunks(max_window_chunks); + while (completed && should_continue && window_begin < keys.size() && + window_begin < std::min(metadata_stop, visitor_stop)) { + const size_t remaining = keys.size() - window_begin; + const size_t window_chunk_count = + std::min(max_window_chunks, size_t{1} + (remaining - 1) / parallel_chunk_size); + const size_t window_size = window_chunk_count > remaining / parallel_chunk_size + ? remaining + : window_chunk_count * parallel_chunk_size; + auto read_window = [window_begin, parallel_chunk_size, &read_chunk, &chunks](size_t begin, size_t end) { + for (size_t relative_begin = begin; relative_begin < end; relative_begin += parallel_chunk_size) { + const size_t relative_end = std::min(end, relative_begin + parallel_chunk_size); + read_chunk(window_begin + relative_begin, + relative_end - relative_begin, + chunks[relative_begin / parallel_chunk_size]); + } + }; + if (query_executor_) { + completed = query_executor_->ParallelForWithChunkSize(window_size, parallel_chunk_size, read_window); + } else { + try { + read_window(0, window_size); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("ordered compact local location read failed: %s", e.what()); + completed = false; + } catch (...) { + KVCM_LOG_ERROR("ordered compact local location read failed with unknown exception"); + completed = false; + } + } + if (!completed) { + break; + } + for (size_t chunk_index = 0; chunk_index < window_chunk_count; ++chunk_index) { + if (!consume_chunk(chunks[chunk_index])) { + should_continue = false; + break; + } + } + window_begin += window_size; + } + + prefix_result.read_key_count = read_key_count.load(std::memory_order_relaxed); + if (!completed || visitor_failed) { + prefix_result.terminal_ec = EC_ERROR; + } else { + prefix_result.valid_key_count = std::min(metadata_stop, visitor_stop); + prefix_result.stopped_by_visitor = visitor_stop <= metadata_stop && visitor_stop < keys.size(); + if (metadata_stop < visitor_stop && metadata_stop < keys.size()) { + prefix_result.terminal_ec = metadata_terminal_ec; + if (prefix_result.terminal_ec != EC_NOENT) { + PREFIX_INDEXER_LOG(ERROR, + "meta indexer ordered prefix get failed, key[%lu] ec[%d]", + keys[metadata_stop], + prefix_result.terminal_ec); + } + } + } + prefix_result.backend_read_wall_time_us = backend_read_wall_time_us.load(std::memory_order_relaxed); + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_indexer, get_io_time_us, prefix_result.backend_read_wall_time_us); + return prefix_result; + } + + const size_t remaining_chunk_count = + first_chunk_size == keys.size() ? 0 : 1 + (keys.size() - first_chunk_size - 1) / parallel_chunk_size; + struct ChunkTerminal { + size_t index = 0; + ErrorCode ec = EC_OK; + bool present = false; + }; + std::vector terminals(1 + remaining_chunk_count); + std::atomic metadata_stop(keys.size()); + std::atomic visitor_stop(keys.size()); + std::atomic read_key_count(0); + + auto reduce_stop_index = [](std::atomic &target, size_t candidate) { + size_t current = target.load(std::memory_order_relaxed); + while (candidate < current && !target.compare_exchange_weak( + current, candidate, std::memory_order_release, std::memory_order_relaxed)) {} + }; + auto current_stop = [&metadata_stop, &visitor_stop]() { + return std::min(metadata_stop.load(std::memory_order_acquire), visitor_stop.load(std::memory_order_acquire)); + }; + auto read_one_chunk = [this, + &keys, + &visitor, + &terminals, + &metadata_stop, + &visitor_stop, + &read_key_count, + &active_backend_reads, + &backend_read_interval_start_us, + &backend_read_wall_time_us, + &reduce_stop_index, + ¤t_stop](size_t chunk_begin, size_t count, size_t terminal_slot) { + if (chunk_begin >= current_stop()) { + return; + } + + PrefixLocationScratchLease scratch_lease(count); + auto &locations = scratch_lease.locations(); + const int64_t backend_read_begin_us = TimestampUtil::GetCurrentTimeUs(); + if (active_backend_reads.fetch_add(1, std::memory_order_acq_rel) == 0) { + backend_read_interval_start_us.store(backend_read_begin_us, std::memory_order_release); + } + auto errors = backend_manager_->GetLocationValuesCompact(nullptr, keys.data() + chunk_begin, count, locations); + const int64_t backend_read_end_us = TimestampUtil::GetCurrentTimeUs(); + const int64_t interval_start_us = backend_read_interval_start_us.load(std::memory_order_acquire); + if (active_backend_reads.fetch_sub(1, std::memory_order_acq_rel) == 1) { + backend_read_wall_time_us.fetch_add(std::max(backend_read_end_us - interval_start_us, 0), + std::memory_order_relaxed); + } + read_key_count.fetch_add(count, std::memory_order_relaxed); + + size_t successful_count = 0; + ErrorCode terminal_ec = EC_OK; + if (errors.size() != count || !locations.IsValid(count)) { + KVCM_LOG_ERROR("compact local location read size mismatch: errors[%zu] locations[%zu] expected[%zu]", + errors.size(), + locations.size(), + count); + terminal_ec = EC_MISMATCH; + } else { + while (successful_count < count && errors[successful_count] == EC_OK) { + ++successful_count; + } + if (successful_count < count) { + terminal_ec = errors[successful_count]; + } + } + + if (successful_count != 0 && visitor) { + const size_t requested_stop = std::min(keys.size(), visitor(chunk_begin, locations, successful_count)); + reduce_stop_index(visitor_stop, requested_stop); + } + if (terminal_ec != EC_OK) { + auto &terminal = terminals[terminal_slot]; + terminal.index = chunk_begin + successful_count; + terminal.ec = terminal_ec; + terminal.present = true; + reduce_stop_index(metadata_stop, terminal.index); + } + }; + auto read_ranges = [&read_one_chunk, ¤t_stop, &keys, first_chunk_size, parallel_chunk_size](size_t begin, + size_t end) { + for (size_t chunk_begin = begin; chunk_begin < end;) { + if (chunk_begin >= current_stop()) { + return; + } + const size_t chunk_end = chunk_begin + std::min(parallel_chunk_size, end - chunk_begin); + const size_t terminal_slot = 1 + (chunk_begin - first_chunk_size) / parallel_chunk_size; + read_one_chunk(chunk_begin, chunk_end - chunk_begin, terminal_slot); + chunk_begin = chunk_end; + } + }; + + bool completed = true; + try { + // Candidate hosts are derived from key zero. Visiting this chunk before + // scheduling the suffix makes every later callback independent and + // allows an early host miss to cancel all suffix metadata reads. + read_one_chunk(0, first_chunk_size, 0); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("first compact local location read failed: %s", e.what()); + completed = false; + } catch (...) { + KVCM_LOG_ERROR("first compact local location read failed with unknown exception"); + completed = false; + } + + if (completed && first_chunk_size < keys.size() && first_chunk_size < current_stop()) { + const size_t remaining_count = keys.size() - first_chunk_size; + auto read_remaining_ranges = [&read_ranges, first_chunk_size](size_t begin, size_t end) { + read_ranges(first_chunk_size + begin, first_chunk_size + end); + }; + if (query_executor_) { + completed = + query_executor_->ParallelForWithChunkSize(remaining_count, parallel_chunk_size, read_remaining_ranges); + } else { + try { + read_remaining_ranges(0, remaining_count); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("serial compact local location read failed: %s", e.what()); + completed = false; + } catch (...) { + KVCM_LOG_ERROR("serial compact local location read failed with unknown exception"); + completed = false; + } + } + } + + prefix_result.read_key_count = read_key_count.load(std::memory_order_relaxed); + if (!completed) { + prefix_result.terminal_ec = EC_ERROR; + } else { + const size_t first_metadata_error = metadata_stop.load(std::memory_order_acquire); + const size_t first_visitor_stop = visitor_stop.load(std::memory_order_acquire); + prefix_result.valid_key_count = std::min(first_metadata_error, first_visitor_stop); + prefix_result.stopped_by_visitor = + first_visitor_stop <= first_metadata_error && first_visitor_stop < keys.size(); + if (first_metadata_error < first_visitor_stop && first_metadata_error < keys.size()) { + const size_t terminal_slot = first_metadata_error < first_chunk_size + ? 0 + : 1 + (first_metadata_error - first_chunk_size) / parallel_chunk_size; + const auto &terminal = terminals[terminal_slot]; + if (!terminal.present || terminal.index != first_metadata_error) { + prefix_result.terminal_ec = EC_MISMATCH; + prefix_result.valid_key_count = 0; + } else { + prefix_result.terminal_ec = terminal.ec; + if (prefix_result.terminal_ec != EC_NOENT) { + PREFIX_INDEXER_LOG(ERROR, + "meta indexer prefix get failed, key[%lu] ec[%d]", + keys[first_metadata_error], + prefix_result.terminal_ec); + } + } + } + } + + prefix_result.backend_read_wall_time_us = backend_read_wall_time_us.load(std::memory_order_relaxed); + KVCM_METRICS_COLLECTOR_SET_METRICS( + service_metrics_collector, meta_indexer, get_io_time_us, prefix_result.backend_read_wall_time_us); + return prefix_result; +} + MetaIndexer::LocationResult MetaIndexer::GetLocations(RequestContext *request_context, const KeyVector &keys, const LocationIdsPerKey &location_ids, LocationsPerKey &out_locations) noexcept { - assert(keys.size() == location_ids.size()); + if (keys.size() != location_ids.size()) { + out_locations.clear(); + KVCM_LOG_ERROR("instance[%s] | GetLocations keys size[%lu] != location_ids size[%lu]", + instance_id_.c_str(), + keys.size(), + location_ids.size()); + return LocationResult(EC_BADARGS); + } if (keys.empty()) { out_locations.clear(); return LocationResult(EC_OK); @@ -759,7 +1878,46 @@ MetaIndexer::LocationResult MetaIndexer::GetLocations(RequestContext *request_co service_metrics_collector, meta_indexer, get_io_time_us, TimestampUtil::GetCurrentTimeUs() - begin_get_io_time); LocationResult result(location_ids); - result.per_location_error_codes = std::move(per_location_ecs); + if (per_location_ecs.size() != keys.size() || out_locations.size() != keys.size()) { + PREFIX_INDEXER_LOG(ERROR, + "GetLocations result size mismatch, keys[%lu], ecs[%lu], locations[%lu]", + keys.size(), + per_location_ecs.size(), + out_locations.size()); + out_locations.assign(keys.size(), CacheLocationVector{}); + for (size_t i = 0; i < keys.size(); ++i) { + out_locations[i].resize(location_ids[i].size()); + result.per_location_error_codes[i].assign(location_ids[i].size(), EC_MISMATCH); + } + } else { + for (size_t i = 0; i < keys.size(); ++i) { + const size_t expected = location_ids[i].size(); + if (per_location_ecs[i].size() != expected || out_locations[i].size() != expected) { + PREFIX_INDEXER_LOG(ERROR, + "GetLocations per-key result size mismatch, key[%ld], ids[%lu], ecs[%lu], " + "locations[%lu]", + keys[i], + expected, + per_location_ecs[i].size(), + out_locations[i].size()); + out_locations[i].assign(expected, CacheLocationConstPtr{}); + result.per_location_error_codes[i].assign(expected, EC_MISMATCH); + continue; + } + result.per_location_error_codes[i] = std::move(per_location_ecs[i]); + for (size_t j = 0; j < expected; ++j) { + if (result.per_location_error_codes[i][j] == EC_OK && + (!out_locations[i][j] || out_locations[i][j]->id() != location_ids[i][j])) { + PREFIX_INDEXER_LOG(ERROR, + "GetLocations invalid EC_OK value, key[%ld], requested id[%s]", + keys[i], + location_ids[i][j].c_str()); + out_locations[i][j].reset(); + result.per_location_error_codes[i][j] = EC_MISMATCH; + } + } + } + } int64_t total_slots = 0; int64_t error_slots = 0; @@ -787,6 +1945,22 @@ MetaIndexer::LocationResult MetaIndexer::GetLocations(RequestContext *request_co return result; } +bool MetaIndexer::ParallelForQuery(std::size_t count, const QueryExecutor::RangeFunction &fn) const noexcept { + if (query_executor_) { + return query_executor_->ParallelFor(count, fn); + } + if (count == 0) { + return true; + } + try { + fn(0, count); + return true; + } catch (const std::exception &e) { + KVCM_LOG_ERROR("serial query callback threw exception: %s", e.what()); + } catch (...) { KVCM_LOG_ERROR("serial query callback threw unknown exception"); } + return false; +} + MetaIndexer::Result MetaIndexer::GetProperties(RequestContext *request_context, const KeyVector &keys, const std::vector &property_names, @@ -925,7 +2099,7 @@ std::vector MetaIndexer::MakeBatches(const KeyVector &keys, std::map> shard_map; for (int32_t i = 0; i < static_cast(keys.size()); ++i) { - const int32_t shard_idx = GetShardIndex(keys[i], mutex_shard_mask_); + const int32_t shard_idx = GetMutexShardIndex(keys[i]); shard_map[shard_idx].push_back(i); } if (shard_map.empty()) { diff --git a/kv_cache_manager/meta/meta_indexer.h b/kv_cache_manager/meta/meta_indexer.h index d4e7982a3..2ea0b9496 100644 --- a/kv_cache_manager/meta/meta_indexer.h +++ b/kv_cache_manager/meta/meta_indexer.h @@ -15,6 +15,7 @@ #include "kv_cache_manager/meta/cache_location.h" #include "kv_cache_manager/meta/common.h" #include "kv_cache_manager/meta/meta_storage_backend_manager.h" +#include "kv_cache_manager/meta/query_executor.h" #include "kv_cache_manager/meta/storage_usage_data.h" #include "kv_cache_manager/meta/types.h" #include "kv_cache_manager/metrics/revisit_interval_histogram.h" @@ -51,6 +52,52 @@ class MetaIndexer { } }; + struct SingleLocationResult { + ErrorCode ec = EC_OK; + std::vector error_codes; + explicit SingleLocationResult(ErrorCode error_code) : ec(error_code) {} + explicit SingleLocationResult(size_t count) : ec(EC_OK), error_codes(count, EC_OK) {} + }; + + struct PrefixLocationResult { + // terminal_ec is the first backend error that precedes the visitor's + // stop point. EC_NOENT is a normal metadata-prefix terminator; other + // values must be propagated by the caller. EC_OK can also mean the + // visitor proved that keys at valid_key_count and beyond are no longer + // needed, allowing already queued suffix reads to be cancelled. + ErrorCode terminal_ec = EC_OK; + size_t valid_key_count = 0; + size_t read_key_count = 0; + // Union wall time of backend calls made by this invocation. This is + // returned as well as published so a multi-pass reducer can aggregate + // all of its scans instead of leaving the request metric at the last + // pass only. Non-progressive backends continue to publish their + // existing GetLocationValues metric and leave this field at zero. + int64_t backend_read_wall_time_us = 0; + bool stopped_by_visitor = false; + }; + + enum class PrefixVisitOrder { + // The first chunk is visited synchronously; suffix callbacks may run + // concurrently and arrive out of key order. + CONCURRENT, + // Backend chunks are still read in bounded parallel windows, but the + // visitor consumes them in key order. Stateful reducers can therefore + // keep bounded state instead of materializing every key. + ORDERED, + }; + + // Called once per successfully read compact chunk. valid_key_count can be + // smaller than locations.size() when the chunk ends at a backend error. + // Return the first absolute key index that no longer needs to be read, or + // the request key count to continue. The first chunk is always visited + // before suffix work so callers can initialize candidate state. Suffix + // callback ordering is controlled by PrefixVisitOrder. + // locations and every view/iterator obtained from it are callback-scoped; + // callers must copy any state that needs to outlive the invocation. + using PrefixLocationVisitor = + std::function; + public: MetaIndexer() = default; ~MetaIndexer(); @@ -59,6 +106,8 @@ class MetaIndexer { // Set revisit interval histogram for tracking cache access patterns. void SetRevisitHistogram(std::shared_ptr histogram); + // Injected once by MetaIndexerManager before Init/traffic begins. + void SetQueryExecutor(std::shared_ptr executor) { query_executor_ = std::move(executor); } // ---------- WRITE ---------- Result Put(RequestContext *request_context, @@ -77,6 +126,23 @@ class MetaIndexer { const LocationModifierFunc &modifier, bool adjust_reclaimed_key_count = true, bool refresh_cache_from_persistent = false) noexcept; + // Targeted upsert RMW that also distinguishes a brand-new key from an + // existing key missing the requested location. This lets ReportEvent + // create or merge locations in one shard-lock/read/write pass while + // keeping max_key_count and key_count exact. + LocationResult ReadModifyWriteTargetLocations(RequestContext *request_context, + const KeyVector &keys, + const LocationIdsPerKey &location_ids, + const LocationModifierFunc &modifier) noexcept; + // Pure-local fast path for one target location per unique key. It keeps + // ids and results flat, avoids per-key temporary maps/vectors, and retains + // the same metadata shard locking and key-count semantics as the generic + // targeted RMW. + SingleLocationResult ReadModifyWriteSingleTargetLocations(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + const SingleLocationModifierFunc &modifier) noexcept; + bool SupportsSingleLocationRmw() const noexcept; // ---------- READ ---------- Result Exist(RequestContext *request_context, const KeyVector &keys, std::vector &out_exists) noexcept; @@ -87,6 +153,18 @@ class MetaIndexer { Result GetLocations(RequestContext *request_context, const KeyVector &keys, CacheLocationMapVector &out_location_maps) noexcept; + // Lightweight all-location view used by GetHostCacheState. For a single + // local backend, VisitLocationValuesForPrefix performs a bounded first + // probe and then reads the still-needed suffix through the shared query + // executor. Other backend modes remain one batched call. + Result + GetLocationValues(RequestContext *request_context, const KeyVector &keys, LocationsPerKey &out_locations) noexcept; + PrefixLocationResult + VisitLocationValuesForPrefix(RequestContext *request_context, + const KeyVector &keys, + const PrefixLocationVisitor &visitor, + PrefixVisitOrder visit_order = PrefixVisitOrder::CONCURRENT) noexcept; + [[nodiscard]] bool SupportsProgressiveLocationValueReads() const noexcept; // Source-of-truth read used by maintenance admission. It never backfills // or touches the optional hot-cache backend. Result GetLocationsFromPersistent(RequestContext *request_context, @@ -113,6 +191,11 @@ class MetaIndexer { ErrorCode SampleReclaimKeys(RequestContext *request_context, const int64_t count, KeyVector &out_keys) const noexcept; + // Reuses the same bounded executor for CPU-only query projection/reduction. + // Directly constructed test/indexer instances without an executor retain + // serial behavior. + bool ParallelForQuery(std::size_t count, const QueryExecutor::RangeFunction &fn) const noexcept; + void PersistMetaData() noexcept; size_t GetKeyCount() const noexcept; size_t GetMaxKeyCount() const noexcept; @@ -135,7 +218,16 @@ class MetaIndexer { private: class ScopedBatchLock; + LocationResult ReadModifyWriteLocationImpl(RequestContext *request_context, + const KeyVector &keys, + const LocationIdsPerKey &location_ids, + const LocationModifierFunc &modifier, + bool adjust_reclaimed_key_count, + bool track_created_key_count, + bool refresh_cache_from_persistent) noexcept; + private: + int32_t GetMutexShardIndex(KeyType key) const noexcept; std::vector MakeBatches(const KeyVector &keys, const LocationIdsPerKey &location_ids, CacheLocationMapVector &locations, @@ -179,7 +271,8 @@ class MetaIndexer { const std::vector &put_global_indexs, const KeyVector &all_keys, RmwStats &stats, - Result &result) noexcept; + Result &result, + bool preserve_existing_updates_when_full = false) noexcept; // Returns {error_count, delete_success_count}. std::pair ExecuteRmwDelete(const std::string &trace_id, RequestContext *request_context, @@ -193,12 +286,14 @@ class MetaIndexer { private: std::vector> mutex_shards_; std::unique_ptr backend_manager_; + std::shared_ptr query_executor_; std::atomic key_count_ = {0}; int64_t last_persist_metadata_time_ = 0; int64_t persist_metadata_interval_time_ms_ = 0; size_t max_key_count_ = MetaIndexerConfig::kDefaultMaxKeyCount; size_t mutex_shard_mask_ = MetaIndexerConfig::kDefaultMutexShardNum - 1; + uint64_t mutex_shard_hash_seed_ = 0; size_t batch_key_size_ = MetaIndexerConfig::kDefaultBatchKeySize; std::string instance_id_; StorageUsageData storage_usage_data_; diff --git a/kv_cache_manager/meta/meta_indexer_manager.cc b/kv_cache_manager/meta/meta_indexer_manager.cc index 36f1cd9c9..35a6073e3 100644 --- a/kv_cache_manager/meta/meta_indexer_manager.cc +++ b/kv_cache_manager/meta/meta_indexer_manager.cc @@ -1,8 +1,12 @@ #include "kv_cache_manager/meta/meta_indexer_manager.h" +#include +#include + #include "kv_cache_manager/common/logger.h" #include "kv_cache_manager/config/meta_indexer_config.h" #include "kv_cache_manager/meta/meta_indexer.h" +#include "kv_cache_manager/meta/query_executor.h" #include "kv_cache_manager/metrics/revisit_interval_histogram.h" namespace kv_cache_manager { @@ -13,6 +17,45 @@ void MetaIndexerManager::SetRevisitHistogramConfig(std::shared_ptr 64 || parallel_threshold == 0 || chunk_size == 0 || + chunk_size > parallel_threshold) { + KVCM_LOG_ERROR("invalid query executor config: workers[%zu] threshold[%zu] chunk_size[%zu]", + worker_count, + parallel_threshold, + chunk_size); + return false; + } + std::scoped_lock write_guard(mutex_); + if (!meta_indexers_.empty()) { + KVCM_LOG_ERROR("query executor must be configured before creating meta indexers"); + return false; + } + constexpr std::size_t kQueueSlotsPerWorker = 64; + try { + query_executor_ = + std::make_shared(worker_count, + parallel_threshold, + chunk_size, + std::max(64, worker_count * kQueueSlotsPerWorker)); + } catch (const std::exception &e) { + KVCM_LOG_ERROR("failed to create query executor: %s", e.what()); + query_executor_.reset(); + return false; + } catch (...) { + KVCM_LOG_ERROR("failed to create query executor with unknown exception"); + query_executor_.reset(); + return false; + } + KVCM_LOG_INFO("configured query executor: workers[%zu] threshold[%zu] chunk_size[%zu]", + worker_count, + parallel_threshold, + chunk_size); + return true; +} + ErrorCode MetaIndexerManager::CreateMetaIndexer(const std::string &instance_id, const std::shared_ptr &config, const std::vector &boundaries) { @@ -28,6 +71,7 @@ ErrorCode MetaIndexerManager::CreateMetaIndexer(const std::string &instance_id, return ErrorCode::EC_EXIST; } indexer = std::make_shared(); + indexer->SetQueryExecutor(query_executor_); auto ec = indexer->Init(instance_id, config); if (ec != ErrorCode::EC_OK) { KVCM_LOG_ERROR("Init meta indexer failed, instance_id: %s", instance_id.c_str()); diff --git a/kv_cache_manager/meta/meta_indexer_manager.h b/kv_cache_manager/meta/meta_indexer_manager.h index 0b19a391f..1e634677c 100644 --- a/kv_cache_manager/meta/meta_indexer_manager.h +++ b/kv_cache_manager/meta/meta_indexer_manager.h @@ -14,6 +14,7 @@ namespace kv_cache_manager { class MetaIndexerConfig; class MetaIndexer; class MetricsRegistry; +class QueryExecutor; class MetaIndexerManager { public: @@ -25,6 +26,11 @@ class MetaIndexerManager { // Must be called before CreateMetaIndexer. void SetRevisitHistogramConfig(std::shared_ptr registry, const std::vector &boundaries); + // Configures the single process-level executor shared by every indexer. + // Must be called before CreateMetaIndexer. worker_count == 1 keeps the + // exact serial behavior while preserving the same code path. + bool ConfigureQueryExecutor(std::size_t worker_count, std::size_t parallel_threshold, std::size_t chunk_size); + ErrorCode CreateMetaIndexer(const std::string &instance_id, const std::shared_ptr &config, const std::vector &boundaries = {}); @@ -50,6 +56,7 @@ class MetaIndexerManager { // Histogram configuration (shared across all indexers) std::shared_ptr metrics_registry_; std::vector revisit_boundaries_; + std::shared_ptr query_executor_; }; } // namespace kv_cache_manager diff --git a/kv_cache_manager/meta/meta_local_backend.cc b/kv_cache_manager/meta/meta_local_backend.cc index 684a0d598..5f3942dd6 100644 --- a/kv_cache_manager/meta/meta_local_backend.cc +++ b/kv_cache_manager/meta/meta_local_backend.cc @@ -2,7 +2,9 @@ #include #include +#include #include +#include #include #include "kv_cache_manager/common/logger.h" @@ -11,6 +13,60 @@ namespace kv_cache_manager { +namespace { + +constexpr size_t kMaxRetainedCompactReadKeys = 16384; + +struct CompactReadScratch { + bool in_use = false; + std::vector key_views; + std::vector handles; + std::vector revisit_intervals; + Cache::BatchOperationScratch cache_batch; +}; + +thread_local CompactReadScratch tls_compact_read_scratch; + +class CompactReadScratchLease { +public: + explicit CompactReadScratchLease(size_t key_count) { + if (key_count <= kMaxRetainedCompactReadKeys && !tls_compact_read_scratch.in_use) { + scratch_ = &tls_compact_read_scratch; + scratch_->in_use = true; + uses_thread_local_ = true; + } else { + local_.emplace(); + scratch_ = &*local_; + } + } + + ~CompactReadScratchLease() { + if (uses_thread_local_) { + scratch_->in_use = false; + } + } + + CompactReadScratch &get() noexcept { return *scratch_; } + +private: + std::optional local_; + CompactReadScratch *scratch_ = nullptr; + bool uses_thread_local_ = false; +}; + +} // namespace + +SingleLocationRmwScratch::~SingleLocationRmwScratch() { ReleaseRetainedHandles(); } + +void SingleLocationRmwScratch::ReleaseRetainedHandles() noexcept { + if (!retained_handle_owner) { + return; + } + retained_handle_owner->ReleaseBatchWithScratch(handles.data(), handles.size(), &cache_batch); + std::fill(handles.begin(), handles.end(), nullptr); + retained_handle_owner = nullptr; +} + std::string MetaLocalBackend::GetStorageType() noexcept { return "local"; } ErrorCode MetaLocalBackend::Init(const std::string &instance_id, @@ -149,13 +205,10 @@ ErrorCode MetaLocalBackend::CreateAndInsertIfAbsent(std::string_view key_sv, return ret; } -ErrorCode MetaLocalBackend::UpdateInPlace(std::string_view key_sv, - const CacheLocationMap &locations, - const PropertyMap &properties) { - Cache::Handle *handle = cache_->Lookup(key_sv); - if (!handle) { - return EC_NOENT; - } +ErrorCode MetaLocalBackend::UpdateHandleInPlace(Cache::Handle *handle, + const CacheLocationMap &locations, + const PropertyMap &properties) { + assert(handle != nullptr); auto *existing = static_cast(cache_->Value(handle)); existing->TouchAccessTime(); ssize_t charge_delta = 0; @@ -190,10 +243,93 @@ ErrorCode MetaLocalBackend::UpdateInPlace(std::string_view key_sv, if (charge_delta != 0) { cache_->AdjustCharge(handle, charge_delta); } - cache_->Release(handle); return EC_OK; } +ErrorCode MetaLocalBackend::UpdateHandleInPlaceSingleLocation(Cache::Handle *handle, + const LocationId &location_id, + CacheLocationConstPtr location, + CacheLocationVector *retired_locations) { + assert(handle != nullptr); + auto *existing = static_cast(cache_->Value(handle)); + existing->TouchAccessTime(); + ssize_t charge_delta = 0; + { + std::unique_lock lock(existing->GetMutex()); + auto &existing_locations = existing->GetMutableLocations(); + auto it = existing_locations.end(); + if (existing_locations.size() == 1) { + auto only = existing_locations.begin(); + if (only->first == location_id) { + it = only; + } + } else if (!existing_locations.empty()) { + it = existing_locations.find(location_id); + } + if (it != existing_locations.end()) { + const ssize_t old_usage = it->second ? static_cast(it->second->EstimateMemUsage()) : 0; + const ssize_t new_usage = location ? static_cast(location->EstimateMemUsage()) : 0; + if (retired_locations) { + assert(retired_locations->size() < retired_locations->capacity()); + retired_locations->push_back(std::move(it->second)); + it->second = std::move(location); + } else { + it->second = std::move(location); + } + charge_delta = new_usage - old_usage; + } else { + const ssize_t new_usage = location ? static_cast(location->EstimateMemUsage()) : 0; + charge_delta = static_cast(sizeof(void *) * 4 + location_id.size()) + new_usage; + existing_locations.emplace(location_id, std::move(location)); + } + } + if (charge_delta != 0) { + cache_->AdjustCharge(handle, charge_delta); + } + return EC_OK; +} + +ErrorCode MetaLocalBackend::UpdateInPlace(std::string_view key_sv, + const CacheLocationMap &locations, + const PropertyMap &properties) { + Cache::Handle *handle = cache_->Lookup(key_sv); + if (!handle) { + return EC_NOENT; + } + const ErrorCode ec = UpdateHandleInPlace(handle, locations, properties); + cache_->Release(handle); + return ec; +} + +ErrorCode MetaLocalBackend::CreateAndInsertSingleLocation(std::string_view key_sv, + const LocationId &location_id, + CacheLocationConstPtr location) { + MetaMemCacheItem *item = MetaMemCacheItem::CreateSingleLocation(location_id, std::move(location)); + item->TouchAccessTime(); + const size_t charge = item->Size(); + Cache::Handle *handle = nullptr; + const ErrorCode ec = cache_->Insert(key_sv, item, cache_item_helper_.get(), charge, &handle); + if (ec != EC_OK) { + MetaMemCacheItem::Deleter(item, nullptr); + } else if (handle) { + cache_->Release(handle); + } + return ec; +} + +ErrorCode MetaLocalBackend::UpsertSingleLocationForOneKey(KeyType key, + const LocationId &location_id, + const CacheLocationConstPtr &location) { + const std::string_view key_view = KeyToView(key); + Cache::Handle *handle = cache_->Lookup(key_view); + if (handle) { + const ErrorCode ec = UpdateHandleInPlaceSingleLocation(handle, location_id, location); + cache_->Release(handle); + return ec; + } + return CreateAndInsertSingleLocation(key_view, location_id, location); +} + // --------------------------------------------------------------------------- // Per-key helpers // --------------------------------------------------------------------------- @@ -278,12 +414,247 @@ std::vector MetaLocalBackend::Upsert(RequestContext * /*request_conte const CacheLocationMapVector &locations, const PropertyMapVector &properties) noexcept { std::vector results(keys.size(), EC_OK); + bool has_duplicate_keys = false; + bool keys_are_sorted = true; + for (size_t i = 1; i < keys.size(); ++i) { + has_duplicate_keys = has_duplicate_keys || keys[i] == keys[i - 1]; + keys_are_sorted = keys_are_sorted && keys[i - 1] <= keys[i]; + } + if (!has_duplicate_keys && !keys_are_sorted) { + std::unordered_set seen_keys; + seen_keys.reserve(keys.size()); + for (const KeyType key : keys) { + if (!seen_keys.insert(key).second) { + has_duplicate_keys = true; + break; + } + } + } + if (has_duplicate_keys) { + // Preserve the historical request-order merge semantics for the + // general backend API. ReportEvent supplies sorted unique keys and + // therefore stays on the batched-LRU fast path below. + for (size_t i = 0; i < keys.size(); ++i) { + results[i] = UpsertForOneKey(keys[i], locations[i], properties[i]); + } + return results; + } + + std::vector key_views(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + key_views[i] = KeyToView(keys[i]); + } + std::vector handles(keys.size(), nullptr); + cache_->LookupBatch(key_views.data(), key_views.size(), handles.data()); + const size_t missing_count = static_cast(std::count(handles.begin(), handles.end(), nullptr)); + if (missing_count > 0 && missing_count < handles.size()) { + // Updating every hit before inserting every miss reorders a mixed + // request. Besides partial-field merge semantics, order is observable + // under strict capacity because in-place charge growth is admitted + // differently from a new insertion. Preserve the original per-key + // behavior for this uncommon shape. Existing handles were already + // acquired in one batch, but are updated/released at their original + // position so a preceding insert still observes the preceding cache + // charge. All-hit updates and all-miss creates retain the fully + // batched-LRU fast paths below. + for (size_t i = 0; i < keys.size(); ++i) { + if (handles[i]) { + results[i] = UpdateHandleInPlace(handles[i], locations[i], properties[i]); + cache_->Release(handles[i]); + } else { + results[i] = CreateAndInsert(key_views[i], locations[i], properties[i]); + } + } + return results; + } + + if (missing_count == 0) { + for (size_t i = 0; i < keys.size(); ++i) { + results[i] = UpdateHandleInPlace(handles[i], locations[i], properties[i]); + } + cache_->ReleaseBatch(handles.data(), handles.size()); + return results; + } + + // The mixed shape returned above, so every handle is null here. Avoid a + // pointless ReleaseBatch shard-group allocation and retain ordered insert + // semantics for the all-new-key path. + assert(missing_count == handles.size()); for (size_t i = 0; i < keys.size(); ++i) { - results[i] = UpsertForOneKey(keys[i], locations[i], properties[i]); + results[i] = CreateAndInsert(key_views[i], locations[i], properties[i]); } return results; } +std::vector MetaLocalBackend::UpsertSingleLocations(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations) noexcept { + SingleLocationRmwScratch scratch; + PrepareSingleLocationRmwScratch(keys.size(), scratch); + std::vector results; + results.reserve(keys.size()); + UpsertSingleLocationsInto(nullptr, keys, location_ids, locations, results, scratch); + return results; +} + +void MetaLocalBackend::PrepareSingleLocationRmwScratch(size_t max_count, SingleLocationRmwScratch &scratch) noexcept { + scratch.ReleaseRetainedHandles(); + scratch.retired_locations.clear(); + scratch.key_views.reserve(max_count); + scratch.handles.reserve(max_count); + scratch.retired_locations.reserve(max_count); + cache_->PrepareBatchOperationScratch(max_count, &scratch.cache_batch); +} + +void MetaLocalBackend::UpsertSingleLocationsInto(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations, + std::vector &results, + SingleLocationRmwScratch &scratch) noexcept { + if (keys.size() != location_ids.size() || keys.size() != locations.size()) { + results.assign(keys.size(), EC_BADARGS); + return; + } + if (keys.empty()) { + results.clear(); + return; + } + results.assign(keys.size(), EC_OK); + for (size_t i = 0; i < keys.size(); ++i) { + if (location_ids[i] == nullptr || !locations[i] || locations[i]->id() != *location_ids[i]) { + // The method is a batch write: do not report EC_OK for valid-looking + // entries when a malformed sibling prevents the batch from being + // applied. This also matches the generic backend adapter. + results.assign(keys.size(), EC_BADARGS); + return; + } + } + + bool has_duplicate_keys = false; + bool keys_are_sorted = true; + for (size_t i = 1; i < keys.size(); ++i) { + has_duplicate_keys = has_duplicate_keys || keys[i] == keys[i - 1]; + keys_are_sorted = keys_are_sorted && keys[i - 1] <= keys[i]; + } + if (!has_duplicate_keys && !keys_are_sorted) { + std::unordered_set seen_keys; + seen_keys.reserve(keys.size()); + for (const KeyType key : keys) { + if (!seen_keys.insert(key).second) { + has_duplicate_keys = true; + break; + } + } + } + if (has_duplicate_keys) { + for (size_t i = 0; i < keys.size(); ++i) { + results[i] = UpsertSingleLocationForOneKey(keys[i], *location_ids[i], locations[i]); + } + return; + } + + scratch.key_views.resize(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + scratch.key_views[i] = KeyToView(keys[i]); + } + scratch.handles.assign(keys.size(), nullptr); + cache_->LookupBatchWithScratch( + scratch.key_views.data(), scratch.key_views.size(), scratch.handles.data(), &scratch.cache_batch); + const size_t missing_count = + static_cast(std::count(scratch.handles.begin(), scratch.handles.end(), nullptr)); + if (missing_count == 0) { + for (size_t i = 0; i < keys.size(); ++i) { + results[i] = UpdateHandleInPlaceSingleLocation(scratch.handles[i], *location_ids[i], locations[i]); + } + cache_->ReleaseBatchWithScratch(scratch.handles.data(), scratch.handles.size(), &scratch.cache_batch); + return; + } + if (missing_count == scratch.handles.size()) { + for (size_t i = 0; i < keys.size(); ++i) { + results[i] = CreateAndInsertSingleLocation(scratch.key_views[i], *location_ids[i], locations[i]); + } + return; + } + + // Preserve request order for the mixed hit/miss shape. Capacity admission + // can observe whether an earlier item was inserted or grew in place. + for (size_t i = 0; i < keys.size(); ++i) { + if (scratch.handles[i]) { + results[i] = UpdateHandleInPlaceSingleLocation(scratch.handles[i], *location_ids[i], locations[i]); + cache_->Release(scratch.handles[i]); + } else { + results[i] = CreateAndInsertSingleLocation(scratch.key_views[i], *location_ids[i], locations[i]); + } + } +} + +void MetaLocalBackend::UpsertSingleLocationsUsingRetainedHandlesInto(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &locations, + const std::vector &read_indices, + std::vector &results, + SingleLocationRmwScratch &scratch) noexcept { + auto fail_and_release = [&results, &scratch, &keys](ErrorCode ec) { + results.assign(keys.size(), ec); + scratch.ReleaseRetainedHandles(); + }; + if (keys.size() != location_ids.size() || keys.size() != locations.size() || keys.size() != read_indices.size()) { + fail_and_release(EC_BADARGS); + return; + } + if (keys.empty()) { + results.clear(); + scratch.ReleaseRetainedHandles(); + return; + } + if (scratch.retained_handle_owner != cache_.get()) { + fail_and_release(EC_BADARGS); + return; + } + + results.assign(keys.size(), EC_OK); + size_t previous_read_index = 0; + for (size_t i = 0; i < keys.size(); ++i) { + const size_t read_index = read_indices[i]; + if (read_index >= scratch.handles.size() || read_index >= scratch.key_views.size() || + (i > 0 && read_index <= previous_read_index) || location_ids[i] == nullptr || !locations[i] || + locations[i]->id() != *location_ids[i] || scratch.key_views[read_index] != KeyToView(keys[i])) { + fail_and_release(EC_BADARGS); + return; + } + previous_read_index = read_index; + } + + // A modifier may skip some read hits. Release those before inserting new + // keys so pinned, unrelated entries cannot change strict-capacity behavior. + size_t selected_position = 0; + for (size_t read_index = 0; read_index < scratch.handles.size(); ++read_index) { + if (selected_position < read_indices.size() && read_indices[selected_position] == read_index) { + ++selected_position; + continue; + } + if (scratch.handles[read_index]) { + cache_->Release(scratch.handles[read_index]); + scratch.handles[read_index] = nullptr; + } + } + + for (size_t i = 0; i < keys.size(); ++i) { + Cache::Handle *handle = scratch.handles[read_indices[i]]; + if (handle) { + results[i] = UpdateHandleInPlaceSingleLocation( + handle, *location_ids[i], std::move(locations[i]), &scratch.retired_locations); + } else { + results[i] = CreateAndInsertSingleLocation( + scratch.key_views[read_indices[i]], *location_ids[i], std::move(locations[i])); + } + } + scratch.ReleaseRetainedHandles(); +} + std::vector MetaLocalBackend::Delete(RequestContext * /*request_context*/, const KeyTypeVec &keys) noexcept { std::vector results(keys.size(), EC_OK); for (size_t i = 0; i < keys.size(); ++i) { @@ -462,25 +833,153 @@ std::vector MetaLocalBackend::GetLocations(RequestContext * /*request return results; } -std::vector> MetaLocalBackend::GetLocations(RequestContext * /*request_context*/, +std::vector MetaLocalBackend::GetLocationValues(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept { + std::vector results(keys.size(), EC_OK); + out_locations.clear(); + out_locations.resize(keys.size()); + std::vector revisit_intervals; + if (revisit_histogram_) { + revisit_intervals.reserve(keys.size()); + } + for (size_t i = 0; i < keys.size(); ++i) { + std::string_view key_sv = KeyToView(keys[i]); + Cache::Handle *handle = cache_->Lookup(key_sv); + if (!handle) { + results[i] = EC_NOENT; + continue; + } + auto *item = static_cast(cache_->Value(handle)); + const int64_t stored_time = item->GetLastAccessTime(); + if (revisit_histogram_ && stored_time > 0) { + revisit_intervals.push_back(TimestampUtil::GetCurrentTimeUs() - stored_time); + } + item->TouchAccessTime(); + { + std::shared_lock lock(item->GetMutex()); + const auto &locations = item->GetLocations(); + auto &values = out_locations[i]; + values.reserve(locations.size()); + for (const auto &[location_id, location] : locations) { + (void)location_id; + values.push_back(location); + } + } + cache_->Release(handle); + } + if (revisit_histogram_) { + revisit_histogram_->ObserveBatch(revisit_intervals); + } + return results; +} + +std::vector MetaLocalBackend::GetLocationValuesCompact(RequestContext * /*request_context*/, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept { + std::vector results(key_count, EC_OK); + out_locations.Clear(key_count, key_count); + if (key_count != 0 && keys == nullptr) { + results.assign(key_count, EC_BADARGS); + for (size_t i = 0; i < key_count; ++i) { + out_locations.FinishKey(); + } + return results; + } + + CompactReadScratchLease scratch_lease(key_count); + auto &scratch = scratch_lease.get(); + auto &revisit_intervals = scratch.revisit_intervals; + revisit_intervals.clear(); + if (revisit_histogram_) { + revisit_intervals.reserve(key_count); + } + auto &key_views = scratch.key_views; + key_views.resize(key_count); + for (size_t i = 0; i < key_count; ++i) { + key_views[i] = KeyToView(keys[i]); + } + auto &handles = scratch.handles; + handles.assign(key_count, nullptr); + cache_->PrepareBatchOperationScratch(key_count, &scratch.cache_batch); + cache_->LookupBatchWithScratch(key_views.data(), key_count, handles.data(), &scratch.cache_batch); + + const int64_t access_time_us = TimestampUtil::GetCurrentTimeUs(); + for (size_t i = 0; i < key_count; ++i) { + Cache::Handle *handle = handles[i]; + if (!handle) { + results[i] = EC_NOENT; + out_locations.FinishKey(); + continue; + } + + auto *item = static_cast(cache_->Value(handle)); + if (revisit_histogram_) { + const int64_t stored_time = item->GetLastAccessTime(); + if (stored_time > 0 && stored_time <= access_time_us) { + revisit_intervals.push_back(access_time_us - stored_time); + } + } + item->TouchAccessTime(access_time_us); + { + std::shared_lock lock(item->GetMutex()); + for (const auto &[location_id, location] : item->GetLocations()) { + (void)location_id; + out_locations.values.push_back(location); + } + } + out_locations.FinishKey(); + } + cache_->ReleaseBatchWithScratch(handles.data(), handles.size(), &scratch.cache_batch); + if (revisit_histogram_) { + revisit_histogram_->ObserveBatch(revisit_intervals); + } + return results; +} + +std::vector> MetaLocalBackend::GetLocations(RequestContext *request_context, const KeyTypeVec &keys, const LocationIdsPerKey &location_ids, LocationsPerKey &out_locations) noexcept { + std::vector ignored_key_error_codes; + return GetLocationsWithKeyStatus(request_context, keys, location_ids, out_locations, ignored_key_error_codes); +} + +std::vector> +MetaLocalBackend::GetLocationsWithKeyStatus(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept { assert(keys.size() == location_ids.size()); std::vector> results(keys.size()); - out_locations.resize(keys.size()); - + out_key_error_codes.assign(keys.size(), EC_OK); + out_locations.assign(keys.size(), CacheLocationVector{}); + + // Targeted RMW commonly reads thousands of one-location keys at once. + // Group the LRU lookups/releases by cache shard instead of taking the same + // shard mutex once per key. Handles remain pinned while the immutable + // CacheLocation pointers are projected below, matching the compact query + // path's lifetime contract. + std::vector key_views(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + key_views[i] = KeyToView(keys[i]); + } + std::vector handles(keys.size(), nullptr); + cache_->LookupBatch(key_views.data(), key_views.size(), handles.data()); + const int64_t access_time_us = TimestampUtil::GetCurrentTimeUs(); for (size_t i = 0; i < keys.size(); ++i) { out_locations[i].resize(location_ids[i].size()); - std::string_view key_sv = KeyToView(keys[i]); - Cache::Handle *handle = cache_->Lookup(key_sv); + Cache::Handle *handle = handles[i]; if (!handle) { + out_key_error_codes[i] = EC_NOENT; results[i].assign(location_ids[i].size(), EC_NOENT); continue; } auto *item = static_cast(cache_->Value(handle)); - item->TouchAccessTime(); + item->TouchAccessTime(access_time_us); results[i].resize(location_ids[i].size()); { std::shared_lock lock(item->GetMutex()); @@ -495,11 +994,139 @@ std::vector> MetaLocalBackend::GetLocations(RequestContex } } } - cache_->Release(handle); } + cache_->ReleaseBatch(handles.data(), handles.size()); + return results; +} + +std::vector +MetaLocalBackend::GetSingleLocationsWithKeyStatus(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes) noexcept { + SingleLocationRmwScratch scratch; + PrepareSingleLocationRmwScratch(keys.size(), scratch); + std::vector results; + results.reserve(keys.size()); + GetSingleLocationsWithKeyStatusInto( + nullptr, keys, location_ids, out_locations, out_key_error_codes, results, scratch); return results; } +void MetaLocalBackend::GetSingleLocationsWithKeyStatusInto(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes, + std::vector &results, + SingleLocationRmwScratch &scratch, + bool retain_handles) noexcept { + GetSingleLocationsWithKeyStatusIntoImpl(request_context, + keys, + location_ids, + &out_locations, + nullptr, + out_key_error_codes, + results, + scratch, + retain_handles); +} + +void MetaLocalBackend::GetSingleLocationViewsWithKeyStatusInto(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationViewVector &out_locations, + std::vector &out_key_error_codes, + std::vector &results, + SingleLocationRmwScratch &scratch) noexcept { + GetSingleLocationsWithKeyStatusIntoImpl(request_context, + keys, + location_ids, + nullptr, + &out_locations, + out_key_error_codes, + results, + scratch, + /*retain_handles=*/true); +} + +void MetaLocalBackend::GetSingleLocationsWithKeyStatusIntoImpl(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector *out_owned_locations, + CacheLocationViewVector *out_borrowed_locations, + std::vector &out_key_error_codes, + std::vector &results, + SingleLocationRmwScratch &scratch, + bool retain_handles) noexcept { + scratch.ReleaseRetainedHandles(); + assert((out_owned_locations == nullptr) != (out_borrowed_locations == nullptr)); + if (out_owned_locations) { + out_owned_locations->assign(keys.size(), CacheLocationConstPtr{}); + } + if (out_borrowed_locations) { + out_borrowed_locations->assign(keys.size(), nullptr); + } + out_key_error_codes.assign(keys.size(), EC_OK); + if (keys.size() != location_ids.size()) { + out_key_error_codes.assign(keys.size(), EC_BADARGS); + results.assign(keys.size(), EC_BADARGS); + return; + } + if (keys.empty()) { + results.clear(); + return; + } + results.assign(keys.size(), EC_NOENT); + scratch.key_views.resize(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + if (location_ids[i] == nullptr) { + out_key_error_codes.assign(keys.size(), EC_BADARGS); + results.assign(keys.size(), EC_BADARGS); + return; + } + scratch.key_views[i] = KeyToView(keys[i]); + } + scratch.handles.assign(keys.size(), nullptr); + cache_->LookupBatchWithScratch( + scratch.key_views.data(), scratch.key_views.size(), scratch.handles.data(), &scratch.cache_batch); + const int64_t access_time_us = TimestampUtil::GetCurrentTimeUs(); + for (size_t i = 0; i < keys.size(); ++i) { + Cache::Handle *handle = scratch.handles[i]; + if (!handle) { + out_key_error_codes[i] = EC_NOENT; + continue; + } + auto *item = static_cast(cache_->Value(handle)); + item->TouchAccessTime(access_time_us); + std::shared_lock lock(item->GetMutex()); + const auto &locations = item->GetLocations(); + auto it = locations.end(); + if (locations.size() == 1) { + auto only = locations.begin(); + if (only->first == *location_ids[i]) { + it = only; + } + } else if (!locations.empty()) { + it = locations.find(*location_ids[i]); + } + if (it != locations.end()) { + if (out_borrowed_locations) { + (*out_borrowed_locations)[i] = it->second.get(); + } else { + (*out_owned_locations)[i] = it->second; + } + results[i] = EC_OK; + } + } + if (retain_handles) { + scratch.retained_handle_owner = cache_.get(); + } else { + cache_->ReleaseBatchWithScratch(scratch.handles.data(), scratch.handles.size(), &scratch.cache_batch); + } +} + std::vector MetaLocalBackend::GetLocationIds(RequestContext * /*request_context*/, const KeyTypeVec &keys, LocationIdsPerKey &out_location_ids) noexcept { @@ -713,6 +1340,14 @@ ErrorCode MetaLocalBackend::GetMetaData(FieldMap & /*field_maps*/) noexcept { re size_t MetaLocalBackend::GetMemUsage() const noexcept { return cache_->GetUsage(); } +bool MetaLocalBackend::GetCacheHashSeed(uint32_t &out_hash_seed) const noexcept { + if (!cache_) { + return false; + } + out_hash_seed = cache_->GetHashSeed(); + return true; +} + int64_t MetaLocalBackend::GetOldestAccessTime() const noexcept { int64_t oldest = INT64_MAX; size_t num_shards = shard_mask_ + 1; diff --git a/kv_cache_manager/meta/meta_local_backend.h b/kv_cache_manager/meta/meta_local_backend.h index 40c5009b2..7a67df1ad 100644 --- a/kv_cache_manager/meta/meta_local_backend.h +++ b/kv_cache_manager/meta/meta_local_backend.h @@ -46,6 +46,12 @@ struct MetaMemCacheItem { int64_t GetLastAccessTime() const { return last_access_time_.load(std::memory_order_relaxed); } void TouchAccessTime() { last_access_time_.store(TimestampUtil::GetCurrentTimeUs(), std::memory_order_relaxed); } + void TouchAccessTime(int64_t access_time_us) { + int64_t current = last_access_time_.load(std::memory_order_relaxed); + while (current < access_time_us && + !last_access_time_.compare_exchange_weak( + current, access_time_us, std::memory_order_relaxed, std::memory_order_relaxed)) {} + } static MetaMemCacheItem *Create(const CacheLocationMap &locations, const PropertyMap &properties) { auto *item = new MetaMemCacheItem(); @@ -59,6 +65,11 @@ struct MetaMemCacheItem { item->properties_ = std::move(properties); return item; } + static MetaMemCacheItem *CreateSingleLocation(const LocationId &location_id, CacheLocationConstPtr location) { + auto *item = new MetaMemCacheItem(); + item->locations_.emplace(location_id, std::move(location)); + return item; + } static void Deleter(void *value, MemoryAllocator * /*allocator*/) { delete static_cast(value); } private: @@ -68,6 +79,27 @@ struct MetaMemCacheItem { std::atomic last_access_time_{0}; }; +// Caller-owned workspace for the pure-local one-location RMW path. It is +// prepared before MetaIndexer acquires metadata shard locks, then reused by +// both the read and write halves of the operation. The fused path may retain +// read handles until its matching write call; the destructor is a final guard +// that releases every handle on early returns. +struct SingleLocationRmwScratch { + SingleLocationRmwScratch() = default; + ~SingleLocationRmwScratch(); + SingleLocationRmwScratch(const SingleLocationRmwScratch &) = delete; + SingleLocationRmwScratch &operator=(const SingleLocationRmwScratch &) = delete; + + void ReleaseRetainedHandles() noexcept; + [[nodiscard]] bool HasRetainedHandles() const noexcept { return retained_handle_owner != nullptr; } + + std::vector key_views; + std::vector handles; + CacheLocationVector retired_locations; + Cache::BatchOperationScratch cache_batch; + Cache *retained_handle_owner = nullptr; +}; + class MetaLocalBackend : public MetaCacheBaseBackend { public: MetaLocalBackend() = default; @@ -98,6 +130,24 @@ class MetaLocalBackend : public MetaCacheBaseBackend { const KeyTypeVec &keys, const CacheLocationMapVector &locations, const PropertyMapVector &properties) noexcept override; + std::vector UpsertSingleLocations(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations) noexcept override; + void PrepareSingleLocationRmwScratch(size_t max_count, SingleLocationRmwScratch &scratch) noexcept; + void UpsertSingleLocationsInto(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept; + void UpsertSingleLocationsUsingRetainedHandlesInto(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &locations, + const std::vector &read_indices, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept; std::vector Delete(RequestContext *request_context, const KeyTypeVec &keys) noexcept override; std::vector DeleteLocations(RequestContext *request_context, const KeyTypeVec &keys, @@ -135,10 +185,47 @@ class MetaLocalBackend : public MetaCacheBaseBackend { std::vector GetLocations(RequestContext *request_context, const KeyTypeVec &keys, CacheLocationMapVector &out_locations) noexcept override; + std::vector GetLocationValues(RequestContext *request_context, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept override; + std::vector GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept override; std::vector> GetLocations(RequestContext *request_context, const KeyTypeVec &keys, const LocationIdsPerKey &location_ids, LocationsPerKey &out_locations) noexcept override; + std::vector> + GetLocationsWithKeyStatus(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept override; + std::vector + GetSingleLocationsWithKeyStatus(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes) noexcept override; + void GetSingleLocationsWithKeyStatusInto(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch, + bool retain_handles = false) noexcept; + // Pure-local RMW-only variant: borrows immutable location values without + // incrementing one shared_ptr control block per key. Callers must retain + // the handles and consume every view before the matching write/release. + void GetSingleLocationViewsWithKeyStatusInto(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationViewVector &out_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept; std::vector GetLocationIds(RequestContext *request_context, const KeyTypeVec &keys, LocationIdsPerKey &out_location_ids) noexcept override; @@ -174,6 +261,7 @@ class MetaLocalBackend : public MetaCacheBaseBackend { size_t GetMemUsage() const noexcept override; int64_t GetOldestAccessTime() const noexcept override; + bool GetCacheHashSeed(uint32_t &out_hash_seed) const noexcept; private: static std::string_view KeyToView(const KeyType &key) { @@ -190,7 +278,27 @@ class MetaLocalBackend : public MetaCacheBaseBackend { CreateAndInsert(std::string_view key_sv, const CacheLocationMap &locations, const PropertyMap &properties); ErrorCode CreateAndInsertIfAbsent(std::string_view key_sv, const CacheLocationMap &locations, const PropertyMap &properties); + ErrorCode + UpdateHandleInPlace(Cache::Handle *handle, const CacheLocationMap &locations, const PropertyMap &properties); + ErrorCode UpdateHandleInPlaceSingleLocation(Cache::Handle *handle, + const LocationId &location_id, + CacheLocationConstPtr location, + CacheLocationVector *retired_locations = nullptr); ErrorCode UpdateInPlace(std::string_view key_sv, const CacheLocationMap &locations, const PropertyMap &properties); + ErrorCode CreateAndInsertSingleLocation(std::string_view key_sv, + const LocationId &location_id, + CacheLocationConstPtr location); + ErrorCode + UpsertSingleLocationForOneKey(KeyType key, const LocationId &location_id, const CacheLocationConstPtr &location); + void GetSingleLocationsWithKeyStatusIntoImpl(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector *out_owned_locations, + CacheLocationViewVector *out_borrowed_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch, + bool retain_handles) noexcept; ErrorCode UpsertForOneKey(KeyType key, const CacheLocationMap &locations, const PropertyMap &properties); ErrorCode DeleteForOneKey(KeyType key); ErrorCode DeleteLocationsForOneKey(KeyType key, const std::vector &location_ids); diff --git a/kv_cache_manager/meta/meta_storage_backend.h b/kv_cache_manager/meta/meta_storage_backend.h index 467bba7a4..c00f12322 100644 --- a/kv_cache_manager/meta/meta_storage_backend.h +++ b/kv_cache_manager/meta/meta_storage_backend.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -80,6 +81,27 @@ class MetaStorageBackend { const CacheLocationMapVector &locations, const PropertyMapVector &properties) noexcept = 0; + // Allocation-light one-location upsert used by pure-local targeted RMW. + // The default adapter preserves backend semantics; local memory overrides + // it to avoid constructing one temporary unordered_map per key. + virtual std::vector UpsertSingleLocations(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations) noexcept { + if (keys.size() != location_ids.size() || keys.size() != locations.size()) { + return std::vector(keys.size(), EC_BADARGS); + } + CacheLocationMapVector location_maps(keys.size()); + PropertyMapVector properties(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + if (location_ids[i] == nullptr) { + return std::vector(keys.size(), EC_BADARGS); + } + location_maps[i].emplace(*location_ids[i], locations[i]); + } + return Upsert(request_context, keys, location_maps, properties); + } + // 删除整个 key 及其所有 locations 和 properties。 // @param request_context 请求上下文；可为 nullptr // @param keys 待删除的 key 列表 @@ -136,6 +158,62 @@ class MetaStorageBackend { const KeyTypeVec &keys, CacheLocationMapVector &out_locations) noexcept = 0; + // Lightweight all-location read for consumers that only need immutable + // CacheLocation values and do not need to look them up again by id. The + // default implementation preserves every backend's existing behavior by + // flattening GetLocations. Local memory overrides it to avoid cloning an + // unordered_map (including every location-id string and hash node) per key. + virtual std::vector GetLocationValues(RequestContext *request_context, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept { + CacheLocationMapVector location_maps; + auto results = GetLocations(request_context, keys, location_maps); + out_locations.clear(); + out_locations.resize(keys.size()); + const std::size_t count = std::min(keys.size(), location_maps.size()); + for (std::size_t i = 0; i < count; ++i) { + auto &values = out_locations[i]; + values.reserve(location_maps[i].size()); + for (const auto &[location_id, location] : location_maps[i]) { + (void)location_id; + values.push_back(location); + } + } + return results; + } + + // Range-based compact variant used by very large prefix queries. The + // generic fallback preserves backend behavior; the local backend overrides + // it so callers avoid both copying the key slice and allocating one vector + // per key. + virtual std::vector GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept { + if (key_count != 0 && keys == nullptr) { + out_locations.Clear(key_count); + for (size_t i = 0; i < key_count; ++i) { + out_locations.FinishKey(); + } + return std::vector(key_count, EC_BADARGS); + } + KeyTypeVec key_vector; + if (key_count != 0) { + key_vector.assign(keys, keys + key_count); + } + LocationsPerKey locations; + auto results = GetLocationValues(request_context, key_vector, locations); + out_locations.Clear(key_count); + const size_t value_count = std::min(key_count, locations.size()); + for (size_t i = 0; i < key_count; ++i) { + if (i < value_count) { + out_locations.values.insert(out_locations.values.end(), locations[i].begin(), locations[i].end()); + } + out_locations.FinishKey(); + } + return results; + } + // 读取指定 location id 对应的 CacheLocation。 // @param request_context 请求上下文；可为 nullptr // @param keys 待查询的 key 列表 @@ -152,6 +230,107 @@ class MetaStorageBackend { const LocationIdsPerKey &location_ids, LocationsPerKey &out_locations) noexcept = 0; + // Read selected locations and preserve the key-level existence result. + // A targeted read alone cannot distinguish a missing key from an existing + // key that does not contain any requested location. RMW callers need that + // distinction to update key_count correctly when an upsert creates a new + // block. Backends that can determine both states in one lookup should + // override this method. The generic fallback only probes key existence for + // ambiguous all-NOENT rows, so existing-location reads remain one request. + virtual std::vector> + GetLocationsWithKeyStatus(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept { + auto results = GetLocations(request_context, keys, location_ids, out_locations); + out_key_error_codes.assign(keys.size(), EC_MISMATCH); + if (results.size() != keys.size() || out_locations.size() != keys.size()) { + return results; + } + + KeyTypeVec ambiguous_keys; + std::vector ambiguous_indices; + for (size_t i = 0; i < keys.size(); ++i) { + if (results[i].size() != location_ids[i].size() || out_locations[i].size() != location_ids[i].size()) { + continue; + } + bool found_location = false; + ErrorCode hard_error = EC_OK; + for (const ErrorCode ec : results[i]) { + if (ec == EC_OK) { + found_location = true; + break; + } + if (ec != EC_NOENT && hard_error == EC_OK) { + hard_error = ec; + } + } + if (found_location) { + out_key_error_codes[i] = EC_OK; + } else if (hard_error != EC_OK) { + out_key_error_codes[i] = hard_error; + } else { + ambiguous_keys.push_back(keys[i]); + ambiguous_indices.push_back(i); + } + } + + if (ambiguous_keys.empty()) { + return results; + } + std::vector exists; + const auto exists_results = Exists(request_context, ambiguous_keys, exists); + if (exists_results.size() != ambiguous_keys.size() || exists.size() != ambiguous_keys.size()) { + return results; + } + for (size_t i = 0; i < ambiguous_keys.size(); ++i) { + const size_t original_index = ambiguous_indices[i]; + out_key_error_codes[original_index] = + exists_results[i] == EC_OK ? (exists[i] ? EC_OK : EC_NOENT) : exists_results[i]; + } + return results; + } + + // Flat one-location form of GetLocationsWithKeyStatus. The default + // adapter is deliberately generic; local memory overrides it so the + // common ReportEvent shape allocates O(1) vectors per batch instead of + // two tiny vectors per key. + virtual std::vector + GetSingleLocationsWithKeyStatus(RequestContext *request_context, + const KeyTypeVec &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes) noexcept { + out_locations.assign(keys.size(), CacheLocationConstPtr{}); + out_key_error_codes.assign(keys.size(), EC_BADARGS); + if (keys.size() != location_ids.size()) { + return std::vector(keys.size(), EC_BADARGS); + } + LocationIdsPerKey ids_per_key(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + if (location_ids[i] == nullptr) { + return std::vector(keys.size(), EC_BADARGS); + } + ids_per_key[i].push_back(*location_ids[i]); + } + LocationsPerKey locations_per_key; + auto nested_results = + GetLocationsWithKeyStatus(request_context, keys, ids_per_key, locations_per_key, out_key_error_codes); + std::vector results(keys.size(), EC_MISMATCH); + if (nested_results.size() != keys.size() || locations_per_key.size() != keys.size() || + out_key_error_codes.size() != keys.size()) { + return results; + } + for (size_t i = 0; i < keys.size(); ++i) { + if (nested_results[i].size() == 1 && locations_per_key[i].size() == 1) { + results[i] = nested_results[i][0]; + out_locations[i] = std::move(locations_per_key[i][0]); + } + } + return results; + } + // 仅获取 key 的 location id 列表（不读取 location body）。 // @param request_context 请求上下文；可为 nullptr // @param keys 待查询的 key 列表 diff --git a/kv_cache_manager/meta/meta_storage_backend_manager.cc b/kv_cache_manager/meta/meta_storage_backend_manager.cc index 6fa5ee478..d3ec47ec4 100644 --- a/kv_cache_manager/meta/meta_storage_backend_manager.cc +++ b/kv_cache_manager/meta/meta_storage_backend_manager.cc @@ -1,7 +1,11 @@ #include "kv_cache_manager/meta/meta_storage_backend_manager.h" +#include #include #include +#include +#include +#include #include #include "kv_cache_manager/common/error_code.h" @@ -11,6 +15,7 @@ #include "kv_cache_manager/common/timestamp_util.h" #include "kv_cache_manager/config/meta_storage_backend_config.h" #include "kv_cache_manager/meta/common.h" +#include "kv_cache_manager/meta/meta_local_backend.h" #include "kv_cache_manager/meta/meta_storage_backend_factory.h" #include "kv_cache_manager/metrics/metrics_collector.h" @@ -25,7 +30,11 @@ std::pair> CollectMissingKeys(const KeyVector &k const std::vector &results) { KeyTypeVec missing_keys; std::vector missing_indices; - for (size_t i = 0; i < keys.size(); ++i) { + // Callers validate the positional contract before recovery. Keep this + // helper defensive as well so a newly added caller cannot index a short + // backend response before its outer layer normalizes the failure. + const size_t result_count = std::min(keys.size(), results.size()); + for (size_t i = 0; i < result_count; ++i) { if (results[i] == EC_NOENT) { missing_keys.push_back(keys[i]); missing_indices.push_back(i); @@ -36,77 +45,101 @@ std::pair> CollectMissingKeys(const KeyVector &k } // namespace MetaStorageBackendManager::~MetaStorageBackendManager() { - // Defensive cleanup in case Close was never called explicitly. - is_closed_.store(true, std::memory_order_release); - if (recover_thread_.joinable()) { - recover_thread_.join(); - } + // Close is idempotent and also owns recovery-thread shutdown, so an owner + // that forgets the explicit lifecycle call cannot leak backend resources. + (void)Close(); } ErrorCode MetaStorageBackendManager::Init(const std::string &instance_id, const std::shared_ptr &config) noexcept { - if (instance_id.empty()) { - KVCM_LOG_ERROR("init meta storage backend manager failed, empty instance id"); - return EC_BADARGS; - } - if (!config) { - KVCM_LOG_ERROR("init meta storage backend manager failed, null storage backend config"); - return EC_BADARGS; - } - instance_id_ = instance_id; + try { + std::lock_guard lifecycle_guard(lifecycle_mutex_); + if (instance_id.empty()) { + KVCM_LOG_ERROR("init meta storage backend manager failed, empty instance id"); + return EC_BADARGS; + } + if (!config) { + KVCM_LOG_ERROR("init meta storage backend manager failed, null storage backend config"); + return EC_BADARGS; + } + if (persistent_backend_ || cache_backend_ || opened_) { + KVCM_LOG_ERROR("meta storage backend manager is already initialized, instance[%s]", instance_id_.c_str()); + return EC_ERROR; + } + + const std::string &storage_uri = config->GetStorageUri(); + if (config->GetStorageType() != META_CACHED_BACKEND_TYPE_STR) { + // Single-backend mode: one backend serves every read/write directly. + auto persistent_backend = MetaStorageBackendFactory::CreateAndInitStorageBackend(instance_id, config); + if (!persistent_backend) { + KVCM_LOG_ERROR("fail to create persistent backend uri[%s]", storage_uri.c_str()); + return EC_ERROR; + } + instance_id_ = instance_id; + persistent_backend_ = std::move(persistent_backend); + KVCM_LOG_INFO("meta storage backend manager init ok in single-backend mode, instance[%s] type[%s]", + instance_id_.c_str(), + config->GetStorageType().c_str()); + return EC_OK; + } - const std::string &storage_uri = config->GetStorageUri(); - if (config->GetStorageType() != META_CACHED_BACKEND_TYPE_STR) { - // Single-backend mode: one backend serves every read/write directly. - persistent_backend_ = MetaStorageBackendFactory::CreateAndInitStorageBackend(instance_id_, config); - if (!persistent_backend_) { + assert((config->GetStorageType() == META_CACHED_BACKEND_TYPE_STR)); + std::string persistent_type; + std::string cache_type; + if (!storage_uri.empty()) { + StandardUri uri = StandardUri::FromUri(storage_uri); + if (!uri.Valid()) { + KVCM_LOG_ERROR("invalid storage uri[%s]", storage_uri.c_str()); + return EC_BADARGS; + } + persistent_type = uri.GetParam("persistent_type"); + cache_type = uri.GetParam("cache_type"); + } + // default to redis / local + persistent_type = persistent_type.empty() ? META_REDIS_BACKEND_TYPE_STR : persistent_type; + cache_type = cache_type.empty() ? META_LOCAL_BACKEND_TYPE_STR : cache_type; + + auto persistent_config = std::make_shared(persistent_type); + persistent_config->SetStorageUri(storage_uri); + auto persistent_backend = MetaStorageBackendFactory::CreatePersistentBackend(instance_id, persistent_config); + if (!persistent_backend) { KVCM_LOG_ERROR("fail to create persistent backend uri[%s]", storage_uri.c_str()); return EC_ERROR; } - KVCM_LOG_INFO("meta storage backend manager init ok in single-backend mode, instance[%s] type[%s]", - instance_id_.c_str(), - config->GetStorageType().c_str()); - return EC_OK; - } - - assert((config->GetStorageType() == META_CACHED_BACKEND_TYPE_STR)); - std::string persistent_type; - std::string cache_type; - if (!storage_uri.empty()) { - StandardUri uri = StandardUri::FromUri(storage_uri); - if (!uri.Valid()) { - KVCM_LOG_ERROR("invalid storage uri[%s]", storage_uri.c_str()); - return EC_BADARGS; + auto cache_config = std::make_shared(cache_type); + cache_config->SetStorageUri(storage_uri); + auto cache_backend = MetaStorageBackendFactory::CreateCacheBackend(instance_id, cache_config); + if (!cache_backend) { + KVCM_LOG_ERROR("fail to create cache backend uri[%s]", storage_uri.c_str()); + return EC_ERROR; } - persistent_type = uri.GetParam("persistent_type"); - cache_type = uri.GetParam("cache_type"); - } - // default to redis / local - persistent_type = persistent_type.empty() ? META_REDIS_BACKEND_TYPE_STR : persistent_type; - cache_type = cache_type.empty() ? META_LOCAL_BACKEND_TYPE_STR : cache_type; - auto persistent_config = std::make_shared(persistent_type); - persistent_config->SetStorageUri(storage_uri); - persistent_backend_ = MetaStorageBackendFactory::CreatePersistentBackend(instance_id_, persistent_config); - if (!persistent_backend_) { - KVCM_LOG_ERROR("fail to create persistent backend uri[%s]", storage_uri.c_str()); - return EC_ERROR; - } - auto cache_config = std::make_shared(cache_type); - cache_config->SetStorageUri(storage_uri); - cache_backend_ = MetaStorageBackendFactory::CreateCacheBackend(instance_id_, cache_config); - if (!cache_backend_) { - KVCM_LOG_ERROR("fail to create cache backend uri[%s]", storage_uri.c_str()); - return EC_ERROR; + // Publish the pair only after both factories have succeeded. A failed + // dual-backend Init can therefore be retried on the same object and + // never exposes a half-configured persistent side to Open(). + instance_id_ = instance_id; + persistent_backend_ = std::move(persistent_backend); + cache_backend_ = std::move(cache_backend); + KVCM_LOG_INFO("meta storage backend manager init ok, instance[%s] cache[%s] persistent[%s]", + instance_id_.c_str(), + cache_type.c_str(), + persistent_type.c_str()); + return EC_OK; + } catch (const std::exception &e) { + KVCM_LOG_ERROR( + "init meta storage backend manager raised, instance[%s] error[%s]", instance_id.c_str(), e.what()); + } catch (...) { + KVCM_LOG_ERROR("init meta storage backend manager raised unknown exception, instance[%s]", instance_id.c_str()); } - KVCM_LOG_INFO("meta storage backend manager init ok, instance[%s] cache[%s] persistent[%s]", - instance_id_.c_str(), - cache_type.c_str(), - persistent_type.c_str()); - return EC_OK; + return EC_ERROR; } ErrorCode MetaStorageBackendManager::Open() noexcept { + std::lock_guard lifecycle_guard(lifecycle_mutex_); + if (opened_) { + KVCM_LOG_ERROR("meta storage backend manager is already open, instance[%s]", instance_id_.c_str()); + return EC_ERROR; + } if (!persistent_backend_) { KVCM_LOG_ERROR("persistent backend not inited! instance[%s]", instance_id_.c_str()); return EC_ERROR; @@ -114,13 +147,20 @@ ErrorCode MetaStorageBackendManager::Open() noexcept { ErrorCode ec = persistent_backend_->Open(); if (ec != EC_OK) { - KVCM_LOG_ERROR("open persistent failed, instance[%s] ec[%d]", instance_id_.c_str(), ec); + // Open implementations may have allocated resources before reporting + // failure. Roll them back even though the manager never publishes an + // opened state. + is_closed_.store(true, std::memory_order_release); + const ErrorCode close_ec = persistent_backend_->Close(); + KVCM_LOG_ERROR( + "open persistent failed, instance[%s] ec[%d] rollback_close_ec[%d]", instance_id_.c_str(), ec, close_ec); return ec; } is_closed_.store(false, std::memory_order_release); if (!cache_backend_) { recover_state_.store(RecoverState::kRunning, std::memory_order_release); + opened_ = true; KVCM_LOG_INFO("meta storage backend manager opened in single-backend mode, instance[%s]", instance_id_.c_str()); return EC_OK; } @@ -128,19 +168,64 @@ ErrorCode MetaStorageBackendManager::Open() noexcept { ec = cache_backend_->Open(); if (ec != EC_OK) { KVCM_LOG_ERROR("open cache failed, instance[%s] ec[%d]", instance_id_.c_str(), ec); + // Open is transactional from the manager's point of view. Do not + // leave the persistent side live when the cache side cannot serve. + // Also close the cache defensively in case its Open partially + // initialized resources before returning an error. + is_closed_.store(true, std::memory_order_release); + ErrorCode cache_close_ec = cache_backend_->Close(); + ErrorCode persistent_close_ec = persistent_backend_->Close(); + if (cache_close_ec != EC_OK || persistent_close_ec != EC_OK) { + KVCM_LOG_ERROR("rollback after cache open failure was incomplete, instance[%s] " + "cache_close_ec[%d] persistent_close_ec[%d]", + instance_id_.c_str(), + cache_close_ec, + persistent_close_ec); + } return ec; } recover_state_.store(RecoverState::kRecover, std::memory_order_release); - recover_thread_ = std::thread(&MetaStorageBackendManager::AsyncRecoverTask, this); + try { + recover_thread_ = std::thread(&MetaStorageBackendManager::AsyncRecoverTask, this); + } catch (const std::exception &e) { + // Open is noexcept, so thread construction must not escape. Roll both + // backends back to a closed state instead of publishing a manager that + // can never finish recovery. + is_closed_.store(true, std::memory_order_release); + ErrorCode cache_close_ec = cache_backend_->Close(); + ErrorCode persistent_close_ec = persistent_backend_->Close(); + KVCM_LOG_ERROR("start async recover failed, instance[%s] error[%s] " + "cache_close_ec[%d] persistent_close_ec[%d]", + instance_id_.c_str(), + e.what(), + cache_close_ec, + persistent_close_ec); + return EC_ERROR; + } catch (...) { + is_closed_.store(true, std::memory_order_release); + ErrorCode cache_close_ec = cache_backend_->Close(); + ErrorCode persistent_close_ec = persistent_backend_->Close(); + KVCM_LOG_ERROR("start async recover failed with unknown exception, instance[%s] " + "cache_close_ec[%d] persistent_close_ec[%d]", + instance_id_.c_str(), + cache_close_ec, + persistent_close_ec); + return EC_ERROR; + } + opened_ = true; KVCM_LOG_INFO("meta storage backend manager opened, instance[%s], async recover started", instance_id_.c_str()); return EC_OK; } ErrorCode MetaStorageBackendManager::Close() noexcept { + std::lock_guard lifecycle_guard(lifecycle_mutex_); is_closed_.store(true, std::memory_order_release); if (recover_thread_.joinable()) { recover_thread_.join(); } + if (!opened_) { + return EC_OK; + } ErrorCode cache_ec = EC_OK; ErrorCode persistent_ec = EC_OK; @@ -158,6 +243,7 @@ ErrorCode MetaStorageBackendManager::Close() noexcept { KVCM_LOG_ERROR("close persistent failed, instance[%s] ec[%d]", instance_id_.c_str(), persistent_ec); return persistent_ec; } + opened_ = false; KVCM_LOG_INFO("meta storage backend manager closed, instance[%s]", instance_id_.c_str()); return EC_OK; } @@ -170,27 +256,102 @@ void MetaStorageBackendManager::AsyncRecoverTask() noexcept { std::string next_cursor; KeyTypeVec scanned_keys; FieldMapVec field_maps; - do { + bool has_pending_batch = false; + bool recovery_complete = false; + while (!recovery_complete) { if (is_closed_.load(std::memory_order_acquire)) { KVCM_LOG_INFO("async recover aborted due to close, instance[%s]", instance_id_.c_str()); return; } - scanned_keys.clear(); - field_maps.clear(); - ErrorCode scan_ec = - persistent_backend_->ListKeys(nullptr, cursor, kRecoverScanBatchSize, next_cursor, scanned_keys); - if (scan_ec != EC_OK) { + if (!has_pending_batch) { + scanned_keys.clear(); + field_maps.clear(); + ErrorCode scan_ec = + persistent_backend_->ListKeys(nullptr, cursor, kRecoverScanBatchSize, next_cursor, scanned_keys); + if (scan_ec != EC_OK) { + ++consecutive_failures; + KVCM_LOG_ERROR("async recover scan failed, instance[%s] cursor[%s] ec[%d] attempt[%d/%d]", + instance_id_.c_str(), + cursor.c_str(), + scan_ec, + consecutive_failures, + kRecoverMaxConsecutiveFailures); + if (consecutive_failures >= kRecoverMaxConsecutiveFailures) { + KVCM_LOG_ERROR("async recover giving up after %d consecutive scan failures, " + "leaving backend in Recover, instance[%s]", + kRecoverMaxConsecutiveFailures, + instance_id_.c_str()); + break; + } + continue; + } + + if (scanned_keys.empty()) { + consecutive_failures = 0; + cursor = next_cursor; + recovery_complete = (cursor == SCAN_BASE_CURSOR); + continue; + } + // Redis SCAN is not a stable snapshot. Once a cursor yields a + // batch, retain those exact keys across Get/backfill retries; + // rescanning the cursor could return a different set and let the + // failed keys disappear before recovery is published complete. + has_pending_batch = true; + } + CacheLocationMapVector locations; + PropertyMapVector properties; + std::vector get_error_codes = persistent_backend_->Get(nullptr, scanned_keys, locations, properties); + if (get_error_codes.size() != scanned_keys.size() || locations.size() != scanned_keys.size() || + properties.size() != scanned_keys.size()) { + KVCM_LOG_ERROR("async recover Get results[%lu] locations[%lu] properties[%lu] mismatch keys[%lu], " + "skip batch", + get_error_codes.size(), + locations.size(), + properties.size(), + scanned_keys.size()); ++consecutive_failures; - KVCM_LOG_ERROR("async recover scan failed, instance[%s] cursor[%s] ec[%d] attempt[%d/%d]", + if (consecutive_failures >= kRecoverMaxConsecutiveFailures) { + KVCM_LOG_ERROR("async recover giving up after %d malformed Get responses, " + "leaving backend in Recover, instance[%s]", + kRecoverMaxConsecutiveFailures, + instance_id_.c_str()); + break; + } + continue; + } + bool get_complete = true; + for (size_t i = 0; i < scanned_keys.size(); ++i) { + if (get_error_codes[i] != EC_OK && get_error_codes[i] != EC_NOENT) { + KVCM_LOG_WARN("async recover key[%ld] get failed ec[%d]", scanned_keys[i], get_error_codes[i]); + get_complete = false; + } + } + if (!get_complete) { + ++consecutive_failures; + if (consecutive_failures >= kRecoverMaxConsecutiveFailures) { + KVCM_LOG_ERROR("async recover giving up after %d incomplete Get responses, " + "leaving backend in Recover, instance[%s]", + kRecoverMaxConsecutiveFailures, + instance_id_.c_str()); + break; + } + continue; + } + + bool backfill_success = false; + const int64_t backfilled_keys = + BackfillKeysToCache(scanned_keys, locations, properties, get_error_codes, &backfill_success); + if (!backfill_success) { + ++consecutive_failures; + KVCM_LOG_ERROR("async recover backfill failed, instance[%s] cursor[%s] attempt[%d/%d]", instance_id_.c_str(), cursor.c_str(), - scan_ec, consecutive_failures, kRecoverMaxConsecutiveFailures); if (consecutive_failures >= kRecoverMaxConsecutiveFailures) { - KVCM_LOG_ERROR("async recover giving up after %d consecutive failures, " - "forcing transition to Running, instance[%s]", + KVCM_LOG_ERROR("async recover giving up after %d consecutive backfill failures, " + "leaving backend in Recover, instance[%s]", kRecoverMaxConsecutiveFailures, instance_id_.c_str()); break; @@ -198,31 +359,26 @@ void MetaStorageBackendManager::AsyncRecoverTask() noexcept { continue; } + total_backfilled_keys += backfilled_keys; consecutive_failures = 0; cursor = next_cursor; - if (scanned_keys.empty()) { - continue; - } - CacheLocationMapVector locations; - PropertyMapVector properties; - std::vector get_error_codes = persistent_backend_->Get(nullptr, scanned_keys, locations, properties); - for (size_t i = 0; i < scanned_keys.size(); ++i) { - if (get_error_codes[i] != EC_OK && get_error_codes[i] != EC_NOENT) { - KVCM_LOG_WARN("async recover key[%ld] get failed ec[%d]", scanned_keys[i], get_error_codes[i]); - } - } - total_backfilled_keys += BackfillKeysToCache(scanned_keys, locations, properties, get_error_codes); - } while (cursor != SCAN_BASE_CURSOR); + recovery_complete = (cursor == SCAN_BASE_CURSOR); + has_pending_batch = false; + } - if (consecutive_failures == 0) { - KVCM_LOG_INFO("async recover completed instance[%s] total_backfilled_keys[%ld]", - instance_id_.c_str(), - total_backfilled_keys); - } else { - KVCM_LOG_WARN("async recover partial, instance[%s] total_backfilled_keys[%ld], forcing transition to Running", - instance_id_.c_str(), - total_backfilled_keys); + if (!recovery_complete) { + // Recover mode is deliberately retained. Reads can still fall back to + // persistent storage, and delete tombstones must remain live so a + // partial cache cannot resurrect keys through a late backfill. + KVCM_LOG_ERROR("async recover incomplete, instance[%s] total_backfilled_keys[%ld], " + "backend remains in Recover", + instance_id_.c_str(), + total_backfilled_keys); + return; } + + KVCM_LOG_INFO( + "async recover completed instance[%s] total_backfilled_keys[%ld]", instance_id_.c_str(), total_backfilled_keys); recover_state_.store(RecoverState::kRunning, std::memory_order_release); { std::lock_guard lock(deleted_keys_mutex_); @@ -230,12 +386,19 @@ void MetaStorageBackendManager::AsyncRecoverTask() noexcept { } } -void MetaStorageBackendManager::EnsureKeyInCache(RequestContext *request_context, const KeyTypeVec &keys) noexcept { +bool MetaStorageBackendManager::EnsureKeyInCache(RequestContext *request_context, const KeyTypeVec &keys) noexcept { if (keys.empty()) { - return; + return true; } std::vector exists_vec; std::vector exists_results = cache_backend_->Exists(request_context, keys, exists_vec); + if (exists_results.size() != keys.size() || exists_vec.size() != keys.size()) { + KVCM_LOG_ERROR("ensure cache Exists results[%lu] values[%lu] mismatch keys[%lu]", + exists_results.size(), + exists_vec.size(), + keys.size()); + return false; + } KeyTypeVec missing_keys; for (size_t i = 0; i < keys.size(); ++i) { if (exists_results[i] != EC_OK || !exists_vec[i]) { @@ -243,51 +406,118 @@ void MetaStorageBackendManager::EnsureKeyInCache(RequestContext *request_context } } if (missing_keys.empty()) { - return; + return true; } CacheLocationMapVector locations; PropertyMapVector properties; std::vector get_results = persistent_backend_->Get(request_context, missing_keys, locations, properties); + if (get_results.size() != missing_keys.size() || locations.size() != missing_keys.size() || + properties.size() != missing_keys.size()) { + KVCM_LOG_ERROR("ensure cache Get results[%lu] locations[%lu] properties[%lu] mismatch keys[%lu]", + get_results.size(), + locations.size(), + properties.size(), + missing_keys.size()); + return false; + } + + // PutIfAbsent prevents a stale persistent read from overwriting a newer + // dual-write that populated the cache after the Exists probe. std::vector put_results = - cache_backend_->Put(request_context, missing_keys, locations, properties, get_results); + cache_backend_->PutIfAbsent(request_context, missing_keys, locations, properties, get_results); + if (put_results.size() != missing_keys.size()) { + KVCM_LOG_ERROR( + "ensure cache PutIfAbsent results[%lu] mismatch keys[%lu]", put_results.size(), missing_keys.size()); + return false; + } + bool hydrated = true; for (size_t i = 0; i < missing_keys.size(); ++i) { - if (put_results[i] != EC_OK && get_results[i] == EC_OK) { - KVCM_LOG_WARN("ensure key[%ld] in cache failed, ec[%d]", missing_keys[i], put_results[i]); + if (get_results[i] == EC_NOENT) { + // The key is genuinely new. The following Upsert can safely + // create it from the request's fields. + continue; + } + if (get_results[i] != EC_OK || (put_results[i] != EC_OK && put_results[i] != EC_EXIST)) { + KVCM_LOG_WARN("ensure key[%ld] in cache failed, get_ec[%d] put_ec[%d]", + missing_keys[i], + get_results[i], + put_results[i]); + hydrated = false; } } + return hydrated; } int64_t MetaStorageBackendManager::BackfillKeysToCache(const KeyTypeVec &keys, const CacheLocationMapVector &locations, const PropertyMapVector &properties, - const std::vector &get_error_codes) noexcept { + const std::vector &get_error_codes, + bool *out_success) noexcept { + if (out_success) { + *out_success = false; + } std::lock_guard lock(deleted_keys_mutex_); // Merge get errors and deleted-key tombstones into a single error vector. - assert(get_error_codes.size() == keys.size()); + if (get_error_codes.size() != keys.size() || locations.size() != keys.size() || properties.size() != keys.size()) { + KVCM_LOG_ERROR("backfill results[%lu] locations[%lu] properties[%lu] mismatch keys[%lu], skip batch", + get_error_codes.size(), + locations.size(), + properties.size(), + keys.size()); + return 0; + } int64_t valid_count = 0; std::vector merged_error_codes = get_error_codes; for (size_t i = 0; i < keys.size(); ++i) { + if (merged_error_codes[i] != EC_OK && merged_error_codes[i] != EC_NOENT) { + KVCM_LOG_ERROR("backfill source read failed key[%ld] ec[%d], skip batch", keys[i], merged_error_codes[i]); + return 0; + } if (merged_error_codes[i] == EC_OK && deleted_keys_.count(keys[i]) > 0) { merged_error_codes[i] = EC_NOENT; } valid_count += (merged_error_codes[i] == EC_OK); } if (valid_count == 0) { + if (out_success) { + *out_success = true; + } return 0; } std::vector put_results = cache_backend_->PutIfAbsent(nullptr, keys, locations, properties, merged_error_codes); + if (put_results.size() != keys.size()) { + KVCM_LOG_ERROR( + "backfill PutIfAbsent results[%lu] mismatch keys[%lu], skip accounting", put_results.size(), keys.size()); + return 0; + } int64_t backfilled_count = 0; + bool write_complete = true; for (size_t i = 0; i < keys.size(); ++i) { - if (put_results[i] == EC_OK) { - ++backfilled_count; - } else if (merged_error_codes[i] == EC_OK && put_results[i] != EC_NOENT) { - KVCM_LOG_WARN("backfill PutIfAbsent failed key[%ld] ec[%d]", keys[i], put_results[i]); + if (merged_error_codes[i] == EC_OK) { + if (put_results[i] == EC_OK) { + ++backfilled_count; + } else if (put_results[i] != EC_EXIST) { + KVCM_LOG_WARN("backfill PutIfAbsent failed key[%ld] ec[%d]", keys[i], put_results[i]); + write_complete = false; + } + } else if (put_results[i] != merged_error_codes[i]) { + // Conditional cache writes must preserve the source error for + // skipped keys. EC_OK here could mean a tombstoned value was + // inserted despite the guard, so never publish this cache. + KVCM_LOG_WARN("backfill PutIfAbsent violated skip contract key[%ld] source_ec[%d] put_ec[%d]", + keys[i], + merged_error_codes[i], + put_results[i]); + write_complete = false; } } + if (out_success) { + *out_success = write_complete; + } return backfilled_count; } @@ -297,6 +527,10 @@ std::vector MetaStorageBackendManager::Put(RequestContext *request_co CacheLocationMapVector &locations = batch.batch_locations; PropertyMapVector &properties = batch.batch_properties; std::vector persistent_results = persistent_backend_->Put(request_context, keys, locations, properties); + if (persistent_results.size() != keys.size()) { + KVCM_LOG_ERROR("persistent Put results[%lu] mismatch keys[%lu]", persistent_results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } if (!cache_backend_) { return persistent_results; } @@ -307,6 +541,10 @@ std::vector MetaStorageBackendManager::Put(RequestContext *request_co KVCM_METRICS_COLLECTOR_SET_METRICS( mc, meta_indexer, cache_backend_put_time_us, TimestampUtil::GetCurrentTimeUs() - cache_begin); } + if (results.size() != keys.size()) { + KVCM_LOG_ERROR("cache Put results[%lu] mismatch keys[%lu]", results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } return results; } @@ -320,10 +558,16 @@ std::vector MetaStorageBackendManager::Upsert(RequestContext *request // Upsert may touch only a subset of fields, so Recover-time hydration // is needed to avoid overwriting unmentioned fields with empty values. if (cache_backend_ && recover_state_.load(std::memory_order_acquire) == RecoverState::kRecover) { - EnsureKeyInCache(request_context, keys); + if (!EnsureKeyInCache(request_context, keys)) { + return std::vector(keys.size(), EC_ERROR); + } } std::vector persistent_results = persistent_backend_->Upsert(request_context, keys, locations, properties); + if (persistent_results.size() != keys.size()) { + KVCM_LOG_ERROR("persistent Upsert results[%lu] mismatch keys[%lu]", persistent_results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } if (!cache_backend_) { return persistent_results; } @@ -334,20 +578,83 @@ std::vector MetaStorageBackendManager::Upsert(RequestContext *request KVCM_METRICS_COLLECTOR_SET_METRICS( mc, meta_indexer, cache_backend_upsert_time_us, TimestampUtil::GetCurrentTimeUs() - cache_begin); } + if (results.size() != keys.size()) { + KVCM_LOG_ERROR("cache Upsert results[%lu] mismatch keys[%lu]", results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } return results; } +std::vector MetaStorageBackendManager::UpsertSingleLocations(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations) noexcept { + if (!SupportsSingleLocationRmw()) { + KVCM_LOG_ERROR("single-location upsert requires a pure local metadata backend"); + return std::vector(keys.size(), EC_UNIMPLEMENTED); + } + return persistent_backend_->UpsertSingleLocations(request_context, keys, location_ids, locations); +} + +void MetaStorageBackendManager::PrepareSingleLocationRmwScratch(size_t max_count, + SingleLocationRmwScratch &scratch) noexcept { + if (!SupportsSingleLocationRmw()) { + return; + } + static_cast(persistent_backend_.get())->PrepareSingleLocationRmwScratch(max_count, scratch); +} + +void MetaStorageBackendManager::UpsertSingleLocationsInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept { + if (!SupportsSingleLocationRmw()) { + KVCM_LOG_ERROR("single-location upsert requires a pure local metadata backend"); + out_results.assign(keys.size(), EC_UNIMPLEMENTED); + return; + } + static_cast(persistent_backend_.get()) + ->UpsertSingleLocationsInto(request_context, keys, location_ids, locations, out_results, scratch); +} + +void MetaStorageBackendManager::UpsertSingleLocationsUsingRetainedHandlesInto( + RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &locations, + const std::vector &read_indices, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept { + if (!SupportsSingleLocationRmw()) { + KVCM_LOG_ERROR("retained-handle single-location upsert requires a pure local metadata backend"); + out_results.assign(keys.size(), EC_UNIMPLEMENTED); + scratch.ReleaseRetainedHandles(); + return; + } + static_cast(persistent_backend_.get()) + ->UpsertSingleLocationsUsingRetainedHandlesInto( + request_context, keys, location_ids, locations, read_indices, out_results, scratch); +} + std::vector MetaStorageBackendManager::Delete(RequestContext *request_context, const KeyVector &keys) noexcept { std::vector persistent_results = persistent_backend_->Delete(request_context, keys); + if (persistent_results.size() != keys.size()) { + KVCM_LOG_ERROR("persistent Delete results[%lu] mismatch keys[%lu]", persistent_results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } if (!cache_backend_) { return persistent_results; } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRecover) { // Tombstone to prevent Recover backfill from resurrecting deleted keys. std::lock_guard lock(deleted_keys_mutex_); - for (const auto &key : keys) { - deleted_keys_.insert(key); + for (size_t i = 0; i < keys.size(); ++i) { + if (persistent_results[i] == EC_OK || persistent_results[i] == EC_NOENT) { + deleted_keys_.insert(keys[i]); + } } } const int64_t cache_begin = TimestampUtil::GetCurrentTimeUs(); @@ -357,6 +664,10 @@ std::vector MetaStorageBackendManager::Delete(RequestContext *request KVCM_METRICS_COLLECTOR_SET_METRICS( mc, meta_indexer, cache_backend_delete_time_us, TimestampUtil::GetCurrentTimeUs() - cache_begin); } + if (results.size() != keys.size()) { + KVCM_LOG_ERROR("cache Delete results[%lu] mismatch keys[%lu]", results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } return results; } @@ -368,17 +679,26 @@ std::vector MetaStorageBackendManager::Delete(RequestContext *request if (keys.empty()) { return {}; } - assert(location_ids.size() == keys.size()); + if (location_ids.size() != keys.size()) { + return std::vector(keys.size(), EC_BADARGS); + } // Partial-delete during Recover: hydrate cache from persistent first so // the conditional mirror write below has the full pre-restart field set // to delete against (and async backfill cannot later overwrite us). if (cache_backend_ && recover_state_.load(std::memory_order_acquire) == RecoverState::kRecover) { - EnsureKeyInCache(request_context, keys); + if (!EnsureKeyInCache(request_context, keys)) { + return std::vector(keys.size(), EC_ERROR); + } } std::vector persistent_results = persistent_backend_->DeleteLocations(request_context, keys, location_ids); + if (persistent_results.size() != keys.size()) { + KVCM_LOG_ERROR( + "persistent DeleteLocations results[%lu] mismatch keys[%lu]", persistent_results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } std::vector results; if (!cache_backend_) { results = std::move(persistent_results); @@ -391,6 +711,10 @@ std::vector MetaStorageBackendManager::Delete(RequestContext *request mc, meta_indexer, cache_backend_delete_time_us, TimestampUtil::GetCurrentTimeUs() - cache_begin); } } + if (results.size() != keys.size()) { + KVCM_LOG_ERROR("cache DeleteLocations results[%lu] mismatch keys[%lu]", results.size(), keys.size()); + return std::vector(keys.size(), EC_ERROR); + } out_reclaimed_count = MaybeReclaimEmptyKeys(request_context, keys, results); return results; @@ -399,9 +723,20 @@ std::vector MetaStorageBackendManager::Delete(RequestContext *request int32_t MetaStorageBackendManager::MaybeReclaimEmptyKeys(RequestContext *request_context, const KeyVector &keys, const std::vector &delete_results) noexcept { + if (delete_results.size() != keys.size()) { + KVCM_LOG_ERROR( + "delete results[%lu] mismatch keys[%lu], skip empty-key reclamation", delete_results.size(), keys.size()); + return 0; + } + KeyVector candidate_keys; + std::unordered_set seen_candidates; + candidate_keys.reserve(keys.size()); for (size_t i = 0; i < keys.size(); ++i) { - if (delete_results[i] == EC_OK) { + // A single block may contribute several location-deletion tasks to one + // RMW batch. Reclaim and account for that block at most once even when + // several of its locations become empty in the same request. + if (delete_results[i] == EC_OK && seen_candidates.insert(keys[i]).second) { candidate_keys.push_back(keys[i]); } } @@ -416,6 +751,14 @@ int32_t MetaStorageBackendManager::MaybeReclaimEmptyKeys(RequestContext *request } else { exists_ecs = persistent_backend_->ExistsLocation(request_context, candidate_keys, has_locations); } + if (exists_ecs.size() != candidate_keys.size() || has_locations.size() != candidate_keys.size()) { + KVCM_LOG_ERROR("ExistsLocation results[%lu] values[%lu] mismatch candidate keys[%lu], " + "skip empty-key reclamation", + exists_ecs.size(), + has_locations.size(), + candidate_keys.size()); + return 0; + } KeyVector reclaimed_keys; for (size_t i = 0; i < candidate_keys.size(); ++i) { @@ -428,6 +771,13 @@ int32_t MetaStorageBackendManager::MaybeReclaimEmptyKeys(RequestContext *request } std::vector whole_ecs = Delete(request_context, reclaimed_keys); + if (whole_ecs.size() != reclaimed_keys.size()) { + KVCM_LOG_ERROR("whole-key delete results[%lu] mismatch reclaimed keys[%lu], " + "skip key-count adjustment", + whole_ecs.size(), + reclaimed_keys.size()); + return 0; + } int32_t reclaimed = 0; for (const ErrorCode ec : whole_ecs) { if (ec == EC_OK || ec == EC_NOENT) { @@ -446,6 +796,16 @@ std::vector MetaStorageBackendManager::Get(RequestContext *request_co } std::vector results = cache_backend_->Get(request_context, keys, out_locations, out_properties); + if (results.size() != keys.size() || out_locations.size() != keys.size() || out_properties.size() != keys.size()) { + KVCM_LOG_ERROR("cache Get results[%lu] locations[%lu] properties[%lu] mismatch keys[%lu]", + results.size(), + out_locations.size(), + out_properties.size(), + keys.size()); + results.assign(keys.size(), EC_ERROR); + out_locations.assign(keys.size(), CacheLocationMap{}); + out_properties.assign(keys.size(), PropertyMap{}); + } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { return results; } @@ -459,13 +819,18 @@ std::vector MetaStorageBackendManager::Get(RequestContext *request_co PropertyMapVector persistent_properties; std::vector persistent_results = persistent_backend_->Get(request_context, missing_keys, persistent_locations, persistent_properties); - if (missing_keys.size() != persistent_locations.size() || missing_keys.size() != persistent_properties.size()) { - KVCM_LOG_ERROR("persistent Get size mismatch: locations[%lu] properties[%lu] vs keys[%lu]", + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_locations.size() || + missing_keys.size() != persistent_properties.size()) { + KVCM_LOG_ERROR("persistent Get size mismatch: results[%lu] locations[%lu] properties[%lu] vs keys[%lu]", + persistent_results.size(), persistent_locations.size(), persistent_properties.size(), missing_keys.size()); for (size_t i = 0; i < missing_keys.size(); ++i) { - results[missing_indices[i]] = EC_ERROR; + const size_t original_idx = missing_indices[i]; + results[original_idx] = EC_ERROR; + out_locations[original_idx].clear(); + out_properties[original_idx].clear(); } return results; } @@ -488,6 +853,14 @@ std::vector MetaStorageBackendManager::GetLocations(RequestContext *r } std::vector results = cache_backend_->GetLocations(request_context, keys, out_location_maps); + if (results.size() != keys.size() || out_location_maps.size() != keys.size()) { + KVCM_LOG_ERROR("cache GetLocations results[%lu] locations[%lu] mismatch keys[%lu]", + results.size(), + out_location_maps.size(), + keys.size()); + results.assign(keys.size(), EC_ERROR); + out_location_maps.assign(keys.size(), CacheLocationMap{}); + } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { return results; } @@ -500,11 +873,15 @@ std::vector MetaStorageBackendManager::GetLocations(RequestContext *r CacheLocationMapVector persistent_locations; std::vector persistent_results = persistent_backend_->GetLocations(request_context, missing_keys, persistent_locations); - if (missing_keys.size() != persistent_locations.size()) { - KVCM_LOG_ERROR( - "persistent_locations size[%lu] mismatch keys's[%lu]", persistent_locations.size(), missing_keys.size()); + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_locations.size()) { + KVCM_LOG_ERROR("persistent GetLocations results[%lu] locations[%lu] mismatch keys[%lu]", + persistent_results.size(), + persistent_locations.size(), + missing_keys.size()); for (size_t i = 0; i < missing_keys.size(); ++i) { - results[missing_indices[i]] = EC_ERROR; + const size_t original_idx = missing_indices[i]; + results[original_idx] = EC_ERROR; + out_location_maps[original_idx].clear(); } return results; } @@ -518,6 +895,57 @@ std::vector MetaStorageBackendManager::GetLocations(RequestContext *r return results; } +std::vector MetaStorageBackendManager::GetLocationValues(RequestContext *request_context, + const KeyVector &keys, + LocationsPerKey &out_locations) noexcept { + if (!cache_backend_) { + return persistent_backend_->GetLocationValues(request_context, keys, out_locations); + } + + std::vector results = cache_backend_->GetLocationValues(request_context, keys, out_locations); + if (results.size() != keys.size() || out_locations.size() != keys.size()) { + KVCM_LOG_ERROR("cache location values results[%lu] locations[%lu] mismatch keys[%lu]", + results.size(), + out_locations.size(), + keys.size()); + // The two arrays form one positional contract. Once either shape is + // broken, even an in-range EC_OK cannot be trusted to describe the + // value at the same index. + results.assign(keys.size(), EC_ERROR); + out_locations.assign(keys.size(), CacheLocationVector{}); + } + if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { + return results; + } + + auto [missing_keys, missing_indices] = CollectMissingKeys(keys, results); + if (missing_keys.empty()) { + return results; + } + + LocationsPerKey persistent_locations; + std::vector persistent_results = + persistent_backend_->GetLocationValues(request_context, missing_keys, persistent_locations); + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_locations.size()) { + KVCM_LOG_ERROR("persistent location values results[%lu] locations[%lu] mismatch keys[%lu]", + persistent_results.size(), + persistent_locations.size(), + missing_keys.size()); + for (size_t i = 0; i < missing_keys.size(); ++i) { + results[missing_indices[i]] = EC_ERROR; + } + return results; + } + for (size_t i = 0; i < missing_keys.size(); ++i) { + const size_t original_idx = missing_indices[i]; + results[original_idx] = persistent_results[i]; + if (persistent_results[i] == EC_OK) { + out_locations[original_idx] = std::move(persistent_locations[i]); + } + } + return results; +} + std::vector MetaStorageBackendManager::GetLocationsFromPersistent( RequestContext *request_context, const KeyVector &keys, CacheLocationMapVector &out_location_maps) noexcept { out_location_maps.clear(); @@ -593,28 +1021,130 @@ std::vector MetaStorageBackendManager::RefreshCacheFromPersistent(Req return results; } +std::vector +MetaStorageBackendManager::GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept { + if (!cache_backend_) { + return persistent_backend_->GetLocationValuesCompact(request_context, keys, key_count, out_locations); + } + + // Cached mode must retain its recovery/fallback behavior. The large-query + // compact fast path is deliberately restricted to the single local backend, + // so use the existing manager API when this method is reached in another + // configuration. + KeyVector key_vector; + if (key_count != 0) { + if (keys == nullptr) { + out_locations.Clear(key_count); + for (size_t i = 0; i < key_count; ++i) { + out_locations.FinishKey(); + } + return std::vector(key_count, EC_BADARGS); + } + key_vector.assign(keys, keys + key_count); + } + LocationsPerKey locations; + auto results = GetLocationValues(request_context, key_vector, locations); + out_locations.Clear(key_count); + const size_t value_count = std::min(key_count, locations.size()); + for (size_t i = 0; i < key_count; ++i) { + if (i < value_count) { + out_locations.values.insert(out_locations.values.end(), locations[i].begin(), locations[i].end()); + } + out_locations.FinishKey(); + } + return results; +} + std::vector> MetaStorageBackendManager::GetLocations(RequestContext *request_context, const KeyVector &keys, const LocationIdsPerKey &location_ids, LocationsPerKey &out_locations) noexcept { + if (keys.size() != location_ids.size()) { + out_locations.assign(keys.size(), CacheLocationVector{}); + return std::vector>(keys.size(), std::vector{EC_BADARGS}); + } if (!cache_backend_) { return persistent_backend_->GetLocations(request_context, keys, location_ids, out_locations); } std::vector> results = cache_backend_->GetLocations(request_context, keys, location_ids, out_locations); + if (results.size() != keys.size() || out_locations.size() != keys.size()) { + KVCM_LOG_ERROR("cache targeted GetLocations results[%lu] locations[%lu] mismatch keys[%lu]", + results.size(), + out_locations.size(), + keys.size()); + out_locations.assign(keys.size(), CacheLocationVector{}); + results.resize(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + results[i].assign(location_ids[i].size(), EC_ERROR); + out_locations[i].assign(location_ids[i].size(), CacheLocationConstPtr{}); + } + return results; + } + for (size_t i = 0; i < keys.size(); ++i) { + if (results[i].size() != location_ids[i].size() || out_locations[i].size() != location_ids[i].size()) { + KVCM_LOG_ERROR("cache targeted GetLocations key[%ld] results[%lu] locations[%lu] mismatch ids[%lu]", + keys[i], + results[i].size(), + out_locations[i].size(), + location_ids[i].size()); + results[i].assign(location_ids[i].size(), EC_ERROR); + out_locations[i].assign(location_ids[i].size(), CacheLocationConstPtr{}); + } + } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { return results; } + // Per-location EC_NOENT does not say whether the cache missed the whole + // key or found the key without that location. Mixed OK/NOENT is + // unambiguously a cache hit and must never fall back to a potentially + // older persistent value. When every requested location is absent, use a + // cheap key-existence probe to distinguish the two cases. + KeyTypeVec ambiguous_keys; + std::vector ambiguous_indices; + for (size_t i = 0; i < keys.size(); ++i) { + if (!results[i].empty() && + std::all_of(results[i].begin(), results[i].end(), [](ErrorCode ec) { return ec == EC_NOENT; })) { + ambiguous_keys.push_back(keys[i]); + ambiguous_indices.push_back(i); + } + } + KeyTypeVec missing_keys; std::vector missing_indices; LocationIdsPerKey missing_location_ids; - for (size_t i = 0; i < keys.size(); ++i) { - if (!results[i].empty() && results[i][0] == EC_NOENT) { - missing_keys.push_back(keys[i]); - missing_indices.push_back(i); - missing_location_ids.push_back(location_ids[i]); + if (!ambiguous_keys.empty()) { + std::vector cache_key_exists; + const std::vector exists_results = + cache_backend_->Exists(request_context, ambiguous_keys, cache_key_exists); + if (exists_results.size() != ambiguous_keys.size() || cache_key_exists.size() != ambiguous_keys.size()) { + KVCM_LOG_ERROR("cache key-existence results[%lu] values[%lu] mismatch ambiguous keys[%lu]", + exists_results.size(), + cache_key_exists.size(), + ambiguous_keys.size()); + for (const size_t original_idx : ambiguous_indices) { + results[original_idx].assign(location_ids[original_idx].size(), EC_ERROR); + out_locations[original_idx].assign(location_ids[original_idx].size(), CacheLocationConstPtr{}); + } + return results; + } + for (size_t i = 0; i < ambiguous_keys.size(); ++i) { + const size_t original_idx = ambiguous_indices[i]; + if (exists_results[i] != EC_OK) { + results[original_idx].assign(location_ids[original_idx].size(), exists_results[i]); + out_locations[original_idx].assign(location_ids[original_idx].size(), CacheLocationConstPtr{}); + continue; + } + if (!cache_key_exists[i]) { + missing_keys.push_back(keys[original_idx]); + missing_indices.push_back(original_idx); + missing_location_ids.push_back(location_ids[original_idx]); + } } } if (missing_keys.empty()) { @@ -624,9 +1154,11 @@ std::vector> MetaStorageBackendManager::GetLocations(Requ LocationsPerKey persistent_locations; std::vector> persistent_results = persistent_backend_->GetLocations(request_context, missing_keys, missing_location_ids, persistent_locations); - if (missing_keys.size() != persistent_results.size()) { - KVCM_LOG_ERROR( - "persistent results size[%lu] mismatch keys's[%lu]", persistent_results.size(), missing_keys.size()); + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_locations.size()) { + KVCM_LOG_ERROR("persistent targeted GetLocations results[%lu] locations[%lu] mismatch keys[%lu]", + persistent_results.size(), + persistent_locations.size(), + missing_keys.size()); for (size_t i = 0; i < missing_keys.size(); ++i) { results[missing_indices[i]].assign(location_ids[missing_indices[i]].size(), EC_ERROR); } @@ -634,12 +1166,205 @@ std::vector> MetaStorageBackendManager::GetLocations(Requ } for (size_t i = 0; i < missing_keys.size(); ++i) { const size_t original_idx = missing_indices[i]; + if (persistent_results[i].size() != missing_location_ids[i].size() || + persistent_locations[i].size() != missing_location_ids[i].size()) { + KVCM_LOG_ERROR("persistent targeted GetLocations key[%ld] results[%lu] locations[%lu] mismatch ids[%lu]", + missing_keys[i], + persistent_results[i].size(), + persistent_locations[i].size(), + missing_location_ids[i].size()); + results[original_idx].assign(location_ids[original_idx].size(), EC_ERROR); + out_locations[original_idx].assign(location_ids[original_idx].size(), CacheLocationConstPtr{}); + continue; + } results[original_idx] = std::move(persistent_results[i]); out_locations[original_idx] = std::move(persistent_locations[i]); } return results; } +std::vector> +MetaStorageBackendManager::GetLocationsWithKeyStatus(RequestContext *request_context, + const KeyVector &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept { + if (keys.size() != location_ids.size()) { + out_locations.assign(keys.size(), CacheLocationVector{}); + out_key_error_codes.assign(keys.size(), EC_BADARGS); + return std::vector>(keys.size(), std::vector{EC_BADARGS}); + } + if (!cache_backend_) { + return persistent_backend_->GetLocationsWithKeyStatus( + request_context, keys, location_ids, out_locations, out_key_error_codes); + } + + std::vector> results = cache_backend_->GetLocationsWithKeyStatus( + request_context, keys, location_ids, out_locations, out_key_error_codes); + auto response_shape_valid = [&keys, &location_ids](const std::vector> &per_location_ecs, + const LocationsPerKey &locations, + const std::vector &per_key_ecs) { + if (per_location_ecs.size() != keys.size() || locations.size() != keys.size() || + per_key_ecs.size() != keys.size()) { + return false; + } + for (size_t i = 0; i < keys.size(); ++i) { + if (per_location_ecs[i].size() != location_ids[i].size() || locations[i].size() != location_ids[i].size()) { + return false; + } + } + return true; + }; + if (!response_shape_valid(results, out_locations, out_key_error_codes)) { + KVCM_LOG_ERROR("cache targeted GetLocationsWithKeyStatus response shape mismatch keys[%lu]", keys.size()); + out_locations.resize(keys.size()); + results.resize(keys.size()); + out_key_error_codes.assign(keys.size(), EC_ERROR); + for (size_t i = 0; i < keys.size(); ++i) { + out_locations[i].assign(location_ids[i].size(), CacheLocationConstPtr{}); + results[i].assign(location_ids[i].size(), EC_ERROR); + } + return results; + } + if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { + return results; + } + + KeyVector missing_keys; + std::vector missing_indices; + LocationIdsPerKey missing_location_ids; + for (size_t i = 0; i < keys.size(); ++i) { + if (out_key_error_codes[i] == EC_NOENT) { + missing_keys.push_back(keys[i]); + missing_indices.push_back(i); + missing_location_ids.push_back(location_ids[i]); + } + } + if (missing_keys.empty()) { + return results; + } + + LocationsPerKey persistent_locations; + std::vector persistent_key_error_codes; + std::vector> persistent_results = persistent_backend_->GetLocationsWithKeyStatus( + request_context, missing_keys, missing_location_ids, persistent_locations, persistent_key_error_codes); + const auto persistent_shape_valid = + [&missing_keys, &missing_location_ids](const std::vector> &per_location_ecs, + const LocationsPerKey &locations, + const std::vector &per_key_ecs) { + if (per_location_ecs.size() != missing_keys.size() || locations.size() != missing_keys.size() || + per_key_ecs.size() != missing_keys.size()) { + return false; + } + for (size_t i = 0; i < missing_keys.size(); ++i) { + if (per_location_ecs[i].size() != missing_location_ids[i].size() || + locations[i].size() != missing_location_ids[i].size()) { + return false; + } + } + return true; + }; + if (!persistent_shape_valid(persistent_results, persistent_locations, persistent_key_error_codes)) { + KVCM_LOG_ERROR("persistent targeted GetLocationsWithKeyStatus response shape mismatch keys[%lu]", + missing_keys.size()); + for (const size_t original_index : missing_indices) { + results[original_index].assign(location_ids[original_index].size(), EC_ERROR); + out_locations[original_index].assign(location_ids[original_index].size(), CacheLocationConstPtr{}); + out_key_error_codes[original_index] = EC_ERROR; + } + return results; + } + for (size_t i = 0; i < missing_keys.size(); ++i) { + const size_t original_index = missing_indices[i]; + results[original_index] = std::move(persistent_results[i]); + out_locations[original_index] = std::move(persistent_locations[i]); + out_key_error_codes[original_index] = persistent_key_error_codes[i]; + } + return results; +} + +std::vector +MetaStorageBackendManager::GetSingleLocationsWithKeyStatus(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes) noexcept { + if (!SupportsSingleLocationRmw()) { + out_locations.assign(keys.size(), CacheLocationConstPtr{}); + out_key_error_codes.assign(keys.size(), EC_UNIMPLEMENTED); + return std::vector(keys.size(), EC_UNIMPLEMENTED); + } + return persistent_backend_->GetSingleLocationsWithKeyStatus( + request_context, keys, location_ids, out_locations, out_key_error_codes); +} + +void MetaStorageBackendManager::GetSingleLocationsWithKeyStatusInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch, + bool retain_handles) noexcept { + if (!SupportsSingleLocationRmw()) { + out_locations.assign(keys.size(), CacheLocationConstPtr{}); + out_key_error_codes.assign(keys.size(), EC_UNIMPLEMENTED); + out_results.assign(keys.size(), EC_UNIMPLEMENTED); + scratch.ReleaseRetainedHandles(); + return; + } + static_cast(persistent_backend_.get()) + ->GetSingleLocationsWithKeyStatusInto(request_context, + keys, + location_ids, + out_locations, + out_key_error_codes, + out_results, + scratch, + retain_handles); +} + +void MetaStorageBackendManager::GetSingleLocationViewsWithKeyStatusInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationViewVector &out_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept { + if (!SupportsSingleLocationRmw()) { + out_locations.assign(keys.size(), nullptr); + out_key_error_codes.assign(keys.size(), EC_UNIMPLEMENTED); + out_results.assign(keys.size(), EC_UNIMPLEMENTED); + scratch.ReleaseRetainedHandles(); + return; + } + static_cast(persistent_backend_.get()) + ->GetSingleLocationViewsWithKeyStatusInto( + request_context, keys, location_ids, out_locations, out_key_error_codes, out_results, scratch); +} + +bool MetaStorageBackendManager::SupportsConcurrentLocationValueReads() const noexcept { + return !cache_backend_ && persistent_backend_ && + persistent_backend_->GetStorageType() == META_LOCAL_BACKEND_TYPE_STR; +} + +bool MetaStorageBackendManager::SupportsSingleLocationRmw() const noexcept { + // The allocation-light operations bypass the older generic virtual + // methods. Restrict the fast path to the concrete production backend so a + // decorator/subclass that overrides those generic methods for additional + // semantics (fault injection, auditing, admission, etc.) is not bypassed. + // Such backends retain correctness through the generic targeted RMW. + return SupportsConcurrentLocationValueReads() && typeid(*persistent_backend_) == typeid(MetaLocalBackend); +} + +bool MetaStorageBackendManager::GetPureLocalCacheHashSeed(uint32_t &out_hash_seed) const noexcept { + if (cache_backend_ || !persistent_backend_) { + return false; + } + const auto *local_backend = dynamic_cast(persistent_backend_.get()); + return local_backend != nullptr && local_backend->GetCacheHashSeed(out_hash_seed); +} + std::vector MetaStorageBackendManager::GetLocationIds(RequestContext *request_context, const KeyVector &keys, LocationIdsPerKey &out_location_ids) noexcept { @@ -648,6 +1373,14 @@ std::vector MetaStorageBackendManager::GetLocationIds(RequestContext } std::vector results = cache_backend_->GetLocationIds(request_context, keys, out_location_ids); + if (results.size() != keys.size() || out_location_ids.size() != keys.size()) { + KVCM_LOG_ERROR("cache location ids results[%lu] ids[%lu] mismatch keys[%lu]", + results.size(), + out_location_ids.size(), + keys.size()); + results.assign(keys.size(), EC_ERROR); + out_location_ids.assign(keys.size(), LocationIdVector{}); + } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { return results; } @@ -660,8 +1393,9 @@ std::vector MetaStorageBackendManager::GetLocationIds(RequestContext LocationIdsPerKey persistent_location_ids; std::vector persistent_results = persistent_backend_->GetLocationIds(request_context, missing_keys, persistent_location_ids); - if (missing_keys.size() != persistent_location_ids.size()) { - KVCM_LOG_ERROR("persistent_location_ids size[%lu] mismatch keys's[%lu]", + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_location_ids.size()) { + KVCM_LOG_ERROR("persistent location ids results[%lu] ids[%lu] mismatch keys[%lu]", + persistent_results.size(), persistent_location_ids.size(), missing_keys.size()); for (size_t i = 0; i < missing_keys.size(); ++i) { @@ -688,6 +1422,14 @@ std::vector MetaStorageBackendManager::GetProperties(RequestContext * } std::vector results = cache_backend_->GetProperties(request_context, keys, field_names, out_properties); + if (results.size() != keys.size() || out_properties.size() != keys.size()) { + KVCM_LOG_ERROR("cache GetProperties results[%lu] properties[%lu] mismatch keys[%lu]", + results.size(), + out_properties.size(), + keys.size()); + results.assign(keys.size(), EC_ERROR); + out_properties.assign(keys.size(), PropertyMap{}); + } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { return results; } @@ -700,11 +1442,15 @@ std::vector MetaStorageBackendManager::GetProperties(RequestContext * PropertyMapVector persistent_properties; std::vector persistent_results = persistent_backend_->GetProperties(request_context, missing_keys, field_names, persistent_properties); - if (missing_keys.size() != persistent_properties.size()) { - KVCM_LOG_ERROR( - "persistent_properties size[%lu] mismatch keys's[%lu]", persistent_properties.size(), missing_keys.size()); + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_properties.size()) { + KVCM_LOG_ERROR("persistent GetProperties results[%lu] properties[%lu] mismatch keys[%lu]", + persistent_results.size(), + persistent_properties.size(), + missing_keys.size()); for (size_t i = 0; i < missing_keys.size(); ++i) { - results[missing_indices[i]] = EC_ERROR; + const size_t original_idx = missing_indices[i]; + results[original_idx] = EC_ERROR; + out_properties[original_idx].clear(); } return results; } @@ -725,6 +1471,14 @@ std::vector MetaStorageBackendManager::Exists(RequestContext *request return persistent_backend_->Exists(request_context, keys, out_is_exist_vec); } std::vector results = cache_backend_->Exists(request_context, keys, out_is_exist_vec); + if (results.size() != keys.size() || out_is_exist_vec.size() != keys.size()) { + KVCM_LOG_ERROR("cache Exists results[%lu] values[%lu] mismatch keys[%lu]", + results.size(), + out_is_exist_vec.size(), + keys.size()); + results.assign(keys.size(), EC_ERROR); + out_is_exist_vec.assign(keys.size(), false); + } if (recover_state_.load(std::memory_order_acquire) == RecoverState::kRunning) { return results; } @@ -744,11 +1498,15 @@ std::vector MetaStorageBackendManager::Exists(RequestContext *request std::vector persistent_exists; std::vector persistent_results = persistent_backend_->Exists(request_context, missing_keys, persistent_exists); - if (missing_keys.size() != persistent_exists.size()) { - KVCM_LOG_ERROR( - "persistent_exists size[%lu] mismatch missing_keys's[%lu]", persistent_exists.size(), missing_keys.size()); + if (missing_keys.size() != persistent_results.size() || missing_keys.size() != persistent_exists.size()) { + KVCM_LOG_ERROR("persistent Exists results[%lu] values[%lu] mismatch keys[%lu]", + persistent_results.size(), + persistent_exists.size(), + missing_keys.size()); for (size_t i = 0; i < missing_keys.size(); ++i) { - results[missing_indices[i]] = EC_ERROR; + const size_t original_idx = missing_indices[i]; + results[original_idx] = EC_ERROR; + out_is_exist_vec[original_idx] = false; } return results; } diff --git a/kv_cache_manager/meta/meta_storage_backend_manager.h b/kv_cache_manager/meta/meta_storage_backend_manager.h index 49bbbc0e4..54f49860c 100644 --- a/kv_cache_manager/meta/meta_storage_backend_manager.h +++ b/kv_cache_manager/meta/meta_storage_backend_manager.h @@ -17,6 +17,7 @@ namespace kv_cache_manager { class MetaStorageBackendConfig; class RequestContext; +struct SingleLocationRmwScratch; // Backend orchestrator with two modes (auto-selected at Init): // * Dual-backend: persistent (source-of-truth) + cache (hot cache). @@ -47,6 +48,24 @@ class MetaStorageBackendManager { // Put / Upsert merge CacheLocations into batch.batch_properties in place. std::vector Put(RequestContext *request_context, BatchMetaData &batch) noexcept; std::vector Upsert(RequestContext *request_context, BatchMetaData &batch) noexcept; + std::vector UpsertSingleLocations(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations) noexcept; + void PrepareSingleLocationRmwScratch(size_t max_count, SingleLocationRmwScratch &scratch) noexcept; + void UpsertSingleLocationsInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + const CacheLocationVector &locations, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept; + void UpsertSingleLocationsUsingRetainedHandlesInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &locations, + const std::vector &read_indices, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept; std::vector Delete(RequestContext *request_context, const KeyVector &keys) noexcept; std::vector Delete(RequestContext *request_context, const KeyVector &keys, @@ -61,15 +80,50 @@ class MetaStorageBackendManager { std::vector GetLocations(RequestContext *request_context, const KeyVector &keys, CacheLocationMapVector &out_location_maps) noexcept; + std::vector + GetLocationValues(RequestContext *request_context, const KeyVector &keys, LocationsPerKey &out_locations) noexcept; + std::vector GetLocationValuesCompact(RequestContext *request_context, + const KeyType *keys, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept; // Read the source-of-truth backend directly without touching the hot cache. // Maintenance admission uses this to revalidate a persistent scan result. std::vector GetLocationsFromPersistent(RequestContext *request_context, const KeyVector &keys, CacheLocationMapVector &out_location_maps) noexcept; + // Refresh complete keys from persistent storage into the hot cache before + // a maintenance RMW. The caller must hold the corresponding shard locks. + // In single-backend mode this is a no-op. + std::vector RefreshCacheFromPersistent(RequestContext *request_context, const KeyVector &keys) noexcept; std::vector> GetLocations(RequestContext *request_context, const KeyVector &keys, const LocationIdsPerKey &location_ids, LocationsPerKey &out_locations) noexcept; + std::vector> GetLocationsWithKeyStatus(RequestContext *request_context, + const KeyVector &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept; + std::vector GetSingleLocationsWithKeyStatus(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes) noexcept; + void GetSingleLocationsWithKeyStatusInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationVector &out_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch, + bool retain_handles = false) noexcept; + void GetSingleLocationViewsWithKeyStatusInto(RequestContext *request_context, + const KeyVector &keys, + const LocationIdRefVector &location_ids, + CacheLocationViewVector &out_locations, + std::vector &out_key_error_codes, + std::vector &out_results, + SingleLocationRmwScratch &scratch) noexcept; std::vector GetLocationIds(RequestContext *request_context, const KeyVector &keys, LocationIdsPerKey &out_location_ids) noexcept; @@ -80,11 +134,6 @@ class MetaStorageBackendManager { std::vector Exists(RequestContext *request_context, const KeyVector &keys, std::vector &out_is_exist_vec) noexcept; - // Refresh complete keys from persistent storage into the hot cache before - // a maintenance RMW. The caller must hold the corresponding shard locks. - // In single-backend mode this is a no-op. - std::vector RefreshCacheFromPersistent(RequestContext *request_context, const KeyVector &keys) noexcept; - // ----- Cross-batch APIs (no shard locks) ----- ErrorCode ListKeys(RequestContext *request_context, const std::string &cursor, @@ -114,14 +163,26 @@ class MetaStorageBackendManager { // Set revisit interval histogram for cache backend (optional, for metrics tracking). void SetRevisitHistogram(std::shared_ptr histogram); + // Only the single local backend is safe and useful to fan out: its cache + // and items are independently sharded/locked and it ignores RequestContext. + // Redis and cached modes retain their existing batched request semantics. + bool SupportsConcurrentLocationValueReads() const noexcept; + bool SupportsSingleLocationRmw() const noexcept; + bool GetPureLocalCacheHashSeed(uint32_t &out_hash_seed) const noexcept; + private: void AsyncRecoverTask() noexcept; int64_t BackfillKeysToCache(const KeyTypeVec &keys, const CacheLocationMapVector &locations, const PropertyMapVector &properties, - const std::vector &get_error_codes) noexcept; - // Hydrate missing keys from persistent into cache during Recover. - void EnsureKeyInCache(RequestContext *request_context, const KeyTypeVec &keys) noexcept; + const std::vector &get_error_codes, + // Reports whether every source entry and + // conditional cache write completed safely. + bool *out_success = nullptr) noexcept; + // Hydrate missing keys from persistent into cache during Recover. Returns + // false when a backend violates the positional response contract or the + // full pre-update value cannot be made available safely. + bool EnsureKeyInCache(RequestContext *request_context, const KeyTypeVec &keys) noexcept; // Delete keys that have no remaining location fields. Returns reclaimed count. int32_t MaybeReclaimEmptyKeys(RequestContext *request_context, const KeyVector &keys, @@ -134,6 +195,11 @@ class MetaStorageBackendManager { std::atomic recover_state_{RecoverState::kRecover}; std::atomic is_closed_{false}; std::thread recover_thread_; + // Serializes lifecycle transitions and prevents assigning a second + // recovery thread over an already-joinable std::thread (which would call + // std::terminate even though Open() is noexcept). + mutable std::mutex lifecycle_mutex_; + bool opened_ = false; mutable std::mutex deleted_keys_mutex_; std::unordered_set deleted_keys_; diff --git a/kv_cache_manager/meta/query_executor.cc b/kv_cache_manager/meta/query_executor.cc new file mode 100644 index 000000000..af8a0318f --- /dev/null +++ b/kv_cache_manager/meta/query_executor.cc @@ -0,0 +1,221 @@ +#include "kv_cache_manager/meta/query_executor.h" + +#include +#include +#include +#include + +#include "kv_cache_manager/common/logger.h" + +namespace kv_cache_manager { + +namespace { + +thread_local const QueryExecutor *current_query_executor = nullptr; + +struct ParallelState { + std::atomic next{0}; + std::atomic failed{false}; + std::mutex mutex; + std::condition_variable condition; + bool accepting_workers = true; + std::size_t active_workers = 0; + + bool TryStartWorker() { + std::lock_guard lock(mutex); + if (!accepting_workers) { + return false; + } + ++active_workers; + return true; + } + + void CompleteWorker() { + std::lock_guard lock(mutex); + if (--active_workers == 0) { + condition.notify_all(); + } + } + + void StopWorkersAndWait() { + std::unique_lock lock(mutex); + accepting_workers = false; + condition.wait(lock, [this] { return active_workers == 0; }); + } +}; + +} // namespace + +QueryExecutor::QueryExecutor(std::size_t worker_count, + std::size_t parallel_threshold, + std::size_t chunk_size, + std::size_t queue_capacity) + : worker_count_(std::max(1, worker_count)) + , parallel_threshold_(std::max(1, parallel_threshold)) + , chunk_size_(std::max(1, chunk_size)) + , queue_capacity_(std::max(1, queue_capacity)) { + workers_.reserve(worker_count_ - 1); + try { + for (std::size_t i = 1; i < worker_count_; ++i) { + workers_.emplace_back([this] { WorkerLoop(); }); + } + } catch (...) { + // A partially constructed vector of joinable std::threads would call + // std::terminate during stack unwinding. Stop and join every worker + // that was created before propagating the construction failure. + { + std::lock_guard lock(mutex_); + stopping_ = true; + } + condition_.notify_all(); + for (auto &worker : workers_) { + if (worker.joinable()) { + worker.join(); + } + } + throw; + } +} + +QueryExecutor::~QueryExecutor() { + { + std::lock_guard lock(mutex_); + stopping_ = true; + } + condition_.notify_all(); + for (auto &worker : workers_) { + if (worker.joinable()) { + worker.join(); + } + } +} + +bool QueryExecutor::TrySubmit(std::function task) const { + { + std::lock_guard lock(mutex_); + if (stopping_ || tasks_.size() >= queue_capacity_) { + return false; + } + tasks_.push_back(std::move(task)); + } + condition_.notify_one(); + return true; +} + +void QueryExecutor::WorkerLoop() { + current_query_executor = this; + while (true) { + std::function task; + { + std::unique_lock lock(mutex_); + condition_.wait(lock, [this] { return stopping_ || !tasks_.empty(); }); + if (stopping_ && tasks_.empty()) { + break; + } + task = std::move(tasks_.front()); + tasks_.pop_front(); + } + task(); + } + current_query_executor = nullptr; +} + +bool QueryExecutor::ParallelFor(std::size_t count, const RangeFunction &fn) const noexcept { + return ParallelForImpl(count, chunk_size_, fn); +} + +bool QueryExecutor::ParallelForWithChunkSize(std::size_t count, + std::size_t chunk_size, + const RangeFunction &fn) const noexcept { + return ParallelForImpl(count, std::max(1, chunk_size), fn); +} + +bool QueryExecutor::ParallelForImpl(std::size_t count, std::size_t chunk_size, const RangeFunction &fn) const noexcept { + if (count == 0) { + return true; + } + if (worker_count_ <= 1 || count < parallel_threshold_ || current_query_executor == this) { + try { + fn(0, count); + return true; + } catch (const std::exception &e) { + KVCM_LOG_ERROR("query executor serial callback threw exception: %s", e.what()); + } catch (...) { KVCM_LOG_ERROR("query executor serial callback threw unknown exception"); } + return false; + } + + const std::size_t chunk_count = 1 + (count - 1) / chunk_size; + const std::size_t parallelism = std::min(worker_count_, chunk_count); + if (parallelism <= 1) { + try { + fn(0, count); + return true; + } catch (const std::exception &e) { + KVCM_LOG_ERROR("query executor callback threw exception: %s", e.what()); + } catch (...) { KVCM_LOG_ERROR("query executor callback threw unknown exception"); } + return false; + } + + std::shared_ptr state; + try { + state = std::make_shared(); + auto fn_holder = std::make_shared(fn); + auto run_ranges = [state, fn_holder, count, chunk_size]() noexcept { + while (true) { + const std::size_t begin = state->next.fetch_add(chunk_size, std::memory_order_relaxed); + if (begin >= count) { + return; + } + const std::size_t end = std::min(count, begin + chunk_size); + try { + (*fn_holder)(begin, end); + } catch (const std::exception &e) { + state->failed.store(true, std::memory_order_relaxed); + KVCM_LOG_ERROR("query executor parallel callback threw exception: %s", e.what()); + } catch (...) { + state->failed.store(true, std::memory_order_relaxed); + KVCM_LOG_ERROR("query executor parallel callback threw unknown exception"); + } + } + }; + + for (std::size_t i = 1; i < parallelism; ++i) { + if (!TrySubmit([state, run_ranges] { + // The caller may have consumed every range while this task was + // waiting behind another request. In that case it cancels the + // queued helper and returns without waiting for a no-op task to + // reach the head of the global queue. + if (!state->TryStartWorker()) { + return; + } + run_ranges(); + state->CompleteWorker(); + })) { + // The caller and any admitted workers consume every range via the + // shared atomic cursor when the bounded queue is full. + break; + } + } + + run_ranges(); + state->StopWorkersAndWait(); + return !state->failed.load(std::memory_order_relaxed); + } catch (const std::exception &e) { + // ParallelFor is noexcept. Allocation or queue growth failure must be + // reported as a failed query instead of terminating the server. Tasks + // admitted before the exception may capture request-local references, + // so drain active helpers before returning. + if (state) { + state->StopWorkersAndWait(); + } + KVCM_LOG_ERROR("query executor failed to schedule parallel query: %s", e.what()); + } catch (...) { + if (state) { + state->StopWorkersAndWait(); + } + KVCM_LOG_ERROR("query executor failed to schedule parallel query with unknown exception"); + } + return false; +} + +} // namespace kv_cache_manager diff --git a/kv_cache_manager/meta/query_executor.h b/kv_cache_manager/meta/query_executor.h new file mode 100644 index 000000000..120938e31 --- /dev/null +++ b/kv_cache_manager/meta/query_executor.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace kv_cache_manager { + +// Process-local, bounded executor for latency-sensitive metadata queries. +// +// The calling RPC thread always participates in ParallelFor. worker_count is +// therefore the maximum parallelism of one query, not the number of background +// threads: an executor configured with N workers owns N - 1 threads. A bounded +// queue prevents concurrent large requests from creating unbounded work; when +// admission fails, the caller and any already-admitted workers finish the +// remaining chunks themselves. +class QueryExecutor { +public: + using RangeFunction = std::function; + + QueryExecutor(std::size_t worker_count, + std::size_t parallel_threshold, + std::size_t chunk_size, + std::size_t queue_capacity); + ~QueryExecutor(); + + QueryExecutor(const QueryExecutor &) = delete; + QueryExecutor &operator=(const QueryExecutor &) = delete; + + // Runs fn over disjoint half-open ranges that cover [0, count). Returns + // false if a callback throws or the parallel work cannot be allocated or + // scheduled. Calls made recursively from this executor's own worker thread + // deliberately fall back to serial execution so a task can never wait for + // the pool that is currently running it. + bool ParallelFor(std::size_t count, const RangeFunction &fn) const noexcept; + // Same bounded executor with a per-call range size. Large local metadata + // scans use this to amortize shard locking without changing the global + // projection chunk configured for other query work. + bool ParallelForWithChunkSize(std::size_t count, std::size_t chunk_size, const RangeFunction &fn) const noexcept; + + [[nodiscard]] std::size_t worker_count() const noexcept { return worker_count_; } + [[nodiscard]] std::size_t parallel_threshold() const noexcept { return parallel_threshold_; } + [[nodiscard]] std::size_t chunk_size() const noexcept { return chunk_size_; } + +private: + bool ParallelForImpl(std::size_t count, std::size_t chunk_size, const RangeFunction &fn) const noexcept; + bool TrySubmit(std::function task) const; + void WorkerLoop(); + +private: + std::size_t worker_count_ = 1; + std::size_t parallel_threshold_ = 1; + std::size_t chunk_size_ = 1; + std::size_t queue_capacity_ = 1; + + mutable std::mutex mutex_; + mutable std::condition_variable condition_; + mutable std::deque> tasks_; + mutable bool stopping_ = false; + std::vector workers_; +}; + +} // namespace kv_cache_manager diff --git a/kv_cache_manager/meta/test/BUILD b/kv_cache_manager/meta/test/BUILD index 24ce66d4c..2f37de328 100644 --- a/kv_cache_manager/meta/test/BUILD +++ b/kv_cache_manager/meta/test/BUILD @@ -1,5 +1,14 @@ package(default_visibility = ["//visibility:public"]) +cc_test( + name = "query_executor_test", + srcs = ["query_executor_test.cc"], + deps = [ + "//kv_cache_manager/common:unittest", + "//kv_cache_manager/meta:query_executor", + ], +) + cc_library( name = "meta_indexer_testlib", srcs = [ @@ -23,8 +32,10 @@ cc_test( copts = ["-fno-access-control"], data = [], deps = [ - "//kv_cache_manager/common:unittest", ":meta_indexer_testlib", + "//kv_cache_manager/common:unittest", + "//kv_cache_manager/metrics:metrics_collector", + "//kv_cache_manager/metrics:metrics_registry", ], ) @@ -66,8 +77,8 @@ cc_library( copts = ["-fno-access-control"], deps = [ "//kv_cache_manager/meta:meta_cache_base_backend", - "//kv_cache_manager/meta:meta_storage_backend", "//kv_cache_manager/meta:meta_redis_backend", + "//kv_cache_manager/meta:meta_storage_backend", "@com_google_googletest//:gtest", ], ) @@ -122,14 +133,14 @@ cc_test( ], copts = ["-fno-access-control"], data = [], - deps = [ - "//kv_cache_manager/common:unittest", - "//kv_cache_manager/meta:meta_redis_backend", - ], tags = [ "manual", "redis", ], + deps = [ + "//kv_cache_manager/common:unittest", + "//kv_cache_manager/meta:meta_redis_backend", + ], ) cc_test( @@ -139,14 +150,14 @@ cc_test( ], copts = ["-fno-access-control"], data = [], - deps = [ - "//kv_cache_manager/common:unittest", - ":meta_indexer_testlib", - ], tags = [ "manual", "redis", ], + deps = [ + ":meta_indexer_testlib", + "//kv_cache_manager/common:unittest", + ], ) cc_test( @@ -204,6 +215,10 @@ cc_test( ], copts = ["-fno-access-control"], data = [], + tags = [ + "manual", + "redis", + ], deps = [ "//kv_cache_manager/common:unittest", "//kv_cache_manager/meta:cache_location", @@ -212,8 +227,4 @@ cc_test( "//kv_cache_manager/meta:meta_storage_backend_manager", "//kv_cache_manager/meta:types", ], - tags = [ - "manual", - "redis", - ], ) diff --git a/kv_cache_manager/meta/test/meta_dummy_backend_test.cc b/kv_cache_manager/meta/test/meta_dummy_backend_test.cc index 9fa369896..96d22fd1f 100644 --- a/kv_cache_manager/meta/test/meta_dummy_backend_test.cc +++ b/kv_cache_manager/meta/test/meta_dummy_backend_test.cc @@ -103,6 +103,35 @@ TEST_F(MetaDummyBackendTest, TestSimple) { ASSERT_EQ(ErrorCode::EC_OK, meta_storage_backend_->Close()); } +TEST_F(MetaDummyBackendTest, TestGenericTargetedLocationsPreserveKeyStatus) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("targeted_status", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + auto location = std::make_shared(); + location->set_id("target"); + CacheLocationMapVector locations(2); + locations[0].emplace("target", location); + PropertyMapVector properties(2); + properties[1].emplace("property_only", "value"); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->Put(nullptr, {1, 2}, locations, properties)); + + LocationsPerKey selected; + std::vector key_error_codes; + const auto per_location_ecs = meta_storage_backend_->GetLocationsWithKeyStatus( + nullptr, {1, 2, 3}, {{"target"}, {"target"}, {"target"}}, selected, key_error_codes); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOENT}), key_error_codes); + EXPECT_EQ((std::vector>{{EC_OK}, {EC_NOENT}, {EC_NOENT}}), per_location_ecs); + ASSERT_EQ(3u, selected.size()); + ASSERT_EQ(1u, selected[0].size()); + ASSERT_TRUE(selected[0][0]); + EXPECT_EQ("target", selected[0][0]->id()); + ASSERT_EQ(1u, selected[1].size()); + EXPECT_FALSE(selected[1][0]); + ASSERT_EQ(1u, selected[2].size()); + EXPECT_FALSE(selected[2][0]); +} + TEST_F(MetaDummyBackendTest, TestInit) { // invalid config ASSERT_NE(ErrorCode::EC_OK, meta_storage_backend_->Init("test_instance_0", /*config*/ nullptr)); diff --git a/kv_cache_manager/meta/test/meta_indexer_manager_test.cc b/kv_cache_manager/meta/test/meta_indexer_manager_test.cc index 2783ae89d..7b4c2786f 100644 --- a/kv_cache_manager/meta/test/meta_indexer_manager_test.cc +++ b/kv_cache_manager/meta/test/meta_indexer_manager_test.cc @@ -8,6 +8,7 @@ #include "kv_cache_manager/meta/meta_indexer_manager.h" #include "kv_cache_manager/meta/meta_local_backend.h" #include "kv_cache_manager/meta/meta_storage_backend.h" +#include "kv_cache_manager/meta/query_executor.h" #include "kv_cache_manager/metrics/metrics_registry.h" namespace kv_cache_manager { @@ -105,6 +106,26 @@ TEST_F(MetaIndexerManagerTest, TestCreateFailed) { ASSERT_EQ(0, manager_->GetIndexerSize()); } +TEST_F(MetaIndexerManagerTest, TestConfigureQueryExecutorBeforeCreatingIndexers) { + EXPECT_FALSE(manager_->ConfigureQueryExecutor(0, 256, 128)); + EXPECT_FALSE(manager_->ConfigureQueryExecutor(65, 256, 128)); + EXPECT_FALSE(manager_->ConfigureQueryExecutor(4, 0, 128)); + EXPECT_FALSE(manager_->ConfigureQueryExecutor(4, 256, 0)); + EXPECT_FALSE(manager_->ConfigureQueryExecutor(4, 128, 256)); + + ASSERT_TRUE(manager_->ConfigureQueryExecutor(4, 256, 128)); + ASSERT_TRUE(manager_->query_executor_); + EXPECT_EQ(4u, manager_->query_executor_->worker_count()); + EXPECT_EQ(256u, manager_->query_executor_->parallel_threshold()); + EXPECT_EQ(128u, manager_->query_executor_->chunk_size()); + + ASSERT_EQ(EC_OK, CreateMetaIndexer("configured", META_LOCAL_BACKEND_TYPE_STR)); + const auto indexer = manager_->GetMetaIndexer("configured"); + ASSERT_TRUE(indexer); + EXPECT_EQ(manager_->query_executor_, indexer->query_executor_); + EXPECT_FALSE(manager_->ConfigureQueryExecutor(2, 64, 32)); +} + TEST_F(MetaIndexerManagerTest, TestDoCleanup) { // 创建多个 indexer std::string id_1 = "1"; diff --git a/kv_cache_manager/meta/test/meta_indexer_test.cc b/kv_cache_manager/meta/test/meta_indexer_test.cc index 00ff288a2..1ddb6ab05 100644 --- a/kv_cache_manager/meta/test/meta_indexer_test.cc +++ b/kv_cache_manager/meta/test/meta_indexer_test.cc @@ -1,10 +1,13 @@ #include +#include #include #include +#include #include #include #include #include +#include #include #include "kv_cache_manager/common/request_context.h" @@ -13,12 +16,16 @@ #include "kv_cache_manager/meta//test/meta_indexer_test_base.h" #include "kv_cache_manager/meta/common.h" #include "kv_cache_manager/meta/meta_indexer.h" +#include "kv_cache_manager/meta/meta_local_backend.h" #include "kv_cache_manager/meta/meta_search_cache.h" #include "kv_cache_manager/meta/meta_storage_backend.h" #include "kv_cache_manager/meta/meta_storage_backend_manager.h" +#include "kv_cache_manager/meta/query_executor.h" #include "kv_cache_manager/meta/storage_usage_data.h" #include "kv_cache_manager/meta/types.h" #include "kv_cache_manager/meta/utils.h" +#include "kv_cache_manager/metrics/metrics_collector.h" +#include "kv_cache_manager/metrics/metrics_registry.h" using namespace kv_cache_manager; @@ -29,6 +36,74 @@ namespace { std::string GetPersistentStorageType(const MetaIndexer &indexer) { return indexer.backend_manager_->persistent_backend_->GetStorageType(); } + +class MalformedLocationReadBackend : public MetaLocalBackend { +public: + enum class Shape { + kShortOuter, + kShortInner, + kNullValueWithOk, + }; + + void SetShape(Shape shape) { shape_ = shape; } + + std::vector GetLocations(RequestContext * /*request_context*/, + const KeyTypeVec & /*keys*/, + CacheLocationMapVector &out_locations) noexcept override { + out_locations.clear(); + return {EC_OK}; + } + + std::vector> GetLocations(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + const LocationIdsPerKey &location_ids, + LocationsPerKey &out_locations) noexcept override { + if (shape_ == Shape::kShortOuter) { + out_locations.clear(); + return {}; + } + if (shape_ == Shape::kShortInner) { + out_locations.assign(keys.size(), CacheLocationVector{}); + return std::vector>(keys.size()); + } + out_locations.resize(keys.size()); + std::vector> result(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) { + out_locations[i].assign(location_ids[i].size(), CacheLocationConstPtr{}); + result[i].assign(location_ids[i].size(), EC_OK); + } + return result; + } + + std::vector GetLocationValues(RequestContext * /*request_context*/, + const KeyTypeVec &keys, + LocationsPerKey &out_locations) noexcept override { + if (shape_ == Shape::kShortOuter) { + out_locations.clear(); + return {EC_OK}; + } + out_locations.assign(keys.size(), CacheLocationVector{}); + return std::vector(keys.size(), EC_OK); + } + + std::vector GetLocationValuesCompact(RequestContext * /*request_context*/, + const KeyType * /*keys*/, + size_t key_count, + CompactLocationsPerKey &out_locations) noexcept override { + if (shape_ == Shape::kShortOuter) { + out_locations.Clear(); + return {EC_OK}; + } + out_locations.Clear(key_count); + for (size_t i = 0; i < key_count; ++i) { + out_locations.FinishKey(); + } + return std::vector(key_count, EC_OK); + } + +private: + Shape shape_ = Shape::kShortOuter; +}; } // namespace class MetaIndexerTest : public MetaIndexerTestBase, public TESTBASE { @@ -116,6 +191,559 @@ TEST_F(MetaIndexerTest, TestProcessErrorCodesRejectsAbnormalResultCount) { EXPECT_EQ((std::vector{EC_MISMATCH, EC_MISMATCH, EC_MISMATCH}), long_result.error_codes); } +TEST_F(MetaIndexerTest, TestParallelLocalLocationValuesMatchSerialAndPreserveErrors) { + constexpr std::size_t kKeyCount = 1024; + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 32)); + const std::string config_str = R"({ + "max_key_count" : 2048, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + KeyVector query_keys = data.keys; + constexpr std::size_t kMissingIndex = 333; + query_keys[kMissingIndex] = 100000; + LocationsPerKey parallel_values; + const auto parallel_result = meta_indexer_->GetLocationValues(request_context_.get(), query_keys, parallel_values); + ASSERT_EQ(EC_PARTIAL_OK, parallel_result.ec); + ASSERT_EQ(kKeyCount, parallel_result.error_codes.size()); + ASSERT_EQ(kKeyCount, parallel_values.size()); + for (std::size_t i = 0; i < kKeyCount; ++i) { + if (i == kMissingIndex) { + EXPECT_EQ(EC_NOENT, parallel_result.error_codes[i]); + EXPECT_TRUE(parallel_values[i].empty()); + continue; + } + EXPECT_EQ(EC_OK, parallel_result.error_codes[i]) << "index=" << i; + ASSERT_EQ(1u, parallel_values[i].size()) << "index=" << i; + ASSERT_TRUE(parallel_values[i].front()); + EXPECT_EQ("loc_" + std::to_string(query_keys[i]), parallel_values[i].front()->id()); + } + + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 1)); + LocationsPerKey serial_values; + const auto serial_result = meta_indexer_->GetLocationValues(request_context_.get(), query_keys, serial_values); + EXPECT_EQ(parallel_result.ec, serial_result.ec); + EXPECT_EQ(parallel_result.error_codes, serial_result.error_codes); + ASSERT_EQ(parallel_values.size(), serial_values.size()); + for (std::size_t i = 0; i < parallel_values.size(); ++i) { + ASSERT_EQ(parallel_values[i].size(), serial_values[i].size()) << "index=" << i; + for (std::size_t j = 0; j < parallel_values[i].size(); ++j) { + ASSERT_TRUE(serial_values[i][j]); + EXPECT_EQ(parallel_values[i][j]->id(), serial_values[i][j]->id()) << "index=" << i; + } + } +} + +TEST_F(MetaIndexerTest, TestCompactPrefixLocationValuesStopsAtFirstMissingChunk) { + constexpr std::size_t kKeyCount = 16384; + constexpr std::size_t kChunkSize = 32; + // Pure-local scans deliberately use a larger bounded metadata-read window + // than the CPU projection chunk so LRU shard locks are amortized. + constexpr std::size_t kLocalReadWindow = 4096; + constexpr std::size_t kMissingIndex = 97; + // A single worker makes the amount of speculative work deterministic. The + // parallel case uses the same range callback and can read at most a bounded + // number of already-claimed chunks beyond the first miss. + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, kChunkSize, /*queue_capacity*/ 1)); + const std::string config_str = R"({ + "max_key_count" : 32768, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + KeyVector query_keys = data.keys; + query_keys[kMissingIndex] = 100000; + std::vector observed(kKeyCount); + const auto visitor = [&query_keys, + &observed](size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + for (size_t i = 0; i < valid_count; ++i) { + if (locations[i].size() == 1) { + observed[begin + i] = *locations[i].begin(); + } + } + return query_keys.size(); + }; + const auto prefix = meta_indexer_->VisitLocationValuesForPrefix(request_context_.get(), query_keys, visitor); + EXPECT_EQ(EC_NOENT, prefix.terminal_ec); + EXPECT_EQ(kMissingIndex, prefix.valid_key_count); + EXPECT_EQ(kLocalReadWindow, prefix.read_key_count); + EXPECT_FALSE(prefix.stopped_by_visitor); + for (std::size_t i = 0; i < kMissingIndex; ++i) { + ASSERT_TRUE(observed[i]) << "index=" << i; + EXPECT_EQ("loc_" + std::to_string(query_keys[i]), observed[i]->id()) << "index=" << i; + } +} + +TEST_F(MetaIndexerTest, TestCompactPrefixLocationValuesHonorsVisitorStop) { + constexpr std::size_t kKeyCount = 16384; + constexpr std::size_t kChunkSize = 32; + constexpr std::size_t kLocalReadWindow = 4096; + constexpr std::size_t kVisitorStop = 97; + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, kChunkSize, /*queue_capacity*/ 1)); + const std::string config_str = R"({ + "max_key_count" : 32768, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + std::atomic visited_key_count(0); + const auto prefix = meta_indexer_->VisitLocationValuesForPrefix( + request_context_.get(), + data.keys, + [&data, &visited_key_count](size_t begin, const CompactLocationsPerKey &, size_t valid_count) { + visited_key_count.fetch_add(valid_count, std::memory_order_relaxed); + return begin <= kVisitorStop && kVisitorStop < begin + valid_count ? kVisitorStop : data.keys.size(); + }); + EXPECT_EQ(EC_OK, prefix.terminal_ec); + EXPECT_EQ(kVisitorStop, prefix.valid_key_count); + EXPECT_TRUE(prefix.stopped_by_visitor); + EXPECT_EQ(kLocalReadWindow, prefix.read_key_count); + EXPECT_EQ(prefix.read_key_count, visited_key_count.load(std::memory_order_relaxed)); +} + +TEST_F(MetaIndexerTest, TestCompactPrefixLocationValuesReturnsEveryAllHitKey) { + constexpr std::size_t kKeyCount = 1024; + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 32)); + const std::string config_str = R"({ + "max_key_count" : 2048, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + std::vector observed(kKeyCount); + const auto visitor = [&data, &observed](size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + for (size_t i = 0; i < valid_count; ++i) { + if (locations[i].size() == 1) { + observed[begin + i] = *locations[i].begin(); + } + } + return data.keys.size(); + }; + const auto prefix = meta_indexer_->VisitLocationValuesForPrefix(request_context_.get(), data.keys, visitor); + EXPECT_EQ(EC_OK, prefix.terminal_ec); + EXPECT_EQ(kKeyCount, prefix.valid_key_count); + EXPECT_EQ(kKeyCount, prefix.read_key_count); + EXPECT_FALSE(prefix.stopped_by_visitor); + for (std::size_t i = 0; i < observed.size(); ++i) { + ASSERT_TRUE(observed[i]) << "index=" << i; + EXPECT_EQ("loc_" + std::to_string(data.keys[i]), observed[i]->id()) << "index=" << i; + } +} + +TEST_F(MetaIndexerTest, TestCompactPrefixLocationValuesOrderedVisitorIsBoundedAndStopsInOrder) { + constexpr std::size_t kKeyCount = 70000; + constexpr std::size_t kStopIndex = 25000; + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 32)); + const std::string config_str = R"({ + "max_key_count" : 131072, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + size_t visited_until = 0; + const auto all = meta_indexer_->VisitLocationValuesForPrefix( + request_context_.get(), + data.keys, + [&data, &visited_until](size_t begin, const CompactLocationsPerKey &, size_t valid_count) { + EXPECT_EQ(visited_until, begin); + visited_until += valid_count; + return data.keys.size(); + }, + MetaIndexer::PrefixVisitOrder::ORDERED); + EXPECT_EQ(EC_OK, all.terminal_ec); + EXPECT_EQ(kKeyCount, all.valid_key_count); + EXPECT_EQ(kKeyCount, visited_until); + + visited_until = 0; + std::vector callback_begins; + const auto stopped = meta_indexer_->VisitLocationValuesForPrefix( + request_context_.get(), + data.keys, + [&visited_until, &callback_begins, kStopIndex]( + size_t begin, const CompactLocationsPerKey &, size_t valid_count) { + EXPECT_EQ(visited_until, begin); + callback_begins.push_back(begin); + visited_until = std::min(kStopIndex, begin + valid_count); + return kStopIndex; + }, + MetaIndexer::PrefixVisitOrder::ORDERED); + EXPECT_EQ(EC_OK, stopped.terminal_ec); + EXPECT_EQ(kStopIndex, stopped.valid_key_count); + EXPECT_EQ(kStopIndex, visited_until); + EXPECT_EQ((std::vector{0, 4096, 20480}), callback_begins); + EXPECT_EQ(69632u, stopped.read_key_count); + EXPECT_TRUE(stopped.stopped_by_visitor); +} + +TEST_F(MetaIndexerTest, TestCompactPrefixLocationValuesClampsOversizedConfiguredChunk) { + constexpr std::size_t kKeyCount = 5000; + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, + /*parallel_threshold*/ 1, + /*chunk_size*/ std::numeric_limits::max(), + /*queue_capacity*/ 4)); + const std::string config_str = R"({ + "max_key_count" : 8192, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + std::vector chunk_begins; + const auto prefix = meta_indexer_->VisitLocationValuesForPrefix( + request_context_.get(), + data.keys, + [&data, &chunk_begins](size_t begin, const CompactLocationsPerKey &locations, size_t valid_count) { + chunk_begins.push_back(begin); + EXPECT_EQ(valid_count, locations.size()); + return data.keys.size(); + }); + EXPECT_EQ(EC_OK, prefix.terminal_ec); + EXPECT_EQ(kKeyCount, prefix.valid_key_count); + EXPECT_EQ(kKeyCount, prefix.read_key_count); + EXPECT_EQ((std::vector{0, 4096}), chunk_begins); +} + +TEST_F(MetaIndexerTest, TestCompactPrefixGetIoMetricExcludesPipelinedVisitorTime) { + constexpr std::size_t kKeyCount = 5000; + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 1, /*parallel_threshold*/ 64, /*chunk_size*/ 32, /*queue_capacity*/ 1)); + const std::string config_str = R"({ + "max_key_count" : 8192, + "mutex_shard_num" : 64, + "batch_key_size" : 128, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + KVData data; + MakeKVData(0, kKeyCount, data); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), data.keys, data.location_maps, data.properties).ec); + + auto metrics_registry = std::make_shared(); + auto metrics_collector = std::make_shared(metrics_registry); + ASSERT_TRUE(metrics_collector->Init()); + RequestContext metrics_context("prefix_metric_test", metrics_collector); + size_t visitor_calls = 0; + const auto begin = std::chrono::steady_clock::now(); + const auto prefix = meta_indexer_->VisitLocationValuesForPrefix( + &metrics_context, data.keys, [&data, &visitor_calls](size_t, const CompactLocationsPerKey &, size_t) { + ++visitor_calls; + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + return data.keys.size(); + }); + const auto elapsed_us = + std::chrono::duration_cast(std::chrono::steady_clock::now() - begin).count(); + + EXPECT_EQ(EC_OK, prefix.terminal_ec); + EXPECT_EQ(2u, visitor_calls); + const auto backend_wall_us = metrics_collector->get_meta_indexer_get_io_time_us_metrics(); + EXPECT_GT(backend_wall_us, 0); + EXPECT_EQ(static_cast(backend_wall_us), prefix.backend_read_wall_time_us); + EXPECT_GE(elapsed_us - static_cast(backend_wall_us), 30000); +} + +TEST_F(MetaIndexerTest, TestCompactPrefixLocationValuesRejectsMalformedShape) { + meta_indexer_->SetQueryExecutor(std::make_shared( + /*worker_count*/ 4, /*parallel_threshold*/ 2, /*chunk_size*/ 2, /*queue_capacity*/ 4)); + const std::string config_str = R"({ + "max_key_count" : 100, + "mutex_shard_num" : 8, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + auto malformed = std::make_unique(); + auto backend_config = std::make_shared(); + ASSERT_EQ(EC_OK, malformed->Init("test", backend_config)); + ASSERT_EQ(EC_OK, malformed->Open()); + malformed->SetShape(MalformedLocationReadBackend::Shape::kShortOuter); + ASSERT_EQ(EC_OK, meta_indexer_->backend_manager_->persistent_backend_->Close()); + meta_indexer_->backend_manager_->persistent_backend_ = std::move(malformed); + meta_indexer_->backend_manager_->cache_backend_.reset(); + + const KeyVector keys{1, 2, 3, 4}; + std::atomic visitor_calls(0); + const auto prefix = meta_indexer_->VisitLocationValuesForPrefix( + request_context_.get(), keys, [&visitor_calls, &keys](size_t, const CompactLocationsPerKey &, size_t) { + visitor_calls.fetch_add(1, std::memory_order_relaxed); + return keys.size(); + }); + EXPECT_EQ(EC_MISMATCH, prefix.terminal_ec); + EXPECT_EQ(0u, prefix.valid_key_count); + EXPECT_EQ(0u, visitor_calls.load(std::memory_order_relaxed)); +} + +TEST_F(MetaIndexerTest, TestReadModifyWriteLocationRejectsMalformedBackendResultShapes) { + const std::string config_str = R"({ + "max_key_count" : 100, + "mutex_shard_num" : 8, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + auto malformed = std::make_unique(); + auto backend_config = std::make_shared(); + ASSERT_EQ(EC_OK, malformed->Init("test", backend_config)); + ASSERT_EQ(EC_OK, malformed->Open()); + auto *malformed_raw = malformed.get(); + ASSERT_EQ(EC_OK, meta_indexer_->backend_manager_->persistent_backend_->Close()); + meta_indexer_->backend_manager_->persistent_backend_ = std::move(malformed); + meta_indexer_->backend_manager_->cache_backend_.reset(); + + const KeyVector keys{123}; + const LocationIdsPerKey location_ids{{"loc"}}; + size_t modifier_calls = 0; + const auto modifier = + [&modifier_calls]( + const std::vector &, const LocationIdVector &ids, size_t, CacheLocationVector &, PropertyMap &) { + ++modifier_calls; + return LocationModifierResult{ModifierAction::MA_SKIP, std::vector(ids.size(), EC_OK)}; + }; + + auto result = meta_indexer_->ReadModifyWriteLocation(request_context_.get(), keys, location_ids, modifier); + EXPECT_EQ(EC_ERROR, result.ec); + ASSERT_EQ(1u, result.per_location_error_codes.size()); + EXPECT_EQ((std::vector{EC_MISMATCH}), result.per_location_error_codes[0]); + EXPECT_EQ(0u, modifier_calls); + + malformed_raw->SetShape(MalformedLocationReadBackend::Shape::kShortInner); + result = meta_indexer_->ReadModifyWriteLocation(request_context_.get(), keys, location_ids, modifier); + EXPECT_EQ(EC_ERROR, result.ec); + ASSERT_EQ(1u, result.per_location_error_codes.size()); + EXPECT_EQ((std::vector{EC_MISMATCH}), result.per_location_error_codes[0]); + EXPECT_EQ(0u, modifier_calls); + + malformed_raw->SetShape(MalformedLocationReadBackend::Shape::kNullValueWithOk); + result = meta_indexer_->ReadModifyWriteLocation(request_context_.get(), keys, location_ids, modifier); + EXPECT_EQ(EC_ERROR, result.ec); + ASSERT_EQ(1u, result.per_location_error_codes.size()); + EXPECT_EQ((std::vector{EC_MISMATCH}), result.per_location_error_codes[0]); + EXPECT_EQ(1u, modifier_calls); + + LocationsPerKey locations; + auto get_result = meta_indexer_->GetLocations(request_context_.get(), keys, location_ids, locations); + EXPECT_EQ(EC_ERROR, get_result.ec); + ASSERT_EQ(1u, get_result.per_location_error_codes.size()); + EXPECT_EQ((std::vector{EC_MISMATCH}), get_result.per_location_error_codes[0]); + ASSERT_EQ(1u, locations.size()); + ASSERT_EQ(1u, locations[0].size()); + EXPECT_FALSE(locations[0][0]); + + CacheLocationMapVector location_maps; + const auto get_all_result = meta_indexer_->GetLocations(request_context_.get(), keys, location_maps); + EXPECT_EQ(EC_ERROR, get_all_result.ec); + EXPECT_EQ((std::vector{EC_MISMATCH}), get_all_result.error_codes); + ASSERT_EQ(1u, location_maps.size()); + EXPECT_TRUE(location_maps[0].empty()); + + malformed_raw->SetShape(MalformedLocationReadBackend::Shape::kShortOuter); + LocationsPerKey location_values; + const KeyVector two_keys{123, 124}; + const auto get_values_result = meta_indexer_->GetLocationValues(request_context_.get(), two_keys, location_values); + EXPECT_EQ(EC_ERROR, get_values_result.ec); + EXPECT_EQ((std::vector{EC_MISMATCH, EC_MISMATCH}), get_values_result.error_codes); + ASSERT_EQ(2u, location_values.size()); + EXPECT_TRUE(location_values[0].empty()); + EXPECT_TRUE(location_values[1].empty()); +} + +TEST_F(MetaIndexerTest, TestReadModifyWriteLocationPreservesPartialModifierResult) { + const std::string config_str = R"({ + "max_key_count" : 100, + "mutex_shard_num" : 8, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + + const KeyVector keys{124}; + const LocationIdsPerKey location_ids{{"good", "bad"}}; + const auto modifier = [](const std::vector &get_ecs, + const LocationIdVector &ids, + size_t, + CacheLocationVector &locations, + PropertyMap &) { + EXPECT_EQ((std::vector{EC_NOENT, EC_NOENT}), get_ecs); + auto good = std::make_shared(); + good->set_id(ids[0]); + good->set_type(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2); + good->set_status(CLS_SERVING); + locations[0] = std::move(good); + return LocationModifierResult{MA_OK, {EC_OK, EC_BADARGS}}; + }; + + const auto result = meta_indexer_->ReadModifyWriteLocation(request_context_.get(), keys, location_ids, modifier); + // The RMW itself succeeded. A per-location validation failure is surfaced + // in the aligned result without turning the whole batch into an + // infrastructure failure. + EXPECT_EQ(EC_OK, result.ec); + ASSERT_EQ(1u, result.per_location_error_codes.size()); + EXPECT_EQ((std::vector{EC_OK, EC_BADARGS}), result.per_location_error_codes[0]); + + LocationsPerKey stored_locations; + const auto get_result = meta_indexer_->GetLocations(request_context_.get(), keys, location_ids, stored_locations); + EXPECT_EQ(EC_PARTIAL_OK, get_result.ec); + EXPECT_EQ((std::vector{EC_OK, EC_NOENT}), get_result.per_location_error_codes[0]); + ASSERT_TRUE(stored_locations[0][0]); + EXPECT_EQ("good", stored_locations[0][0]->id()); + EXPECT_FALSE(stored_locations[0][1]); +} + +TEST_F(MetaIndexerTest, TestSingleTargetRmwPreservesCapacityAndExistingKeySemantics) { + const std::string config_str = R"({ + "max_key_count" : 2, + "mutex_shard_num" : 2, + "batch_key_size" : 100, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(config_str)); + ASSERT_TRUE(meta_indexer_->SupportsSingleLocationRmw()); + + const std::string target_id = "target"; + const std::string sibling_id = "sibling"; + auto old_target = std::make_shared(); + old_target->set_id(target_id); + auto sibling = std::make_shared(); + sibling->set_id(sibling_id); + CacheLocationMapVector seed_locations(2); + seed_locations[0].emplace(target_id, old_target); + seed_locations[1].emplace(sibling_id, sibling); + PropertyMapVector seed_properties(2); + ASSERT_EQ(EC_OK, meta_indexer_->Put(request_context_.get(), {1, 2}, seed_locations, seed_properties).ec); + ASSERT_EQ(2u, meta_indexer_->GetKeyCount()); + + const KeyVector keys{1, 2, 3}; + const LocationIdRefVector target_ids{&target_id, &target_id, &target_id}; + std::vector replacements(keys.size()); + const long old_target_use_count = old_target.use_count(); + const auto modifier = [&replacements, &old_target, old_target_use_count](ErrorCode get_ec, + const LocationId &location_id, + size_t key_index, + const CacheLocation *existing_location, + CacheLocationConstPtr &out_location) { + if (key_index == 0) { + EXPECT_EQ(EC_OK, get_ec); + EXPECT_EQ(old_target.get(), existing_location); + // The fused local read borrows the immutable value from the pinned + // cache item; it must not increment the shared_ptr control block. + EXPECT_EQ(old_target_use_count, old_target.use_count()); + } else { + EXPECT_EQ(EC_NOENT, get_ec); + EXPECT_EQ(nullptr, existing_location); + } + auto replacement = std::make_shared(); + replacement->set_id(location_id); + replacement->set_status(CLS_SERVING); + replacements[key_index] = replacement; + out_location = std::move(replacement); + return ModifierResult{MA_OK, EC_OK}; + }; + + const auto result = + meta_indexer_->ReadModifyWriteSingleTargetLocations(request_context_.get(), keys, target_ids, modifier); + EXPECT_EQ(EC_PARTIAL_OK, result.ec); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOSPC}), result.error_codes); + EXPECT_EQ(2u, meta_indexer_->GetKeyCount()); + + CacheLocationMapVector stored_locations; + const auto get_result = meta_indexer_->GetLocations(request_context_.get(), keys, stored_locations); + EXPECT_EQ(EC_PARTIAL_OK, get_result.ec); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOENT}), get_result.error_codes); + ASSERT_EQ(1u, stored_locations[0].size()); + EXPECT_EQ(replacements[0], stored_locations[0].at(target_id)); + ASSERT_EQ(2u, stored_locations[1].size()); + EXPECT_EQ(sibling, stored_locations[1].at(sibling_id)); + EXPECT_EQ(replacements[1], stored_locations[1].at(target_id)); + EXPECT_TRUE(stored_locations[2].empty()); + + size_t skip_calls = 0; + const auto skip_result = + meta_indexer_->ReadModifyWriteSingleTargetLocations(request_context_.get(), + {3}, + LocationIdRefVector{&target_id}, + [&skip_calls](ErrorCode get_ec, + const LocationId &, + size_t, + const CacheLocation *existing_location, + CacheLocationConstPtr &) { + ++skip_calls; + EXPECT_EQ(EC_NOENT, get_ec); + EXPECT_EQ(nullptr, existing_location); + return ModifierResult{MA_SKIP, EC_OK}; + }); + EXPECT_EQ(EC_OK, skip_result.ec); + EXPECT_EQ((std::vector{EC_OK}), skip_result.error_codes); + EXPECT_EQ(1u, skip_calls); + EXPECT_EQ(2u, meta_indexer_->GetKeyCount()); + + size_t duplicate_modifier_calls = 0; + const auto duplicate_result = meta_indexer_->ReadModifyWriteSingleTargetLocations( + request_context_.get(), + {4, 4}, + LocationIdRefVector{&target_id, &target_id}, + [&duplicate_modifier_calls]( + ErrorCode, const LocationId &, size_t, const CacheLocation *, CacheLocationConstPtr &) { + ++duplicate_modifier_calls; + return ModifierResult{MA_SKIP, EC_OK}; + }); + EXPECT_EQ(EC_BADARGS, duplicate_result.ec); + EXPECT_TRUE(duplicate_result.error_codes.empty()); + EXPECT_EQ(0u, duplicate_modifier_calls); +} + // Verifies the invariants of MakeBatches() that callers rely on, regardless // of the exact shard distribution (which is now hash-driven and therefore // not deterministic across keys): @@ -151,7 +779,7 @@ TEST_F(MetaIndexerTest, TestMakeBatches) { for (size_t j = 0; j < batch.batch_keys.size(); ++j) { const int32_t origin_idx = batch.batch_indexs[j]; ASSERT_EQ(keys[origin_idx], batch.batch_keys[j]); - const int32_t shard = GetShardIndex(batch.batch_keys[j], 7); + const int32_t shard = meta_indexer_->GetMutexShardIndex(batch.batch_keys[j]); ASSERT_TRUE(shards_in_batch.count(shard) > 0) << "key " << batch.batch_keys[j] << " hashed to shard " << shard << " but the batch only locked shards declared in batch_shard_indexs"; @@ -166,6 +794,36 @@ TEST_F(MetaIndexerTest, TestMakeBatches) { ASSERT_EQ(expected_indexs, covered_indexs); } +TEST_F(MetaIndexerTest, TestPureLocalMutexShardsReuseLruHashSeed) { + std::string configStr = R"({ + "max_key_count" : 100, + "mutex_shard_num" : 16, + "batch_key_size" : 4, + "meta_storage_backend_config" : { "storage_type" : "local" }, + "meta_cache_policy_config" : { "capacity" : 0 } + })"; + ASSERT_EQ(EC_OK, InitIndexer(configStr)); + + auto *local_backend = dynamic_cast(meta_indexer_->backend_manager_->persistent_backend_.get()); + ASSERT_NE(nullptr, local_backend); + uint32_t lru_hash_seed = 0; + ASSERT_TRUE(local_backend->GetCacheHashSeed(lru_hash_seed)); + ASSERT_EQ(static_cast(lru_hash_seed), meta_indexer_->mutex_shard_hash_seed_); + + const KeyVector keys = {KeyType{0}, + KeyType{1}, + KeyType{2}, + KeyType{17}, + KeyType{1'000}, + KeyType{94'422}, + std::numeric_limits::max()}; + for (KeyType key : keys) { + const uint64_t lru_hash = Hash64(reinterpret_cast(&key), sizeof(key), lru_hash_seed); + EXPECT_EQ(static_cast(lru_hash & meta_indexer_->mutex_shard_mask_), + meta_indexer_->GetMutexShardIndex(key)); + } +} + TEST_F(MetaIndexerTest, TestMakeBatches2) { std::string configStr = R"({ "max_key_count" : 100, @@ -204,7 +862,7 @@ TEST_F(MetaIndexerTest, TestMakeBatches2) { for (size_t j = 0; j < batch.batch_keys.size(); ++j) { const int32_t origin_idx = batch.batch_indexs[j]; ASSERT_EQ(keys[origin_idx], batch.batch_keys[j]); - const int32_t shard = GetShardIndex(batch.batch_keys[j], 15); + const int32_t shard = meta_indexer_->GetMutexShardIndex(batch.batch_keys[j]); ASSERT_TRUE(shards_in_batch.count(shard) > 0); ASSERT_EQ(std::to_string(keys[origin_idx]), batch.batch_properties[j].at("uri")); covered_indexs.push_back(origin_idx); diff --git a/kv_cache_manager/meta/test/meta_local_backend_test.cc b/kv_cache_manager/meta/test/meta_local_backend_test.cc index fc577cc51..741a1d197 100644 --- a/kv_cache_manager/meta/test/meta_local_backend_test.cc +++ b/kv_cache_manager/meta/test/meta_local_backend_test.cc @@ -90,6 +90,222 @@ TEST_F(MetaLocalBackendTest, TestSimple) { ASSERT_EQ(EC_OK, meta_storage_backend_->Close()); } +TEST_F(MetaLocalBackendTest, TestTargetedLocationsPreserveKeyStatus) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("targeted_status", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + auto location = std::make_shared(); + location->set_id("target"); + CacheLocationMapVector locations(2); + locations[0].emplace("target", location); + PropertyMapVector properties(2); + properties[1].emplace("property_only", "value"); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->Put(nullptr, {1, 2}, locations, properties)); + + auto stale = std::make_shared(); + stale->set_id("stale"); + LocationsPerKey selected(3, CacheLocationVector{stale}); + std::vector key_error_codes; + const auto per_location_ecs = meta_storage_backend_->GetLocationsWithKeyStatus( + nullptr, {1, 2, 3}, {{"target"}, {"target"}, {"target"}}, selected, key_error_codes); + + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOENT}), key_error_codes); + EXPECT_EQ((std::vector>{{EC_OK}, {EC_NOENT}, {EC_NOENT}}), per_location_ecs); + ASSERT_EQ(3u, selected.size()); + ASSERT_EQ(1u, selected[0].size()); + ASSERT_TRUE(selected[0][0]); + EXPECT_EQ("target", selected[0][0]->id()); + ASSERT_EQ(1u, selected[1].size()); + EXPECT_FALSE(selected[1][0]); + ASSERT_EQ(1u, selected[2].size()); + EXPECT_FALSE(selected[2][0]); +} + +TEST_F(MetaLocalBackendTest, TestSingleLocationFastPathPreservesKeyStatusAndMetadata) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("single_location_fast_path", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + const std::string target_id = "target"; + const std::string sibling_id = "sibling"; + auto old_target = std::make_shared(); + old_target->set_id(target_id); + auto sibling = std::make_shared(); + sibling->set_id(sibling_id); + CacheLocationMapVector seed_locations(2); + seed_locations[0].emplace(target_id, old_target); + seed_locations[0].emplace(sibling_id, sibling); + PropertyMapVector seed_properties(2); + seed_properties[1].emplace("property_only", "preserved"); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->Put(nullptr, {1, 2}, seed_locations, seed_properties)); + + const LocationIdRefVector target_ids{&target_id, &target_id, &target_id}; + auto stale = std::make_shared(); + stale->set_id("stale"); + CacheLocationVector selected(3, stale); + std::vector key_error_codes; + EXPECT_EQ((std::vector{EC_OK, EC_NOENT, EC_NOENT}), + meta_storage_backend_->GetSingleLocationsWithKeyStatus( + nullptr, {1, 2, 3}, target_ids, selected, key_error_codes)); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOENT}), key_error_codes); + ASSERT_EQ(3u, selected.size()); + EXPECT_EQ(old_target, selected[0]); + EXPECT_FALSE(selected[1]); + EXPECT_FALSE(selected[2]); + + auto replacement = std::make_shared(); + replacement->set_id(target_id); + replacement->set_status(CLS_SERVING); + auto property_key_location = std::make_shared(); + property_key_location->set_id(target_id); + auto new_key_location = std::make_shared(); + new_key_location->set_id(target_id); + EXPECT_EQ( + (std::vector{EC_OK, EC_OK, EC_OK}), + meta_storage_backend_->UpsertSingleLocations( + nullptr, {1, 2, 3}, target_ids, CacheLocationVector{replacement, property_key_location, new_key_location})); + + CacheLocationMapVector stored_locations; + PropertyMapVector stored_properties; + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_OK}), + meta_storage_backend_->Get(nullptr, {1, 2, 3}, stored_locations, stored_properties)); + ASSERT_EQ(2u, stored_locations[0].size()); + EXPECT_EQ(replacement, stored_locations[0].at(target_id)); + EXPECT_EQ(sibling, stored_locations[0].at(sibling_id)); + ASSERT_EQ(1u, stored_locations[1].size()); + EXPECT_EQ(property_key_location, stored_locations[1].at(target_id)); + EXPECT_EQ("preserved", stored_properties[1].at("property_only")); + ASSERT_EQ(1u, stored_locations[2].size()); + EXPECT_EQ(new_key_location, stored_locations[2].at(target_id)); +} + +TEST_F(MetaLocalBackendTest, TestSingleLocationRmwReusesAndReleasesReadHandles) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("single_location_retained_handles", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + const std::string target_id = "target"; + auto old_target = std::make_shared(); + old_target->set_id(target_id); + CacheLocationMapVector seed_locations(2); + seed_locations[0].emplace(target_id, old_target); + PropertyMapVector seed_properties(2); + seed_properties[1].emplace("property_only", "preserved"); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->Put(nullptr, {1, 2}, seed_locations, seed_properties)); + + auto *backend = GetLocalBackend(); + SingleLocationRmwScratch scratch; + backend->PrepareSingleLocationRmwScratch(3, scratch); + const KeyTypeVec read_keys{1, 2, 3}; + const LocationIdRefVector read_ids{&target_id, &target_id, &target_id}; + CacheLocationViewVector selected_views; + std::vector key_ecs; + std::vector location_ecs; + const long old_target_use_count = old_target.use_count(); + backend->GetSingleLocationViewsWithKeyStatusInto( + nullptr, read_keys, read_ids, selected_views, key_ecs, location_ecs, scratch); + EXPECT_TRUE(scratch.HasRetainedHandles()); + EXPECT_EQ((std::vector{EC_OK, EC_NOENT, EC_NOENT}), location_ecs); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOENT}), key_ecs); + ASSERT_EQ(3u, selected_views.size()); + EXPECT_EQ(old_target.get(), selected_views[0]); + EXPECT_EQ(nullptr, selected_views[1]); + EXPECT_EQ(old_target_use_count, old_target.use_count()); + + auto replacement = std::make_shared(); + replacement->set_id(target_id); + replacement->set_status(CLS_SERVING); + auto new_key_location = std::make_shared(); + new_key_location->set_id(target_id); + std::vector write_ecs; + CacheLocationVector replacements{replacement, new_key_location}; + backend->UpsertSingleLocationsUsingRetainedHandlesInto( + nullptr, {1, 3}, {&target_id, &target_id}, replacements, {0, 2}, write_ecs, scratch); + EXPECT_EQ((std::vector{EC_OK, EC_OK}), write_ecs); + EXPECT_FALSE(scratch.HasRetainedHandles()); + ASSERT_EQ(1u, scratch.retired_locations.size()); + EXPECT_EQ(old_target, scratch.retired_locations[0]); + scratch.retired_locations.clear(); + + CacheLocationMapVector stored_locations; + PropertyMapVector stored_properties; + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_OK}), + meta_storage_backend_->Get(nullptr, {1, 2, 3}, stored_locations, stored_properties)); + EXPECT_EQ(replacement, stored_locations[0].at(target_id)); + EXPECT_TRUE(stored_locations[1].empty()); + EXPECT_EQ("preserved", stored_properties[1].at("property_only")); + EXPECT_EQ(new_key_location, stored_locations[2].at(target_id)); + + // Invalid subset metadata must fail the whole write and release the read + // handle, so an early validation return cannot pin an LRU entry. + CacheLocationVector selected; + backend->GetSingleLocationsWithKeyStatusInto(nullptr, + {1}, + {&target_id}, + selected, + key_ecs, + location_ecs, + scratch, + /*retain_handles=*/true); + ASSERT_TRUE(scratch.HasRetainedHandles()); + CacheLocationVector invalid_replacement{replacement}; + backend->UpsertSingleLocationsUsingRetainedHandlesInto( + nullptr, {1}, {&target_id}, invalid_replacement, {1}, write_ecs, scratch); + EXPECT_EQ((std::vector{EC_BADARGS}), write_ecs); + EXPECT_FALSE(scratch.HasRetainedHandles()); +} + +TEST_F(MetaLocalBackendTest, TestSingleLocationFastPathValidatesBatchAndPreservesDuplicateOrder) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("single_location_edge_cases", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + const std::string first_id = "first"; + const std::string second_id = "second"; + auto first_old = std::make_shared(); + first_old->set_id(first_id); + auto second = std::make_shared(); + second->set_id(second_id); + auto first_new = std::make_shared(); + first_new->set_id(first_id); + first_new->set_status(CLS_SERVING); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_OK}), + meta_storage_backend_->UpsertSingleLocations(nullptr, + {10, 10, 10}, + LocationIdRefVector{&first_id, &second_id, &first_id}, + CacheLocationVector{first_old, second, first_new})); + + CacheLocationMapVector stored_locations; + EXPECT_EQ((std::vector{EC_OK}), meta_storage_backend_->GetLocations(nullptr, {10}, stored_locations)); + ASSERT_EQ(2u, stored_locations[0].size()); + EXPECT_EQ(first_new, stored_locations[0].at(first_id)); + EXPECT_EQ(second, stored_locations[0].at(second_id)); + + const std::string mismatched_id = "mismatched"; + auto malformed = std::make_shared(); + malformed->set_id(mismatched_id); + EXPECT_EQ( + (std::vector{EC_BADARGS, EC_BADARGS}), + meta_storage_backend_->UpsertSingleLocations( + nullptr, {20, 21}, LocationIdRefVector{&first_id, &first_id}, CacheLocationVector{first_old, malformed})); + std::vector exists; + EXPECT_EQ((std::vector{EC_OK, EC_OK}), meta_storage_backend_->Exists(nullptr, {20, 21}, exists)); + EXPECT_EQ((std::vector{false, false}), exists); + + EXPECT_EQ((std::vector{EC_BADARGS}), + meta_storage_backend_->UpsertSingleLocations( + nullptr, {30}, LocationIdRefVector{nullptr}, CacheLocationVector{first_old})); + EXPECT_TRUE(meta_storage_backend_->UpsertSingleLocations(nullptr, {}, {}, {}).empty()); + + CacheLocationVector empty_locations{first_old}; + std::vector empty_key_error_codes{EC_ERROR}; + EXPECT_TRUE( + meta_storage_backend_->GetSingleLocationsWithKeyStatus(nullptr, {}, {}, empty_locations, empty_key_error_codes) + .empty()); + EXPECT_TRUE(empty_locations.empty()); + EXPECT_TRUE(empty_key_error_codes.empty()); +} + TEST_F(MetaLocalBackendTest, TestInit) { // invalid config ASSERT_EQ(EC_BADARGS, meta_storage_backend_->Init("test_instance_0", /*config*/ nullptr)); @@ -201,6 +417,155 @@ TEST_F(MetaLocalBackendTest, TestUpsert) { ASSERT_EQ(EC_OK, meta_storage_backend_->Close()); } +TEST_F(MetaLocalBackendTest, TestUpsertPreservesRequestOrderForDuplicateKeys) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("test_duplicate_upsert", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + // The public backend API historically applies duplicate keys in request + // order. In particular, a later partial update to a key first created by + // the same batch must merge with, rather than replace, its earlier fields. + ASSERT_EQ((std::vector{EC_OK, EC_OK, EC_OK}), + UpsertWithFieldMaps(meta_storage_backend_.get(), + {7, 8, 7}, + {{{PROPERTY_URI, "uri7"}}, {{PROPERTY_URI, "uri8"}}, {{PROPERTY_HIT_COUNT, "700"}}})); + AssertGetProperties(meta_storage_backend_.get(), + {7, 8}, + {PROPERTY_URI, PROPERTY_HIT_COUNT}, + {EC_OK, EC_OK}, + {{{PROPERTY_URI, "uri7"}, {PROPERTY_HIT_COUNT, "700"}}, {{PROPERTY_URI, "uri8"}}}); + + ASSERT_EQ(EC_OK, meta_storage_backend_->Close()); +} + +TEST_F(MetaLocalBackendTest, TestUpsertPreservesRequestOrderForMixedCapacityBatch) { + auto config = std::make_shared(); + config->SetStorageUri("local://?capacity=1&num_shard_bits=0"); + auto backend = std::make_shared(); + ASSERT_EQ(EC_OK, backend->Init("test_mixed_capacity_upsert", config)); + ASSERT_EQ(EC_OK, backend->Open()); + + ASSERT_EQ((std::vector{EC_OK}), PutWithFieldMaps(backend.get(), {2}, {{{PROPERTY_URI, "existing"}}})); + + // UpdateInPlace historically does not reject charge growth. Therefore the + // request order below first admits key 1 while the cache has room, then + // expands existing key 2. Reordering all existing updates ahead of all + // missing inserts would incorrectly reject the earlier key 1. + const std::string large_insert(600 * 1024, 'i'); + const std::string large_update(600 * 1024, 'u'); + EXPECT_EQ((std::vector{EC_OK, EC_OK}), + UpsertWithFieldMaps( + backend.get(), {1, 2}, {{{PROPERTY_URI, large_insert}}, {{PROPERTY_HIT_COUNT, large_update}}})); + + AssertGetProperties( + backend.get(), + {1, 2}, + {PROPERTY_URI, PROPERTY_HIT_COUNT}, + {EC_OK, EC_OK}, + {{{PROPERTY_URI, large_insert}}, {{PROPERTY_URI, "existing"}, {PROPERTY_HIT_COUNT, large_update}}}); + ASSERT_EQ(EC_OK, backend->Close()); + + auto reverse_backend = std::make_shared(); + ASSERT_EQ(EC_OK, reverse_backend->Init("test_reverse_mixed_capacity_upsert", config)); + ASSERT_EQ(EC_OK, reverse_backend->Open()); + ASSERT_EQ((std::vector{EC_OK}), + PutWithFieldMaps(reverse_backend.get(), {2}, {{{PROPERTY_URI, "existing"}}})); + EXPECT_EQ((std::vector{EC_OK, EC_NOSPC}), + UpsertWithFieldMaps(reverse_backend.get(), + {2, 1}, + {{{PROPERTY_HIT_COUNT, large_update}}, {{PROPERTY_URI, large_insert}}})); + AssertGetProperties(reverse_backend.get(), + {1, 2}, + {PROPERTY_URI, PROPERTY_HIT_COUNT}, + {EC_NOENT, EC_OK}, + {{}, {{PROPERTY_URI, "existing"}, {PROPERTY_HIT_COUNT, large_update}}}); + ASSERT_EQ(EC_OK, reverse_backend->Close()); +} + +TEST_F(MetaLocalBackendTest, TestBatchedUpsertShapesMatchSequentialReference) { + auto make_backend = [](const std::string &instance_id) { + auto config = std::make_shared(); + auto backend = std::make_shared(); + EXPECT_EQ(EC_OK, backend->Init(instance_id, config)); + EXPECT_EQ(EC_OK, backend->Open()); + return backend; + }; + auto optimized = make_backend("test_batched_upsert_reference_optimized"); + auto reference = make_backend("test_batched_upsert_reference_sequential"); + + KeyTypeVec seed_keys; + FieldMapVec seed_fields; + for (KeyType key = 0; key < 12; ++key) { + seed_keys.push_back(key); + seed_fields.push_back({{PROPERTY_URI, "seed_" + std::to_string(key)}}); + } + ASSERT_EQ(PutWithFieldMaps(reference.get(), seed_keys, seed_fields), + PutWithFieldMaps(optimized.get(), seed_keys, seed_fields)); + + std::set observed_keys(seed_keys.begin(), seed_keys.end()); + auto apply_and_compare = [&](const KeyTypeVec &keys, const FieldMapVec &fields) { + SCOPED_TRACE(::testing::PrintToString(keys)); + CacheLocationMapVector reference_locations; + PropertyMapVector reference_properties; + SplitFieldMaps(fields, reference_locations, reference_properties); + std::vector reference_results(keys.size(), EC_OK); + for (size_t i = 0; i < keys.size(); ++i) { + reference_results[i] = reference->UpsertForOneKey(keys[i], reference_locations[i], reference_properties[i]); + } + EXPECT_EQ(reference_results, UpsertWithFieldMaps(optimized.get(), keys, fields)); + observed_keys.insert(keys.begin(), keys.end()); + + const KeyTypeVec all_keys(observed_keys.begin(), observed_keys.end()); + CacheLocationMapVector optimized_locations; + CacheLocationMapVector sequential_locations; + PropertyMapVector optimized_properties; + PropertyMapVector sequential_properties; + const auto optimized_ec = optimized->Get(nullptr, all_keys, optimized_locations, optimized_properties); + const auto sequential_ec = reference->Get(nullptr, all_keys, sequential_locations, sequential_properties); + EXPECT_EQ(sequential_ec, optimized_ec); + EXPECT_EQ(sequential_locations, optimized_locations); + for (auto &properties : optimized_properties) { + properties.erase(PROPERTY_LRU_TIME); + } + for (auto &properties : sequential_properties) { + properties.erase(PROPERTY_LRU_TIME); + } + EXPECT_EQ(sequential_properties, optimized_properties); + EXPECT_EQ(reference->GetMemUsage(), optimized->GetMemUsage()); + }; + + KeyTypeVec all_hit_keys; + FieldMapVec all_hit_fields; + for (KeyType key = 0; key < 12; ++key) { + all_hit_keys.push_back(key); + all_hit_fields.push_back({{"hit_" + std::to_string(key), "value"}}); + } + apply_and_compare(all_hit_keys, all_hit_fields); + + KeyTypeVec all_miss_keys; + FieldMapVec all_miss_fields; + for (KeyType key = 100; key < 112; ++key) { + all_miss_keys.push_back(key); + all_miss_fields.push_back({{PROPERTY_URI, "new_" + std::to_string(key)}}); + } + apply_and_compare(all_miss_keys, all_miss_fields); + + apply_and_compare({200, 0, 201, 1, 202, 2}, + {{{PROPERTY_URI, "mixed_200"}}, + {{PROPERTY_HIT_COUNT, "mixed_0"}}, + {{PROPERTY_URI, "mixed_201"}}, + {{PROPERTY_HIT_COUNT, "mixed_1"}}, + {{PROPERTY_URI, "mixed_202"}}, + {{PROPERTY_HIT_COUNT, "mixed_2"}}}); + apply_and_compare({300, 300, 3, 300}, + {{{PROPERTY_URI, "duplicate_first"}}, + {{PROPERTY_HIT_COUNT, "duplicate_second"}}, + {{PROPERTY_HIT_COUNT, "existing"}}, + {{"final_field", "duplicate_final"}}}); + + EXPECT_EQ(EC_OK, optimized->Close()); + EXPECT_EQ(EC_OK, reference->Close()); +} + TEST_F(MetaLocalBackendTest, TestDelete) { ASSERT_EQ(EC_OK, meta_storage_backend_->Init("test_instance_0", meta_storage_backend_config_)); ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); @@ -928,6 +1293,108 @@ TEST_F(MetaLocalBackendTest, TestConditionalDeleteFields) { // --------------------------------------------------------------------------- // Concurrent read-write stress test for MetaMemCacheItem fields_ mutex // --------------------------------------------------------------------------- +TEST_F(MetaLocalBackendTest, TestGetLocationValuesPreservesKeyOrderAndSharedValues) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("test_location_values", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + auto make_location = [](const std::string &id) { + auto location = std::make_shared(); + location->set_id(id); + location->set_status(CacheLocationStatus::CLS_SERVING); + location->set_type(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2); + location->set_location_specs({LocationSpec("tp0", "event_report://host/mem")}); + return location; + }; + auto location_a = make_location("location-a"); + auto location_b = make_location("location-b"); + CacheLocationMapVector locations(2); + locations[0].emplace(location_a->id(), location_a); + locations[0].emplace(location_b->id(), location_b); + PropertyMapVector properties(2); + properties[1][PROPERTY_URI] = "property-only-key"; + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->Put(nullptr, {11, 22}, locations, properties)); + + LocationsPerKey values; + EXPECT_EQ((std::vector{EC_OK, EC_NOENT, EC_OK, EC_OK}), + meta_storage_backend_->GetLocationValues(nullptr, {11, 33, 22, 11}, values)); + ASSERT_EQ(4u, values.size()); + EXPECT_TRUE(values[1].empty()); + EXPECT_TRUE(values[2].empty()); + for (const std::size_t index : {std::size_t{0}, std::size_t{3}}) { + ASSERT_EQ(2u, values[index].size()); + std::set ids; + for (const auto &location : values[index]) { + ASSERT_TRUE(location); + ids.insert(location->id()); + EXPECT_TRUE(location == location_a || location == location_b); + } + EXPECT_EQ((std::set{"location-a", "location-b"}), ids); + } + + ASSERT_EQ(EC_OK, meta_storage_backend_->Close()); +} + +TEST_F(MetaLocalBackendTest, TestGetLocationValuesCompactPreservesOffsetsAndCanBeReused) { + ASSERT_EQ(EC_OK, meta_storage_backend_->Init("test_compact_location_values", meta_storage_backend_config_)); + ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); + + auto make_location = [](const std::string &id) { + auto location = std::make_shared(); + location->set_id(id); + location->set_status(CacheLocationStatus::CLS_SERVING); + location->set_type(DataStorageType::DATA_STORAGE_TYPE_EVENT_REPORT_L2); + location->set_location_specs({LocationSpec("tp0", "event_report://host/mem")}); + return location; + }; + auto location_a = make_location("location-a"); + auto location_b = make_location("location-b"); + CacheLocationMapVector locations(2); + locations[0].emplace(location_a->id(), location_a); + locations[0].emplace(location_b->id(), location_b); + PropertyMapVector properties(2); + properties[1][PROPERTY_URI] = "property-only-key"; + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->Put(nullptr, {11, 22}, locations, properties)); + + const KeyType keys[] = {11, 33, 22, 11}; + CompactLocationsPerKey compact; + EXPECT_EQ((std::vector{EC_OK, EC_NOENT, EC_OK, EC_OK}), + meta_storage_backend_->GetLocationValuesCompact(nullptr, keys, std::size(keys), compact)); + ASSERT_TRUE(compact.IsValid(std::size(keys))); + EXPECT_EQ((std::vector{0, 2, 2, 2, 4}), compact.offsets); + EXPECT_TRUE(compact[1].empty()); + EXPECT_TRUE(compact[2].empty()); + for (const std::size_t index : {std::size_t{0}, std::size_t{3}}) { + ASSERT_EQ(2u, compact[index].size()); + std::set ids; + for (const auto &location : compact[index]) { + ASSERT_TRUE(location); + ids.insert(location->id()); + EXPECT_TRUE(location == location_a || location == location_b); + } + EXPECT_EQ((std::set{"location-a", "location-b"}), ids); + } + + // Reusing an output object must discard every offset and value left by the + // previous, longer request. + const KeyType shorter_keys[] = {22, 11}; + EXPECT_EQ((std::vector{EC_OK, EC_OK}), + meta_storage_backend_->GetLocationValuesCompact(nullptr, shorter_keys, std::size(shorter_keys), compact)); + ASSERT_TRUE(compact.IsValid(std::size(shorter_keys))); + EXPECT_EQ((std::vector{0, 0, 2}), compact.offsets); + EXPECT_TRUE(compact[0].empty()); + EXPECT_EQ(2u, compact[1].size()); + + EXPECT_EQ((std::vector{EC_BADARGS, EC_BADARGS}), + meta_storage_backend_->GetLocationValuesCompact(nullptr, nullptr, 2, compact)); + ASSERT_TRUE(compact.IsValid(2)); + EXPECT_TRUE(compact[0].empty()); + EXPECT_TRUE(compact[1].empty()); + + ASSERT_EQ(EC_OK, meta_storage_backend_->Close()); +} + TEST_F(MetaLocalBackendTest, TestConcurrentReadWrite) { ASSERT_EQ(EC_OK, meta_storage_backend_->Init("test_instance_concurrent", meta_storage_backend_config_)); ASSERT_EQ(EC_OK, meta_storage_backend_->Open()); @@ -991,6 +1458,17 @@ TEST_F(MetaLocalBackendTest, TestConcurrentReadWrite) { auto get_loc_ids_ec = meta_storage_backend_->GetLocationIds(nullptr, {kTestKey}, loc_ids); ASSERT_EQ(1u, get_loc_ids_ec.size()); ASSERT_EQ(EC_OK, get_loc_ids_ec[0]); + + // Lightweight host-query projection must be safe while location + // entries are concurrently inserted and removed. + LocationsPerKey location_values; + auto get_values_ec = meta_storage_backend_->GetLocationValues(nullptr, {kTestKey}, location_values); + ASSERT_EQ(1u, get_values_ec.size()); + ASSERT_EQ(EC_OK, get_values_ec[0]); + ASSERT_EQ(1u, location_values.size()); + for (const auto &location : location_values[0]) { + ASSERT_TRUE(location); + } } }; diff --git a/kv_cache_manager/meta/test/meta_storage_backend_manager_test.cc b/kv_cache_manager/meta/test/meta_storage_backend_manager_test.cc index 19ba73855..c757ea1aa 100644 --- a/kv_cache_manager/meta/test/meta_storage_backend_manager_test.cc +++ b/kv_cache_manager/meta/test/meta_storage_backend_manager_test.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "kv_cache_manager/common/request_context.h" @@ -16,6 +17,303 @@ namespace kv_cache_manager { +namespace { + +class MalformedMetaCacheBackend : public MetaLocalBackend { +public: + std::vector Get(RequestContext *, + const KeyTypeVec &, + CacheLocationMapVector &out_locations, + PropertyMapVector &out_properties) noexcept override { + out_locations.clear(); + out_properties.clear(); + return {EC_OK}; + } + + std::vector + GetLocations(RequestContext *, const KeyTypeVec &, CacheLocationMapVector &out_locations) noexcept override { + out_locations.clear(); + return {EC_OK}; + } + + std::vector + GetLocationValues(RequestContext *, const KeyTypeVec &, LocationsPerKey &out_locations) noexcept override { + out_locations.clear(); + return {EC_OK}; + } + + std::vector> GetLocations(RequestContext *, + const KeyTypeVec &, + const LocationIdsPerKey &, + LocationsPerKey &out_locations) noexcept override { + out_locations.clear(); + return {}; + } + + std::vector> + GetLocationsWithKeyStatus(RequestContext *, + const KeyTypeVec &, + const LocationIdsPerKey &, + LocationsPerKey &out_locations, + std::vector &out_key_error_codes) noexcept override { + out_locations.clear(); + out_key_error_codes = {EC_OK}; + return {}; + } + + std::vector + GetLocationIds(RequestContext *, const KeyTypeVec &, LocationIdsPerKey &out_location_ids) noexcept override { + out_location_ids.clear(); + return {}; + } + + std::vector GetProperties(RequestContext *, + const KeyTypeVec &, + const std::vector &, + PropertyMapVector &out_properties) noexcept override { + out_properties.clear(); + return {EC_OK}; + } + + std::vector + Exists(RequestContext *, const KeyTypeVec &, std::vector &out_exists) noexcept override { + out_exists.clear(); + return {EC_OK}; + } +}; + +class RecoverContractCacheBackend : public MetaLocalBackend { +public: + explicit RecoverContractCacheBackend(std::vector put_results = {}) + : put_results_(std::move(put_results)) {} + + std::vector + Exists(RequestContext *, const KeyTypeVec &keys, std::vector &out_exists) noexcept override { + out_exists.assign(keys.size(), false); + return std::vector(keys.size(), EC_OK); + } + + std::vector PutIfAbsent(RequestContext *, + const KeyTypeVec &keys, + const CacheLocationMapVector &, + const PropertyMapVector &, + const std::vector &) noexcept override { + if (!put_results_.empty()) { + return put_results_; + } + return std::vector(keys.size(), EC_OK); + } + +private: + std::vector put_results_; +}; + +class MalformedPersistentGetBackend : public MetaLocalBackend { +public: + std::vector Get(RequestContext *, + const KeyTypeVec &, + CacheLocationMapVector &out_locations, + PropertyMapVector &out_properties) noexcept override { + out_locations.clear(); + out_properties.clear(); + return {EC_OK}; + } +}; + +class ScriptedRecoverPersistentBackend : public MetaLocalBackend { +public: + ScriptedRecoverPersistentBackend(int scan_failures, int get_failures, bool malformed_get) + : scan_failures_(scan_failures), get_failures_(get_failures), malformed_get_(malformed_get) {} + + ErrorCode ListKeys(RequestContext *, + const std::string &, + const int64_t, + std::string &out_next_cursor, + KeyTypeVec &out_keys) noexcept override { + ++list_calls_; + if (list_calls_ <= scan_failures_) { + return EC_ERROR; + } + out_next_cursor = SCAN_BASE_CURSOR; + out_keys = {101}; + return EC_OK; + } + + std::vector Get(RequestContext *, + const KeyTypeVec &keys, + CacheLocationMapVector &out_locations, + PropertyMapVector &out_properties) noexcept override { + ++get_calls_; + if (malformed_get_) { + out_locations.clear(); + out_properties.clear(); + return std::vector(keys.size(), EC_OK); + } + out_locations.resize(keys.size()); + out_properties.resize(keys.size()); + if (get_calls_ <= get_failures_) { + return std::vector(keys.size(), EC_ERROR); + } + return std::vector(keys.size(), EC_OK); + } + + int list_calls() const { return list_calls_; } + int get_calls() const { return get_calls_; } + +private: + int scan_failures_ = 0; + int get_failures_ = 0; + bool malformed_get_ = false; + int list_calls_ = 0; + int get_calls_ = 0; +}; + +class FlakyRecoverCacheBackend : public RecoverContractCacheBackend { +public: + explicit FlakyRecoverCacheBackend(int put_failures) : put_failures_(put_failures) {} + + std::vector PutIfAbsent(RequestContext *, + const KeyTypeVec &keys, + const CacheLocationMapVector &, + const PropertyMapVector &, + const std::vector &) noexcept override { + ++put_calls_; + return std::vector(keys.size(), put_calls_ <= put_failures_ ? EC_ERROR : EC_OK); + } + + int put_calls() const { return put_calls_; } + +private: + int put_failures_ = 0; + int put_calls_ = 0; +}; + +class MalformedPersistentWriteBackend : public MetaLocalBackend { +public: + std::vector Put(RequestContext *, + const KeyTypeVec &, + const CacheLocationMapVector &, + const PropertyMapVector &) noexcept override { + return {EC_OK}; + } + + std::vector Upsert(RequestContext *, + const KeyTypeVec &, + const CacheLocationMapVector &, + const PropertyMapVector &) noexcept override { + return {EC_OK}; + } + + std::vector Delete(RequestContext *, const KeyTypeVec &) noexcept override { return {EC_OK}; } + + std::vector + DeleteLocations(RequestContext *, const KeyTypeVec &, const LocationIdsPerKey &) noexcept override { + return {EC_OK}; + } +}; + +class WellFormedPersistentWriteBackend : public MetaLocalBackend { +public: + std::vector Put(RequestContext *, + const KeyTypeVec &keys, + const CacheLocationMapVector &, + const PropertyMapVector &) noexcept override { + return std::vector(keys.size(), EC_OK); + } + + std::vector Upsert(RequestContext *, + const KeyTypeVec &keys, + const CacheLocationMapVector &, + const PropertyMapVector &) noexcept override { + return std::vector(keys.size(), EC_OK); + } + + std::vector Delete(RequestContext *, const KeyTypeVec &keys) noexcept override { + return std::vector(keys.size(), EC_OK); + } + + std::vector + DeleteLocations(RequestContext *, const KeyTypeVec &keys, const LocationIdsPerKey &) noexcept override { + return std::vector(keys.size(), EC_OK); + } +}; + +class MalformedCacheWriteBackend : public MetaLocalBackend { +public: + std::vector Put(RequestContext *, + const KeyTypeVec &, + const CacheLocationMapVector &, + const PropertyMapVector &, + const std::vector &) noexcept override { + return {EC_OK}; + } + + std::vector Upsert(RequestContext *, + const KeyTypeVec &, + const CacheLocationMapVector &, + const PropertyMapVector &, + const std::vector &) noexcept override { + return {EC_OK}; + } + + std::vector + Delete(RequestContext *, const KeyTypeVec &, const std::vector &) noexcept override { + return {EC_OK}; + } + + std::vector DeleteLocations(RequestContext *, + const KeyTypeVec &, + const LocationIdsPerKey &, + const std::vector &) noexcept override { + return {EC_OK}; + } +}; + +class MixedDeletePersistentBackend : public WellFormedPersistentWriteBackend { +public: + std::vector Delete(RequestContext *, const KeyTypeVec &keys) noexcept override { + std::vector results(keys.size(), EC_OK); + if (!results.empty()) { + results.front() = EC_ERROR; + } + return results; + } +}; + +class PassthroughDeleteCacheBackend : public RecoverContractCacheBackend { +public: + std::vector + Delete(RequestContext *, const KeyTypeVec &, const std::vector &previous_error_codes) noexcept override { + return previous_error_codes; + } +}; + +struct BackendLifecycleCalls { + ErrorCode open_result = EC_OK; + int open_calls = 0; + int close_calls = 0; +}; + +class LifecycleMetaLocalBackend : public MetaLocalBackend { +public: + explicit LifecycleMetaLocalBackend(std::shared_ptr calls) : calls_(std::move(calls)) {} + + ErrorCode Open() noexcept override { + ++calls_->open_calls; + return calls_->open_result; + } + + ErrorCode Close() noexcept override { + ++calls_->close_calls; + return EC_OK; + } + +private: + std::shared_ptr calls_; +}; + +} // namespace + class MetaStorageBackendManagerTest : public TESTBASE { public: void SetUp() override { request_context_ = std::make_shared("test_trace_id"); } @@ -146,6 +444,404 @@ TEST_F(MetaStorageBackendManagerTest, TestInitDualBackend) { ASSERT_EQ(EC_OK, mgr.Close()); } +TEST_F(MetaStorageBackendManagerTest, TestInitIsTransactionalAndOneShot) { + auto invalid_cache_config = std::make_shared(); + invalid_cache_config->SetStorageType(META_CACHED_BACKEND_TYPE_STR); + invalid_cache_config->SetStorageUri( + "file:///tmp/kvcm_invalid_cache?persistent_type=dummy&cache_type=unknown_backend"); + + MetaStorageBackendManager mgr; + EXPECT_EQ(EC_ERROR, mgr.Init("failed_dual", invalid_cache_config)); + EXPECT_FALSE(mgr.persistent_backend_); + EXPECT_FALSE(mgr.cache_backend_); + EXPECT_TRUE(mgr.instance_id_.empty()); + + const std::string path = GetPrivateTestRuntimeDataPath() + "mgr_transactional_retry"; + std::filesystem::remove(path); + ASSERT_EQ(EC_OK, mgr.Init("valid_after_failure", MakeSingleConfig(path))); + auto *const persistent_backend = mgr.persistent_backend_.get(); + ASSERT_NE(nullptr, persistent_backend); + + EXPECT_EQ(EC_ERROR, mgr.Init("must_not_replace", MakeDualConfig(path + "_other"))); + EXPECT_EQ("valid_after_failure", mgr.instance_id_); + EXPECT_EQ(persistent_backend, mgr.persistent_backend_.get()); + EXPECT_FALSE(mgr.cache_backend_); +} + +TEST_F(MetaStorageBackendManagerTest, TestCacheOpenFailureRollsBackBothBackends) { + auto persistent_calls = std::make_shared(); + auto cache_calls = std::make_shared(); + cache_calls->open_result = EC_ERROR; + + MetaStorageBackendManager mgr; + mgr.instance_id_ = "rollback_instance"; + mgr.persistent_backend_ = std::make_unique(persistent_calls); + mgr.cache_backend_ = std::make_unique(cache_calls); + + EXPECT_EQ(EC_ERROR, mgr.Open()); + EXPECT_TRUE(mgr.is_closed_.load(std::memory_order_acquire)); + EXPECT_EQ(1, persistent_calls->open_calls); + EXPECT_EQ(1, cache_calls->open_calls); + EXPECT_EQ(1, persistent_calls->close_calls); + EXPECT_EQ(1, cache_calls->close_calls); + EXPECT_FALSE(mgr.recover_thread_.joinable()); +} + +TEST_F(MetaStorageBackendManagerTest, TestPersistentOpenFailureRollsBackBackend) { + auto persistent_calls = std::make_shared(); + persistent_calls->open_result = EC_ERROR; + + MetaStorageBackendManager mgr; + mgr.instance_id_ = "persistent_rollback_instance"; + mgr.persistent_backend_ = std::make_unique(persistent_calls); + + EXPECT_EQ(EC_ERROR, mgr.Open()); + EXPECT_TRUE(mgr.is_closed_.load(std::memory_order_acquire)); + EXPECT_FALSE(mgr.opened_); + EXPECT_EQ(1, persistent_calls->open_calls); + EXPECT_EQ(1, persistent_calls->close_calls); +} + +TEST_F(MetaStorageBackendManagerTest, TestRepeatedOpenIsRejectedWithoutReopeningBackend) { + auto persistent_calls = std::make_shared(); + + MetaStorageBackendManager mgr; + mgr.instance_id_ = "repeat_open_instance"; + mgr.persistent_backend_ = std::make_unique(persistent_calls); + + ASSERT_EQ(EC_OK, mgr.Open()); + EXPECT_TRUE(mgr.opened_); + EXPECT_EQ(EC_ERROR, mgr.Open()); + EXPECT_EQ(1, persistent_calls->open_calls); + + EXPECT_EQ(EC_OK, mgr.Close()); + EXPECT_FALSE(mgr.opened_); + EXPECT_EQ(1, persistent_calls->close_calls); +} + +TEST_F(MetaStorageBackendManagerTest, TestCloseIsIdempotent) { + auto persistent_calls = std::make_shared(); + + MetaStorageBackendManager mgr; + mgr.instance_id_ = "repeat_close_instance"; + mgr.persistent_backend_ = std::make_unique(persistent_calls); + + ASSERT_EQ(EC_OK, mgr.Open()); + ASSERT_EQ(EC_OK, mgr.Close()); + EXPECT_EQ(EC_OK, mgr.Close()); + EXPECT_EQ(1, persistent_calls->close_calls); +} + +TEST_F(MetaStorageBackendManagerTest, TestDestructorClosesOpenedBackendExactlyOnce) { + auto persistent_calls = std::make_shared(); + + { + MetaStorageBackendManager mgr; + mgr.instance_id_ = "destructor_close_instance"; + mgr.persistent_backend_ = std::make_unique(persistent_calls); + + ASSERT_EQ(EC_OK, mgr.Open()); + EXPECT_EQ(0, persistent_calls->close_calls); + } + + EXPECT_EQ(1, persistent_calls->close_calls); +} + +TEST_F(MetaStorageBackendManagerTest, TestConcurrentLocationValueReadsAreLocalOnly) { + MetaStorageBackendManager mgr; + EXPECT_FALSE(mgr.SupportsConcurrentLocationValueReads()); + EXPECT_FALSE(mgr.SupportsSingleLocationRmw()); + + mgr.persistent_backend_ = std::make_unique(); + EXPECT_TRUE(mgr.SupportsConcurrentLocationValueReads()); + EXPECT_TRUE(mgr.SupportsSingleLocationRmw()); + + mgr.cache_backend_ = std::make_unique(); + EXPECT_FALSE(mgr.SupportsConcurrentLocationValueReads()); + EXPECT_FALSE(mgr.SupportsSingleLocationRmw()); + + mgr.cache_backend_.reset(); + mgr.persistent_backend_ = std::make_unique(); + EXPECT_TRUE(mgr.SupportsConcurrentLocationValueReads()); + EXPECT_FALSE(mgr.SupportsSingleLocationRmw()); + + mgr.persistent_backend_ = std::make_unique(); + EXPECT_FALSE(mgr.SupportsConcurrentLocationValueReads()); + EXPECT_FALSE(mgr.SupportsSingleLocationRmw()); +} + +TEST_F(MetaStorageBackendManagerTest, TestPureLocalHashSeedIsNotExposedForCachedOrNonLocalBackends) { + MetaStorageBackendManager mgr; + uint32_t hash_seed = 0; + EXPECT_FALSE(mgr.GetPureLocalCacheHashSeed(hash_seed)); + + auto backend_config = std::make_shared(); + auto local_backend = std::make_unique(); + ASSERT_EQ(EC_OK, local_backend->Init("hash_seed_local", backend_config)); + uint32_t expected_hash_seed = 0; + ASSERT_TRUE(local_backend->GetCacheHashSeed(expected_hash_seed)); + mgr.persistent_backend_ = std::move(local_backend); + ASSERT_TRUE(mgr.GetPureLocalCacheHashSeed(hash_seed)); + EXPECT_EQ(expected_hash_seed, hash_seed); + + mgr.cache_backend_ = std::make_unique(); + EXPECT_FALSE(mgr.GetPureLocalCacheHashSeed(hash_seed)); + + mgr.cache_backend_.reset(); + mgr.persistent_backend_ = std::make_unique(); + EXPECT_FALSE(mgr.GetPureLocalCacheHashSeed(hash_seed)); +} + +TEST_F(MetaStorageBackendManagerTest, TestMalformedCacheReadShapesFailClosed) { + MetaStorageBackendManager mgr; + mgr.persistent_backend_ = std::make_unique(); + mgr.cache_backend_ = std::make_unique(); + mgr.recover_state_.store(MetaStorageBackendManager::RecoverState::kRunning); + + const KeyVector keys{1, 2}; + CacheLocationMapVector all_locations; + PropertyMapVector all_properties; + const auto get_results = mgr.Get(request_context_.get(), keys, all_locations, all_properties); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), get_results); + EXPECT_EQ(2u, all_locations.size()); + EXPECT_EQ(2u, all_properties.size()); + + const auto all_location_results = mgr.GetLocations(request_context_.get(), keys, all_locations); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), all_location_results); + EXPECT_EQ(2u, all_locations.size()); + + const LocationIdsPerKey requested_ids{{"a"}, {"b", "c"}}; + LocationsPerKey locations; + const auto per_location = mgr.GetLocations(request_context_.get(), keys, requested_ids, locations); + ASSERT_EQ(2u, per_location.size()); + EXPECT_EQ((std::vector{EC_ERROR}), per_location[0]); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), per_location[1]); + ASSERT_EQ(2u, locations.size()); + EXPECT_EQ(1u, locations[0].size()); + EXPECT_EQ(2u, locations[1].size()); + + std::vector key_error_codes; + const auto with_key_status = + mgr.GetLocationsWithKeyStatus(request_context_.get(), keys, requested_ids, locations, key_error_codes); + ASSERT_EQ(2u, with_key_status.size()); + EXPECT_EQ((std::vector{EC_ERROR}), with_key_status[0]); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), with_key_status[1]); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), key_error_codes); + ASSERT_EQ(2u, locations.size()); + EXPECT_EQ(1u, locations[0].size()); + EXPECT_EQ(2u, locations[1].size()); + + LocationIdsPerKey location_ids; + const auto per_key = mgr.GetLocationIds(request_context_.get(), keys, location_ids); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), per_key); + EXPECT_EQ(2u, location_ids.size()); + + LocationsPerKey location_values; + const auto value_results = mgr.GetLocationValues(request_context_.get(), keys, location_values); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), value_results); + EXPECT_EQ(2u, location_values.size()); + + const auto property_results = mgr.GetProperties(request_context_.get(), keys, {"field"}, all_properties); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), property_results); + EXPECT_EQ(2u, all_properties.size()); + + std::vector exists; + const auto exists_results = mgr.Exists(request_context_.get(), keys, exists); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), exists_results); + EXPECT_EQ((std::vector{false, false}), exists); +} + +TEST_F(MetaStorageBackendManagerTest, TestRecoverHydrationMalformedShapesFailClosed) { + const KeyVector keys{1, 2}; + + MetaStorageBackendManager malformed_exists; + malformed_exists.persistent_backend_ = std::make_unique(); + malformed_exists.cache_backend_ = std::make_unique(); + malformed_exists.recover_state_.store(MetaStorageBackendManager::RecoverState::kRecover); + BatchMetaData upsert = MakeBatch(keys); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_exists.Upsert(request_context_.get(), upsert)); + + MetaStorageBackendManager malformed_get; + malformed_get.persistent_backend_ = std::make_unique(); + malformed_get.cache_backend_ = std::make_unique(); + EXPECT_FALSE(malformed_get.EnsureKeyInCache(request_context_.get(), keys)); + + MetaStorageBackendManager malformed_put; + malformed_put.cache_backend_ = std::make_unique(std::vector{EC_OK}); + const CacheLocationMapVector locations(keys.size()); + const PropertyMapVector properties(keys.size()); + bool backfill_success = true; + EXPECT_EQ(0, malformed_put.BackfillKeysToCache(keys, locations, properties, {EC_OK, EC_OK}, &backfill_success)); + EXPECT_FALSE(backfill_success); + backfill_success = true; + EXPECT_EQ( + 0, + malformed_put.BackfillKeysToCache( + keys, CacheLocationMapVector(1), properties, std::vector{EC_OK, EC_OK}, &backfill_success)); + EXPECT_FALSE(backfill_success); +} + +TEST_F(MetaStorageBackendManagerTest, TestRecoverFailureKeepsFallbackAndTombstones) { + MetaStorageBackendManager mgr; + auto persistent = std::make_unique(/*scan_failures*/ 3, + /*get_failures*/ 0, + /*malformed_get*/ false); + auto *persistent_ptr = persistent.get(); + mgr.persistent_backend_ = std::move(persistent); + mgr.cache_backend_ = std::make_unique(); + mgr.recover_state_.store(MetaStorageBackendManager::RecoverState::kRecover); + mgr.deleted_keys_.insert(404); + + mgr.AsyncRecoverTask(); + + EXPECT_EQ(3, persistent_ptr->list_calls()); + EXPECT_EQ(MetaStorageBackendManager::RecoverState::kRecover, mgr.GetRecoverState()); + EXPECT_EQ(1u, mgr.deleted_keys_.count(404)); +} + +TEST_F(MetaStorageBackendManagerTest, TestRecoverMalformedGetDoesNotPublishPartialCache) { + MetaStorageBackendManager mgr; + auto persistent = std::make_unique(/*scan_failures*/ 0, + /*get_failures*/ 0, + /*malformed_get*/ true); + auto *persistent_ptr = persistent.get(); + mgr.persistent_backend_ = std::move(persistent); + mgr.cache_backend_ = std::make_unique(); + mgr.recover_state_.store(MetaStorageBackendManager::RecoverState::kRecover); + + mgr.AsyncRecoverTask(); + + EXPECT_EQ(3, persistent_ptr->get_calls()); + EXPECT_EQ(MetaStorageBackendManager::RecoverState::kRecover, mgr.GetRecoverState()); +} + +TEST_F(MetaStorageBackendManagerTest, TestRecoverRetriesSameBatchUntilFullyBackfilled) { + MetaStorageBackendManager mgr; + auto persistent = std::make_unique(/*scan_failures*/ 0, + /*get_failures*/ 1, + /*malformed_get*/ false); + auto *persistent_ptr = persistent.get(); + auto cache = std::make_unique(/*put_failures*/ 1); + auto *cache_ptr = cache.get(); + mgr.persistent_backend_ = std::move(persistent); + mgr.cache_backend_ = std::move(cache); + mgr.recover_state_.store(MetaStorageBackendManager::RecoverState::kRecover); + mgr.deleted_keys_.insert(404); + + mgr.AsyncRecoverTask(); + + // First Get fails, then the first cache fill fails. Neither attempt may + // rescan or advance the cursor; only the third complete attempt publishes + // the retained batch and clears Recover-time tombstones. + EXPECT_EQ(1, persistent_ptr->list_calls()); + EXPECT_EQ(3, persistent_ptr->get_calls()); + EXPECT_EQ(2, cache_ptr->put_calls()); + EXPECT_EQ(MetaStorageBackendManager::RecoverState::kRunning, mgr.GetRecoverState()); + EXPECT_TRUE(mgr.deleted_keys_.empty()); +} + +TEST_F(MetaStorageBackendManagerTest, TestMalformedWriteShapesFailClosed) { + const KeyVector keys{1, 2}; + BatchMetaData batch = MakeBatch(keys); + const LocationIdsPerKey location_ids{{"a"}, {"b"}}; + + MetaStorageBackendManager malformed_persistent; + malformed_persistent.persistent_backend_ = std::make_unique(); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_persistent.Put(request_context_.get(), batch)); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_persistent.Upsert(request_context_.get(), batch)); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_persistent.Delete(request_context_.get(), keys)); + int32_t reclaimed = -1; + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), + malformed_persistent.Delete(request_context_.get(), keys, location_ids, reclaimed)); + EXPECT_EQ(0, reclaimed); + + MetaStorageBackendManager malformed_cache; + malformed_cache.persistent_backend_ = std::make_unique(); + malformed_cache.cache_backend_ = std::make_unique(); + malformed_cache.recover_state_.store(MetaStorageBackendManager::RecoverState::kRunning); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_cache.Put(request_context_.get(), batch)); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_cache.Upsert(request_context_.get(), batch)); + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), malformed_cache.Delete(request_context_.get(), keys)); + reclaimed = -1; + EXPECT_EQ((std::vector{EC_ERROR, EC_ERROR}), + malformed_cache.Delete(request_context_.get(), keys, location_ids, reclaimed)); + EXPECT_EQ(0, reclaimed); +} + +TEST_F(MetaStorageBackendManagerTest, TestRecoverDeleteTombstonesOnlyCommittedKeys) { + MetaStorageBackendManager mgr; + mgr.persistent_backend_ = std::make_unique(); + mgr.cache_backend_ = std::make_unique(); + mgr.recover_state_.store(MetaStorageBackendManager::RecoverState::kRecover); + + const KeyVector keys{1, 2}; + EXPECT_EQ((std::vector{EC_ERROR, EC_OK}), mgr.Delete(request_context_.get(), keys)); + EXPECT_EQ(0u, mgr.deleted_keys_.count(1)); + EXPECT_EQ(1u, mgr.deleted_keys_.count(2)); +} + +TEST_F(MetaStorageBackendManagerTest, TestTargetedRecoveryReadDoesNotOverwriteCacheHitWithPersistentData) { + MetaStorageBackendManager mgr; + auto backend_config = std::make_shared(); + mgr.persistent_backend_ = std::make_unique(); + mgr.cache_backend_ = std::make_unique(); + ASSERT_EQ(EC_OK, mgr.persistent_backend_->Init("targeted_persistent", backend_config)); + ASSERT_EQ(EC_OK, mgr.cache_backend_->Init("targeted_cache", backend_config)); + ASSERT_EQ(EC_OK, mgr.persistent_backend_->Open()); + ASSERT_EQ(EC_OK, mgr.cache_backend_->Open()); + mgr.recover_state_.store(MetaStorageBackendManager::RecoverState::kRecover); + + const KeyVector persistent_keys{77, 78}; + CacheLocationMapVector persistent_locations(2); + persistent_locations[0].emplace("missing", MakeLocation("missing", "persistent_missing")); + persistent_locations[0].emplace("present", MakeLocation("present", "persistent_stale")); + persistent_locations[1].emplace("only_persistent", MakeLocation("only_persistent", "persistent_fallback")); + PropertyMapVector properties(2); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), + mgr.persistent_backend_->Put(request_context_.get(), persistent_keys, persistent_locations, properties)); + + CacheLocationMapVector cache_locations(1); + cache_locations[0].emplace("present", MakeLocation("present", "cache_current")); + ASSERT_EQ((std::vector{EC_OK}), + mgr.cache_backend_->Put(request_context_.get(), {77}, cache_locations, PropertyMapVector(1))); + + LocationsPerKey locations; + const LocationIdsPerKey requested_ids{{"missing", "present"}, {"only_persistent"}}; + const auto results = mgr.GetLocations(request_context_.get(), persistent_keys, requested_ids, locations); + ASSERT_EQ(2u, results.size()); + EXPECT_EQ((std::vector{EC_NOENT, EC_OK}), results[0]); + EXPECT_EQ((std::vector{EC_OK}), results[1]); + ASSERT_EQ(2u, locations.size()); + ASSERT_EQ(2u, locations[0].size()); + EXPECT_FALSE(locations[0][0]); + ASSERT_TRUE(locations[0][1]); + EXPECT_EQ("cache_current", locations[0][1]->location_specs().front().uri()); + ASSERT_EQ(1u, locations[1].size()); + ASSERT_TRUE(locations[1][0]); + EXPECT_EQ("persistent_fallback", locations[1][0]->location_specs().front().uri()); + + locations.clear(); + std::vector key_error_codes; + const KeyVector status_keys{77, 78, 79}; + const LocationIdsPerKey status_ids{{"missing", "present"}, {"only_persistent"}, {"absent"}}; + const auto status_results = + mgr.GetLocationsWithKeyStatus(request_context_.get(), status_keys, status_ids, locations, key_error_codes); + ASSERT_EQ(3u, status_results.size()); + EXPECT_EQ((std::vector{EC_NOENT, EC_OK}), status_results[0]); + EXPECT_EQ((std::vector{EC_OK}), status_results[1]); + EXPECT_EQ((std::vector{EC_NOENT}), status_results[2]); + EXPECT_EQ((std::vector{EC_OK, EC_OK, EC_NOENT}), key_error_codes); + ASSERT_EQ(3u, locations.size()); + ASSERT_TRUE(locations[0][1]); + EXPECT_EQ("cache_current", locations[0][1]->location_specs().front().uri()); + ASSERT_TRUE(locations[1][0]); + EXPECT_EQ("persistent_fallback", locations[1][0]->location_specs().front().uri()); + EXPECT_FALSE(locations[2][0]); + + ASSERT_EQ(EC_OK, mgr.cache_backend_->Close()); + ASSERT_EQ(EC_OK, mgr.persistent_backend_->Close()); +} + // --- Put/Get: CacheLocation serialization round-trip -------------------------- TEST_F(MetaStorageBackendManagerTest, TestPutAndGetLocationsRoundTrip) { @@ -184,6 +880,16 @@ TEST_F(MetaStorageBackendManagerTest, TestPutAndGetLocationsRoundTrip) { ASSERT_EQ("uri_" + std::to_string(keys[i]), it->second->location_specs().front().uri()); } + LocationsPerKey location_values; + auto get_value_ecs = mgr.GetLocationValues(request_context_.get(), {1, 404, 3}, location_values); + ASSERT_EQ((std::vector{EC_OK, EC_NOENT, EC_OK}), get_value_ecs); + ASSERT_EQ(3u, location_values.size()); + ASSERT_EQ(1u, location_values[0].size()); + EXPECT_EQ("loc_1", location_values[0].front()->id()); + EXPECT_TRUE(location_values[1].empty()); + ASSERT_EQ(1u, location_values[2].size()); + EXPECT_EQ("loc_3", location_values[2].front()->id()); + // Block-level properties should be preserved alongside the location fields. PropertyMapVector field_maps; auto field_ecs = mgr.GetProperties(nullptr, keys, {"p0"}, field_maps); @@ -702,6 +1408,44 @@ TEST_F(MetaStorageBackendManagerTest, TestMaybeReclaimEmptyKeysAfterLastLocation ASSERT_EQ(EC_OK, mgr.Close()); } +TEST_F(MetaStorageBackendManagerTest, TestMaybeReclaimCountsDuplicateKeyOnce) { + const std::string path = GetPrivateTestRuntimeDataPath() + "mgr_reclaim_duplicate_key"; + std::filesystem::remove(path); + MetaStorageBackendManager mgr; + ASSERT_EQ(EC_OK, mgr.Init("inst_reclaim_duplicate", MakeDualConfig(path))); + ASSERT_EQ(EC_OK, mgr.Open()); + WaitRunning(mgr); + + constexpr KeyType key = 31; + BatchMetaData batch; + batch.batch_keys = {key}; + batch.batch_indexs = {0}; + batch.batch_locations.resize(1); + batch.batch_properties.resize(1); + batch.batch_locations[0].emplace("loc_31_a", MakeLocation("loc_31_a", "uri_31_a")); + batch.batch_locations[0].emplace("loc_31_b", MakeLocation("loc_31_b", "uri_31_b")); + ASSERT_EQ((std::vector{EC_OK}), mgr.Put(request_context_.get(), batch)); + + // MetaSearcher flattens one location-deletion task per entry, so deleting + // the final two locations of one block legitimately supplies the key twice. + // The physical key and key-count metadata must nevertheless be reclaimed + // exactly once. + const KeyVector duplicate_keys = {key, key}; + const LocationIdsPerKey location_ids = {{"loc_31_a"}, {"loc_31_b"}}; + int32_t reclaimed = 0; + const auto delete_ecs = mgr.Delete(nullptr, duplicate_keys, location_ids, reclaimed); + ASSERT_EQ((std::vector{EC_OK, EC_OK}), delete_ecs); + EXPECT_EQ(1, reclaimed); + + std::vector exists; + const auto exists_ecs = mgr.Exists(nullptr, {key}, exists); + ASSERT_EQ((std::vector{EC_OK}), exists_ecs); + ASSERT_EQ(1u, exists.size()); + EXPECT_FALSE(exists[0]); + + ASSERT_EQ(EC_OK, mgr.Close()); +} + // --- Multi-key, multi-location: gradual deletion until key reclaimed ---------- TEST_F(MetaStorageBackendManagerTest, TestMultiKeyMultiLocationGradualDeletion) { diff --git a/kv_cache_manager/meta/test/query_executor_test.cc b/kv_cache_manager/meta/test/query_executor_test.cc new file mode 100644 index 000000000..58e24cd7f --- /dev/null +++ b/kv_cache_manager/meta/test/query_executor_test.cc @@ -0,0 +1,258 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kv_cache_manager/common/unittest.h" +#include "kv_cache_manager/meta/query_executor.h" + +namespace kv_cache_manager { +namespace { + +using namespace std::chrono_literals; + +TEST(QueryExecutorTest, EmptyAndBelowThresholdCallsStayOnCaller) { + QueryExecutor executor(/*worker_count*/ 4, + /*parallel_threshold*/ 10, + /*chunk_size*/ 2, + /*queue_capacity*/ 8); + std::size_t call_count = 0; + std::thread::id callback_thread; + const std::thread::id caller_thread = std::this_thread::get_id(); + + EXPECT_TRUE(executor.ParallelFor(0, [&](std::size_t, std::size_t) { ++call_count; })); + EXPECT_EQ(0u, call_count); + EXPECT_TRUE(executor.ParallelFor(9, [&](std::size_t begin, std::size_t end) { + ++call_count; + callback_thread = std::this_thread::get_id(); + EXPECT_EQ(0u, begin); + EXPECT_EQ(9u, end); + })); + EXPECT_EQ(1u, call_count); + EXPECT_EQ(caller_thread, callback_thread); +} + +TEST(QueryExecutorTest, ParallelRangesCoverEveryIndexExactlyOnce) { + QueryExecutor executor(/*worker_count*/ 4, + /*parallel_threshold*/ 1, + /*chunk_size*/ 1, + /*queue_capacity*/ 16); + constexpr std::size_t kCount = 512; + std::vector> visits(kCount); + for (auto &visit : visits) { + visit.store(0, std::memory_order_relaxed); + } + + std::mutex threads_mutex; + std::condition_variable threads_cv; + std::set callback_threads; + ASSERT_TRUE(executor.ParallelFor(kCount, [&](std::size_t begin, std::size_t end) { + { + std::unique_lock lock(threads_mutex); + callback_threads.insert(std::this_thread::get_id()); + threads_cv.notify_all(); + if (callback_threads.size() == 1) { + threads_cv.wait_for(lock, 2s, [&] { return callback_threads.size() >= 2; }); + } + } + for (std::size_t i = begin; i < end; ++i) { + visits[i].fetch_add(1, std::memory_order_relaxed); + } + })); + + EXPECT_GE(callback_threads.size(), 2u); + EXPECT_LE(callback_threads.size(), 4u); + for (std::size_t i = 0; i < kCount; ++i) { + EXPECT_EQ(1u, visits[i].load(std::memory_order_relaxed)) << "index=" << i; + } +} + +TEST(QueryExecutorTest, PerCallChunkSizeOverridesConfiguredRangeSize) { + QueryExecutor executor(/*worker_count*/ 4, + /*parallel_threshold*/ 1, + /*chunk_size*/ 1, + /*queue_capacity*/ 16); + constexpr std::size_t kCount = 100; + constexpr std::size_t kChunkSize = 17; + std::vector> visits(kCount); + for (auto &visit : visits) { + visit.store(0, std::memory_order_relaxed); + } + std::mutex ranges_mutex; + std::vector> ranges; + ASSERT_TRUE(executor.ParallelForWithChunkSize(kCount, kChunkSize, [&](size_t begin, size_t end) { + { + std::lock_guard lock(ranges_mutex); + ranges.emplace_back(begin, end); + } + for (size_t i = begin; i < end; ++i) { + visits[i].fetch_add(1, std::memory_order_relaxed); + } + })); + + EXPECT_EQ((kCount + kChunkSize - 1) / kChunkSize, ranges.size()); + for (const auto &[begin, end] : ranges) { + EXPECT_EQ(0u, begin % kChunkSize); + EXPECT_GT(end, begin); + EXPECT_LE(end - begin, kChunkSize); + } + for (size_t i = 0; i < kCount; ++i) { + EXPECT_EQ(1u, visits[i].load(std::memory_order_relaxed)) << "index=" << i; + } +} + +TEST(QueryExecutorTest, CallbackExceptionIsReportedWithoutDroppingOtherRanges) { + QueryExecutor executor(/*worker_count*/ 4, + /*parallel_threshold*/ 1, + /*chunk_size*/ 1, + /*queue_capacity*/ 16); + constexpr std::size_t kCount = 128; + std::vector> visits(kCount); + for (auto &visit : visits) { + visit.store(0, std::memory_order_relaxed); + } + + EXPECT_FALSE(executor.ParallelFor(kCount, [&](std::size_t begin, std::size_t end) { + for (std::size_t i = begin; i < end; ++i) { + visits[i].fetch_add(1, std::memory_order_relaxed); + if (i == 37) { + throw std::runtime_error("expected test exception"); + } + } + })); + for (std::size_t i = 0; i < kCount; ++i) { + EXPECT_EQ(1u, visits[i].load(std::memory_order_relaxed)) << "index=" << i; + } +} + +TEST(QueryExecutorTest, NestedParallelForDoesNotDeadlockOrDuplicateWork) { + QueryExecutor executor(/*worker_count*/ 4, + /*parallel_threshold*/ 1, + /*chunk_size*/ 1, + /*queue_capacity*/ 16); + constexpr std::size_t kOuterCount = 32; + constexpr std::size_t kInnerCount = 8; + std::vector> visits(kOuterCount * kInnerCount); + for (auto &visit : visits) { + visit.store(0, std::memory_order_relaxed); + } + + ASSERT_TRUE(executor.ParallelFor(kOuterCount, [&](std::size_t outer_begin, std::size_t outer_end) { + for (std::size_t outer = outer_begin; outer < outer_end; ++outer) { + ASSERT_TRUE(executor.ParallelFor(kInnerCount, [&](std::size_t inner_begin, std::size_t inner_end) { + for (std::size_t inner = inner_begin; inner < inner_end; ++inner) { + visits[outer * kInnerCount + inner].fetch_add(1, std::memory_order_relaxed); + } + })); + } + })); + for (std::size_t i = 0; i < visits.size(); ++i) { + EXPECT_EQ(1u, visits[i].load(std::memory_order_relaxed)) << "index=" << i; + } +} + +TEST(QueryExecutorTest, CompletedCallerDoesNotWaitForQueuedHelper) { + QueryExecutor executor(/*worker_count*/ 2, + /*parallel_threshold*/ 1, + /*chunk_size*/ 1, + /*queue_capacity*/ 8); + std::mutex blocker_mutex; + std::condition_variable blocker_cv; + std::size_t blocked_callbacks = 0; + bool release_blocker = false; + + std::thread blocker([&] { + EXPECT_TRUE(executor.ParallelFor(2, [&](std::size_t, std::size_t) { + std::unique_lock lock(blocker_mutex); + ++blocked_callbacks; + blocker_cv.notify_all(); + blocker_cv.wait(lock, [&] { return release_blocker; }); + })); + }); + bool both_callbacks_blocked = false; + { + std::unique_lock lock(blocker_mutex); + both_callbacks_blocked = blocker_cv.wait_for(lock, 2s, [&] { return blocked_callbacks == 2; }); + } + if (!both_callbacks_blocked) { + { + std::lock_guard lock(blocker_mutex); + release_blocker = true; + } + blocker_cv.notify_all(); + blocker.join(); + FAIL() << "query executor background worker did not enter the blocking callback"; + } + + std::mutex fast_mutex; + std::condition_variable fast_cv; + bool fast_done = false; + std::thread fast([&] { + EXPECT_TRUE(executor.ParallelFor(2, [](std::size_t, std::size_t) {})); + { + std::lock_guard lock(fast_mutex); + fast_done = true; + } + fast_cv.notify_one(); + }); + { + std::unique_lock lock(fast_mutex); + EXPECT_TRUE(fast_cv.wait_for(lock, 1s, [&] { return fast_done; })); + } + { + std::lock_guard lock(blocker_mutex); + release_blocker = true; + } + blocker_cv.notify_all(); + fast.join(); + blocker.join(); +} + +TEST(QueryExecutorTest, ConcurrentRequestsRemainExactWithSaturatedQueue) { + QueryExecutor executor(/*worker_count*/ 4, + /*parallel_threshold*/ 1, + /*chunk_size*/ 3, + /*queue_capacity*/ 1); + constexpr std::size_t kThreadCount = 16; + constexpr std::size_t kRounds = 20; + constexpr std::size_t kCount = 257; + std::atomic start{false}; + std::vector threads; + threads.reserve(kThreadCount); + for (std::size_t thread_index = 0; thread_index < kThreadCount; ++thread_index) { + threads.emplace_back([&, thread_index] { + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + for (std::size_t round = 0; round < kRounds; ++round) { + std::vector> visits(kCount); + for (auto &visit : visits) { + visit.store(0, std::memory_order_relaxed); + } + EXPECT_TRUE(executor.ParallelFor(kCount, + [&](std::size_t begin, std::size_t end) { + for (std::size_t i = begin; i < end; ++i) { + visits[i].fetch_add(1, std::memory_order_relaxed); + } + })) + << "thread=" << thread_index << " round=" << round; + for (std::size_t i = 0; i < kCount; ++i) { + EXPECT_EQ(1u, visits[i].load(std::memory_order_relaxed)) + << "thread=" << thread_index << " round=" << round << " index=" << i; + } + } + }); + } + start.store(true, std::memory_order_release); + for (auto &thread : threads) { + thread.join(); + } +} + +} // namespace +} // namespace kv_cache_manager diff --git a/kv_cache_manager/meta/types.h b/kv_cache_manager/meta/types.h index 9893e27d1..1e1c334a2 100644 --- a/kv_cache_manager/meta/types.h +++ b/kv_cache_manager/meta/types.h @@ -45,8 +45,16 @@ using PropertyMapVector = std::vector; // ---------- Location primitives ---------- using LocationId = std::string; using LocationIdVector = std::vector; +// Borrowed ids used by the pure-local one-location RMW fast path. The owner +// (normally a MetaSearcher task vector) must outlive the synchronous call. +using LocationIdRefVector = std::vector; using LocationIdsPerKey = std::vector; using LocationsPerKey = std::vector; +// Non-owning immutable values used only by the synchronous pure-local RMW +// path. The retained cache handles and metadata shard locks keep the backing +// MetaMemCacheItem (and its location map) alive until the matching write or +// explicit handle release. These views must never escape that interval. +using CacheLocationViewVector = std::vector; // A maintenance scan reads keys and their locations from the authoritative // backend without updating online access/LRU state. The three vectors are @@ -98,6 +106,17 @@ using LocationModifierFunc = std::function; +// Allocation-light targeted-upsert modifier for the common one-location-per- +// key ReportEvent shape. `existing_location` is a borrowed immutable view +// protected by the synchronous RMW's retained cache handle and shard lock; +// `out_location` receives the independently-owned replacement when MA_OK is +// returned. MA_DELETE is not supported by this specialized path. +using SingleLocationModifierFunc = std::function; + // ---------- Batch primitives ---------- // Single-batch view consumed by MetaStorageBackendManager. Fields with suffix // `[j]` are parallel arrays indexed by position within the batch. diff --git a/kv_cache_manager/meta/utils.h b/kv_cache_manager/meta/utils.h index 76cc44304..32dea6237 100644 --- a/kv_cache_manager/meta/utils.h +++ b/kv_cache_manager/meta/utils.h @@ -12,17 +12,23 @@ namespace kv_cache_manager { +inline constexpr uint64_t kDefaultMetaShardHashSeed = 0x9E3779B97F4A7C15ULL; + inline uint64_t HashKey(KeyType key) noexcept { - constexpr uint64_t kSeed = 0x9E3779B97F4A7C15ULL; - return Hash64(reinterpret_cast(&key), sizeof(key), kSeed); + return Hash64(reinterpret_cast(&key), sizeof(key), kDefaultMetaShardHashSeed); } // Maps a key to a shard index using a 64-bit hash so that adjacent keys // (which often share low bits) are distributed evenly across shards. // `shard_mask` MUST equal `shard_num - 1` where `shard_num` is a power of two. -inline int32_t GetShardIndex(KeyType key, size_t shard_mask) noexcept { +inline int32_t GetShardIndex(KeyType key, size_t shard_mask, uint64_t hash_seed) noexcept { assert(((shard_mask + 1) & shard_mask) == 0); - return static_cast(HashKey(key) & static_cast(shard_mask)); + return static_cast(Hash64(reinterpret_cast(&key), sizeof(key), hash_seed) & + static_cast(shard_mask)); +} + +inline int32_t GetShardIndex(KeyType key, size_t shard_mask) noexcept { + return GetShardIndex(key, shard_mask, kDefaultMetaShardHashSeed); } FieldMap SerializeToFieldMap(const CacheLocationMap &locations, const PropertyMap &properties); @@ -33,8 +39,7 @@ ErrorCode DeserializeLocations(const FieldMap &field_map, CacheLocationMap &out_ void ExtractLocationIds(const FieldMap &field_map, std::vector &out_location_ids); -std::vector -AppendPrefixToKeys(const std::string &cache_key_prefix, const KeyTypeVec &keys); +std::vector AppendPrefixToKeys(const std::string &cache_key_prefix, const KeyTypeVec &keys); bool StripPrefixInKeys(const std::string &cache_key_prefix, const std::string &instance_id, diff --git a/kv_cache_manager/metrics/kmonitor_metrics_reporter.cc b/kv_cache_manager/metrics/kmonitor_metrics_reporter.cc index 3086039e2..bfee5fde4 100644 --- a/kv_cache_manager/metrics/kmonitor_metrics_reporter.cc +++ b/kv_cache_manager/metrics/kmonitor_metrics_reporter.cc @@ -50,10 +50,6 @@ struct KmonitorMetricsReporter::Context { DECLARE_METRICS(service, error_qps); DECLARE_METRICS(service, request_queue_size); - DECLARE_METRICS(event_report, qps); - DECLARE_METRICS(event_report, request_rt_us); - DECLARE_METRICS(event_report, error_qps); - // manager metrics metrics DECLARE_METRICS(manager, request_key_count); DECLARE_METRICS(manager, prefix_match_len); @@ -72,6 +68,8 @@ struct KmonitorMetricsReporter::Context { // meta searcher metrics DECLARE_METRICS(meta_searcher, indexer_get_time_us); + DECLARE_METRICS(meta_searcher, host_projection_time_us); + DECLARE_METRICS(meta_searcher, host_prefix_reduce_time_us); DECLARE_METRICS(meta_searcher, indexer_read_modify_write_block_time_us); DECLARE_METRICS(meta_searcher, indexer_read_modify_write_location_time_us); DECLARE_METRICS(meta_searcher, index_serialize_time_us); @@ -319,10 +317,6 @@ bool KmonitorMetricsReporter::InitMetrics() { REGISTER_QPS_METRIC(service, error_qps); REGISTER_GAUGE_METRIC(service, request_queue_size); - REGISTER_QPS_METRIC(event_report, qps); - REGISTER_GAUGE_METRIC(event_report, request_rt_us); - REGISTER_QPS_METRIC(event_report, error_qps); - // manager metrics REGISTER_GAUGE_METRIC(manager, request_key_count); REGISTER_GAUGE_METRIC(manager, prefix_match_len); @@ -344,6 +338,8 @@ bool KmonitorMetricsReporter::InitMetrics() { // meta searcher metrics REGISTER_GAUGE_METRIC(meta_searcher, indexer_get_time_us); + REGISTER_GAUGE_METRIC(meta_searcher, host_projection_time_us); + REGISTER_GAUGE_METRIC(meta_searcher, host_prefix_reduce_time_us); REGISTER_GAUGE_METRIC(meta_searcher, indexer_read_modify_write_block_time_us); REGISTER_GAUGE_METRIC(meta_searcher, indexer_read_modify_write_location_time_us); REGISTER_GAUGE_METRIC(meta_searcher, index_serialize_time_us); @@ -541,6 +537,8 @@ void KmonitorMetricsReporter::ReportPerQuery(MetricsCollector *collector) { // meta searcher metrics REPORT_COLLECTED_METRICS(meta_searcher, indexer_get_time_us); + REPORT_COLLECTED_METRICS(meta_searcher, host_projection_time_us); + REPORT_COLLECTED_METRICS(meta_searcher, host_prefix_reduce_time_us); REPORT_STEAL_METRICS(meta_searcher, indexer_read_modify_write_block_time_us); REPORT_STEAL_METRICS(meta_searcher, indexer_read_modify_write_location_time_us); REPORT_STEAL_METRICS(meta_searcher, index_serialize_time_us); @@ -574,11 +572,14 @@ void KmonitorMetricsReporter::ReportPerQuery(MetricsCollector *collector) { } else if (dynamic_cast(collector)) { auto *p = dynamic_cast(collector); const kmonitor::MetricsTags tags = ctx_->GetKmonitorTags(p->GetMetricsTags()); - REPORT_METRICS(event_report, qps, 1.0); - REPORT_METRICS(event_report, request_rt_us, p->GetRequestRtUsSample()); + REPORT_METRICS(service, qps, 1.0); + REPORT_METRICS(service, query_rt_us, p->GetRequestRtUsSample()); const double error_code = p->GetErrorCodeSample(); - REPORT_METRICS_WHEN(event_report, error_qps, 1.0, !CommonUtil::IsZeroDouble(error_code)); + REPORT_METRICS_WHEN(service, error_qps, 1.0, !CommonUtil::IsZeroDouble(error_code)); + if (p->HasRequestKeyCountSample()) { + REPORT_METRICS(manager, request_key_count, p->GetRequestKeyCountSample()); + } } else if (dynamic_cast(collector)) { const auto *p = dynamic_cast(collector); const kmonitor::MetricsTags tags = ctx_->GetKmonitorTags(p->GetMetricsTags()); diff --git a/kv_cache_manager/metrics/local_metrics_reporter.cc b/kv_cache_manager/metrics/local_metrics_reporter.cc index ad715a0f2..c76f3145b 100644 --- a/kv_cache_manager/metrics/local_metrics_reporter.cc +++ b/kv_cache_manager/metrics/local_metrics_reporter.cc @@ -65,15 +65,15 @@ void LocalMetricsReporter::ReportPerQuery(MetricsCollector *collector) { } while (false); } else if (dynamic_cast(collector)) { auto *p = dynamic_cast(collector); - Counter request_counter; - COPY_METRICS_(p, event_report, request_counter, request_counter); - ++request_counter; + Counter service_query_counter; + COPY_METRICS_(p, service, query_counter, service_query_counter); + ++service_query_counter; const double error_code = p->GetErrorCodeSample(); if (!CommonUtil::IsZeroDouble(error_code)) { - Counter error_counter; - COPY_METRICS_(p, event_report, error_counter, error_counter); - ++error_counter; + Counter service_error_counter; + COPY_METRICS_(p, service, error_counter, service_error_counter); + ++service_error_counter; } } else if (dynamic_cast(collector)) { auto *p = dynamic_cast(collector); diff --git a/kv_cache_manager/metrics/metrics_collector.cc b/kv_cache_manager/metrics/metrics_collector.cc index 316842ac5..3eb7f4ee3 100644 --- a/kv_cache_manager/metrics/metrics_collector.cc +++ b/kv_cache_manager/metrics/metrics_collector.cc @@ -78,6 +78,8 @@ DEFINE_METRICS_NAME_FOR_MANAGER(batch_update_location_time_us); REGISTER_METRICS_W_TAGS_GAUGE_(metrics_registry_, meta_searcher, name, metrics_tags_) DEFINE_METRICS_NAME_FOR_META_SEARCHER(indexer_get_time_us); +DEFINE_METRICS_NAME_FOR_META_SEARCHER(host_projection_time_us); +DEFINE_METRICS_NAME_FOR_META_SEARCHER(host_prefix_reduce_time_us); DEFINE_METRICS_NAME_FOR_META_SEARCHER(indexer_read_modify_write_block_time_us); DEFINE_METRICS_NAME_FOR_META_SEARCHER(indexer_read_modify_write_location_time_us); DEFINE_METRICS_NAME_FOR_META_SEARCHER(index_serialize_time_us); @@ -150,6 +152,8 @@ bool ServiceMetricsCollector::Init() { // meta searcher metrics REGISTER_GAUGE_METRICS_FOR_META_SEARCHER(indexer_get_time_us); + REGISTER_GAUGE_METRICS_FOR_META_SEARCHER(host_projection_time_us); + REGISTER_GAUGE_METRICS_FOR_META_SEARCHER(host_prefix_reduce_time_us); REGISTER_GAUGE_METRICS_FOR_META_SEARCHER(indexer_read_modify_write_block_time_us); REGISTER_GAUGE_METRICS_FOR_META_SEARCHER(indexer_read_modify_write_location_time_us); REGISTER_GAUGE_METRICS_FOR_META_SEARCHER(index_serialize_time_us); @@ -186,16 +190,22 @@ bool ServiceMetricsCollector::Init() { /* ---------------- EventReportMetricsCollector -------------------- */ -#define DEFINE_METRICS_NAME_FOR_EVENT_REPORT(name) DEFINE_METRICS_NAME_(EventReportMetricsCollector, event_report, name) -#define REGISTER_COUNTER_METRICS_FOR_EVENT_REPORT(name) \ - REGISTER_METRICS_W_TAGS_COUNTER_(metrics_registry_, event_report, name, metrics_tags_) -#define REGISTER_GAUGE_METRICS_FOR_EVENT_REPORT(name) \ - REGISTER_METRICS_W_TAGS_GAUGE_(metrics_registry_, event_report, name, metrics_tags_) +#define DEFINE_METRICS_NAME_FOR_EVENT_REPORT_SERVICE(name) \ + DEFINE_METRICS_NAME_(EventReportMetricsCollector, service, name) +#define REGISTER_COUNTER_METRICS_FOR_EVENT_REPORT_SERVICE(name) \ + REGISTER_METRICS_W_TAGS_COUNTER_(metrics_registry_, service, name, metrics_tags_) +#define REGISTER_GAUGE_METRICS_FOR_EVENT_REPORT_SERVICE(name) \ + REGISTER_METRICS_W_TAGS_GAUGE_(metrics_registry_, service, name, metrics_tags_) +#define DEFINE_METRICS_NAME_FOR_EVENT_REPORT_MANAGER(name) \ + DEFINE_METRICS_NAME_(EventReportMetricsCollector, manager, name) +#define REGISTER_GAUGE_METRICS_FOR_EVENT_REPORT_MANAGER(name) \ + REGISTER_METRICS_W_TAGS_GAUGE_(metrics_registry_, manager, name, metrics_tags_) -DEFINE_METRICS_NAME_FOR_EVENT_REPORT(request_counter); -DEFINE_METRICS_NAME_FOR_EVENT_REPORT(request_rt_us); -DEFINE_METRICS_NAME_FOR_EVENT_REPORT(error_code); -DEFINE_METRICS_NAME_FOR_EVENT_REPORT(error_counter); +DEFINE_METRICS_NAME_FOR_EVENT_REPORT_SERVICE(query_counter); +DEFINE_METRICS_NAME_FOR_EVENT_REPORT_SERVICE(query_rt_us); +DEFINE_METRICS_NAME_FOR_EVENT_REPORT_SERVICE(error_code); +DEFINE_METRICS_NAME_FOR_EVENT_REPORT_SERVICE(error_counter); +DEFINE_METRICS_NAME_FOR_EVENT_REPORT_MANAGER(request_key_count); EventReportMetricsCollector::EventReportMetricsCollector(std::shared_ptr metrics_registry, MetricsTags metrics_tags) noexcept @@ -203,20 +213,22 @@ EventReportMetricsCollector::EventReportMetricsCollector(std::shared_ptr &intervals_us) { + if (intervals_us.empty()) { + return; + } + + std::vector bucket_deltas(boundaries_.size() + 1, 0); + uint64_t sum_delta = 0; + uint64_t count_delta = 0; + for (const int64_t interval_us : intervals_us) { + if (interval_us <= 0) { + continue; + } + const double interval_s = static_cast(interval_us) / 1e6; + for (size_t i = 0; i < boundaries_.size(); ++i) { + if (boundaries_[i] >= interval_s) { + ++bucket_deltas[i]; + } + } + ++bucket_deltas.back(); + sum_delta += static_cast(interval_us); + ++count_delta; + } + + if (count_delta == 0) { + return; + } + for (size_t i = 0; i < bucket_deltas.size(); ++i) { + if (bucket_deltas[i] != 0) { + bucket_counters_[i] += bucket_deltas[i]; + } + } + sum_counter_ += sum_delta; + count_counter_ += count_delta; +} + std::vector RevisitIntervalHistogram::GetBucketCounts() const { std::vector counts; counts.reserve(bucket_counters_.size()); diff --git a/kv_cache_manager/metrics/revisit_interval_histogram.h b/kv_cache_manager/metrics/revisit_interval_histogram.h index d6e656dc8..9007cc858 100644 --- a/kv_cache_manager/metrics/revisit_interval_histogram.h +++ b/kv_cache_manager/metrics/revisit_interval_histogram.h @@ -47,6 +47,13 @@ class RevisitIntervalHistogram { // Thread-safe: uses only atomic operations. void Observe(int64_t interval_us); + // Records a batch with the same histogram semantics as repeated Observe() + // calls, but aggregates bucket deltas locally before touching the shared + // counters. Large GetHostCacheState reads otherwise make every worker + // update the same counters for every block, creating avoidable cache-line + // contention inside the metadata I/O timer. + void ObserveBatch(const std::vector &intervals_us); + // Get bucket boundaries (for testing/debugging). const std::vector &GetBoundaries() const { return boundaries_; } diff --git a/kv_cache_manager/metrics/test/kmonitor_metrics_reporter_test.cc b/kv_cache_manager/metrics/test/kmonitor_metrics_reporter_test.cc index 7acc1a5b2..6e8b033f3 100644 --- a/kv_cache_manager/metrics/test/kmonitor_metrics_reporter_test.cc +++ b/kv_cache_manager/metrics/test/kmonitor_metrics_reporter_test.cc @@ -52,11 +52,11 @@ TEST_F(KmonitorMetricsReporterTest, TestReportPerQuery) { { EventReportMetricsCollector collector(metrics_registry_, {{"event_type", "block_snapshot"}}); ASSERT_TRUE(collector.Init()); - SET_METRICS_(&collector, event_report, request_rt_us, 123.); - SET_METRICS_(&collector, event_report, error_code, 1.); + SET_METRICS_(&collector, service, query_rt_us, 123.); + SET_METRICS_(&collector, service, error_code, 1.); EXPECT_NO_FATAL_FAILURE(reporter_->ReportPerQuery(&collector)); - EXPECT_EQ(1, collector.get_event_report_request_counter_metrics()); - EXPECT_EQ(1, collector.get_event_report_error_counter_metrics()); + EXPECT_EQ(1, collector.get_service_query_counter_metrics()); + EXPECT_EQ(1, collector.get_service_error_counter_metrics()); } { diff --git a/kv_cache_manager/metrics/test/local_metrics_reporter_test.cc b/kv_cache_manager/metrics/test/local_metrics_reporter_test.cc index cf73af484..974c7fb48 100644 --- a/kv_cache_manager/metrics/test/local_metrics_reporter_test.cc +++ b/kv_cache_manager/metrics/test/local_metrics_reporter_test.cc @@ -173,12 +173,12 @@ TEST_F(LocalMetricsReporterTest, TestReportPerQuery02) { ServiceMetricsCollector collector(metrics_registry_); collector.Init(); - EXPECT_EQ(3 + 5 + 14 + 6 + 23, metrics_registry_->GetSize()); + EXPECT_EQ(3 + 5 + 14 + 6 + 25, metrics_registry_->GetSize()); { reporter_->ReportPerQuery(&collector); - EXPECT_EQ(3 + 5 + 14 + 6 + 23, metrics_registry_->GetSize()); + EXPECT_EQ(3 + 5 + 14 + 6 + 25, metrics_registry_->GetSize()); std::uint64_t v; GET_METRICS_(&collector, service, query_counter, v); @@ -195,7 +195,7 @@ TEST_F(LocalMetricsReporterTest, TestReportPerQuery02) { reporter_->ReportPerQuery(&collector); - EXPECT_EQ(3 + 5 + 14 + 6 + 23, metrics_registry_->GetSize()); + EXPECT_EQ(3 + 5 + 14 + 6 + 25, metrics_registry_->GetSize()); std::uint64_t v; GET_METRICS_(&collector, service, query_counter, v); @@ -212,16 +212,16 @@ TEST_F(LocalMetricsReporterTest, TestReportPerQueryEventReport) { reporter_->ReportPerQuery(&collector); std::uint64_t value = 0; - GET_METRICS_(&collector, event_report, request_counter, value); + GET_METRICS_(&collector, service, query_counter, value); EXPECT_EQ(1, value); - GET_METRICS_(&collector, event_report, error_counter, value); + GET_METRICS_(&collector, service, error_counter, value); EXPECT_EQ(0, value); - SET_METRICS_(&collector, event_report, error_code, 10.); + SET_METRICS_(&collector, service, error_code, 10.); reporter_->ReportPerQuery(&collector); - GET_METRICS_(&collector, event_report, request_counter, value); + GET_METRICS_(&collector, service, query_counter, value); EXPECT_EQ(2, value); - GET_METRICS_(&collector, event_report, error_counter, value); + GET_METRICS_(&collector, service, error_counter, value); EXPECT_EQ(1, value); } @@ -236,14 +236,14 @@ TEST_F(LocalMetricsReporterTest, ConcurrentEventCollectorsDoNotShareErrorSamples failure.SetRequestSample(20.0, 1.0); // Both objects point at the same tagged registry gauge. Deliberately leave // that shared gauge in the failed state before reporting the success. - SET_METRICS_(&success, event_report, error_code, 0.0); - SET_METRICS_(&failure, event_report, error_code, 1.0); + SET_METRICS_(&success, service, error_code, 0.0); + SET_METRICS_(&failure, service, error_code, 1.0); reporter_->ReportPerQuery(&success); reporter_->ReportPerQuery(&failure); - EXPECT_EQ(2u, success.get_event_report_request_counter_metrics()); - EXPECT_EQ(1u, success.get_event_report_error_counter_metrics()); + EXPECT_EQ(2u, success.get_service_query_counter_metrics()); + EXPECT_EQ(1u, success.get_service_error_counter_metrics()); } TEST_F(LocalMetricsReporterTest, ServiceCallGuardCopiesRequestOutcomeToEventReportMetrics) { @@ -262,10 +262,10 @@ TEST_F(LocalMetricsReporterTest, ServiceCallGuardCopiesRequestOutcomeToEventRepo std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - EXPECT_GT(snapshot_collector->get_event_report_request_rt_us_metrics(), 0.); - EXPECT_DOUBLE_EQ(1., snapshot_collector->get_event_report_error_code_metrics()); - EXPECT_EQ(1, snapshot_collector->get_event_report_request_counter_metrics()); - EXPECT_EQ(1, snapshot_collector->get_event_report_error_counter_metrics()); + EXPECT_GT(snapshot_collector->get_service_query_rt_us_metrics(), 0.); + EXPECT_DOUBLE_EQ(1., snapshot_collector->get_service_error_code_metrics()); + EXPECT_EQ(1, snapshot_collector->get_service_query_counter_metrics()); + EXPECT_EQ(1, snapshot_collector->get_service_error_counter_metrics()); } TEST_F(LocalMetricsReporterTest, EventReportMetricsUseRequestLocalStatusInsteadOfSharedServiceGauge) { @@ -283,9 +283,9 @@ TEST_F(LocalMetricsReporterTest, EventReportMetricsUseRequestLocalStatusInsteadO request_context.GetMetricsCollectorsVehicle().AddMetricsCollector(heartbeat_collector); { ServiceCallGuard guard(cache_manager_.get(), &request_context, reporter_.get()); } - EXPECT_DOUBLE_EQ(0., heartbeat_collector->get_event_report_error_code_metrics()); - EXPECT_EQ(1, heartbeat_collector->get_event_report_request_counter_metrics()); - EXPECT_EQ(0, heartbeat_collector->get_event_report_error_counter_metrics()); + EXPECT_DOUBLE_EQ(0., heartbeat_collector->get_service_error_code_metrics()); + EXPECT_EQ(1, heartbeat_collector->get_service_query_counter_metrics()); + EXPECT_EQ(0, heartbeat_collector->get_service_error_counter_metrics()); } TEST_F(LocalMetricsReporterTest, TestReportInterval00) { diff --git a/kv_cache_manager/metrics/test/metrics_collector_test.cc b/kv_cache_manager/metrics/test/metrics_collector_test.cc index a856d13d3..b21dca175 100644 --- a/kv_cache_manager/metrics/test/metrics_collector_test.cc +++ b/kv_cache_manager/metrics/test/metrics_collector_test.cc @@ -27,17 +27,18 @@ TEST_F(MetricsCollectorTest, EventReportMetricsTest) { auto collector = std::make_shared(metrics_registry_, tags); ASSERT_TRUE(collector->Init()); EXPECT_EQ(tags, collector->GetMetricsTags()); - EXPECT_EQ(4, metrics_registry_->GetSize()); - - EXPECT_EQ(0, GET(collector, event_report, request_counter)); - EXPECT_DOUBLE_EQ(0., GET(collector, event_report, request_rt_us)); - EXPECT_DOUBLE_EQ(0., GET(collector, event_report, error_code)); - EXPECT_EQ(0, GET(collector, event_report, error_counter)); - - SET_METRICS_(collector, event_report, request_rt_us, 123.); - SET_METRICS_(collector, event_report, error_code, 10.); - EXPECT_DOUBLE_EQ(123., GET(collector, event_report, request_rt_us)); - EXPECT_DOUBLE_EQ(10., GET(collector, event_report, error_code)); + EXPECT_EQ(5, metrics_registry_->GetSize()); + + EXPECT_EQ(0, GET(collector, service, query_counter)); + EXPECT_DOUBLE_EQ(0., GET(collector, service, query_rt_us)); + EXPECT_DOUBLE_EQ(0., GET(collector, service, error_code)); + EXPECT_EQ(0, GET(collector, service, error_counter)); + EXPECT_DOUBLE_EQ(0., GET(collector, manager, request_key_count)); + + SET_METRICS_(collector, service, query_rt_us, 123.); + SET_METRICS_(collector, service, error_code, 10.); + EXPECT_DOUBLE_EQ(123., GET(collector, service, query_rt_us)); + EXPECT_DOUBLE_EQ(10., GET(collector, service, error_code)); } // Test MetaIndexer metrics functionality @@ -131,6 +132,8 @@ TEST_F(MetricsCollectorTest, MetaSearcherMetricsTest) { ASSERT_NE(nullptr, p); EXPECT_DOUBLE_EQ(GET(p, meta_searcher, indexer_get_time_us), 0.); + EXPECT_DOUBLE_EQ(GET(p, meta_searcher, host_projection_time_us), 0.); + EXPECT_DOUBLE_EQ(GET(p, meta_searcher, host_prefix_reduce_time_us), 0.); EXPECT_DOUBLE_EQ(GET(p, meta_searcher, indexer_read_modify_write_block_time_us), 0.); EXPECT_DOUBLE_EQ(GET(p, meta_searcher, indexer_read_modify_write_location_time_us), 0.); EXPECT_DOUBLE_EQ(GET(p, meta_searcher, index_serialize_time_us), 0.); @@ -146,13 +149,19 @@ TEST_F(MetricsCollectorTest, MetaSearcherMetricsTest) { // Test time measurement for indexer get KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(p, MetaSearcherIndexerGet); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(p, MetaSearcherHostProjection); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(p, MetaSearcherHostPrefixReduce); KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(p, MetaSearcherIndexerReadModifyWriteBlock); KVCM_METRICS_COLLECTOR_CHRONO_MARK_BEGIN(p, MetaSearcherIndexerReadModifyWriteLocation); usleep(1000); // 1ms KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(p, MetaSearcherIndexerGet); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(p, MetaSearcherHostProjection); + KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(p, MetaSearcherHostPrefixReduce); KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(p, MetaSearcherIndexerReadModifyWriteBlock); KVCM_METRICS_COLLECTOR_CHRONO_MARK_END(p, MetaSearcherIndexerReadModifyWriteLocation); EXPECT_GE(GET(p, meta_searcher, indexer_get_time_us), 1000.0); + EXPECT_GE(GET(p, meta_searcher, host_projection_time_us), 1000.0); + EXPECT_GE(GET(p, meta_searcher, host_prefix_reduce_time_us), 1000.0); EXPECT_GE(GET(p, meta_searcher, indexer_read_modify_write_block_time_us), 1000.0); EXPECT_GE(GET(p, meta_searcher, indexer_read_modify_write_location_time_us), 1000.0); } diff --git a/kv_cache_manager/metrics/test/prometheus_exporter_test.cc b/kv_cache_manager/metrics/test/prometheus_exporter_test.cc index bd21910e5..ab97041d6 100644 --- a/kv_cache_manager/metrics/test/prometheus_exporter_test.cc +++ b/kv_cache_manager/metrics/test/prometheus_exporter_test.cc @@ -59,24 +59,26 @@ TEST_F(PrometheusExporterTest, MetricsWithTags) { << output; } -TEST_F(PrometheusExporterTest, EventReportSnapshotMetricsExposeIndependentTagsAndValues) { - MetricsTags tags = {{"event_type", "block_snapshot"}, +TEST_F(PrometheusExporterTest, EventReportSnapshotMetricsReuseServiceTagsAndValues) { + MetricsTags tags = {{"api_name", "ReportEvent"}, + {"event_type", "block_snapshot"}, {"instance_group", "group_a"}, {"instance_id", "instance_a"}, {"type", "event_report_l2"}}; EventReportMetricsCollector collector(registry_, tags); ASSERT_TRUE(collector.Init()); Counter request_counter; - COPY_METRICS_(&collector, event_report, request_counter, request_counter); + COPY_METRICS_(&collector, service, query_counter, request_counter); ++request_counter; - SET_METRICS_(&collector, event_report, request_rt_us, 321.); + SET_METRICS_(&collector, service, query_rt_us, 321.); const std::string output = PrometheusExporter::Expose(*registry_); - const std::string labels = "{event_type=\"block_snapshot\",instance_group=\"group_a\",instance_id=\"instance_a\"," - "type=\"event_report_l2\"}"; - EXPECT_NE(output.find("kvcm_event_report_request_counter" + labels + " 1"), std::string::npos) << output; - EXPECT_NE(output.find("kvcm_event_report_request_rt_us" + labels + " 321"), std::string::npos) << output; - EXPECT_EQ(output.find("kvcm_event_report_error_counter"), std::string::npos) << output; + const std::string labels = + "{api_name=\"ReportEvent\",event_type=\"block_snapshot\",instance_group=\"group_a\"," + "instance_id=\"instance_a\",type=\"event_report_l2\"}"; + EXPECT_NE(output.find("kvcm_service_query_counter" + labels + " 1"), std::string::npos) << output; + EXPECT_NE(output.find("kvcm_service_query_rt_us" + labels + " 321"), std::string::npos) << output; + EXPECT_EQ(output.find("kvcm_event_report_request_counter"), std::string::npos) << output; } TEST_F(PrometheusExporterTest, MultipleTagSets) { diff --git a/kv_cache_manager/metrics/test/revisit_interval_histogram_test.cc b/kv_cache_manager/metrics/test/revisit_interval_histogram_test.cc index b41a5113f..3c192dfb0 100644 --- a/kv_cache_manager/metrics/test/revisit_interval_histogram_test.cc +++ b/kv_cache_manager/metrics/test/revisit_interval_histogram_test.cc @@ -217,6 +217,46 @@ TEST_F(RevisitIntervalHistogramTest, ManyObservations) { EXPECT_EQ(counts[3], 1000); // <= +Inf } +TEST_F(RevisitIntervalHistogramTest, ObserveBatchMatchesIndividualObservations) { + const std::vector boundaries = {1.0, 5.0, 10.0}; + RevisitIntervalHistogram individual; + RevisitIntervalHistogram batched; + ASSERT_TRUE(individual.Init(registry_, boundaries, "individual")); + ASSERT_TRUE(batched.Init(registry_, boundaries, "batched")); + + const std::vector intervals = { + -1, + 0, + 500000, + 1000000, + 3000000, + 5000000, + 7000000, + 10000000, + 50000000, + }; + for (const int64_t interval : intervals) { + individual.Observe(interval); + } + batched.ObserveBatch(intervals); + + EXPECT_EQ(individual.GetBucketCounts(), batched.GetBucketCounts()); + EXPECT_EQ(individual.GetSum(), batched.GetSum()); + EXPECT_EQ(individual.GetCount(), batched.GetCount()); +} + +TEST_F(RevisitIntervalHistogramTest, ObserveBatchIgnoresEmptyAndNonPositiveBatch) { + RevisitIntervalHistogram hist; + ASSERT_TRUE(hist.Init(registry_, {1.0, 5.0, 10.0}, "test_instance")); + + hist.ObserveBatch({}); + hist.ObserveBatch({-100, 0, -1}); + + EXPECT_EQ(0u, hist.GetCount()); + EXPECT_EQ(0u, hist.GetSum()); + EXPECT_EQ(std::vector({0, 0, 0, 0}), hist.GetBucketCounts()); +} + // 测试 le 标签格式化（整数边界不含多余零） TEST_F(RevisitIntervalHistogramTest, LeLabelFormatting) { RevisitIntervalHistogram hist; diff --git a/kv_cache_manager/protocol/protobuf/meta_service.proto b/kv_cache_manager/protocol/protobuf/meta_service.proto index 4a7626bd8..a63c64c32 100644 --- a/kv_cache_manager/protocol/protobuf/meta_service.proto +++ b/kv_cache_manager/protocol/protobuf/meta_service.proto @@ -3,6 +3,10 @@ syntax = "proto3"; package kv_cache_manager.proto.meta; option cc_generic_services = false; +// HTTP handlers keep request/response messages on one request-scoped arena. +// This is a C++ allocation policy only; it does not change the wire or JSON +// representation of any MetaService message. +option cc_enable_arenas = true; enum ErrorCode { UNSPECIFIED = 0; @@ -80,7 +84,9 @@ message NodeRegisterEventParams { } message BlockAddEventParams { - string block_key = 1; // int64 as string + // Signed int64 decimal, or unsigned uint64 decimal mapped to the same + // internal 64-bit key (e.g. UINT64_MAX is equivalent to -1). + string block_key = 1; string uri = 2; // deprecated, use specs instead string medium = 3; // cache tier/namespace, e.g. "mem", "disk", "gpu", "hbm" repeated LocationSpec specs = 4; @@ -520,11 +526,14 @@ message GetHostCacheStateRequest { QueryType query_type = 3; repeated int64 block_cache_keys = 4; // 请求 prompt 的有序 block cache keys repeated string medium = 5; // 要匹配的介质列表；为空时考虑所有介质 + int32 p2p_host_count = 6; // 继续计算 P2P 的 local top host 数；未传或 0 表示只计算 local } message HostCacheMatch { string host_ip_port = 1; // 必须与 Master 中 workerStatus.getIpPort() 的格式完全一致 - int64 prefix_match_blocks = 2; // 连续命中的 block 数（调用方自己 ×block_size 算 tokens） + int64 local = 2; // 本地前缀匹配的 block 数 + int64 p2p_1_fetch = 3; // P2P 实际拉取的 spec 所属的去重 block key 数 + int64 p2p_1_total_match = 4; // p2p 后最终的前缀匹配的block数 } message GetHostCacheStateResponse { diff --git a/kv_cache_manager/py_connector/common/manager_client.py b/kv_cache_manager/py_connector/common/manager_client.py index 8dd51c6eb..b2c9321f8 100644 --- a/kv_cache_manager/py_connector/common/manager_client.py +++ b/kv_cache_manager/py_connector/common/manager_client.py @@ -17,6 +17,14 @@ _LEADER_DISCOVERY_TIMEOUT_SECONDS = 5.0 +class KvCacheManagerHTTPError(requests.HTTPError, AssertionError): + """A non-200 Manager response, compatible with the legacy assertion API.""" + + +class KvCacheManagerProtocolError(requests.RequestException, AssertionError): + """The Manager returned HTTP 200 with a malformed API envelope.""" + + class KvCacheManagerClient: @classmethod def from_connector_config( @@ -60,40 +68,43 @@ def __init__(self, base_url, *, instance_id="", auto_discover_leader=False, lead self.session = requests.Session() self.headers = {'Accept': 'application/json', 'Content-Type': 'application/json'} + self._resource_lock = threading.Lock() + self._session_close_claimed = False + self._service_discovery_close_claimed = False + self._service_discovery_close_deferred = False + self._refresh_worker_cleanup_reached = False + # Constructor rollback uses the normal close path, so initialize its + # synchronization fields before invoking any discovery plugin code. + self._refresh_event = threading.Event() + self._closed = threading.Event() + self._refresh_thread = None # Resolve service-discovery URLs before issuing any HTTP requests. Keep the # discovery object alive so each leader refresh can start from a fresh endpoint. self._service_discovery: Optional[ServiceDiscovery] = None resolved_url = base_url - if not self._is_http_url(base_url): - self._service_discovery = create_service_discovery(base_url) - if self._service_discovery is None: - self.session.close() - raise ValueError( - f"failed to create service discovery from manager address {base_url!r}" - ) - - try: + try: + if not self._is_http_url(base_url): + self._service_discovery = create_service_discovery(base_url) + if self._service_discovery is None: + raise ValueError( + f"failed to create service discovery from manager address {base_url!r}" + ) endpoint = self._service_discovery.get_one_endpoint() - except Exception as e: - self._close_service_discovery() - self.session.close() - raise RuntimeError( - f"failed to resolve manager endpoints from {base_url!r}" - ) from e - if endpoint is None: - self._close_service_discovery() - self.session.close() - raise RuntimeError( - f"service discovery returned no manager endpoints for {base_url!r}" - ) + if endpoint is None: + raise RuntimeError( + f"service discovery returned no manager endpoints for {base_url!r}" + ) - resolved_url = self._endpoint_url(endpoint) - logger.info( - "Service discovery (%s) resolved manager endpoint: %s", - self._service_discovery.get_type(), - resolved_url, - ) + resolved_url = self._endpoint_url(endpoint) + logger.info( + "Service discovery (%s) resolved manager endpoint: %s", + self._service_discovery.get_type(), + resolved_url, + ) + except BaseException: + self._rollback_construction() + raise self.base_url = resolved_url.rstrip('/') @@ -109,10 +120,7 @@ def __init__(self, base_url, *, instance_id="", auto_discover_leader=False, lead self._min_discover_interval = min_discover_interval_seconds self._route_lock = threading.Lock() - self._refresh_event = threading.Event() - self._closed = threading.Event() self._last_route_refresh_time = 0.0 # time.monotonic() - self._refresh_thread = None if self._auto_discover_leader: try: @@ -124,10 +132,16 @@ def __init__(self, base_url, *, instance_id="", auto_discover_leader=False, lead # One route-refresh thread serves both modes. Leader discovery wakes # periodically; service-discovery-only mode waits for transport failures. if self._auto_discover_leader or self._service_discovery is not None: - self._refresh_thread = threading.Thread( - target=self._route_refresh_loop, daemon=True, - name="kvcm-route-refresh") - self._refresh_thread.start() + try: + self._refresh_thread = threading.Thread( + target=self._route_refresh_loop, daemon=True, + name="kvcm-route-refresh") + self._refresh_thread.start() + except BaseException: + # The caller cannot close an object whose constructor failed. + # Roll back both resources even when OS thread admission fails. + self._rollback_construction() + raise @staticmethod def _is_http_url(url): @@ -239,29 +253,32 @@ def _resolve_discovery_url(self, force_refresh=False): def _route_refresh_loop(self): """Background daemon for periodic leader and event-driven route refresh.""" - while not self._closed.is_set(): - timeout = ( - self._discovery_refresh_interval - if self._auto_discover_leader - else None - ) - refresh_requested = self._refresh_event.wait(timeout=timeout) - self._refresh_event.clear() - if self._closed.is_set(): - break - # Min interval protection: wait remaining time instead of skipping - remaining = self._min_discover_interval - ( - time.monotonic() - self._last_route_refresh_time - ) - if remaining > 0: - if self._closed.wait(timeout=remaining): + try: + while not self._closed.is_set(): + timeout = ( + self._discovery_refresh_interval + if self._auto_discover_leader + else None + ) + refresh_requested = self._refresh_event.wait(timeout=timeout) + self._refresh_event.clear() + if self._closed.is_set(): break - try: - self._refresh_manager_route( - force_service_refresh=refresh_requested, + # Min interval protection: wait remaining time instead of skipping + remaining = self._min_discover_interval - ( + time.monotonic() - self._last_route_refresh_time ) - except Exception as e: - logger.warning("Background manager route refresh failed: %s", e) + if remaining > 0: + if self._closed.wait(timeout=remaining): + break + try: + self._refresh_manager_route( + force_service_refresh=refresh_requested, + ) + except Exception as e: + logger.warning("Background manager route refresh failed: %s", e) + finally: + self._close_deferred_service_discovery() def _make_request(self, method, endpoint, data=None): """Helper method to make HTTP requests to the service""" @@ -286,17 +303,34 @@ def _make_request(self, method, endpoint, data=None): return response - def _check_response(self, endpoint, response, response_data): - """Validate API response, raise AssertionError on failure.""" + def _check_response(self, endpoint, response, response_data, + check_business_status=True): + """Validate transport/envelope and optionally the Manager status.""" if response.status_code != 200: - raise AssertionError(f"Request to {endpoint} failed with status code {response.status_code}") + raise KvCacheManagerHTTPError( + f"Request to {endpoint} failed with status code {response.status_code}", + response=response, + ) - if 'header' not in response_data: - raise AssertionError(f"Response from {endpoint} missing 'header' field") + if not isinstance(response_data, dict): + raise KvCacheManagerProtocolError( + f"Response from {endpoint} is not a JSON object" + ) + header = response_data.get('header') + if not isinstance(header, dict): + raise KvCacheManagerProtocolError( + f"Response from {endpoint} missing a valid 'header' field" + ) + status = header.get('status') + if not isinstance(status, dict) or not status.get('code'): + raise KvCacheManagerProtocolError( + f"Response from {endpoint} missing a valid 'header.status' field" + ) - if response_data['header']['status']['code'] != "OK": + if check_business_status and status['code'] != "OK": raise AssertionError( - f"Request to {endpoint} failed with error: {response_data['header']['status']['message']}") + f"Request to {endpoint} failed with error: " + f"{status.get('message', '')}") def _make_api_request(self, endpoint, data=None, check_response=True): """Helper method to make POST requests to API endpoints and optionally validate response""" @@ -320,6 +354,16 @@ def _make_api_request(self, endpoint, data=None, check_response=True): response_data = response.json() + # Validate transport and the common envelope before inspecting the + # status for leader routing. This remains mandatory even when callers + # ask to receive a non-OK business status verbatim. + self._check_response( + endpoint, + response, + response_data, + check_business_status=False, + ) + # SERVER_NOT_LEADER handling: rediscover leader and retry with backoff if self._auto_discover_leader and self._get_status_code(response_data) == 'SERVER_NOT_LEADER': if retries_left > 0: @@ -389,11 +433,75 @@ def get_cluster_info(self, data, check_response=True): """Get cluster info including leader endpoint (leader discovery API)""" return self._make_api_request('/api/getClusterInfo', data, check_response) + def _close_session_once(self): + with self._resource_lock: + if self._session_close_claimed: + return + self._session_close_claimed = True + self.session.close() + + def _rollback_construction(self): + try: + self.close() + except BaseException: + logger.error( + "Failed to release a partially constructed manager client", + exc_info=True, + ) + + def _request_service_discovery_close(self, refresh_thread): + discovery = None + with self._resource_lock: + if ( + self._service_discovery_close_claimed + or self._service_discovery_close_deferred + ): + return + if ( + refresh_thread is not None + and refresh_thread.is_alive() + and not self._refresh_worker_cleanup_reached + ): + self._service_discovery_close_deferred = True + return + self._service_discovery_close_claimed = True + discovery = self._service_discovery + self._service_discovery = None + if discovery is not None: + discovery.close() + + def _close_deferred_service_discovery(self): + discovery = None + with self._resource_lock: + self._refresh_worker_cleanup_reached = True + if ( + not self._service_discovery_close_deferred + or self._service_discovery_close_claimed + ): + return + self._service_discovery_close_deferred = False + self._service_discovery_close_claimed = True + discovery = self._service_discovery + self._service_discovery = None + if discovery is not None: + try: + discovery.close() + except Exception: + logger.error( + "Failed to close service discovery after route-refresh exit", + exc_info=True, + ) + def close(self): """Close the HTTP session, discovery client, and background refresh thread.""" self._closed.set() self._refresh_event.set() - if self._refresh_thread and self._refresh_thread.is_alive(): - self._refresh_thread.join(timeout=5) - self.session.close() - self._close_service_discovery() + refresh_thread = self._refresh_thread + if ( + refresh_thread is not None + and refresh_thread is not threading.current_thread() + and refresh_thread.is_alive() + ): + refresh_thread.join(timeout=5) + self._close_session_once() + self._request_service_discovery_close(refresh_thread) diff --git a/kv_cache_manager/py_connector/test/test_manager_client.py b/kv_cache_manager/py_connector/test/test_manager_client.py index 0cfa05705..981e8bc98 100644 --- a/kv_cache_manager/py_connector/test/test_manager_client.py +++ b/kv_cache_manager/py_connector/test/test_manager_client.py @@ -7,7 +7,10 @@ import requests -from kv_cache_manager.py_connector.common.manager_client import KvCacheManagerClient +from kv_cache_manager.py_connector.common.manager_client import ( + KvCacheManagerClient, + KvCacheManagerProtocolError, +) from kv_cache_manager.py_connector.common.service_discovery import ServiceEndpoint @@ -98,6 +101,50 @@ def test_unavailable_discovery_implementation_fails_fast(self, mock_create): with self.assertRaisesRegex(ValueError, "failed to create service discovery"): KvCacheManagerClient("custom://manager-service") + @patch( + "kv_cache_manager.py_connector.common.manager_client.create_service_discovery" + ) + @patch("kv_cache_manager.py_connector.common.manager_client.requests.Session") + def test_discovery_factory_exception_closes_session( + self, + mock_session_cls, + mock_create, + ): + session = MagicMock() + mock_session_cls.return_value = session + mock_create.side_effect = RuntimeError("discovery plugin failed") + + with self.assertRaisesRegex(RuntimeError, "discovery plugin failed"): + KvCacheManagerClient("custom://manager-service") + + session.close.assert_called_once_with() + + @patch( + "kv_cache_manager.py_connector.common.manager_client.create_service_discovery" + ) + @patch("kv_cache_manager.py_connector.common.manager_client.requests.Session") + def test_discovery_post_resolution_exception_rolls_back_all_resources( + self, + mock_session_cls, + mock_create, + ): + session = MagicMock() + mock_session_cls.return_value = session + discovery = MagicMock() + discovery.get_one_endpoint.return_value = ServiceEndpoint( + ip="10.0.0.1", + port=8080, + host="10.0.0.1:8080", + ) + discovery.get_type.side_effect = RuntimeError("discovery metadata failed") + mock_create.return_value = discovery + + with self.assertRaisesRegex(RuntimeError, "discovery metadata failed"): + KvCacheManagerClient("custom://manager-service") + + session.close.assert_called_once_with() + discovery.close.assert_called_once_with() + @patch( "kv_cache_manager.py_connector.common.manager_client.create_service_discovery" ) @@ -226,6 +273,65 @@ def test_leader_discovery_uses_dedicated_timeout(self, mock_post): client.close() +class TestResponseClassification(unittest.TestCase): + """Transport/protocol failures must be distinct from Manager business errors.""" + + def setUp(self): + self.client = KvCacheManagerClient( + "http://10.0.0.1:8080", + auto_discover_leader=False, + ) + + def tearDown(self): + self.client.close() + + def test_non_200_response_is_transport_failure(self): + self.client.session.post = MagicMock( + return_value=_make_mock_response({}, status_code=503) + ) + + with self.assertRaises(requests.HTTPError): + self.client.register_instance({"trace_id": "test"}) + + def test_malformed_api_envelope_is_protocol_failure(self): + for payload in ([], {}, {"header": {}}, {"header": {"status": {}}}): + with self.subTest(payload=payload): + self.client.session.post = MagicMock( + return_value=_make_mock_response(payload) + ) + with self.assertRaises(KvCacheManagerProtocolError): + self.client.register_instance({"trace_id": "test"}) + + def test_manager_business_error_keeps_assertion_contract(self): + self.client.session.post = MagicMock( + return_value=_make_mock_response({ + "header": { + "status": { + "code": "INVALID_ARGUMENT", + "message": "bad request", + } + } + }) + ) + + with self.assertRaises(AssertionError) as caught: + self.client.register_instance({"trace_id": "test"}) + self.assertNotIsInstance(caught.exception, requests.RequestException) + + def test_check_response_false_still_validates_transport_and_envelope(self): + for response, error_type in ( + (_make_mock_response({}, status_code=503), requests.HTTPError), + (_make_mock_response([]), KvCacheManagerProtocolError), + ): + with self.subTest(error_type=error_type.__name__): + self.client.session.post = MagicMock(return_value=response) + with self.assertRaises(error_type): + self.client.report_event( + {"trace_id": "test"}, + check_response=False, + ) + + class TestLeaderDiscoveryInit(unittest.TestCase): """Tests for leader discovery during __init__.""" @@ -679,6 +785,73 @@ def test_close_without_discovery(self): client.close() # Should not raise self.assertTrue(client._closed.is_set()) + @patch( + "kv_cache_manager.py_connector.common.manager_client.create_service_discovery" + ) + @patch("kv_cache_manager.py_connector.common.manager_client.requests.Session") + def test_refresh_thread_start_failure_rolls_back_resources( + self, + mock_session_cls, + mock_create_discovery, + ): + session = MagicMock() + mock_session_cls.return_value = session + discovery = MagicMock() + discovery.get_one_endpoint.return_value = ServiceEndpoint( + ip="10.0.0.1", + port=8080, + host="10.0.0.1:8080", + ) + discovery.get_type.return_value = "Test" + mock_create_discovery.return_value = discovery + + with patch( + "kv_cache_manager.py_connector.common.manager_client.threading.Thread.start", + side_effect=RuntimeError("thread admission failed"), + ): + with self.assertRaisesRegex(RuntimeError, "thread admission failed"): + KvCacheManagerClient( + "custom://manager-service", + auto_discover_leader=False, + ) + + session.close.assert_called_once_with() + discovery.close.assert_called_once_with() + + def test_close_defers_discovery_cleanup_until_refresh_worker_exits(self): + client = KvCacheManagerClient( + "http://10.0.0.1:8080", + auto_discover_leader=False, + ) + discovery = MagicMock() + client._service_discovery = discovery + worker_entered = threading.Event() + release_worker = threading.Event() + + def blocked_refresh_worker(): + try: + worker_entered.set() + release_worker.wait(timeout=5) + finally: + client._close_deferred_service_discovery() + + refresh_thread = threading.Thread(target=blocked_refresh_worker) + client._refresh_thread = refresh_thread + refresh_thread.start() + self.assertTrue(worker_entered.wait(timeout=1)) + + with patch.object(refresh_thread, "join", return_value=None): + client.close() + + discovery.close.assert_not_called() + release_worker.set() + refresh_thread.join(timeout=1) + self.assertFalse(refresh_thread.is_alive()) + discovery.close.assert_called_once_with() + + client.close() + discovery.close.assert_called_once_with() + class TestThreadSafety(unittest.TestCase): """Tests for thread-safe Manager route refresh dedup.""" diff --git a/kv_cache_manager/service/grpc_service/meta_service_grpc.cc b/kv_cache_manager/service/grpc_service/meta_service_grpc.cc index fa1ba43cd..4d9fb68cd 100644 --- a/kv_cache_manager/service/grpc_service/meta_service_grpc.cc +++ b/kv_cache_manager/service/grpc_service/meta_service_grpc.cc @@ -116,28 +116,7 @@ grpc::Status MetaServiceGRpc::ReportEvent(grpc::ServerContext *context, const proto::meta::ReportEventRequest *request, proto::meta::ReportEventResponse *response) { std::string metrics_type; - std::shared_ptr metrics_collector; - switch (request->storage_type()) { - case proto::meta::ST_EVENT_REPORT_L1P5: - metrics_type = kEventReportL1P5MetricsType; - metrics_collector = GetTypedMetricsCollectorForReportEvent(request->instance_id(), metrics_type); - break; - case proto::meta::ST_EVENT_REPORT_L2: - metrics_type = kEventReportL2MetricsType; - metrics_collector = GetTypedMetricsCollectorForReportEvent(request->instance_id(), metrics_type); - break; - default: - metrics_collector = get_metrics_collector_from_map_for_ReportEvent(request->instance_id()); - break; - } - if (metrics_collector == nullptr) { - KVCM_LOG_ERROR("get ReportEvent metrics collector failed"); - auto *header = response->mutable_header(); - auto *status = header->mutable_status(); - status->set_code(proto::meta::INSTANCE_NOT_EXIST); - status->set_message("get ReportEvent metrics collector failed"); - return grpc::Status::OK; - } + auto metrics_collector = ResolveReportEventMetricsCollector(*request, metrics_type); API_CONTEXT_INIT(metrics_collector, ExtractIpFromPeer, context->peer()) AttachReportEventTypeMetricsCollectors(*request, metrics_type, request_context); meta_service_impl_->ReportEvent(request_context, request, response); diff --git a/kv_cache_manager/service/http_service/BUILD b/kv_cache_manager/service/http_service/BUILD index fea794d67..632957b32 100644 --- a/kv_cache_manager/service/http_service/BUILD +++ b/kv_cache_manager/service/http_service/BUILD @@ -36,6 +36,7 @@ cc_library( "//kv_cache_manager/service:meta_service_impl", "//kv_cache_manager/service:meta_service_metrics_base", "//kv_cache_manager/service/util:proto_message_json_util", + "//kv_cache_manager/service/util:report_event_json_parser", "@com_google_protobuf//:protobuf", "@com_google_protobuf//:protobuf_headers", "@yalantinglibs//:ylt", diff --git a/kv_cache_manager/service/http_service/coro_http_service.h b/kv_cache_manager/service/http_service/coro_http_service.h index 03301c301..058b58484 100644 --- a/kv_cache_manager/service/http_service/coro_http_service.h +++ b/kv_cache_manager/service/http_service/coro_http_service.h @@ -3,10 +3,12 @@ #include #include #include +#include #include #include #include +#include "google/protobuf/arena.h" #include "google/protobuf/message.h" #include "kv_cache_manager/service/util/proto_message_json_util.h" #include "ylt/coro_http/coro_http_server.hpp" @@ -43,6 +45,11 @@ class CoroHttpService { HandlerType GetHandler( std::function>( ServiceType *, coro_http::coro_http_connection *, PbRequestMessage *, PbResponseMessage *)> callback); + template + HandlerType GetArenaHandler( + std::function>( + ServiceType *, coro_http::coro_http_connection *, PbRequestMessage *, PbResponseMessage *)> callback, + bool (*request_parser)(char *, size_t, PbRequestMessage *) = nullptr); private: std::unordered_map get_handlers_{}; @@ -61,7 +68,7 @@ CoroHttpService::HandlerType CoroHttpService::GetHandler( std::string json_res; - if (!ProtoMessageJsonUtil::FromJson(std::string(req.get_body()), &pb_req)) { + if (!ProtoMessageJsonUtil::FromJson(req.get_body(), &pb_req)) { json_res = "{}"; res.set_status_and_content(coro_http::status_type::bad_request, json_res); co_return; @@ -81,4 +88,60 @@ CoroHttpService::HandlerType CoroHttpService::GetHandler( }; } +template +CoroHttpService::HandlerType CoroHttpService::GetArenaHandler( + std::function>( + ServiceType *, coro_http::coro_http_connection *, PbRequestMessage *, PbResponseMessage *)> callback, + bool (*request_parser)(char *, size_t, PbRequestMessage *)) { + return [this, callback, request_parser](coro_http::coro_http_request &req, + coro_http::coro_http_response &res) -> async_simple::coro::Lazy { + // Large ReportEvent requests contain tens of thousands of nested + // protobuf messages. Keeping them on a request-scoped arena avoids a + // separate malloc/free for every EventItem/spec and releases all + // parser-owned objects in one pass when the synchronous handler + // returns. Neither message escapes this coroutine. + const std::string_view body = req.get_body(); + google::protobuf::ArenaOptions arena_options; + if (body.size() >= 32 * 1024) { + // Protobuf 3.13 defaults to 256-byte/8-KiB arena blocks. A multi-MiB + // ReportEvent would otherwise allocate and free hundreds of tiny + // blocks. Keep small heartbeats on the defaults while allowing + // large requests to grow geometrically to 1 MiB blocks. + arena_options.start_block_size = 64 * 1024; + arena_options.max_block_size = 1024 * 1024; + } + google::protobuf::Arena arena(arena_options); + auto *pb_req = google::protobuf::Arena::CreateMessage(&arena); + auto *pb_res = google::protobuf::Arena::CreateMessage(&arena); + std::string json_res; + + // cinatra 0.5.5 backs get_body() with the connection's mutable + // std::string. The handler is synchronous with respect to that body, + // and neither the request nor a DOM view escapes this coroutine, so a + // specialized parser may safely decode it in place. Generic handlers + // continue to receive the immutable length-aware view. + const bool parsed = + request_parser + ? request_parser(body.empty() ? nullptr : const_cast(body.data()), body.size(), pb_req) + : ProtoMessageJsonUtil::FromJson(body, pb_req); + if (!parsed) { + json_res = "{}"; + res.set_status_and_content(coro_http::status_type::bad_request, json_res); + co_return; + } + + callback(static_cast(this), req.get_conn(), pb_req, pb_res); + + json_res.reserve(512); + if (!ProtoMessageJsonUtil::ToJson(pb_res, json_res)) { + json_res = "{}"; + res.set_status_and_content(coro_http::status_type::internal_server_error, json_res); + co_return; + } + res.add_header("Content-Type", "application/json"); + res.set_status_and_content(coro_http::status_type::ok, json_res); + co_return; + }; +} + } // namespace kv_cache_manager diff --git a/kv_cache_manager/service/http_service/meta_service_http.cc b/kv_cache_manager/service/http_service/meta_service_http.cc index b48662ed6..8d45efeab 100644 --- a/kv_cache_manager/service/http_service/meta_service_http.cc +++ b/kv_cache_manager/service/http_service/meta_service_http.cc @@ -10,6 +10,7 @@ #include "kv_cache_manager/protocol/protobuf/meta_service.pb.h" #include "kv_cache_manager/service/meta_service_impl.h" #include "kv_cache_manager/service/util/common.h" +#include "kv_cache_manager/service/util/report_event_json_parser.h" namespace kv_cache_manager { @@ -43,7 +44,10 @@ void MetaServiceHttp::RegisterHandler() { REGISTER_HTTP_HANDLER_FOR_META_SERVICE(Post, removeCache, RemoveCache, Common, RemoveCache); REGISTER_HTTP_HANDLER_FOR_META_SERVICE(Post, trimCache, TrimCache, Common, TrimCache); REGISTER_HTTP_HANDLER_FOR_META_SERVICE(Post, getClusterInfo, GetClusterInfo, GetClusterInfo, GetClusterInfo); - REGISTER_HTTP_HANDLER_FOR_META_SERVICE(Post, reportEvent, ReportEvent, ReportEvent, ReportEvent); + RegisterPostHandler( + "/api/reportEvent", + GetArenaHandler( + &MetaServiceHttp::ReportEvent, &ReportEventJsonParser::FromMutableNullTerminatedJson)); REGISTER_HTTP_HANDLER_FOR_META_SERVICE( Post, getHostCacheState, GetHostCacheState, GetHostCacheState, GetHostCacheState); } @@ -179,28 +183,7 @@ void MetaServiceHttp::ReportEvent(coro_http::coro_http_connection *http_conn, proto::meta::ReportEventRequest *request, proto::meta::ReportEventResponse *response) { std::string metrics_type; - std::shared_ptr metrics_collector; - switch (request->storage_type()) { - case proto::meta::ST_EVENT_REPORT_L1P5: - metrics_type = kEventReportL1P5MetricsType; - metrics_collector = GetTypedMetricsCollectorForReportEvent(request->instance_id(), metrics_type); - break; - case proto::meta::ST_EVENT_REPORT_L2: - metrics_type = kEventReportL2MetricsType; - metrics_collector = GetTypedMetricsCollectorForReportEvent(request->instance_id(), metrics_type); - break; - default: - metrics_collector = get_metrics_collector_from_map_for_ReportEvent(request->instance_id()); - break; - } - if (metrics_collector == nullptr) { - KVCM_LOG_ERROR("get ReportEvent metrics collector failed"); - auto *header = response->mutable_header(); - auto *status = header->mutable_status(); - status->set_code(proto::meta::INSTANCE_NOT_EXIST); - status->set_message("get ReportEvent metrics collector failed"); - return; - } + auto metrics_collector = ResolveReportEventMetricsCollector(*request, metrics_type); API_CONTEXT_INIT(metrics_collector, GetHttpClientIp, http_conn) std::string first_event_type = "N/A"; std::string first_block_key = "N/A"; @@ -217,14 +200,14 @@ void MetaServiceHttp::ReportEvent(coro_http::coro_http_connection *http_conn, } } } - KVCM_LOG_INFO("[traceId: %s] ReportEvent called, instance_id: %s, host_ip_port: %s, event_count: %d, " - "first_event_type: %s, first_block_key: %s", - request->trace_id().c_str(), - request->instance_id().c_str(), - request->host_ip_port().c_str(), - request->events_size(), - first_event_type.c_str(), - first_block_key.c_str()); + KVCM_LOG_DEBUG("[traceId: %s] ReportEvent called, instance_id: %s, host_ip_port: %s, event_count: %d, " + "first_event_type: %s, first_block_key: %s", + request->trace_id().c_str(), + request->instance_id().c_str(), + request->host_ip_port().c_str(), + request->events_size(), + first_event_type.c_str(), + first_block_key.c_str()); AttachReportEventTypeMetricsCollectors(*request, metrics_type, request_context); meta_service_impl_->ReportEvent(request_context, request, response); } @@ -233,10 +216,10 @@ void MetaServiceHttp::GetHostCacheState(coro_http::coro_http_connection *http_co proto::meta::GetHostCacheStateRequest *request, proto::meta::GetHostCacheStateResponse *response) { API_CONTEXT_GET_COLLECTOR_AND_INIT_HTTP(GetHostCacheState, __NOTHING__); - KVCM_LOG_INFO("[traceId: %s] GetHostCacheState called, instance_id: %s, block_cache_keys_count: %d", - request->trace_id().c_str(), - request->instance_id().c_str(), - request->block_cache_keys_size()); + KVCM_LOG_DEBUG("[traceId: %s] GetHostCacheState called, instance_id: %s, block_cache_keys_count: %d", + request->trace_id().c_str(), + request->instance_id().c_str(), + request->block_cache_keys_size()); meta_service_impl_->GetHostCacheState(request_context, request, response); } diff --git a/kv_cache_manager/service/meta_service_impl.cc b/kv_cache_manager/service/meta_service_impl.cc index 7ce2c957b..0865c79ac 100644 --- a/kv_cache_manager/service/meta_service_impl.cc +++ b/kv_cache_manager/service/meta_service_impl.cc @@ -1,5 +1,6 @@ #include "kv_cache_manager/service/meta_service_impl.h" +#include #include #include #include @@ -111,6 +112,7 @@ namespace kv_cache_manager { namespace { constexpr const char *kReportEventFullAccessLogEnv = "KVCM_REPORT_EVENT_FULL_ACCESS_LOG"; +constexpr const char *kGetHostCacheStateFullAccessLogEnv = "KVCM_GET_HOST_CACHE_STATE_FULL_ACCESS_LOG"; std::string BuildProtoMessageDebugJson(const google::protobuf::Message *message) { std::string debug_json; @@ -120,6 +122,8 @@ std::string BuildProtoMessageDebugJson(const google::protobuf::Message *message) bool IsReportEventFullAccessLogEnabled() { return EnvUtil::GetEnv(kReportEventFullAccessLogEnv, false); } +bool IsGetHostCacheStateFullAccessLogEnabled() { return EnvUtil::GetEnv(kGetHostCacheStateFullAccessLogEnv, false); } + const char *FirstBlockKeyFromEvent(const proto::meta::EventItem &event) { if (event.has_block_add()) { return event.block_add().block_key().c_str(); @@ -248,6 +252,59 @@ std::string BuildReportEventResponseAccessLogSummary(const proto::meta::ReportEv return sb.GetString(); } +std::string BuildGetHostCacheStateRequestAccessLogSummary(const proto::meta::GetHostCacheStateRequest *request) { + if (IsGetHostCacheStateFullAccessLogEnabled()) { + return BuildProtoMessageDebugJson(request); + } + + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + writer.StartObject(); + writer.Key("trace_id"); + writer.String(request->trace_id().c_str()); + writer.Key("instance_id"); + writer.String(request->instance_id().c_str()); + writer.Key("query_type"); + writer.String(proto::meta::QueryType_Name(request->query_type()).c_str()); + writer.Key("key_count"); + writer.Int(request->block_cache_keys_size()); + if (request->block_cache_keys_size() > 0) { + writer.Key("first_key"); + writer.Int64(request->block_cache_keys(0)); + writer.Key("last_key"); + writer.Int64(request->block_cache_keys(request->block_cache_keys_size() - 1)); + } + writer.Key("medium_count"); + writer.Int(request->medium_size()); + writer.EndObject(); + return sb.GetString(); +} + +std::string BuildGetHostCacheStateResponseAccessLogSummary(const proto::meta::GetHostCacheStateResponse *response) { + if (IsGetHostCacheStateFullAccessLogEnabled()) { + return BuildProtoMessageDebugJson(response); + } + + int64_t max_prefix_match_blocks = 0; + for (const auto &host : response->hosts()) { + max_prefix_match_blocks = std::max(max_prefix_match_blocks, host.local()); + } + const auto &status = response->header().status(); + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + writer.StartObject(); + writer.Key("status_code"); + writer.String(proto::meta::ErrorCode_Name(status.code()).c_str()); + writer.Key("status_message"); + writer.String(status.message().c_str()); + writer.Key("returned_host_count"); + writer.Int(response->hosts_size()); + writer.Key("max_prefix_match_blocks"); + writer.Int64(max_prefix_match_blocks); + writer.EndObject(); + return sb.GetString(); +} + } // namespace MetaServiceImpl::MetaServiceImpl(std::shared_ptr cache_manager, @@ -487,8 +544,13 @@ void MetaServiceImpl::GetCacheLocationsByBackend(RequestContext *request_context std::vector backend_selectors; backend_selectors.reserve(request->backend_selectors_size()); for (const auto &sel : request->backend_selectors()) { + DataStorageType backend_type = DataStorageType::DATA_STORAGE_TYPE_UNKNOWN; + // StorageType is an open protobuf enum while DataStorageType uses an + // uint8_t underlying type. Explicit conversion prevents an unknown + // wire value such as 263 from truncating to L1P5 (7). + ProtoConvert::DataStorageTypeFromProto(sel.backend_type(), backend_type); backend_selectors.push_back({ - static_cast(sel.backend_type()), + backend_type, static_cast(sel.strategy()), }); } @@ -518,9 +580,9 @@ void MetaServiceImpl::GetCacheLocationsByBackend(RequestContext *request_context status->set_code(proto::meta::OK); request_context->set_status_code(status->code()); status->set_message("Batch cache locations retrieved successfully"); - KVCM_LOG_INFO("[traceId: %s] GetCacheLocationsByBackend succeeded, returned %zu keys", - request->trace_id().c_str(), - batch_result.size()); + KVCM_LOG_DEBUG("[traceId: %s] GetCacheLocationsByBackend succeeded, returned %zu keys", + request->trace_id().c_str(), + batch_result.size()); } SET_SPAN_TRACER_STR_IN_HEADER(request_context); } @@ -877,11 +939,11 @@ void MetaServiceImpl::ReportEvent(RequestContext *request_context, BuildReportEventResponseAccessLogSummary(response)); auto *header = response->mutable_header(); - KVCM_LOG_INFO("[traceId: %s] ReportEvent called, instance_id: %s, host_ip_port: %s, event_count: %d", - request->trace_id().c_str(), - request->instance_id().c_str(), - request->host_ip_port().c_str(), - request->events_size()); + KVCM_LOG_DEBUG("[traceId: %s] ReportEvent called, instance_id: %s, host_ip_port: %s, event_count: %d", + request->trace_id().c_str(), + request->instance_id().c_str(), + request->host_ip_port().c_str(), + request->events_size()); auto ec = cache_manager_->ReportEvent(request_context, request, response); // Partial failures are logged once with bounded per-type/error counts by @@ -902,7 +964,10 @@ void MetaServiceImpl::GetHostCacheState(RequestContext *request_context, const proto::meta::GetHostCacheStateRequest *request, proto::meta::GetHostCacheStateResponse *response) { SPAN_TRACER(request_context); - API_CALL_GUARD("GetHostCacheState", true); + API_CALL_GUARD_WITH_DEBUG("GetHostCacheState", + true, + BuildGetHostCacheStateRequestAccessLogSummary(request), + BuildGetHostCacheStateResponseAccessLogSummary(response)); auto *header = response->mutable_header(); auto *status = header->mutable_status(); std::string invalid_fields = "missing or invalid fields: "; @@ -916,21 +981,27 @@ void MetaServiceImpl::GetHostCacheState(RequestContext *request_context, SET_SPAN_TRACER_STR_IN_HEADER(request_context); return; } + if (request->p2p_host_count() < 0) { + CHECK_REQUIRED_FIELDS_VALIDATION("GetHostCacheState", "p2p_host_count (must be >= 0)", true); + SET_SPAN_TRACER_STR_IN_HEADER(request_context); + return; + } CacheManager::KeyVector keys(request->block_cache_keys().begin(), request->block_cache_keys().end()); std::vector mediums(request->medium().begin(), request->medium().end()); - KVCM_LOG_INFO("[traceId: %s] GetHostCacheState called, instance_id: %s, block_cache_keys_count: %d", - request->trace_id().c_str(), - request->instance_id().c_str(), - request->block_cache_keys_size()); + KVCM_LOG_DEBUG("[traceId: %s] GetHostCacheState called, instance_id: %s, block_cache_keys_count: %d", + request->trace_id().c_str(), + request->instance_id().c_str(), + request->block_cache_keys_size()); auto [ec, host_matches] = cache_manager_->GetHostCacheState(request_context, request->instance_id(), static_cast(request->query_type()), keys, - mediums); + mediums, + static_cast(request->p2p_host_count())); if (ec != EC_OK) { status->set_code(ToMetaPbError(ec)); request_context->set_status_code(status->code()); @@ -940,14 +1011,16 @@ void MetaServiceImpl::GetHostCacheState(RequestContext *request_context, for (const auto &match : host_matches) { auto *host_match = response->add_hosts(); host_match->set_host_ip_port(match.host_ip_port); - host_match->set_prefix_match_blocks(match.prefix_match_blocks); + host_match->set_local(match.local); + host_match->set_p2p_1_fetch(match.p2p_1_fetch); + host_match->set_p2p_1_total_match(match.p2p_1_total_match); } status->set_code(proto::meta::OK); request_context->set_status_code(status->code()); status->set_message("Host cache state retrieved successfully"); - KVCM_LOG_INFO("[traceId: %s] GetHostCacheState succeeded, returned %d hosts", - request->trace_id().c_str(), - response->hosts_size()); + KVCM_LOG_DEBUG("[traceId: %s] GetHostCacheState succeeded, returned %d hosts", + request->trace_id().c_str(), + response->hosts_size()); } SET_SPAN_TRACER_STR_IN_HEADER(request_context); } diff --git a/kv_cache_manager/service/meta_service_metrics_base.cc b/kv_cache_manager/service/meta_service_metrics_base.cc index 1975ca88f..54a52c4c4 100644 --- a/kv_cache_manager/service/meta_service_metrics_base.cc +++ b/kv_cache_manager/service/meta_service_metrics_base.cc @@ -2,6 +2,7 @@ #include #include +#include #include "kv_cache_manager/common/request_context.h" #include "kv_cache_manager/config/registry_manager.h" @@ -46,6 +47,7 @@ void MetaServiceMetricsBase::InitMetrics() { MAKE_SERVICE_METRICS_COLLECTOR(RegisterInstance); MAKE_SERVICE_METRICS_COLLECTOR(GetInstanceInfo); MAKE_SERVICE_METRICS_COLLECTOR(GetClusterInfo); + MAKE_SERVICE_METRICS_COLLECTOR(ReportEvent); // GetClusterInfo 的全局 collector 也预置到 MAP 中，以空 instance_id 为 key KVCM_METRICS_COLLECTOR_MAP_(GetClusterInfo)[""] = KVCM_METRICS_COLLECTOR_(GetClusterInfo); } @@ -160,8 +162,11 @@ std::shared_ptr MetaServiceMetricsBase::GetEventTypeMetricsCol if (instance_group.empty()) { return nullptr; } - MetricsTags tags = { - {"instance_group", instance_group}, {"instance_id", instance_id}, {"type", type}, {"event_type", event_type}}; + MetricsTags tags = {{"api_name", "ReportEvent"}, + {"instance_group", instance_group}, + {"instance_id", instance_id}, + {"type", type}, + {"event_type", event_type}}; auto collector = std::make_shared(metrics_registry_, std::move(tags)); if (!collector->Init()) { return nullptr; @@ -175,6 +180,27 @@ std::shared_ptr MetaServiceMetricsBase::GetTypedMetricsCollect return GetEventTypeMetricsCollectorFromMap(instance_id, type, event_type); } +std::shared_ptr +MetaServiceMetricsBase::ResolveReportEventMetricsCollector(const proto::meta::ReportEventRequest &request, + std::string &out_metrics_type) { + out_metrics_type.clear(); + std::shared_ptr collector; + switch (request.storage_type()) { + case proto::meta::ST_EVENT_REPORT_L1P5: + out_metrics_type = kEventReportL1P5MetricsType; + collector = GetTypedMetricsCollectorForReportEvent(request.instance_id(), out_metrics_type); + break; + case proto::meta::ST_EVENT_REPORT_L2: + out_metrics_type = kEventReportL2MetricsType; + collector = GetTypedMetricsCollectorForReportEvent(request.instance_id(), out_metrics_type); + break; + default: + collector = get_metrics_collector_from_map_for_ReportEvent(request.instance_id()); + break; + } + return collector ? std::move(collector) : KVCM_METRICS_COLLECTOR_(ReportEvent); +} + void MetaServiceMetricsBase::AttachReportEventTypeMetricsCollectors(const proto::meta::ReportEventRequest &request, const std::string &type, RequestContext *request_context) { @@ -182,21 +208,45 @@ void MetaServiceMetricsBase::AttachReportEventTypeMetricsCollectors(const proto: return; } + static constexpr std::array kEventTypeTags = { + "unknown", "node_register", "block_add", "block_delete", "host_down", "heartbeat", "block_snapshot"}; + static constexpr std::array kEnableEventTypeMetrics = { + false, false, true, true, false, false, true}; uint32_t event_type_mask = 0; + std::array request_key_counts{}; for (const auto &event : request.events()) { const int event_type = static_cast(event.event_type()); - event_type_mask |= - 1U << ((event_type >= proto::meta::EVENT_NODE_REGISTER && event_type <= proto::meta::EVENT_BLOCK_SNAPSHOT) - ? event_type - : 0); + const int bounded_event_type = + (event_type >= proto::meta::EVENT_NODE_REGISTER && event_type <= proto::meta::EVENT_BLOCK_SNAPSHOT) + ? event_type + : 0; + event_type_mask |= 1U << bounded_event_type; + // Match request_key_count semantics used by the other manager APIs: + // count keys in the request payload, regardless of later validation, + // deduplication, or persistence outcomes. + switch (bounded_event_type) { + case proto::meta::EVENT_BLOCK_ADD: + request_key_counts[bounded_event_type] += event.has_block_add() ? 1 : 0; + break; + case proto::meta::EVENT_BLOCK_DELETE: + request_key_counts[bounded_event_type] += event.has_block_delete() ? 1 : 0; + break; + case proto::meta::EVENT_BLOCK_SNAPSHOT: + request_key_counts[bounded_event_type] += + event.has_block_snapshot() ? static_cast(event.block_snapshot().blocks_size()) : 0; + break; + default: + break; + } } - static constexpr std::array kEventTypeTags = { - "unknown", "node_register", "block_add", "block_delete", "host_down", "heartbeat", "block_snapshot"}; for (size_t event_type = 0; event_type < kEventTypeTags.size(); ++event_type) { if ((event_type_mask & (1U << event_type)) == 0) { continue; } + if (!kEnableEventTypeMetrics[event_type]) { + continue; + } auto shared_collector = GetTypedMetricsCollectorForReportEventType(request.instance_id(), type, kEventTypeTags[event_type]); auto event_collector = std::dynamic_pointer_cast(shared_collector); @@ -204,8 +254,9 @@ void MetaServiceMetricsBase::AttachReportEventTypeMetricsCollectors(const proto: // The cached object owns the registry handles. Each request gets a // lightweight view with the same handles but private sample state, // avoiding both registry re-registration and cross-request races. - request_context->GetMetricsCollectorsVehicle().AddMetricsCollector( - std::make_shared(*event_collector)); + auto request_collector = std::make_shared(*event_collector); + request_collector->SetRequestKeyCountSample(request_key_counts[event_type]); + request_context->GetMetricsCollectorsVehicle().AddMetricsCollector(std::move(request_collector)); } } } diff --git a/kv_cache_manager/service/meta_service_metrics_base.h b/kv_cache_manager/service/meta_service_metrics_base.h index dce9eb9f4..72091baa8 100644 --- a/kv_cache_manager/service/meta_service_metrics_base.h +++ b/kv_cache_manager/service/meta_service_metrics_base.h @@ -81,6 +81,11 @@ class MetaServiceMetricsBase { std::shared_ptr GetTypedMetricsCollectorForReportEventType(const std::string &instance_id, const std::string &type, const std::string &event_type); + // Metrics lookup must never become a functional precondition for the API: + // malformed or unknown-instance requests still need to reach ReportEvent's + // canonical validation path and return its status. + std::shared_ptr ResolveReportEventMetricsCollector(const proto::meta::ReportEventRequest &request, + std::string &out_metrics_type); void AttachReportEventTypeMetricsCollectors(const proto::meta::ReportEventRequest &request, const std::string &type, RequestContext *request_context); @@ -91,6 +96,7 @@ class MetaServiceMetricsBase { KVCM_DECLARE_METRICS_COLLECTOR_(RegisterInstance); KVCM_DECLARE_METRICS_COLLECTOR_(GetInstanceInfo); KVCM_DECLARE_METRICS_COLLECTOR_(GetClusterInfo); + KVCM_DECLARE_METRICS_COLLECTOR_(ReportEvent); KVCM_DECLARE_METRICS_COLLECTOR_MAP_(GetCacheMeta); KVCM_DECLARE_METRICS_COLLECTOR_MAP_(GetCacheLocation); KVCM_DECLARE_METRICS_COLLECTOR_MAP_(GetCacheLocationsByBackend); diff --git a/kv_cache_manager/service/server.cc b/kv_cache_manager/service/server.cc index e003780d5..93a68ce97 100644 --- a/kv_cache_manager/service/server.cc +++ b/kv_cache_manager/service/server.cc @@ -82,6 +82,9 @@ bool Server::Init(const ServerConfig &config) { config_.GetCacheReclaimerWorkerSize(), async_delete_config, config_.GetSchedulePlanMigrationWorkerBudget(), + config_.GetMetaQueryWorkerCount(), + config_.GetMetaQueryParallelThreshold(), + config_.GetMetaQueryChunkSize(), cache_gc_config)) { KVCM_LOG_ERROR("cache manager init failed"); return false; diff --git a/kv_cache_manager/service/server_config.cc b/kv_cache_manager/service/server_config.cc index 2ce9d0509..a5e7f9d14 100644 --- a/kv_cache_manager/service/server_config.cc +++ b/kv_cache_manager/service/server_config.cc @@ -14,6 +14,22 @@ namespace kv_cache_manager { +namespace { + +bool ParseUint32Setting(const std::string &value, uint32_t &out) { + try { + std::size_t parsed_length = 0; + const auto parsed = std::stoull(value, &parsed_length); + if (parsed_length != value.size() || parsed > std::numeric_limits::max()) { + return false; + } + out = static_cast(parsed); + return true; + } catch (...) { return false; } +} + +} // namespace + std::vector ServerConfig::ParseRevisitIntervalBuckets(const std::string &buckets_str) { auto boundaries = StringUtil::ParseBucketBoundaries(buckets_str); if (!buckets_str.empty() && boundaries.empty()) { @@ -122,6 +138,18 @@ std::unordered_map ServerConfig::kSe return false; } }}, + {"kvcm.meta_query.worker_count", + [](const std::string &value, ServerConfig *config) { + return ParseUint32Setting(value, config->meta_query_worker_count_); + }}, + {"kvcm.meta_query.parallel_threshold", + [](const std::string &value, ServerConfig *config) { + return ParseUint32Setting(value, config->meta_query_parallel_threshold_); + }}, + {"kvcm.meta_query.chunk_size", + [](const std::string &value, ServerConfig *config) { + return ParseUint32Setting(value, config->meta_query_chunk_size_); + }}, {"kvcm.cache_reclaimer.key_sampling_size_total", [](const std::string &value, ServerConfig *config) { config->cache_reclaimer_key_sampling_size_total_ = std::stoull(value); @@ -271,6 +299,9 @@ void ServerConfig::UpdateDefaultConfig() { leader_elector_loop_interval_ms_ = 100; schedule_plan_executor_thread_count_ = 2; schedule_plan_migration_worker_budget_ = 1; + meta_query_worker_count_ = 4; + meta_query_parallel_threshold_ = 256; + meta_query_chunk_size_ = 128; cache_reclaimer_key_sampling_size_total_ = 1000; cache_reclaimer_key_sampling_size_per_task_ = 100; cache_reclaimer_del_batch_size_ = 100; @@ -424,6 +455,13 @@ bool ServerConfig::Check() { return false; } + if (meta_query_worker_count_ == 0 || meta_query_worker_count_ > 64 || meta_query_parallel_threshold_ == 0 || + meta_query_chunk_size_ == 0 || meta_query_chunk_size_ > meta_query_parallel_threshold_) { + fprintf(stderr, + "Meta query executor requires 1 <= worker_count <= 64 and 0 < chunk_size <= parallel_threshold\n"); + return false; + } + if (cache_gc_enabled_ && (cache_gc_scan_interval_ms_ <= 0 || cache_gc_round_pause_ms_ <= 0 || cache_gc_scan_batch_size_ == 0 || cache_gc_scan_batch_size_ > static_cast(std::numeric_limits::max()) || diff --git a/kv_cache_manager/service/server_config.h b/kv_cache_manager/service/server_config.h index 629d39017..6dfd1e2ee 100644 --- a/kv_cache_manager/service/server_config.h +++ b/kv_cache_manager/service/server_config.h @@ -34,6 +34,9 @@ class ServerConfig { const std::string &startup_config() { return startup_config_; } int32_t GetSchedulePlanExecutorThreadCount() { return schedule_plan_executor_thread_count_; } uint32_t GetSchedulePlanMigrationWorkerBudget() const { return schedule_plan_migration_worker_budget_; } + uint32_t GetMetaQueryWorkerCount() const { return meta_query_worker_count_; } + uint32_t GetMetaQueryParallelThreshold() const { return meta_query_parallel_threshold_; } + uint32_t GetMetaQueryChunkSize() const { return meta_query_chunk_size_; } uint64_t GetCacheReclaimerKeySamplingSizeTotal() { return cache_reclaimer_key_sampling_size_total_; } uint64_t GetCacheReclaimerKeySamplingSizePerTask() { return cache_reclaimer_key_sampling_size_per_task_; } uint64_t GetCacheReclaimerDelBatchSize() { return cache_reclaimer_del_batch_size_; } @@ -97,6 +100,9 @@ class ServerConfig { std::string startup_config_; int32_t schedule_plan_executor_thread_count_ = 0; uint32_t schedule_plan_migration_worker_budget_ = 0; + uint32_t meta_query_worker_count_ = 0; + uint32_t meta_query_parallel_threshold_ = 0; + uint32_t meta_query_chunk_size_ = 0; uint64_t cache_reclaimer_key_sampling_size_total_ = 0; uint64_t cache_reclaimer_key_sampling_size_per_task_ = 0; uint64_t cache_reclaimer_del_batch_size_ = 0; diff --git a/kv_cache_manager/service/test/meta_service_metrics_base_test.cc b/kv_cache_manager/service/test/meta_service_metrics_base_test.cc index a9c53d9a6..a03db562e 100644 --- a/kv_cache_manager/service/test/meta_service_metrics_base_test.cc +++ b/kv_cache_manager/service/test/meta_service_metrics_base_test.cc @@ -96,6 +96,24 @@ TEST_F(MetaServiceMetricsBaseTest, TypedReportEventCollectorUsesTypeTagAndStable ASSERT_EQ(l1p5, l1p5_cached); } +TEST_F(MetaServiceMetricsBaseTest, ReportEventMetricsFallbackDoesNotGateRequestValidation) { + proto::meta::ReportEventRequest request; + request.set_instance_id("unknown-instance"); + request.set_storage_type(proto::meta::ST_EVENT_REPORT_L2); + + std::string metrics_type; + auto collector = base_->ResolveReportEventMetricsCollector(request, metrics_type); + ASSERT_NE(nullptr, collector); + EXPECT_EQ("event_report_l2", metrics_type); + EXPECT_EQ((MetricsTags{{"api_name", "ReportEvent"}}), collector->GetMetricsTags()); + + request.clear_instance_id(); + request.set_storage_type(proto::meta::ST_UNSPECIFIED); + auto invalid_request_collector = base_->ResolveReportEventMetricsCollector(request, metrics_type); + EXPECT_EQ(collector, invalid_request_collector); + EXPECT_TRUE(metrics_type.empty()); +} + TEST_F(MetaServiceMetricsBaseTest, InvalidateCollectorCacheRemovesTypedReportEventEntries) { SeedInstance("inst1", "grp1"); @@ -124,7 +142,8 @@ TEST_F(MetaServiceMetricsBaseTest, ReportEventTypeCollectorUsesBoundedTagsAndSta auto snapshot = base_->GetTypedMetricsCollectorForReportEventType("inst1", "event_report_l2", "block_snapshot"); ASSERT_NE(nullptr, snapshot); ASSERT_NE(nullptr, dynamic_cast(snapshot.get())); - MetricsTags expected_tags = {{"instance_group", "grp1"}, + MetricsTags expected_tags = {{"api_name", "ReportEvent"}, + {"instance_group", "grp1"}, {"instance_id", "inst1"}, {"type", "event_report_l2"}, {"event_type", "block_snapshot"}}; @@ -133,11 +152,14 @@ TEST_F(MetaServiceMetricsBaseTest, ReportEventTypeCollectorUsesBoundedTagsAndSta base_->GetTypedMetricsCollectorForReportEventType("inst1", "event_report_l2", "block_snapshot")); } -TEST_F(MetaServiceMetricsBaseTest, AttachesOneCollectorPerDistinctEventTypeAndBoundsUnknownValues) { +TEST_F(MetaServiceMetricsBaseTest, AttachesCollectorsOnlyForBlockMutationEventTypes) { SeedInstance("inst1", "grp1"); proto::meta::ReportEventRequest request; request.set_instance_id("inst1"); - request.add_events()->set_event_type(proto::meta::EVENT_BLOCK_SNAPSHOT); + auto *snapshot_event = request.add_events(); + snapshot_event->set_event_type(proto::meta::EVENT_BLOCK_SNAPSHOT); + snapshot_event->mutable_block_snapshot()->add_blocks()->set_block_key("1"); + snapshot_event->mutable_block_snapshot()->add_blocks()->set_block_key("2"); request.add_events()->set_event_type(proto::meta::EVENT_BLOCK_SNAPSHOT); request.add_events()->set_event_type(proto::meta::EVENT_HEARTBEAT); request.add_events()->set_event_type(static_cast(99)); @@ -146,13 +168,19 @@ TEST_F(MetaServiceMetricsBaseTest, AttachesOneCollectorPerDistinctEventTypeAndBo base_->AttachReportEventTypeMetricsCollectors(request, "event_report_l2", &request_context); const auto collectors = request_context.GetMetricsCollectorsVehicle().GetMetricsCollectors(); - ASSERT_EQ(3, collectors.size()); + ASSERT_EQ(1, collectors.size()); std::set event_types; for (const auto &collector : collectors) { - ASSERT_NE(nullptr, dynamic_cast(collector.get())); - event_types.insert(collector->GetMetricsTags().at("event_type")); + auto *event_collector = dynamic_cast(collector.get()); + ASSERT_NE(nullptr, event_collector); + const auto &event_type = collector->GetMetricsTags().at("event_type"); + event_types.insert(event_type); + if (event_type == "block_snapshot") { + EXPECT_TRUE(event_collector->HasRequestKeyCountSample()); + EXPECT_DOUBLE_EQ(2., event_collector->GetRequestKeyCountSample()); + } } - EXPECT_EQ((std::set{"block_snapshot", "heartbeat", "unknown"}), event_types); + EXPECT_EQ((std::set{"block_snapshot"}), event_types); } TEST_F(MetaServiceMetricsBaseTest, AttachedEventCollectorsHaveRequestLocalSamplesAndSharedCounters) { @@ -184,8 +212,8 @@ TEST_F(MetaServiceMetricsBaseTest, AttachedEventCollectorsHaveRequestLocalSample Counter first_counter; Counter second_counter; - first->copy_event_report_request_counter_metrics(first_counter); - second->copy_event_report_request_counter_metrics(second_counter); + first->copy_service_query_counter_metrics(first_counter); + second->copy_service_query_counter_metrics(second_counter); ++first_counter; EXPECT_EQ(1u, second_counter.Get()); } diff --git a/kv_cache_manager/service/test/server_config_test.cc b/kv_cache_manager/service/test/server_config_test.cc index 8af89344c..7047e5403 100644 --- a/kv_cache_manager/service/test/server_config_test.cc +++ b/kv_cache_manager/service/test/server_config_test.cc @@ -22,6 +22,9 @@ TEST_F(ServerConfigTest, TestSimple) { ASSERT_TRUE(config.Check()); ASSERT_EQ(2, config.GetSchedulePlanExecutorThreadCount()); ASSERT_EQ(1u, config.GetSchedulePlanMigrationWorkerBudget()); + ASSERT_EQ(4u, config.GetMetaQueryWorkerCount()); + ASSERT_EQ(256u, config.GetMetaQueryParallelThreshold()); + ASSERT_EQ(128u, config.GetMetaQueryChunkSize()); ASSERT_EQ(60000, config.GetCacheReclaimerInflightDeleteTimeoutMs()); ASSERT_EQ(100000, config.GetCacheReclaimerPendingLocationLimitPerGroupType()); ASSERT_EQ(64ULL * 1024 * 1024 * 1024, config.GetCacheReclaimerPendingBytesLimitPerGroupType()); @@ -170,6 +173,42 @@ TEST_F(ServerConfigTest, TestSchedulePlanMigrationWorkerBudget) { } } +TEST_F(ServerConfigTest, TestMetaQueryExecutorConfig) { + { + ServerConfig config; + std::unordered_map environ{ + {"kvcm.meta_query.worker_count", "8"}, + {"kvcm.meta_query.parallel_threshold", "512"}, + {"kvcm.meta_query.chunk_size", "64"}, + }; + ASSERT_TRUE(config.Parse("", environ)); + EXPECT_TRUE(config.Check()); + EXPECT_EQ(8u, config.GetMetaQueryWorkerCount()); + EXPECT_EQ(512u, config.GetMetaQueryParallelThreshold()); + EXPECT_EQ(64u, config.GetMetaQueryChunkSize()); + } + for (const auto &invalid : std::vector>{ + {"kvcm.meta_query.worker_count", "0"}, + {"kvcm.meta_query.worker_count", "65"}, + {"kvcm.meta_query.parallel_threshold", "0"}, + {"kvcm.meta_query.chunk_size", "0"}, + }) { + ServerConfig config; + ASSERT_TRUE(config.Parse("", {{invalid.first, invalid.second}})); + EXPECT_FALSE(config.Check()) << invalid.first << "=" << invalid.second; + } + { + ServerConfig config; + ASSERT_TRUE( + config.Parse("", {{"kvcm.meta_query.parallel_threshold", "128"}, {"kvcm.meta_query.chunk_size", "129"}})); + EXPECT_FALSE(config.Check()); + } + for (const auto &invalid_value : {"invalid", "12x", "-1", "4294967296"}) { + ServerConfig config; + EXPECT_FALSE(config.Parse("", {{"kvcm.meta_query.worker_count", invalid_value}})) << invalid_value; + } +} + TEST_F(ServerConfigTest, TestCacheReclaimerAsyncDeleteConfig) { ServerConfig config; std::unordered_map environ{ diff --git a/kv_cache_manager/service/util/BUILD b/kv_cache_manager/service/util/BUILD index dc530b98a..c8e7e6d90 100644 --- a/kv_cache_manager/service/util/BUILD +++ b/kv_cache_manager/service/util/BUILD @@ -16,6 +16,17 @@ cc_library( ], ) +cc_library( + name = "report_event_json_parser", + srcs = ["report_event_json_parser.cc"], + hdrs = ["report_event_json_parser.h"], + deps = [ + ":proto_message_json_util", + "//kv_cache_manager/protocol/protobuf:service_cc_proto", + "@rapidjson", + ], +) + cc_library( name = "manager_message_proto_util", srcs = ["manager_message_proto_util.cc"], @@ -26,8 +37,8 @@ cc_library( "//kv_cache_manager/config", "//kv_cache_manager/config:meta_cache_policy_config", "//kv_cache_manager/data_storage", - "//kv_cache_manager/meta:cache_location", "//kv_cache_manager/manager:cache_location_view", + "//kv_cache_manager/meta:cache_location", "//kv_cache_manager/protocol/protobuf:service_cc_proto", ], ) @@ -64,15 +75,14 @@ cc_library( cc_library( name = "fault_injector", srcs = [ - "fault_injector.cc" + "fault_injector.cc", ], hdrs = [ - "fault_injector.h" + "fault_injector.h", ], deps = [ "//kv_cache_manager/common:logger", ], - ) cc_library( diff --git a/kv_cache_manager/service/util/common.h b/kv_cache_manager/service/util/common.h index cf8d84924..9629f27b4 100644 --- a/kv_cache_manager/service/util/common.h +++ b/kv_cache_manager/service/util/common.h @@ -34,7 +34,7 @@ std::string ExtractIpFromPeer(const std::string &peer); #define API_CONTEXT_GET_AND_INIT_COLLECTOR(method, return_value) \ auto metrics_collector = get_metrics_collector_from_map_for_##method(request->instance_id()); \ if (metrics_collector == nullptr) { \ - KVCM_LOG_ERROR("get " #method " metrics collector failed"); \ + KVCM_LOG_ERROR("get " #method " metrics collector failed, instance_id: %s", request->instance_id().c_str()); \ auto *header = response->mutable_header(); \ auto *status = header->mutable_status(); \ status->set_code(proto::meta::INSTANCE_NOT_EXIST); \ diff --git a/kv_cache_manager/service/util/manager_message_proto_util.cc b/kv_cache_manager/service/util/manager_message_proto_util.cc index 92a6f7731..f5c583077 100644 --- a/kv_cache_manager/service/util/manager_message_proto_util.cc +++ b/kv_cache_manager/service/util/manager_message_proto_util.cc @@ -150,15 +150,19 @@ void ProtoConvert::StorageFromProto(const proto::admin::StorageConfig *proto_sto case proto::admin::StorageConfig::kEventReport: { EventReportStorageSpec spec; const auto &v = proto_storage_config->event_report(); - if (v.heartbeat_timeout_ms() > 0) + // Proto3 scalar zero means "not supplied" for backward + // compatibility. Preserve any non-zero value so negative inputs reach + // StorageConfig validation instead of silently falling back to a + // valid-looking default. + if (v.heartbeat_timeout_ms() != 0) spec.set_heartbeat_timeout_ms(v.heartbeat_timeout_ms()); - if (v.cleanup_grace_ms() > 0) + if (v.cleanup_grace_ms() != 0) spec.set_cleanup_grace_ms(v.cleanup_grace_ms()); - if (v.liveness_check_interval_ms() > 0) + if (v.liveness_check_interval_ms() != 0) spec.set_liveness_check_interval_ms(v.liveness_check_interval_ms()); - if (v.snapshot_min_interval_ms() > 0) + if (v.snapshot_min_interval_ms() != 0) spec.set_snapshot_min_interval_ms(v.snapshot_min_interval_ms()); - if (v.snapshot_delta_drain_timeout_ms() > 0) + if (v.snapshot_delta_drain_timeout_ms() != 0) spec.set_snapshot_delta_drain_timeout_ms(v.snapshot_delta_drain_timeout_ms()); storage_config.set_storage_spec(std::make_shared(spec)); DataStorageType event_report_type = DataStorageType::DATA_STORAGE_TYPE_UNKNOWN; @@ -246,7 +250,8 @@ void ProtoConvert::CacheConfigToProto(const CacheConfig &cache_config_info, auto *method_configs = proto_migration_strategy->mutable_method_configs(); method_configs->mutable_copy()->set_enabled(migration_strategy->methods().copy().enabled()); method_configs->mutable_mark()->set_enabled(migration_strategy->methods().mark().enabled()); - method_configs->mutable_mark()->mutable_timeout_ms()->set_value(migration_strategy->methods().mark().timeout_ms()); + method_configs->mutable_mark()->mutable_timeout_ms()->set_value( + migration_strategy->methods().mark().timeout_ms()); proto_migration_strategy->set_retention( static_cast(migration_strategy->retention())); } @@ -328,13 +333,13 @@ void ProtoConvert::CacheConfigFromProto(const proto::admin::CacheConfig *proto_c methods.mutable_copy().set_enabled(proto_migration_strategy.method_configs().copy().enabled()); methods.mutable_mark().set_enabled(proto_migration_strategy.method_configs().mark().enabled()); if (proto_migration_strategy.method_configs().mark().has_timeout_ms()) { - methods.mutable_mark().set_timeout_ms(proto_migration_strategy.method_configs().mark().timeout_ms().value()); + methods.mutable_mark().set_timeout_ms( + proto_migration_strategy.method_configs().mark().timeout_ms().value()); } else { methods.mutable_mark().set_timeout_ms(MigrationMarkMethod::kDefaultTimeoutMs); } migration_strategy->set_methods(methods); - migration_strategy->set_retention( - static_cast(proto_migration_strategy.retention())); + migration_strategy->set_retention(static_cast(proto_migration_strategy.retention())); migration_strategies.push_back(migration_strategy); } cache_config_info.set_migration_strategies(migration_strategies); diff --git a/kv_cache_manager/service/util/manager_message_proto_util.h b/kv_cache_manager/service/util/manager_message_proto_util.h index aa46a5687..7258e36c2 100644 --- a/kv_cache_manager/service/util/manager_message_proto_util.h +++ b/kv_cache_manager/service/util/manager_message_proto_util.h @@ -319,6 +319,10 @@ template void ProtoConvert::DataStorageTypeFromProto(const T proto_data_storage_type, DataStorageType &data_storage_type_info) { static_assert(std::is_same_v || std::is_same_v, "T must be either proto::meta::DataStorage or proto::admin::DataStorage"); + // Protobuf enums are open on the wire. Always initialize the output so an + // unknown numeric value fails closed instead of leaving callers with an + // indeterminate DataStorageType. + data_storage_type_info = DataStorageType::DATA_STORAGE_TYPE_UNKNOWN; switch (proto_data_storage_type) { case T::ST_UNSPECIFIED: { data_storage_type_info = DataStorageType::DATA_STORAGE_TYPE_UNKNOWN; diff --git a/kv_cache_manager/service/util/proto_message_json_util.cc b/kv_cache_manager/service/util/proto_message_json_util.cc index b21f99d49..5e22524a2 100644 --- a/kv_cache_manager/service/util/proto_message_json_util.cc +++ b/kv_cache_manager/service/util/proto_message_json_util.cc @@ -34,12 +34,13 @@ bool ProtoMessageJsonUtil::ToJson(const ::google::protobuf::Message *message, st return status.ok(); } -bool ProtoMessageJsonUtil::FromJson(const std::string &json, ::google::protobuf::Message *message) { +bool ProtoMessageJsonUtil::FromJson(std::string_view json, ::google::protobuf::Message *message) { if (!message) { return false; } static ::google::protobuf::util::JsonParseOptions option = CreateJsonParseOption(); - auto status = google::protobuf::util::JsonStringToMessage(json, message, option); + const google::protobuf::StringPiece input(json.data(), static_cast(json.size())); + auto status = google::protobuf::util::JsonStringToMessage(input, message, option); if (!status.ok()) { // TODO: change to return in response KVCM_LOG_WARN("json parse error, message: %s", status.error_message().data()); @@ -47,4 +48,4 @@ bool ProtoMessageJsonUtil::FromJson(const std::string &json, ::google::protobuf: return status.ok(); } -} // namespace kv_cache_manager \ No newline at end of file +} // namespace kv_cache_manager diff --git a/kv_cache_manager/service/util/proto_message_json_util.h b/kv_cache_manager/service/util/proto_message_json_util.h index 5e358f603..c8a9cb29c 100644 --- a/kv_cache_manager/service/util/proto_message_json_util.h +++ b/kv_cache_manager/service/util/proto_message_json_util.h @@ -2,13 +2,14 @@ #include #include +#include namespace kv_cache_manager { class ProtoMessageJsonUtil { public: static bool ToJson(const ::google::protobuf::Message *message, std::string &json); - static bool FromJson(const std::string &json, ::google::protobuf::Message *message); + static bool FromJson(std::string_view json, ::google::protobuf::Message *message); }; -} // namespace kv_cache_manager \ No newline at end of file +} // namespace kv_cache_manager diff --git a/kv_cache_manager/service/util/report_event_json_parser.cc b/kv_cache_manager/service/util/report_event_json_parser.cc new file mode 100644 index 000000000..ed215b6bd --- /dev/null +++ b/kv_cache_manager/service/util/report_event_json_parser.cc @@ -0,0 +1,717 @@ +#include "kv_cache_manager/service/util/report_event_json_parser.h" + +#include +#include +#include +#include +#include +#include + +#if defined(__aarch64__) +#include +#elif defined(__x86_64__) || defined(__i386__) +#include +#endif + +#include "kv_cache_manager/service/util/proto_message_json_util.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace kv_cache_manager { +namespace { + +using JsonValue = rapidjson::Value; + +constexpr size_t kLargeRequestThreshold = 32 * 1024; +constexpr size_t kDefaultJsonPoolChunkSize = 64 * 1024; +constexpr size_t kMaxJsonPoolChunkSize = 4 * 1024 * 1024; +constexpr size_t kDefaultJsonStackSize = 256; +constexpr size_t kMinLargeJsonStackSize = 64 * 1024; +constexpr size_t kMaxJsonStackSize = 1024 * 1024; +constexpr size_t kMaxReusableMutableJsonCapacity = 4 * 1024 * 1024; + +struct ThreadLocalMutableJsonBuffer { + std::string value; + bool in_use = false; +}; + +class MutableJsonBufferLease { +public: + explicit MutableJsonBufferLease(size_t size) { + if (size <= kMaxReusableMutableJsonCapacity && !thread_local_buffer_.in_use) { + thread_local_buffer_.in_use = true; + value_ = &thread_local_buffer_.value; + reusable_ = true; + if (value_->capacity() < size) { + // Avoid std::string's geometric growth retaining more than the + // explicit per-worker cap after alternating payload sizes. + std::string replacement; + replacement.reserve(size); + value_->swap(replacement); + } + } else { + value_ = &fallback_; + fallback_.reserve(size); + } + } + + MutableJsonBufferLease(const MutableJsonBufferLease &) = delete; + MutableJsonBufferLease &operator=(const MutableJsonBufferLease &) = delete; + + ~MutableJsonBufferLease() { + if (reusable_) { + value_->clear(); + thread_local_buffer_.in_use = false; + } + } + + std::string &value() { return *value_; } + +private: + static thread_local ThreadLocalMutableJsonBuffer thread_local_buffer_; + std::string fallback_; + std::string *value_ = nullptr; + bool reusable_ = false; +}; + +thread_local ThreadLocalMutableJsonBuffer MutableJsonBufferLease::thread_local_buffer_; + +struct AsciiScanResult { + bool is_ascii; + bool has_nul; +}; + +AsciiScanResult FinishAsciiScan(std::string_view input, bool has_nul) { + constexpr uint64_t kHighBits = 0x8080808080808080ULL; + constexpr uint64_t kLowBits = 0x0101010101010101ULL; + while (input.size() >= sizeof(uint64_t)) { + uint64_t word; + std::memcpy(&word, input.data(), sizeof(word)); + if ((word & kHighBits) != 0) { + return {false, has_nul}; + } + if (((word - kLowBits) & ~word & kHighBits) != 0) { + has_nul = true; + } + input.remove_prefix(sizeof(word)); + } + for (const unsigned char byte : input) { + if ((byte & 0x80U) != 0) { + return {false, has_nul}; + } + if (byte == 0) { + has_nul = true; + } + } + return {true, has_nul}; +} + +#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) +__attribute__((target("avx2"))) AsciiScanResult ScanAsciiAvx2(std::string_view input) { + bool has_nul = false; + const __m256i zero = _mm256_setzero_si256(); + while (input.size() >= 32) { + const __m256i bytes = + _mm256_loadu_si256(reinterpret_cast(static_cast(input.data()))); + if (_mm256_movemask_epi8(bytes) != 0) { + return {false, has_nul}; + } + if (_mm256_movemask_epi8(_mm256_cmpeq_epi8(bytes, zero)) != 0) { + has_nul = true; + } + input.remove_prefix(32); + } + return FinishAsciiScan(input, has_nul); +} +#endif + +AsciiScanResult ScanAscii(std::string_view input) { + bool has_nul = false; +#if defined(__aarch64__) + const uint8x16_t high_bit = vdupq_n_u8(0x80U); + while (input.size() >= 16) { + const uint8x16_t bytes = vld1q_u8(reinterpret_cast(input.data())); + if (vmaxvq_u8(vandq_u8(bytes, high_bit)) != 0) { + return {false, has_nul}; + } + if (vminvq_u8(bytes) == 0) { + has_nul = true; + } + input.remove_prefix(16); + } +#elif (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) + if (__builtin_cpu_supports("avx2")) { + return ScanAsciiAvx2(input); + } +#if defined(__SSE2__) + const __m128i zero = _mm_setzero_si128(); + while (input.size() >= 16) { + const __m128i bytes = + _mm_loadu_si128(reinterpret_cast(static_cast(input.data()))); + if (_mm_movemask_epi8(bytes) != 0) { + return {false, has_nul}; + } + if (_mm_movemask_epi8(_mm_cmpeq_epi8(bytes, zero)) != 0) { + has_nul = true; + } + input.remove_prefix(16); + } +#endif +#endif + return FinishAsciiScan(input, has_nul); +} + +template +inline bool NameIs(const JsonValue &name, const char (&expected)[ExpectedSize]) { + static_assert(ExpectedSize > 1); + return name.GetStringLength() == ExpectedSize - 1 && std::memcmp(name.GetString(), expected, ExpectedSize - 1) == 0; +} + +template +inline bool NameIs(const JsonValue &name, const char (&snake_case)[SnakeSize], const char (&camel_case)[CamelSize]) { + static_assert(SnakeSize > 1 && CamelSize > 1); + const size_t actual_size = name.GetStringLength(); + const char *actual = name.GetString(); + return (actual_size == SnakeSize - 1 && std::memcmp(actual, snake_case, SnakeSize - 1) == 0) || + (actual_size == CamelSize - 1 && std::memcmp(actual, camel_case, CamelSize - 1) == 0); +} + +template +bool SetString(const JsonValue &value, Setter &&setter) { + if (!value.IsString()) { + return false; + } + setter(value.GetString(), value.GetStringLength()); + return true; +} + +bool ParseStorageType(const JsonValue &value, proto::meta::StorageType &out) { + if (value.IsInt()) { + switch (value.GetInt()) { + case proto::meta::ST_UNSPECIFIED: + case proto::meta::ST_3FS: + case proto::meta::ST_MOONCAKE: + case proto::meta::ST_TAIRMEMPOOL: + case proto::meta::ST_NFS: + case proto::meta::ST_VCNS_3FS: + case proto::meta::ST_DUMMY: + case proto::meta::ST_EVENT_REPORT_L1P5: + case proto::meta::ST_EVENT_REPORT_L2: + out = static_cast(value.GetInt()); + return true; + default: + return false; + } + } + if (!value.IsString()) { + return false; + } + const std::string_view name(value.GetString(), value.GetStringLength()); + if (name == "ST_UNSPECIFIED") { + out = proto::meta::ST_UNSPECIFIED; + } else if (name == "ST_3FS") { + out = proto::meta::ST_3FS; + } else if (name == "ST_MOONCAKE") { + out = proto::meta::ST_MOONCAKE; + } else if (name == "ST_TAIRMEMPOOL") { + out = proto::meta::ST_TAIRMEMPOOL; + } else if (name == "ST_NFS") { + out = proto::meta::ST_NFS; + } else if (name == "ST_VCNS_3FS") { + out = proto::meta::ST_VCNS_3FS; + } else if (name == "ST_DUMMY") { + out = proto::meta::ST_DUMMY; + } else if (name == "ST_EVENT_REPORT_L1P5") { + out = proto::meta::ST_EVENT_REPORT_L1P5; + } else if (name == "ST_EVENT_REPORT_L2") { + out = proto::meta::ST_EVENT_REPORT_L2; + } else { + return false; + } + return true; +} + +bool ParseEventType(const JsonValue &value, proto::meta::ReportEventType &out) { + if (value.IsInt()) { + switch (value.GetInt()) { + case proto::meta::EVENT_UNSPECIFIED: + case proto::meta::EVENT_NODE_REGISTER: + case proto::meta::EVENT_BLOCK_ADD: + case proto::meta::EVENT_BLOCK_DELETE: + case proto::meta::EVENT_HOST_DOWN: + case proto::meta::EVENT_HEARTBEAT: + case proto::meta::EVENT_BLOCK_SNAPSHOT: + out = static_cast(value.GetInt()); + return true; + default: + return false; + } + } + if (!value.IsString()) { + return false; + } + const std::string_view name(value.GetString(), value.GetStringLength()); + if (name == "EVENT_UNSPECIFIED") { + out = proto::meta::EVENT_UNSPECIFIED; + } else if (name == "EVENT_NODE_REGISTER") { + out = proto::meta::EVENT_NODE_REGISTER; + } else if (name == "EVENT_BLOCK_ADD") { + out = proto::meta::EVENT_BLOCK_ADD; + } else if (name == "EVENT_BLOCK_DELETE") { + out = proto::meta::EVENT_BLOCK_DELETE; + } else if (name == "EVENT_HOST_DOWN") { + out = proto::meta::EVENT_HOST_DOWN; + } else if (name == "EVENT_HEARTBEAT") { + out = proto::meta::EVENT_HEARTBEAT; + } else if (name == "EVENT_BLOCK_SNAPSHOT") { + out = proto::meta::EVENT_BLOCK_SNAPSHOT; + } else { + return false; + } + return true; +} + +bool ParseLocationSpec(const JsonValue &value, proto::meta::LocationSpec *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_name = false; + bool seen_uri = false; + for (const auto &member : value.GetObject()) { + if (NameIs(member.name, "name")) { + if (seen_name || + !SetString(member.value, [out](const char *data, size_t size) { out->set_name(data, size); })) { + return false; + } + seen_name = true; + } else if (NameIs(member.name, "uri")) { + if (seen_uri || + !SetString(member.value, [out](const char *data, size_t size) { out->set_uri(data, size); })) { + return false; + } + seen_uri = true; + } + } + return true; +} + +template +bool ParseSpecs(const JsonValue &value, RepeatedMessage *out, AddFunc &&add) { + if (!value.IsArray()) { + return false; + } + out->Reserve(static_cast(value.Size())); + for (const auto &entry : value.GetArray()) { + if (!ParseLocationSpec(entry, add())) { + return false; + } + } + return true; +} + +bool ParseNodeRegister(const JsonValue &value, proto::meta::NodeRegisterEventParams *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_mediums = false; + for (const auto &member : value.GetObject()) { + if (!NameIs(member.name, "mediums")) { + continue; + } + if (seen_mediums || !member.value.IsArray()) { + return false; + } + seen_mediums = true; + out->mutable_mediums()->Reserve(static_cast(member.value.Size())); + for (const auto &medium : member.value.GetArray()) { + if (!medium.IsString()) { + return false; + } + out->add_mediums(medium.GetString(), medium.GetStringLength()); + } + } + return true; +} + +bool ParseBlockAdd(const JsonValue &value, proto::meta::BlockAddEventParams *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_block_key = false; + bool seen_uri = false; + bool seen_medium = false; + bool seen_specs = false; + for (const auto &member : value.GetObject()) { + if (NameIs(member.name, "block_key", "blockKey")) { + if (seen_block_key || + !SetString(member.value, [out](const char *data, size_t size) { out->set_block_key(data, size); })) { + return false; + } + seen_block_key = true; + } else if (NameIs(member.name, "uri")) { + if (seen_uri || + !SetString(member.value, [out](const char *data, size_t size) { out->set_uri(data, size); })) { + return false; + } + seen_uri = true; + } else if (NameIs(member.name, "medium")) { + if (seen_medium || + !SetString(member.value, [out](const char *data, size_t size) { out->set_medium(data, size); })) { + return false; + } + seen_medium = true; + } else if (NameIs(member.name, "specs")) { + if (seen_specs || !ParseSpecs(member.value, out->mutable_specs(), [out] { return out->add_specs(); })) { + return false; + } + seen_specs = true; + } + } + return true; +} + +bool ParseBlockDelete(const JsonValue &value, proto::meta::BlockDeleteEventParams *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_block_key = false; + bool seen_medium = false; + bool seen_spec_names = false; + for (const auto &member : value.GetObject()) { + if (NameIs(member.name, "block_key", "blockKey")) { + if (seen_block_key || + !SetString(member.value, [out](const char *data, size_t size) { out->set_block_key(data, size); })) { + return false; + } + seen_block_key = true; + } else if (NameIs(member.name, "medium")) { + if (seen_medium || + !SetString(member.value, [out](const char *data, size_t size) { out->set_medium(data, size); })) { + return false; + } + seen_medium = true; + } else if (NameIs(member.name, "spec_names", "specNames")) { + if (seen_spec_names || !member.value.IsArray()) { + return false; + } + seen_spec_names = true; + out->mutable_spec_names()->Reserve(static_cast(member.value.Size())); + for (const auto &name : member.value.GetArray()) { + if (!name.IsString()) { + return false; + } + out->add_spec_names(name.GetString(), name.GetStringLength()); + } + } + } + return true; +} + +bool ParseSnapshotItem(const JsonValue &value, proto::meta::BlockSnapshotItem *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_block_key = false; + bool seen_medium = false; + bool seen_specs = false; + for (const auto &member : value.GetObject()) { + if (NameIs(member.name, "block_key", "blockKey")) { + if (seen_block_key || + !SetString(member.value, [out](const char *data, size_t size) { out->set_block_key(data, size); })) { + return false; + } + seen_block_key = true; + } else if (NameIs(member.name, "medium")) { + if (seen_medium || + !SetString(member.value, [out](const char *data, size_t size) { out->set_medium(data, size); })) { + return false; + } + seen_medium = true; + } else if (NameIs(member.name, "specs")) { + if (seen_specs || !ParseSpecs(member.value, out->mutable_specs(), [out] { return out->add_specs(); })) { + return false; + } + seen_specs = true; + } + } + return true; +} + +bool ParseBlockSnapshot(const JsonValue &value, proto::meta::BlockSnapshotEventParams *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_medium = false; + bool seen_blocks = false; + for (const auto &member : value.GetObject()) { + if (NameIs(member.name, "medium")) { + if (seen_medium || + !SetString(member.value, [out](const char *data, size_t size) { out->set_medium(data, size); })) { + return false; + } + seen_medium = true; + } else if (NameIs(member.name, "blocks")) { + if (seen_blocks || !member.value.IsArray()) { + return false; + } + seen_blocks = true; + out->mutable_blocks()->Reserve(static_cast(member.value.Size())); + for (const auto &block : member.value.GetArray()) { + if (!ParseSnapshotItem(block, out->add_blocks())) { + return false; + } + } + } + } + return true; +} + +bool ParseHeartbeat(const JsonValue &value, proto::meta::HeartbeatEventParams *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_system_status = false; + for (const auto &member : value.GetObject()) { + if (!NameIs(member.name, "system_status", "systemStatus")) { + continue; + } + if (seen_system_status || !member.value.IsObject()) { + return false; + } + seen_system_status = true; + auto *status = out->mutable_system_status(); + for (const auto &entry : member.value.GetObject()) { + if (!entry.value.IsString()) { + return false; + } + std::string key(entry.name.GetString(), entry.name.GetStringLength()); + // JsonStringToMessage rejects duplicate protobuf map keys. Do not + // silently apply last-value-wins here merely because RapidJSON's + // DOM preserves duplicate object members: request acceptance must + // not depend on whether the specialized parser was selected. + if (status->find(key) != status->end()) { + return false; + } + (*status)[std::move(key)] = std::string(entry.value.GetString(), entry.value.GetStringLength()); + } + } + return true; +} + +bool ParseEvent(const JsonValue &value, proto::meta::EventItem *out) { + if (!value.IsObject() || !out) { + return false; + } + bool seen_event_type = false; + bool seen_params = false; + for (const auto &member : value.GetObject()) { + if (NameIs(member.name, "event_type", "eventType")) { + proto::meta::ReportEventType event_type; + if (seen_event_type || !ParseEventType(member.value, event_type)) { + return false; + } + out->set_event_type(event_type); + seen_event_type = true; + } else if (NameIs(member.name, "node_register", "nodeRegister")) { + if (seen_params || !ParseNodeRegister(member.value, out->mutable_node_register())) { + return false; + } + seen_params = true; + } else if (NameIs(member.name, "block_add", "blockAdd")) { + if (seen_params || !ParseBlockAdd(member.value, out->mutable_block_add())) { + return false; + } + seen_params = true; + } else if (NameIs(member.name, "block_delete", "blockDelete")) { + if (seen_params || !ParseBlockDelete(member.value, out->mutable_block_delete())) { + return false; + } + seen_params = true; + } else if (NameIs(member.name, "host_down", "hostDown")) { + if (seen_params || !member.value.IsObject()) { + return false; + } + out->mutable_host_down(); + seen_params = true; + } else if (NameIs(member.name, "heartbeat")) { + if (seen_params || !ParseHeartbeat(member.value, out->mutable_heartbeat())) { + return false; + } + seen_params = true; + } else if (NameIs(member.name, "block_snapshot", "blockSnapshot")) { + if (seen_params || !ParseBlockSnapshot(member.value, out->mutable_block_snapshot())) { + return false; + } + seen_params = true; + } + } + return true; +} + +bool ParseRequest(const JsonValue &root, proto::meta::ReportEventRequest *out) { + if (!root.IsObject() || !out) { + return false; + } + bool seen_trace_id = false; + bool seen_instance_id = false; + bool seen_host_ip_port = false; + bool seen_events = false; + bool seen_storage_type = false; + for (const auto &member : root.GetObject()) { + if (NameIs(member.name, "trace_id", "traceId")) { + if (seen_trace_id || + !SetString(member.value, [out](const char *data, size_t size) { out->set_trace_id(data, size); })) { + return false; + } + seen_trace_id = true; + } else if (NameIs(member.name, "instance_id", "instanceId")) { + if (seen_instance_id || + !SetString(member.value, [out](const char *data, size_t size) { out->set_instance_id(data, size); })) { + return false; + } + seen_instance_id = true; + } else if (NameIs(member.name, "host_ip_port", "hostIpPort")) { + if (seen_host_ip_port || + !SetString(member.value, [out](const char *data, size_t size) { out->set_host_ip_port(data, size); })) { + return false; + } + seen_host_ip_port = true; + } else if (NameIs(member.name, "events")) { + if (seen_events || !member.value.IsArray()) { + return false; + } + seen_events = true; + out->mutable_events()->Reserve(static_cast(member.value.Size())); + for (const auto &event : member.value.GetArray()) { + if (!ParseEvent(event, out->add_events())) { + return false; + } + } + } else if (NameIs(member.name, "storage_type", "storageType")) { + proto::meta::StorageType storage_type; + if (seen_storage_type || !ParseStorageType(member.value, storage_type)) { + return false; + } + out->set_storage_type(storage_type); + seen_storage_type = true; + } + } + return true; +} + +} // namespace + +bool ReportEventJsonParser::TryFromJson(std::string_view json, proto::meta::ReportEventRequest *message) { + if (!message || json.empty()) { + return false; + } + message->Clear(); + const bool is_large = json.size() >= kLargeRequestThreshold; + const size_t pool_chunk_size = is_large ? std::min(json.size(), kMaxJsonPoolChunkSize) : kDefaultJsonPoolChunkSize; + const size_t stack_size = + is_large ? std::clamp(json.size() / 8, kMinLargeJsonStackSize, kMaxJsonStackSize) : kDefaultJsonStackSize; + rapidjson::MemoryPoolAllocator<> allocator(pool_chunk_size); + rapidjson::Document document(&allocator, stack_size); + const AsciiScanResult ascii = ScanAscii(json); + if (is_large && ascii.is_ascii && !ascii.has_nul) { + // RapidJSON's read-only parser copies every decoded key/value into the + // DOM pool before ParseRequest copies it into protobuf. A single + // contiguous mutable copy lets in-situ parsing reference/unescape the + // strings in that buffer instead, removing thousands of small DOM + // string copies. The buffer outlives ParseRequest below. Raw NUL bytes + // stay on the length-aware path so an early C-string terminator can + // never make a malformed body look valid. + MutableJsonBufferLease mutable_json(json.size()); + mutable_json.value().assign(json.data(), json.size()); + document.ParseInsitu(mutable_json.value().data()); + if (document.HasParseError()) { + return false; + } + return ParseRequest(document, message); + } else if (ascii.is_ascii) { + // ASCII is a strict subset of UTF-8. Skipping RapidJSON's per-codepoint + // validator avoids a full branch-heavy decode on the overwhelmingly + // common URI/key payload without accepting any invalid byte sequence. + document.Parse(json.data(), json.size()); + } else { + document.Parse(json.data(), json.size()); + } + return !document.HasParseError() && ParseRequest(document, message); +} + +bool ReportEventJsonParser::FromJson(std::string_view json, proto::meta::ReportEventRequest *message) { + if (!message) { + return false; + } + if (TryFromJson(json, message)) { + return true; + } + message->Clear(); + return ProtoMessageJsonUtil::FromJson(json, message); +} + +bool ReportEventJsonParser::FromMutableNullTerminatedJson(char *json, + size_t size, + proto::meta::ReportEventRequest *message) { + if (!message || !json || size == 0) { + return false; + } + + const std::string_view view(json, size); + if (size < kLargeRequestThreshold) { + // The immutable parser already performs its own ASCII/UTF-8 scan. + // Avoid scanning small heartbeat/register bodies twice merely because + // the HTTP handler also supports the large mutable-body fast path. + return FromJson(view, message); + } + const AsciiScanResult ascii = ScanAscii(view); + if (!ascii.is_ascii || ascii.has_nul) { + // Non-ASCII and raw-NUL inputs retain the existing length-aware + // validation path. They are rare enough that preserving one shared + // compatibility path is preferable to plumbing scan state through + // both decoders. + return FromJson(view, message); + } + + // cinatra 0.5.5 stores the request body in std::string and exposes a view, + // so the byte immediately after the view is its terminator. Keep the check + // here as a defensive guard against a future transport implementation that + // no longer satisfies the explicit API contract above. + if (json[size] != '\0') { + return FromJson(view, message); + } + + message->Clear(); + const size_t pool_chunk_size = std::min(size, kMaxJsonPoolChunkSize); + const size_t stack_size = std::clamp(size / 8, kMinLargeJsonStackSize, kMaxJsonStackSize); + rapidjson::MemoryPoolAllocator<> allocator(pool_chunk_size); + rapidjson::Document document(&allocator, stack_size); + document.ParseInsitu(json); + if (document.HasParseError()) { + // Both parsers require valid JSON. The mutable source may already have + // been changed by RapidJSON, so it cannot be passed to the protobuf + // fallback; a syntax error is not a compatibility fallback case. + return false; + } + if (ParseRequest(document, message)) { + return true; + } + + // The fast converter intentionally delegates rare protobuf-JSON spellings + // such as null fields and unknown enum names. In-situ parsing has changed + // the source buffer, but the DOM still represents the complete JSON value; + // serialize that value only on this rare path before invoking the generic + // protobuf parser. Unknown fields, duplicate members and escaped strings + // remain represented in the DOM and therefore preserve fallback behavior. + rapidjson::StringBuffer normalized_json; + rapidjson::Writer writer(normalized_json); + if (!document.Accept(writer)) { + return false; + } + message->Clear(); + return ProtoMessageJsonUtil::FromJson(std::string_view(normalized_json.GetString(), normalized_json.GetSize()), + message); +} + +} // namespace kv_cache_manager diff --git a/kv_cache_manager/service/util/report_event_json_parser.h b/kv_cache_manager/service/util/report_event_json_parser.h new file mode 100644 index 000000000..f8fa01363 --- /dev/null +++ b/kv_cache_manager/service/util/report_event_json_parser.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include "kv_cache_manager/protocol/protobuf/meta_service.pb.h" + +namespace kv_cache_manager { + +// Allocation-bounded JSON decoder for the high-volume ReportEvent HTTP API. +// Canonical ReportEvent payloads are decoded directly into the destination +// protobuf (and therefore its Arena). Less common protobuf-JSON spellings are +// deliberately delegated to ProtoMessageJsonUtil so compatibility remains +// defined by protobuf rather than by this fast path. +class ReportEventJsonParser { +public: + static bool FromJson(std::string_view json, proto::meta::ReportEventRequest *message); + + // HTTP-specific entry point for cinatra's mutable, NUL-terminated request + // body. Large ASCII payloads are parsed in place, avoiding a second full + // body copy. `json[size]` must be readable and equal to '\0'; callers that + // cannot provide that contract must use FromJson instead. + static bool FromMutableNullTerminatedJson(char *json, size_t size, proto::meta::ReportEventRequest *message); + + // Exposed for focused tests and benchmarks. False means "use the generic + // protobuf parser"; it does not necessarily mean that the JSON is invalid. + static bool TryFromJson(std::string_view json, proto::meta::ReportEventRequest *message); +}; + +} // namespace kv_cache_manager diff --git a/kv_cache_manager/service/util/service_call_guard.cc b/kv_cache_manager/service/util/service_call_guard.cc index 4fb4a8378..b6be36944 100644 --- a/kv_cache_manager/service/util/service_call_guard.cc +++ b/kv_cache_manager/service/util/service_call_guard.cc @@ -148,9 +148,15 @@ ServiceCallGuard::~ServiceCallGuard() { // event samples from this request's private context instead, // and write immediately before reporting to minimize the shared // registry gauge race window. - SET_METRICS_(event_metrics_collector, event_report, request_rt_us, request_rt_us); - SET_METRICS_(event_metrics_collector, event_report, error_code, error_code); + SET_METRICS_(event_metrics_collector, service, query_rt_us, request_rt_us); + SET_METRICS_(event_metrics_collector, service, error_code, error_code); event_metrics_collector->SetRequestSample(request_rt_us, error_code); + if (event_metrics_collector->HasRequestKeyCountSample()) { + SET_METRICS_(event_metrics_collector, + manager, + request_key_count, + event_metrics_collector->GetRequestKeyCountSample()); + } } metrics_reporter_->ReportPerQuery(mc.get()); } diff --git a/kv_cache_manager/service/util/test/BUILD b/kv_cache_manager/service/util/test/BUILD index eaaf1465f..80e55a5ae 100644 --- a/kv_cache_manager/service/util/test/BUILD +++ b/kv_cache_manager/service/util/test/BUILD @@ -1,7 +1,7 @@ -package(default_visibility = ["//visibility:private"]) - load("//bazel:tf_proto.bzl", "tf_proto_library_cc") +package(default_visibility = ["//visibility:private"]) + cc_test( name = "ProtoMessageJsonUtilTest", srcs = [ @@ -14,6 +14,8 @@ cc_test( "//kv_cache_manager/protocol/protobuf:service_cc_proto", "//kv_cache_manager/service/util:manager_message_proto_util", "//kv_cache_manager/service/util:proto_message_json_util", + "//kv_cache_manager/service/util:report_event_json_parser", + "@com_google_protobuf//:protobuf", ], ) diff --git a/kv_cache_manager/service/util/test/proto_message_json_util_test.cc b/kv_cache_manager/service/util/test/proto_message_json_util_test.cc index dc44fb441..df4753e19 100644 --- a/kv_cache_manager/service/util/test/proto_message_json_util_test.cc +++ b/kv_cache_manager/service/util/test/proto_message_json_util_test.cc @@ -1,8 +1,13 @@ #include +#include +#include +#include +#include "google/protobuf/util/message_differencer.h" #include "kv_cache_manager/common/unittest.h" #include "kv_cache_manager/protocol/protobuf/meta_service.pb.h" #include "kv_cache_manager/service/util/manager_message_proto_util.h" +#include "kv_cache_manager/service/util/report_event_json_parser.h" #include "service/util/proto_message_json_util.h" #include "service/util/test/service_util_test.pb.h" @@ -275,6 +280,272 @@ TEST_F(ProtoMessageJsonUtilTest, TestFromJsonError) { } } +TEST_F(ProtoMessageJsonUtilTest, TestFromJsonHonorsNonNullTerminatedViewBounds) { + const std::string json = R"({"int32Value":123,"stringValue":"bounded"})"; + const std::string prefix = "ignored-prefix"; + const std::string backing = prefix + json + "!invalid-trailing-bytes"; + const std::string_view bounded(backing.data() + prefix.size(), json.size()); + + SimpleMessage msg; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(bounded, &msg)); + EXPECT_EQ(123, msg.int32value()); + EXPECT_EQ("bounded", msg.stringvalue()); + + EXPECT_FALSE(ProtoMessageJsonUtil::FromJson(std::string_view(), &msg)); + EXPECT_FALSE(ProtoMessageJsonUtil::FromJson(bounded, nullptr)); +} + +TEST_F(ProtoMessageJsonUtilTest, TestReportEventFastJsonParserMatchesGenericParser) { + const std::string json = R"json({ + "trace_id":"trace-fast", + "instanceId":"instance-fast", + "host_ip_port":"10.0.0.8:8080", + "events":[ + {"eventType":"EVENT_NODE_REGISTER","nodeRegister":{"mediums":["mem","disk"],"ignored":1}}, + {"event_type":"EVENT_BLOCK_ADD","block_add":{"blockKey":"-1","uri":"legacy://uri","medium":"mem","specs":[{"name":"tp0","uri":"event_report://host/mem"},{"name":"tp1","uri":"event_report://host/mem?part=1"}]}}, + {"event_type":3,"blockDelete":{"block_key":"2","medium":"disk","specNames":["tp0","tp1"]}}, + {"event_type":"EVENT_HOST_DOWN","hostDown":{"ignored":{"nested":true}}}, + {"event_type":"EVENT_HEARTBEAT","heartbeat":{"systemStatus":{"state":"ready","load":"7"}}}, + {"event_type":"EVENT_BLOCK_SNAPSHOT","blockSnapshot":{"medium":"legacy","blocks":[{"blockKey":"3","medium":"mem","specs":[{"name":"tp0","uri":"event_report://host/mem?block=3"}]}]}} + ], + "storageType":8, + "ignored_top_level":{"deep":[1,2,3]} + })json"; + + proto::meta::ReportEventRequest generic; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(json, &generic)); + + proto::meta::ReportEventRequest fast; + fast.set_trace_id("must-be-cleared"); + ASSERT_TRUE(ReportEventJsonParser::TryFromJson(json, &fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, fast)); + + const std::string prefix = "ignored-prefix"; + const std::string backing = prefix + json + "!invalid-trailing-bytes"; + const std::string_view bounded(backing.data() + prefix.size(), json.size()); + proto::meta::ReportEventRequest bounded_fast; + ASSERT_TRUE(ReportEventJsonParser::TryFromJson(bounded, &bounded_fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, bounded_fast)); + + std::string mutable_small = json; + proto::meta::ReportEventRequest mutable_small_fast; + ASSERT_TRUE(ReportEventJsonParser::FromMutableNullTerminatedJson( + mutable_small.data(), mutable_small.size(), &mutable_small_fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, mutable_small_fast)); + + // Force the HTTP-only in-situ path with the same complete event matrix. + // The existing small body above exercises the immutable compatibility + // path, while production ReportEvent batches are normally larger than the + // 32-KiB threshold and mutate cinatra's request-owned buffer directly. + std::string large_json = json; + ASSERT_EQ('}', large_json.back()); + large_json.pop_back(); + large_json += R"json(,"ignored_large_padding":")json"; + large_json.append(40 * 1024, 'p'); + large_json += R"json("})json"; + proto::meta::ReportEventRequest generic_large; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(large_json, &generic_large)); + std::string mutable_large = large_json; + proto::meta::ReportEventRequest mutable_large_fast; + ASSERT_TRUE(ReportEventJsonParser::FromMutableNullTerminatedJson( + mutable_large.data(), mutable_large.size(), &mutable_large_fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic_large, mutable_large_fast)); + EXPECT_NE(large_json, mutable_large); +} + +TEST_F(ProtoMessageJsonUtilTest, TestReportEventFastJsonParserFallsBackForCompatibleRareShapes) { + const std::string json = R"json({ + "trace_id":null, + "instance_id":"fallback-instance", + "host_ip_port":"host:8080", + "events":[{"event_type":"FUTURE_EVENT_NAME","heartbeat":{"system_status":{}}}], + "storage_type":"ST_EVENT_REPORT_L2" + })json"; + + proto::meta::ReportEventRequest fast_attempt; + EXPECT_FALSE(ReportEventJsonParser::TryFromJson(json, &fast_attempt)); + + proto::meta::ReportEventRequest generic; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(json, &generic)); + proto::meta::ReportEventRequest with_fallback; + ASSERT_TRUE(ReportEventJsonParser::FromJson(json, &with_fallback)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, with_fallback)); + EXPECT_EQ(proto::meta::EVENT_UNSPECIFIED, with_fallback.events(0).event_type()); + + std::string mutable_json = json; + mutable_json.pop_back(); + mutable_json += R"json(,"ignored_padding":")json"; + mutable_json.append(40 * 1024, 'p'); + mutable_json += R"json("})json"; + proto::meta::ReportEventRequest generic_large; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(mutable_json, &generic_large)); + proto::meta::ReportEventRequest mutable_with_fallback; + ASSERT_TRUE(ReportEventJsonParser::FromMutableNullTerminatedJson( + mutable_json.data(), mutable_json.size(), &mutable_with_fallback)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic_large, mutable_with_fallback)); + + EXPECT_FALSE(ReportEventJsonParser::FromJson("{\"events\":[", &with_fallback)); + EXPECT_FALSE(ReportEventJsonParser::FromJson("{}", nullptr)); +} + +TEST_F(ProtoMessageJsonUtilTest, TestReportEventFastJsonParserValidatesNonAsciiInput) { + const std::string unicode_json = R"json({ + "trace_id":"追踪", + "instance_id":"实例", + "host_ip_port":"host:8080", + "events":[{"event_type":"EVENT_BLOCK_ADD","block_add":{ + "block_key":"1","medium":"内存","specs":[{"name":"分片","uri":"event_report://host/内存?标签=值"}] + }}], + "storage_type":"ST_EVENT_REPORT_L2" + })json"; + proto::meta::ReportEventRequest generic; + proto::meta::ReportEventRequest fast; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(unicode_json, &generic)); + ASSERT_TRUE(ReportEventJsonParser::TryFromJson(unicode_json, &fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, fast)); + + std::string invalid_json = "{\"trace_id\":\""; + invalid_json.push_back(static_cast(0xff)); + invalid_json += "\",\"instance_id\":\"i\",\"host_ip_port\":\"h\",\"events\":[],\"storage_type\":8}"; + EXPECT_FALSE(ReportEventJsonParser::TryFromJson(invalid_json, &fast)); + EXPECT_FALSE(ReportEventJsonParser::FromJson(invalid_json, &fast)); +} + +TEST_F(ProtoMessageJsonUtilTest, TestReportEventLargeInSituJsonParserPreservesEscapesAndRejectsRawNul) { + std::string json = R"json({ + "trace_id":"quote:\" slash:\\ newline:\n nul:\u0000 snowman:\u2603", + "instanceId":"large-insitu-instance", + "host_ip_port":"host:8080", + "events":[{"eventType":"EVENT_BLOCK_ADD","blockAdd":{ + "blockKey":"7","medium":"mem","specs":[{ + "name":"spec\u005f0","uri":"event_report://host/mem?escaped=a%5C%22b" + }] + }}], + "storageType":"ST_EVENT_REPORT_L2", + "ignored_padding":")json"; + json.append(40 * 1024, 'p'); + json += R"json("})json"; + ASSERT_GT(json.size(), 32U * 1024U); + + proto::meta::ReportEventRequest generic; + proto::meta::ReportEventRequest fast; + ASSERT_TRUE(ProtoMessageJsonUtil::FromJson(json, &generic)); + for (int iteration = 0; iteration < 4; ++iteration) { + ASSERT_TRUE(ReportEventJsonParser::TryFromJson(json, &fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, fast)); + } + + std::string mutable_json = json; + proto::meta::ReportEventRequest mutable_fast; + ASSERT_TRUE( + ReportEventJsonParser::FromMutableNullTerminatedJson(mutable_json.data(), mutable_json.size(), &mutable_fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, mutable_fast)); + + std::string non_terminated_view = json + "!"; + proto::meta::ReportEventRequest non_terminated_fast; + ASSERT_TRUE(ReportEventJsonParser::FromMutableNullTerminatedJson( + non_terminated_view.data(), json.size(), &non_terminated_fast)); + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, non_terminated_fast)); + EXPECT_EQ('!', non_terminated_view[json.size()]); + EXPECT_NE(std::string::npos, fast.trace_id().find('\0')); + EXPECT_EQ("spec_0", fast.events(0).block_add().specs(0).name()); + + std::string raw_nul = json; + raw_nul.insert(raw_nul.size() - 1, 1, '\0'); + EXPECT_FALSE(ReportEventJsonParser::TryFromJson(raw_nul, &fast)); + EXPECT_FALSE(ReportEventJsonParser::FromJson(raw_nul, &fast)); + EXPECT_FALSE(ReportEventJsonParser::FromMutableNullTerminatedJson(raw_nul.data(), raw_nul.size(), &fast)); +} + +TEST_F(ProtoMessageJsonUtilTest, TestReportEventJsonParserCompatibilityCorpusMatchesGenericParser) { + struct CompatibilityCase { + const char *name; + std::string json; + }; + const std::vector cases{ + {"empty", R"json({})json"}, + {"canonical", + R"json({"trace_id":"t","instance_id":"i","host_ip_port":"h:1","events":[{"event_type":"EVENT_BLOCK_ADD","block_add":{"block_key":"1","medium":"mem","specs":[{"name":"tp0","uri":"event_report://h:1/mem?size=1"}]}}],"storage_type":"ST_EVENT_REPORT_L2"})json"}, + {"camel_case_and_numeric_enums", + R"json({"traceId":"t","instanceId":"i","hostIpPort":"h:1","events":[{"eventType":5,"heartbeat":{"systemStatus":{"state":"ready"}}}],"storageType":8})json"}, + {"known_null_fields", + R"json({"trace_id":null,"instance_id":"i","host_ip_port":"h:1","events":null,"storage_type":null})json"}, + {"unknown_enum_names", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":"EVENT_ADDED_LATER","heartbeat":{"system_status":{}}}],"storage_type":"ST_ADDED_LATER"})json"}, + {"numeric_string_fields", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":"EVENT_BLOCK_ADD","block_add":{"block_key":18446744073709551615,"medium":"mem","specs":[]}}],"storage_type":8})json"}, + {"duplicate_aliases", + R"json({"trace_id":"first","traceId":"second","instance_id":"i","host_ip_port":"h:1","events":[],"storage_type":8})json"}, + {"duplicate_map_keys", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":5,"heartbeat":{"system_status":{"state":"first","state":"second"}}}],"storage_type":8})json"}, + {"escaped_duplicate_map_keys", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":5,"heartbeat":{"system_status":{"state":"first","\u0073tate":"second"}}}],"storage_type":8})json"}, + {"unknown_numeric_enums", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":99,"heartbeat":{}}],"storage_type":99})json"}, + {"multiple_oneof_members", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":2,"heartbeat":{},"block_add":{"block_key":"1","medium":"mem","specs":[]}}],"storage_type":8})json"}, + {"payload_before_event_type", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"block_add":{"specs":[{"uri":"event_report://h:1/mem","name":"tp0"}],"medium":"mem","block_key":"1"},"event_type":2}],"storage_type":8})json"}, + {"duplicate_nested_field_aliases", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":2,"block_add":{"block_key":"1","blockKey":"2","medium":"mem","specs":[]}}],"storage_type":8})json"}, + {"duplicate_unknown_members", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[],"storage_type":8,"future":1,"future":2})json"}, + {"unknown_nested_values", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[{"event_type":4,"host_down":{"future":{"array":[1,true,null,{"x":"y"}]}}}],"storage_type":8,"future_top":{"deep":{"value":1}}})json"}, + {"escaped_unicode", + R"json({"trace_id":"pair:\ud83d\ude80 nul:\u0000","instance_id":"\u5b9e\u4f8b","host_ip_port":"h:1","events":[{"event_type":1,"node_register":{"mediums":["m\u00e9m"]}}],"storage_type":8})json"}, + {"unpaired_surrogate", + R"json({"trace_id":"bad:\ud800","instance_id":"i","host_ip_port":"h:1","events":[],"storage_type":8})json"}, + {"wrong_known_field_types", + R"json({"trace_id":7,"instance_id":"i","host_ip_port":"h:1","events":{},"storage_type":true})json"}, + {"null_oneof_and_repeated_entries", + R"json({"instance_id":"i","host_ip_port":"h:1","events":[null,{"event_type":2,"block_add":null},{"event_type":2,"block_add":{"block_key":"1","medium":"mem","specs":[null]}}],"storage_type":8})json"}, + }; + + for (const auto &test_case : cases) { + SCOPED_TRACE(test_case.name); + proto::meta::ReportEventRequest generic; + const bool generic_ok = ProtoMessageJsonUtil::FromJson(test_case.json, &generic); + + proto::meta::ReportEventRequest compatible; + compatible.set_trace_id("must-be-cleared"); + const bool compatible_ok = ReportEventJsonParser::FromJson(test_case.json, &compatible); + EXPECT_EQ(generic_ok, compatible_ok); + if (generic_ok && compatible_ok) { + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic, compatible)); + } + + // Exercise the HTTP-only in-situ path as well. Appending an ignored + // field keeps the protobuf meaning unchanged while forcing the body + // above the mutable parser's 32-KiB threshold. This is especially + // important for rare shapes: after RapidJSON mutates the source, the + // compatibility fallback must reconstruct exactly the same protobuf + // semantics from the complete DOM. + ASSERT_FALSE(test_case.json.empty()); + ASSERT_EQ('}', test_case.json.back()); + std::string large_json = test_case.json; + large_json.pop_back(); + if (large_json.back() != '{') { + large_json.push_back(','); + } + large_json += R"json("ignored_padding":")json"; + large_json.append(40 * 1024, 'p'); + large_json += R"json("})json"; + + proto::meta::ReportEventRequest generic_large; + const bool generic_large_ok = ProtoMessageJsonUtil::FromJson(large_json, &generic_large); + EXPECT_EQ(generic_ok, generic_large_ok); + + proto::meta::ReportEventRequest mutable_compatible; + const bool mutable_ok = ReportEventJsonParser::FromMutableNullTerminatedJson( + large_json.data(), large_json.size(), &mutable_compatible); + EXPECT_EQ(generic_large_ok, mutable_ok); + if (generic_large_ok && mutable_ok) { + EXPECT_TRUE(google::protobuf::util::MessageDifferencer::Equals(generic_large, mutable_compatible)); + } + } +} + TEST_F(ProtoMessageJsonUtilTest, TestFromJsonEnum) { { // normal EnumMessage msg; diff --git a/package/etc/default_server_config.conf b/package/etc/default_server_config.conf index fb1a14f83..ab413b598 100644 --- a/package/etc/default_server_config.conf +++ b/package/etc/default_server_config.conf @@ -72,6 +72,12 @@ kvcm.schedule_plan_executor_thread_count=8 # 0 < migration_worker_budget < executor_thread_count，以保证至少一个 worker 可处理回收和系统任务 kvcm.schedule_plan_migration_worker_budget=3 +# GetHostCacheState 大请求的独立有界 query executor。worker_count 包含 RPC caller， +# 4 表示 caller + 3 个后台线程；小于 threshold 的请求保持串行。 +kvcm.meta_query.worker_count=4 +kvcm.meta_query.parallel_threshold=256 +kvcm.meta_query.chunk_size=128 + # 删除请求在 delay 结束后允许继续抵扣水位的最长时间；只关闭 credit，不中止底层 I/O kvcm.cache_reclaimer.inflight_delete_timeout_ms=60000 diff --git a/package/script/start_server.sh b/package/script/start_server.sh index 3babf9da5..a707fc9dd 100644 --- a/package/script/start_server.sh +++ b/package/script/start_server.sh @@ -12,6 +12,57 @@ DEFAULT_SERVER_CONFIG=$CONFIG_PATH/default_server_config.conf DEFAULT_LOGGER_CONFIG=$CONFIG_PATH/default_logger_config.conf BINARY=$BINARY_PATH/kv_cache_manager_bin +function configure_jemalloc() { + if [ "${KVCM_USE_JEMALLOC:-1}" = "0" ]; then + echo "jemalloc disabled by KVCM_USE_JEMALLOC=0" + return 0 + fi + + local arch + arch=$(uname -m) + local candidates=() + if [ -n "${KVCM_JEMALLOC_PATH:-}" ]; then + candidates+=("$KVCM_JEMALLOC_PATH") + fi + case "$arch" in + x86_64 | amd64) + candidates+=( + "/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" + "/usr/lib64/libjemalloc.so.2" + ) + ;; + aarch64 | arm64) + candidates+=( + "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2" + "/usr/lib64/libjemalloc.so.2" + ) + ;; + *) + echo "unsupported architecture for jemalloc auto-detection: $arch" >&2 + return 0 + ;; + esac + + local jemalloc_path="" + local candidate + for candidate in "${candidates[@]}"; do + if [ -r "$candidate" ]; then + jemalloc_path=$candidate + break + fi + done + if [ -z "$jemalloc_path" ]; then + echo "jemalloc library not found for architecture $arch; continue with the default allocator" >&2 + return 0 + fi + + case ":${LD_PRELOAD// /:}:" in + *":$jemalloc_path:"*) ;; + *) export LD_PRELOAD="$jemalloc_path${LD_PRELOAD:+:$LD_PRELOAD}" ;; + esac + echo "jemalloc enabled: LD_PRELOAD=$LD_PRELOAD" +} + function install_kvcm_ops() { python3 -m pip install "$KVCM_OPS_WHEEL_PATH" } @@ -22,8 +73,9 @@ function start_server() { } function main() { + configure_jemalloc install_kvcm_ops start_server "$@" } -main "$@" \ No newline at end of file +main "$@" diff --git a/tools/scripts/report_event_load.py b/tools/scripts/report_event_load.py index 6373050c5..844d96d5e 100644 --- a/tools/scripts/report_event_load.py +++ b/tools/scripts/report_event_load.py @@ -446,7 +446,7 @@ def build_get_payload( def parse_host_prefixes(response: Dict) -> Dict[str, int]: return { - item["host_ip_port"]: int(item["prefix_match_blocks"]) + item["host_ip_port"]: int(item["local"]) for item in response.get("hosts", []) }