[ci] add coverage reporting workflow

.github/workflows/coverage.yml

@@ -0,0 +1,266 @@
+name: coverage
+permissions:
+  contents: read
+  actions: write
+on:
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+    inputs:
+      runs-on:
+        description: "Runner type"
+        type: choice
+        options:
+          - ubuntu-24.04-arm
+          - ubuntu-latest
+          - aliyun-ecs-x64
+        default: ubuntu-latest
+      rebuildDiskCache:
+        description: "Rebuild disk cache"
+        type: boolean
+        default: false
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest' }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  coverage-github:
+    outputs:
+      coverage-artifact-url: ${{ steps.upload_coverage_artifacts.outputs.artifact-url }}
+    if: ${{ (inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest') == 'ubuntu-latest' || (inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest') == 'ubuntu-24.04-arm' }}
+    name: coverage
+    runs-on: ${{ inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest' }}
+    container:
+      image: ghcr.io/alibaba/tair-kvcache-kvcm-dev:2026_02_13_12_03_24230b1
+      volumes:
+        - /:/host_root/
+      options: --privileged
+    steps:
+      # https://github.com/actions/runner-images/issues/2840
+      - name: Free up disk space
+        run: |
+          echo "Disk space before cleanup:"
+          df -h
+          rm -rf /host_root/usr/share/dotnet
+          rm -rf /host_root/usr/local/lib/android
+          rm -rf /host_root/opt/ghc
+          echo "Disk space after cleanup:"
+          df -h
+
+      - &checkout_step
+        name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - &setup_bazel_step
+        name: setup_bazel
+        uses: bazel-contrib/setup-bazel@0.18.0
+        with:
+          bazelisk-cache: true
+          disk-cache: ${{ runner.os }}-${{ github.workflow }}
+          repository-cache: true
+          cache-save: ${{ inputs.rebuildDiskCache || false }}
+
+      - &clean_disk_cache_step
+        name: clean_disk_cache
+        if: ${{ inputs.rebuildDiskCache || false }}
+        run: |
+          rm -rf ~/.cache/bazel-disk
+
+      - &install_coverage_tools_step
+        name: install_coverage_tools
+        run: |
+          set -e
+          set -x
+          if ! command -v genhtml >/dev/null 2>&1; then
+            if command -v apt-get >/dev/null 2>&1; then
+              apt-get update -qq
+              DEBIAN_FRONTEND=noninteractive apt-get install -y -qq lcov
+            elif command -v yum >/dev/null 2>&1; then
+              yum install -y -q lcov
+            else
+              echo "No supported package manager found for installing lcov/genhtml" >&2
+              exit 1
+            fi
+          fi
+          genhtml --version
+
+      - &bazel_coverage_step
+        name: bazel_coverage
+        env:
+          COVERAGE_BASE_REF: ${{ github.event.pull_request.base.sha || 'origin/main' }}
+          COVERAGE_XTRACE: "1"
+        run: |
+          set -e
+          tools/coverage/run_coverage.sh \
+            --base-ref "${COVERAGE_BASE_REF:-origin/main}" \
+            --head-ref HEAD \
+            --output-dir coverage \
+            --include-prefix kv_cache_manager/ \
+            --jobs 8 \
+            --local-test-jobs 8 \
+            --test-timeout 900 \
+            --test-output errors \
+            --fetch-main \
+            -- \
+            //kv_cache_manager/... \
+            //integration_test/...
+
+      - &delete_old_disk_cache_step
+        name: delete_old_disk_cache
+        if: ${{ inputs.rebuildDiskCache || false }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -x
+          ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')
+          curl -sLo /usr/local/bin/jq "https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-linux-${ARCH}" && chmod +x /usr/local/bin/jq
+          GH_VER=2.89.0
+          curl -sL "https://github.com/cli/cli/releases/download/v${GH_VER}/gh_${GH_VER}_linux_${ARCH}.tar.gz" | tar xz --strip-components=1 -C /usr/local
+          RUNNER_ARCH_LOWER=$(echo "${{ runner.arch }}" | tr '[:upper:]' '[:lower:]')
+          DISK_CACHE_NAME="disk-${{ runner.os }}-${{ github.workflow }}"
+          gh cache list --repo ${{ github.repository }} --ref "${{ github.ref }}" --json id,key --limit 100 | \
+            jq -r --arg arch "$RUNNER_ARCH_LOWER" --arg name "$DISK_CACHE_NAME" \
+              '.[] | select((.key | contains($arch)) and (.key | contains($name))) | .id' | \
+            xargs -I {} gh cache delete {} --repo ${{ github.repository }} || true
+
+      - &append_coverage_summary_step
+        name: Append coverage summary
+        if: ${{ always() && hashFiles('coverage/coverage-summary.md') != '' }}
+        run: |
+          cat coverage/coverage-summary.md >> "${GITHUB_STEP_SUMMARY}"
+          if [ -f coverage/html/index.html ]; then
+            {
+              echo ""
+              echo "HTML coverage report: \`coverage/html/index.html\` in the uploaded artifact."
+            } >> "${GITHUB_STEP_SUMMARY}"
+          fi
+
+      - &upload_coverage_artifacts_step
+        name: Upload coverage artifacts
+        id: upload_coverage_artifacts
+        uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: coverage-report
+          path: |
+            coverage/**
+            bazel-testlogs/**/*.xml
+          if-no-files-found: warn
+          overwrite: true
+
+  coverage-aliyun:
+    outputs:
+      coverage-artifact-url: ${{ steps.upload_coverage_artifacts.outputs.artifact-url }}
+    if: ${{ (inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest') == 'aliyun-ecs-x64' }}
+    name: coverage
+    runs-on: aliyun-ecs-x64
+    container:
+      image: ghcr.io/alibaba/tair-kvcache-kvcm-dev:2026_02_13_12_03_24230b1
+      options: --privileged
+    steps:
+      - *checkout_step
+      - *setup_bazel_step
+      - *clean_disk_cache_step
+      - *install_coverage_tools_step
+      - *bazel_coverage_step
+      - *delete_old_disk_cache_step
+      - *append_coverage_summary_step
+      - *upload_coverage_artifacts_step
+
+  comment-pr-coverage:
+    if: ${{ always() && github.event_name == 'pull_request' && (needs.coverage-github.outputs.coverage-artifact-url != '' || needs.coverage-aliyun.outputs.coverage-artifact-url != '') }}
+    needs:
+      - coverage-github
+      - coverage-aliyun
+    name: comment_pr_coverage
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      issues: write
+    steps:
+      - name: Download coverage artifact
+        uses: actions/download-artifact@v6
+        continue-on-error: true
+        with:
+          name: coverage-report
+          path: coverage-artifact
+
+      - name: Comment PR coverage
+        uses: actions/github-script@v8
+        env:
+          COVERAGE_ARTIFACT_URL: ${{ needs.coverage-github.outputs.coverage-artifact-url || needs.coverage-aliyun.outputs.coverage-artifact-url }}
+        with:
+          script: |
+            const fs = require('fs');
+
+            const marker = '<!-- tair-kvcache-coverage-report -->';
+            const summaryPath = 'coverage-artifact/coverage/coverage-summary.md';
+            if (!fs.existsSync(summaryPath)) {
+              core.warning(`Coverage summary not found at ${summaryPath}; skip coverage comment.`);
+              return;
+            }
+
+            let summary = fs.readFileSync(summaryPath, 'utf8').trim();
+            const maxSummaryLength = 60000;
+            if (summary.length > maxSummaryLength) {
+              summary = `${summary.slice(0, maxSummaryLength)}\n\n_Comment truncated; see the uploaded artifact for the full report._`;
+            }
+
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const artifactUrl = process.env.COVERAGE_ARTIFACT_URL || '';
+            const links = artifactUrl
+              ? `HTML report and raw LCOV: [coverage-report](${artifactUrl})`
+              : `Workflow run: [${context.runId}](${runUrl})`;
+            const body = [
+              marker,
+              summary,
+              '',
+              links,
+              `Workflow run: [${context.runId}](${runUrl})`,
+            ].join('\n');
+
+            const issue_number = context.payload.pull_request?.number;
+            if (!issue_number) {
+              core.warning('No pull request number found; skip coverage comment.');
+              return;
+            }
+
+            try {
+              const { owner, repo } = context.repo;
+              const comments = await github.paginate(github.rest.issues.listComments, {
+                owner,
+                repo,
+                issue_number,
+                per_page: 100,
+              });
+              const existing = comments.find((comment) => comment.body?.includes(marker));
+
+              if (existing) {
+                await github.rest.issues.updateComment({
+                  owner,
+                  repo,
+                  comment_id: existing.id,
+                  body,
+                });
+              } else {
+                await github.rest.issues.createComment({
+                  owner,
+                  repo,
+                  issue_number,
+                  body,
+                });
+              }
+            } catch (error) {
+              core.warning(`Failed to write PR coverage comment: ${error.message}`);
+            }
+
+  validate-runner-selection:
+    if: ${{ (inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest') != 'ubuntu-latest' && (inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest') != 'ubuntu-24.04-arm' && (inputs.runs-on || vars.COVERAGE_RUNS_ON || 'ubuntu-latest') != 'aliyun-ecs-x64' }}
+    name: validate_runner_selection
+    runs-on: ubuntu-latest
+    steps:
+      - name: fail_invalid_runner_selection
+        run: |
+          echo "::error::Invalid runner selection. Set runs-on or COVERAGE_RUNS_ON to one of: ubuntu-latest, ubuntu-24.04-arm, aliyun-ecs-x64."
+          exit 1

.github/workflows/scheduled-cache-rebuild.yml

@@ -23,3 +23,13 @@ jobs:
             -f rebuildDiskCache=true
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Trigger coverage workflow with cache rebuild
+        run: |
+          gh workflow run coverage.yml \
+            --repo ${{ github.repository }} \
+            --ref main \
+            -f runs-on=aliyun-ecs-x64 \
+            -f rebuildDiskCache=true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -6,6 +6,7 @@ bin/*
 lib/*
 var/*
 logs/*
+/coverage/
 kv_cache_manager/optimizer/analysis/result/*
 kv_cache_manager/optimizer/logs/*
 __pycache__/
@@ -15,4 +16,4 @@ __pycache__/
 !.bazeliskrc
 !.aoneci
 compile_commands.json
-stub_source
+stub_source

docs/develop/README.md

@@ -51,6 +51,17 @@ bazelisk info --announce_rc repository_cache
   - 需要本地启动一个Redis或Valkey。
   - ```bazelisk test //kv_cache_manager/common/test:redis_client_real_service_test //kv_cache_manager/meta/test:meta_redis_backend_real_service_test //kv_cache_manager/meta/test:meta_storage_backend_manager_real_redis_test //kv_cache_manager/meta/test:meta_indexer_redis_test //kv_cache_manager/manager/test:MetaSearcherRedisTest //kv_cache_manager/config/test:registry_manager_redis_backend_test --test_tag_filters=redis```
 - 启用ASAN：上述命令后添加 ```--config=debug --config=asan --test_env ASAN_OPTIONS=detect_odr_violation=0```
+- 覆盖率：
+  - ```tools/coverage/run_coverage.sh --base-ref origin/main --head-ref HEAD --output-dir coverage --include-prefix kv_cache_manager/ --jobs 8 --local-test-jobs 8 -- //kv_cache_manager/... //integration_test/...```
+  - 脚本会执行 Bazel LCOV 采集、归一化负 hit count、生成全量/增量覆盖率摘要，并在安装了 lcov/genhtml 时生成 HTML 报告。
+  - 输出位于 ```coverage/```，包含 ```lcov.info```、```coverage-summary.md```、```coverage-summary.json``` 和 ```html/index.html```。
+  - 如本地未安装 ```genhtml``` 且只需要 LCOV/摘要，可添加 ```--no-html```。
+  - ```gcov_json_isolated.sh``` 用于规避 Bazel 6.4 C++ coverage collector 在并发 gcov json 采集时共享 ```*.gcov.json.gz``` 中间文件导致的竞态。
+  - Bazel 6.4 默认不向 gcov 传 ```-b```；当前开发镜像的 GCC 10 支持通过 ```COVERAGE_GCOV_OPTIONS=-b``` 生成 ```BRDA/BRF/BRH``` 分支覆盖率记录。
+  - CI 会上传 ```coverage/lcov.info```、```coverage/coverage-summary.md```、```coverage/coverage-summary.json``` 和 ```coverage/html/```。
+  - PR 触发的 coverage 会更新同一条固定评论；评论 job 仅下载 coverage artifact，不检出或执行 PR 代码。
+  - ```coverage``` workflow 使用 setup-bazel 的 Bazelisk cache、repository cache 和独立 disk cache。PR 只读 cache；仅 nightly cache rebuild 或手动以 ```rebuildDiskCache=true``` 触发时保存并替换 main 分支的旧 disk cache。
+  - 可通过 workflow_dispatch 的 ```runs-on``` 或仓库变量 ```COVERAGE_RUNS_ON``` 选择 ```ubuntu-latest```、```ubuntu-24.04-arm``` 或 ```aliyun-ecs-x64```。
 ### 测试资源清理
 
 测试结束后会自动清理资源。测试工作目录位于 bazel runfiles 目录中，不会污染源代码目录。如果测试异常退出，可能需要手动清理：
@@ -158,4 +169,4 @@ githooks中已经添加了C++等语言的格式化脚本，请确保开发环境
 提交前检查和 commit message 格式见 [Commit 要求](commit_requirements.md)。
 
 ## CI
-可参考```.github/workflows```目录下的配置。```test-opensrc``` 在一个 ```normal_test``` job 中运行普通单元测试和集成测试（包含默认配置下的客户端测试目标），ASAN 测试使用独立 job。
+可参考```.github/workflows```目录下的配置。```test-opensrc``` 在一个 ```normal_test``` job 中运行普通单元测试和集成测试（包含默认配置下的客户端测试目标），ASAN 测试使用独立 job。```coverage``` 负责全量 LCOV、增量覆盖率和 HTML 报告。