[manager/service/client] add cache meta detail API

docs/api/meta_service.md

@@ -227,3 +227,82 @@ curl -g -vvv -X POST http://localhost:6382/api/getCacheMeta \
     "detail_level": 1
 }'
 ```
+
+## Get Cache Meta Detail
+```bash
+curl -g -vvv -X POST http://localhost:6382/api/getCacheMetaDetail \
+  -H "Content-Type: application/json" \
+  -H "Accept: application/json" \
+  -d '{
+    "trace_id": "trace_id_131",
+    "instance_id": "test_instance",
+    "block_keys": [123, 456],
+    "block_mask": {
+        "offset": 0
+    },
+    "detail_level": 1
+}'
+```
+
+Example response:
+```json
+{
+  "header": {
+    "status": {
+      "code": "OK",
+      "message": "Cache metadata detail retrieved successfully"
+    },
+    "request_id": "request_id"
+  },
+  "items": [
+    {
+      "status": {
+        "code": "OK"
+      },
+      "request_index": 0,
+      "block_key": 123,
+      "prev_block_key": "",
+      "properties": {
+        "BP#prev_key": ""
+      },
+      "locations": [
+        {
+          "location_id": "loc_a",
+          "status": "CLS_SERVING",
+          "type": "ST_3FS",
+          "spec_size": 2,
+          "create_time": 1710000000000000,
+          "location_specs": [
+            {"name": "tp0", "uri": "3fs://cluster/root/key_123_tp0?offset=0&size=1024"},
+            {"name": "tp1", "uri": "3fs://cluster/root/key_123_tp1?offset=0&size=1024"}
+          ]
+        },
+        {
+          "location_id": "loc_b",
+          "status": "CLS_WRITING",
+          "type": "ST_NFS",
+          "spec_size": 1,
+          "create_time": 1710000001000000,
+          "location_specs": [
+            {"name": "tp0", "uri": "file://nfs/root/key_123_tp0?offset=0&size=1024"}
+          ]
+        }
+      ]
+    },
+    {
+      "status": {
+        "code": "OK"
+      },
+      "request_index": 1,
+      "block_key": 456,
+      "locations": [
+        {
+          "status": "CLS_NOT_FOUND"
+        }
+      ]
+    }
+  ]
+}
+```
+
+This diagnostic API returns raw metadata for every unmasked requested key. It does not apply location selection, data-file existence filtering, or lazy prune. `prev_block_key` is a convenience projection of `properties["BP#prev_key"]`; the raw property is intentionally retained. A missing key is reported as a successful item with `CLS_NOT_FOUND`. Backend or deserialization failures are reported in that item's `status` while other successful items are still returned. If every item fails to read, the request returns an overall error.

integration_test/meta_service/grpc_interface_test.py

@@ -7,6 +7,7 @@
     RegisterInstanceRequest,
     GetInstanceInfoRequest,
     GetCacheLocationRequest, 
+    GetCacheMetaDetailRequest,
     StartWriteCacheRequest,
     FinishWriteCacheRequest,
     RemoveCacheRequest,
@@ -76,6 +77,18 @@ def get_cache_location(self, data, check_response=True):
                     f"Request to get_cache_location failed with error: {response_dict['header']['status']['message']}")
         return response_dict
 
+    def get_cache_meta_detail(self, data, check_response=True):
+        """Get full raw metadata detail for specified block keys"""
+        request = self._convert_dict_to_proto(GetCacheMetaDetailRequest, data)
+        response = self._stub.GetCacheMetaDetail(request, timeout=self._timeout)
+        response_dict = self._convert_proto_to_dict(response)
+        if check_response:
+            if response_dict['header']['status']['code'] != "OK":
+                raise AssertionError(
+                    f"Request to get_cache_meta_detail failed with error: "
+                    f"{response_dict['header']['status']['message']}")
+        return response_dict
+
     def start_write_cache(self, data, check_response=True):
         """Start writing cache data"""
         request = self._convert_dict_to_proto(StartWriteCacheRequest, data)

integration_test/meta_service/http_interface_test.py

@@ -53,6 +53,10 @@ def get_cache_location(self, data, check_response=True):
         """Get cache location for specified block keys"""
         return self._make_api_request('/api/getCacheLocation', data, check_response)
 
+    def get_cache_meta_detail(self, data, check_response=True):
+        """Get full raw metadata detail for specified block keys"""
+        return self._make_api_request('/api/getCacheMetaDetail', data, check_response)
+
     def start_write_cache(self, data, check_response=True):
         """Start writing cache data"""
         return self._make_api_request('/api/startWriteCache', data, check_response)

integration_test/meta_service/meta_interface_cases.py

@@ -41,6 +41,11 @@ def get_cache_location(self, data, check_response=True) -> Dict:
         """Get cache location for specified block keys"""
         return {}
 
+    @abc.abstractmethod
+    def get_cache_meta_detail(self, data, check_response=True) -> Dict:
+        """Get full raw metadata detail for specified block keys"""
+        return {}
+
     @abc.abstractmethod
     def start_write_cache(self, data, check_response=True) -> Dict:
         """Start writing cache data"""
@@ -149,7 +154,35 @@ def test_basic_smoke(self):
 
         self._client.finish_write_cache(finish_write_data)
 
-        # Step 4: Get cache location to verify it was added correctly
+        # Step 4: Query raw metadata detail for one existing and one missing key.
+        detail_data = {
+            "trace_id": self._trace_id,
+            "instance_id": self._instance_id,
+            "block_keys": [123, 999],
+            "block_mask": {
+                "offset": 0
+            },
+            "detail_level": 1
+        }
+        detail_response = self._client.get_cache_meta_detail(detail_data)
+        self.assertIn('items', detail_response)
+        self.assertEqual(2, len(detail_response['items']))
+
+        found_item, missing_item = detail_response['items']
+        self.assertEqual(0, found_item['request_index'])
+        self.assertEqual(123, int(found_item['block_key']))
+        self.assertEqual('OK', found_item['status']['code'])
+        self.assertGreater(len(found_item['locations']), 0)
+        self.assertTrue(any(location['status'] == 'CLS_SERVING'
+                            for location in found_item['locations']))
+
+        self.assertEqual(1, missing_item['request_index'])
+        self.assertEqual(999, int(missing_item['block_key']))
+        self.assertEqual('OK', missing_item['status']['code'])
+        self.assertEqual(['CLS_NOT_FOUND'],
+                         [location['status'] for location in missing_item['locations']])
+
+        # Step 5: Get cache location to verify it was added correctly
         get_location_data = {
             "trace_id": self._trace_id,
             "query_type": "QT_PREFIX_MATCH",

kv_cache_manager/client/include/common.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <cstdint>
 #include <map>
 #include <memory>
 #include <string>
@@ -83,6 +84,47 @@ struct Metas {
     std::vector<std::string> metas;
 };
 
+enum class CacheMetaLocationStatus : int32_t {
+    CLS_NOT_FOUND = 0,
+    CLS_NEW = 1,
+    CLS_WRITING = 2,
+    CLS_SERVING = 3,
+    CLS_DELETING = 4,
+};
+
+enum class CacheMetaStorageType : int32_t {
+    ST_UNSPECIFIED = 0,
+    ST_3FS = 1,
+    ST_MOONCAKE = 2,
+    ST_TAIRMEMPOOL = 3,
+    ST_NFS = 4,
+    ST_VCNS_3FS = 5,
+    ST_DUMMY = 6,
+    ST_EVENT_REPORT_L1P5 = 7,
+    ST_EVENT_REPORT_L2 = 8,
+};
+
+struct CacheMetaLocationDetail {
+    std::string location_id;
+    CacheMetaLocationStatus status{CacheMetaLocationStatus::CLS_NOT_FOUND};
+    CacheMetaStorageType storage_type{CacheMetaStorageType::ST_UNSPECIFIED};
+    int32_t spec_size{0};
+    int64_t create_time{0};
+    Location location_specs;
+};
+
+struct CacheMetaDetailItem {
+    ClientErrorCode error_code{ClientErrorCode::ER_OK};
+    std::string error_message;
+    int32_t request_index{0};
+    int64_t block_key{0};
+    std::string prev_block_key;
+    std::map<std::string, std::string> properties;
+    std::vector<CacheMetaLocationDetail> locations;
+};
+
+using CacheMetaDetails = std::vector<CacheMetaDetailItem>;
+
 using BlockMaskVector = std::vector<bool>;
 using BlockMaskOffset = size_t;
 using BlockMask = std::variant<BlockMaskVector, BlockMaskOffset>;
@@ -189,4 +231,4 @@ struct TransferTraceInfo {
     std::vector<std::string> block_ids; // block_ids.size() must be equal to block_buffer.size()
 };
 
-} // namespace kv_cache_manager
+} // namespace kv_cache_manager

kv_cache_manager/client/include/manager_client.h

@@ -43,6 +43,12 @@ class ManagerClient {
                                                         const BlockMask &block_mask,
                                                         int32_t detail_level) = 0;
 
+    virtual std::pair<ClientErrorCode, CacheMetaDetails> MatchMetaDetail(const std::string &trace_id,
+                                                                         const std::vector<int64_t> &keys,
+                                                                         const std::vector<int64_t> &tokens,
+                                                                         const BlockMask &block_mask,
+                                                                         int32_t detail_level) = 0;
+
     virtual ClientErrorCode RemoveCache(const std::string &trace_id,
                                         const std::vector<int64_t> &keys,
                                         const std::vector<int64_t> &tokens,
@@ -59,4 +65,4 @@ class ManagerClient {
     virtual void Shutdown() = 0;
 };
 
-} // namespace kv_cache_manager
+} // namespace kv_cache_manager

kv_cache_manager/client/include/meta_client.h

@@ -42,6 +42,12 @@ class MetaClient {
                                                         const BlockMask &block_mask,
                                                         int32_t detail_level) = 0;
 
+    virtual std::pair<ClientErrorCode, CacheMetaDetails> MatchMetaDetail(const std::string &trace_id,
+                                                                         const std::vector<int64_t> &keys,
+                                                                         const std::vector<int64_t> &tokens,
+                                                                         const BlockMask &block_mask,
+                                                                         int32_t detail_level) = 0;
+
     virtual std::pair<ClientErrorCode, int64_t> MatchLocationLen(const std::string &trace_id,
                                                                  QueryType query_type,
                                                                  const std::vector<int64_t> &keys,
@@ -60,4 +66,4 @@ class MetaClient {
     virtual ClientErrorCode Init(const std::string &config, const InitParams &init_params) = 0;
     virtual void Shutdown() = 0;
 };
-} // namespace kv_cache_manager
+} // namespace kv_cache_manager

kv_cache_manager/client/src/internal/stub/grpc_stub.cc

@@ -5,6 +5,8 @@
 #include <chrono>
 #include <grpcpp/grpcpp.h>
 #include <type_traits>
+#include <unordered_map>
+#include <utility>
 
 #include "kv_cache_manager/client/src/internal/util/debug_string_util.h"
 #include "kv_cache_manager/common/logger.h"
@@ -63,6 +65,8 @@
 
 namespace {
 
+kv_cache_manager::ClientErrorCode ToClientError(kv_cache_manager::proto::meta::ErrorCode service_error);
+
 kv_cache_manager::Locations GenLocations(
     const google::protobuf::RepeatedPtrField<::kv_cache_manager::proto::meta::CacheLocation> &proto_locations) {
     kv_cache_manager::Locations locations;
@@ -78,6 +82,42 @@ kv_cache_manager::Locations GenLocations(
     return locations;
 }
 
+kv_cache_manager::CacheMetaDetails
+GenCacheMetaDetails(const google::protobuf::RepeatedPtrField<::kv_cache_manager::proto::meta::CacheMetaDetailItem>
+                        &proto_cache_meta_details) {
+    kv_cache_manager::CacheMetaDetails cache_meta_details;
+    cache_meta_details.reserve(proto_cache_meta_details.size());
+    for (const auto &proto_item : proto_cache_meta_details) {
+        kv_cache_manager::CacheMetaDetailItem item;
+        if (proto_item.has_status() && proto_item.status().code() != kv_cache_manager::proto::meta::OK) {
+            item.error_code = ToClientError(proto_item.status().code());
+            item.error_message = proto_item.status().message();
+        }
+        item.request_index = proto_item.request_index();
+        item.block_key = proto_item.block_key();
+        item.prev_block_key = proto_item.prev_block_key();
+        for (const auto &[property_name, property_value] : proto_item.properties()) {
+            item.properties[property_name] = property_value;
+        }
+        item.locations.reserve(proto_item.locations_size());
+        for (const auto &proto_location : proto_item.locations()) {
+            kv_cache_manager::CacheMetaLocationDetail location;
+            location.location_id = proto_location.location_id();
+            location.status = static_cast<kv_cache_manager::CacheMetaLocationStatus>(proto_location.status());
+            location.storage_type = static_cast<kv_cache_manager::CacheMetaStorageType>(proto_location.type());
+            location.spec_size = proto_location.spec_size();
+            location.create_time = proto_location.create_time();
+            location.location_specs.reserve(proto_location.location_specs_size());
+            for (const auto &proto_spec : proto_location.location_specs()) {
+                location.location_specs.push_back({proto_spec.name(), proto_spec.uri()});
+            }
+            item.locations.push_back(std::move(location));
+        }
+        cache_meta_details.push_back(std::move(item));
+    }
+    return cache_meta_details;
+}
+
 kv_cache_manager::ClientErrorCode
 GenCacheLocation(const kv_cache_manager::Locations &locations,
                  google::protobuf::RepeatedPtrField<::kv_cache_manager::proto::meta::CacheLocation> *proto_locations) {
@@ -322,6 +362,27 @@ std::pair<ClientErrorCode, Metas> GrpcStub::GetCacheMeta(const std::string &trac
     return {ER_OK, {locations, metas}};
 }
 
+std::pair<ClientErrorCode, CacheMetaDetails> GrpcStub::GetCacheMetaDetail(const std::string &trace_id,
+                                                                          const std::string &instance_id,
+                                                                          const KeyVector &keys,
+                                                                          const TokenIdsVector &tokens,
+                                                                          const BlockMask &block_mask,
+                                                                          int32_t detail_level) {
+    auto stub = GET_AND_CHECK_STUB_WITH_TYPE();
+    proto::meta::GetCacheMetaDetailRequest request;
+    SetKeysAndTokens(request, trace_id, instance_id, keys, tokens);
+    ProtoConvert::BlockMaskToProto(block_mask, request.mutable_block_mask());
+    request.set_detail_level(detail_level);
+    grpc::ClientContext context;
+    proto::meta::GetCacheMetaDetailResponse response;
+    auto grpc_status = stub->GetCacheMetaDetail(&context, request, &response);
+    CHECK_GRPC_STATUS_WITH_TYPE(grpc_status);
+    CHECK_COMMON_HEADER_WITH_TYPE(response);
+    auto cache_meta_details = GenCacheMetaDetails(response.items());
+    KVCM_LOG_DEBUG("get cache meta detail success, items: %lu", cache_meta_details.size());
+    return {ER_OK, cache_meta_details};
+}
+
 std::pair<ClientErrorCode, Locations> GrpcStub::GetCacheLocation(const std::string &trace_id,
                                                                  const std::string &instance_id,
                                                                  QueryType query_type,

kv_cache_manager/client/src/internal/stub/grpc_stub.h

@@ -36,6 +36,13 @@ class GrpcStub : public Stub {
                                                    const BlockMask &block_mask,
                                                    int32_t detail_level) override;
 
+    std::pair<ClientErrorCode, CacheMetaDetails> GetCacheMetaDetail(const std::string &trace_id,
+                                                                    const std::string &instance_id,
+                                                                    const KeyVector &keys,
+                                                                    const TokenIdsVector &tokens,
+                                                                    const BlockMask &block_mask,
+                                                                    int32_t detail_level) override;
+
     std::pair<ClientErrorCode, Locations>
     GetCacheLocation(const std::string &trace_id,
                      const std::string &instance_id,

kv_cache_manager/client/src/internal/stub/stub.h

@@ -40,6 +40,13 @@ class Stub {
                                                            const BlockMask &block_mask,
                                                            int32_t detail_level) = 0;
 
+    virtual std::pair<ClientErrorCode, CacheMetaDetails> GetCacheMetaDetail(const std::string &trace_id,
+                                                                            const std::string &instance_id,
+                                                                            const KeyVector &keys,
+                                                                            const TokenIdsVector &tokens,
+                                                                            const BlockMask &block_mask,
+                                                                            int32_t detail_level) = 0;
+
     virtual std::pair<ClientErrorCode, Locations>
     GetCacheLocation(const std::string &trace_id,
                      const std::string &instance_id,