[manager/service/client] preserve partial cache meta details

Author: wangxiyu191

docs/api/meta_service.md

@@ -256,6 +256,9 @@ Example response:
   },
   "items": [
     {
+      "status": {
+        "code": "OK"
+      },
       "request_index": 0,
       "block_key": 123,
       "prev_block_key": "",
@@ -287,6 +290,9 @@ Example response:
       ]
     },
     {
+      "status": {
+        "code": "OK"
+      },
       "request_index": 1,
       "block_key": 456,
       "locations": [
@@ -299,4 +305,4 @@ Example response:
 }
 ```
 
-This diagnostic API returns raw metadata for every unmasked requested key. It does not apply location selection, data-file existence filtering, or lazy prune.
+This diagnostic API returns raw metadata for every unmasked requested key. It does not apply location selection, data-file existence filtering, or lazy prune. `prev_block_key` is a convenience projection of `properties["BP#prev_key"]`; the raw property is intentionally retained. A missing key is reported as a successful item with `CLS_NOT_FOUND`. Backend or deserialization failures are reported in that item's `status` while other successful items are still returned. If every item fails to read, the request returns an overall error.

integration_test/meta_service/meta_interface_cases.py

@@ -154,7 +154,35 @@ def test_basic_smoke(self):
 
         self._client.finish_write_cache(finish_write_data)
 
-        # Step 4: Get cache location to verify it was added correctly
+        # Step 4: Query raw metadata detail for one existing and one missing key.
+        detail_data = {
+            "trace_id": self._trace_id,
+            "instance_id": self._instance_id,
+            "block_keys": [123, 999],
+            "block_mask": {
+                "offset": 0
+            },
+            "detail_level": 1
+        }
+        detail_response = self._client.get_cache_meta_detail(detail_data)
+        self.assertIn('items', detail_response)
+        self.assertEqual(2, len(detail_response['items']))
+
+        found_item, missing_item = detail_response['items']
+        self.assertEqual(0, found_item['request_index'])
+        self.assertEqual(123, int(found_item['block_key']))
+        self.assertEqual('OK', found_item['status']['code'])
+        self.assertGreater(len(found_item['locations']), 0)
+        self.assertTrue(any(location['status'] == 'CLS_SERVING'
+                            for location in found_item['locations']))
+
+        self.assertEqual(1, missing_item['request_index'])
+        self.assertEqual(999, int(missing_item['block_key']))
+        self.assertEqual('OK', missing_item['status']['code'])
+        self.assertEqual(['CLS_NOT_FOUND'],
+                         [location['status'] for location in missing_item['locations']])
+
+        # Step 5: Get cache location to verify it was added correctly
         get_location_data = {
             "trace_id": self._trace_id,
             "query_type": "QT_PREFIX_MATCH",

kv_cache_manager/client/include/common.h

@@ -92,16 +92,30 @@ enum class CacheMetaLocationStatus : int32_t {
     CLS_DELETING = 4,
 };
 
+enum class CacheMetaStorageType : int32_t {
+    ST_UNSPECIFIED = 0,
+    ST_3FS = 1,
+    ST_MOONCAKE = 2,
+    ST_TAIRMEMPOOL = 3,
+    ST_NFS = 4,
+    ST_VCNS_3FS = 5,
+    ST_DUMMY = 6,
+    ST_EVENT_REPORT_L1P5 = 7,
+    ST_EVENT_REPORT_L2 = 8,
+};
+
 struct CacheMetaLocationDetail {
     std::string location_id;
     CacheMetaLocationStatus status{CacheMetaLocationStatus::CLS_NOT_FOUND};
-    int32_t storage_type{0};
+    CacheMetaStorageType storage_type{CacheMetaStorageType::ST_UNSPECIFIED};
     int32_t spec_size{0};
     int64_t create_time{0};
     Location location_specs;
 };
 
 struct CacheMetaDetailItem {
+    ClientErrorCode error_code{ClientErrorCode::ER_OK};
+    std::string error_message;
     int32_t request_index{0};
     int64_t block_key{0};
     std::string prev_block_key;

kv_cache_manager/client/src/internal/stub/grpc_stub.cc

@@ -65,6 +65,8 @@
 
 namespace {
 
+kv_cache_manager::ClientErrorCode ToClientError(kv_cache_manager::proto::meta::ErrorCode service_error);
+
 kv_cache_manager::Locations GenLocations(
     const google::protobuf::RepeatedPtrField<::kv_cache_manager::proto::meta::CacheLocation> &proto_locations) {
     kv_cache_manager::Locations locations;
@@ -80,13 +82,17 @@ kv_cache_manager::Locations GenLocations(
     return locations;
 }
 
-kv_cache_manager::CacheMetaDetails GenCacheMetaDetails(
-    const google::protobuf::RepeatedPtrField<::kv_cache_manager::proto::meta::CacheMetaDetailItem>
-        &proto_cache_meta_details) {
+kv_cache_manager::CacheMetaDetails
+GenCacheMetaDetails(const google::protobuf::RepeatedPtrField<::kv_cache_manager::proto::meta::CacheMetaDetailItem>
+                        &proto_cache_meta_details) {
     kv_cache_manager::CacheMetaDetails cache_meta_details;
     cache_meta_details.reserve(proto_cache_meta_details.size());
     for (const auto &proto_item : proto_cache_meta_details) {
         kv_cache_manager::CacheMetaDetailItem item;
+        if (proto_item.has_status() && proto_item.status().code() != kv_cache_manager::proto::meta::OK) {
+            item.error_code = ToClientError(proto_item.status().code());
+            item.error_message = proto_item.status().message();
+        }
         item.request_index = proto_item.request_index();
         item.block_key = proto_item.block_key();
         item.prev_block_key = proto_item.prev_block_key();
@@ -97,9 +103,8 @@ kv_cache_manager::CacheMetaDetails GenCacheMetaDetails(
         for (const auto &proto_location : proto_item.locations()) {
             kv_cache_manager::CacheMetaLocationDetail location;
             location.location_id = proto_location.location_id();
-            location.status =
-                static_cast<kv_cache_manager::CacheMetaLocationStatus>(proto_location.status());
-            location.storage_type = static_cast<int32_t>(proto_location.type());
+            location.status = static_cast<kv_cache_manager::CacheMetaLocationStatus>(proto_location.status());
+            location.storage_type = static_cast<kv_cache_manager::CacheMetaStorageType>(proto_location.type());
             location.spec_size = proto_location.spec_size();
             location.create_time = proto_location.create_time();
             location.location_specs.reserve(proto_location.location_specs_size());

kv_cache_manager/manager/cache_location_view.h

@@ -88,6 +88,7 @@ struct CacheLocationMetaDetail {
 };
 
 struct CacheKeyMetaDetail {
+    ErrorCode error_code = EC_OK;
     size_t request_index = 0;
     KeyType block_key = 0;
     PropertyMap properties;

kv_cache_manager/manager/cache_manager.cc

@@ -770,13 +770,25 @@ std::pair<ErrorCode, CacheMetaDetailVec> CacheManager::GetCacheMetaDetail(Reques
         CacheKeyMetaDetail item;
         item.request_index = request_indices[idx];
         item.block_key = query_keys[idx];
-        if (idx < properties.size()) {
-            item.properties = properties[idx];
+
+        if (idx >= per_key_ecs.size()) {
+            item.error_code = EC_MISMATCH;
+            details.push_back(std::move(item));
+            continue;
+        }
+        if (per_key_ecs[idx] != EC_OK && per_key_ecs[idx] != EC_NOENT) {
+            item.error_code = per_key_ecs[idx];
+            details.push_back(std::move(item));
+            continue;
         }
+        if (idx >= location_maps.size() || idx >= properties.size()) {
+            item.error_code = EC_MISMATCH;
+            details.push_back(std::move(item));
+            continue;
+        }
+        item.properties = properties[idx];
 
-        const bool key_not_found =
-            idx >= per_key_ecs.size() || per_key_ecs[idx] == ErrorCode::EC_NOENT || idx >= location_maps.size() ||
-            location_maps[idx].empty();
+        const bool key_not_found = per_key_ecs[idx] == ErrorCode::EC_NOENT || location_maps[idx].empty();
         if (key_not_found) {
             CacheLocationMetaDetail not_found;
             not_found.status = CacheLocationStatus::CLS_NOT_FOUND;
@@ -816,6 +828,12 @@ std::pair<ErrorCode, CacheMetaDetailVec> CacheManager::GetCacheMetaDetail(Reques
         details.push_back(std::move(item));
     }
 
+    const auto first_item_error =
+        std::find_if(details.begin(), details.end(), [](const auto &item) { return item.error_code != EC_OK; });
+    if (first_item_error != details.end() &&
+        std::all_of(details.begin(), details.end(), [](const auto &item) { return item.error_code != EC_OK; })) {
+        return {first_item_error->error_code, std::move(details)};
+    }
     return {EC_OK, std::move(details)};
 }
 

kv_cache_manager/manager/test/cache_manager_test.cc

@@ -82,6 +82,38 @@ ErrorCode ReadError_stub(void * /*obj*/,
 }
 } // namespace mark_query_read_error_stub
 
+namespace raw_meta_partial_error_stub {
+ErrorCode BatchGetRawMeta_stub(void * /*obj*/,
+                               RequestContext * /*request_context*/,
+                               const KeyVector &keys,
+                               CacheLocationMapVector &out_location_maps,
+                               PropertyMapVector &out_properties,
+                               std::vector<ErrorCode> &out_error_codes) {
+    out_location_maps.assign(keys.size(), {});
+    out_properties.assign(keys.size(), {});
+    out_error_codes.assign(keys.size(), EC_OK);
+
+    for (size_t idx = 0; idx < keys.size(); ++idx) {
+        if (keys[idx] == 22) {
+            out_error_codes[idx] = EC_TIMEOUT;
+            continue;
+        }
+        if (keys[idx] == 33) {
+            out_error_codes[idx] = EC_NOENT;
+            continue;
+        }
+        auto location = std::make_shared<CacheLocation>("location_ok",
+                                                        CacheLocationStatus::CLS_SERVING,
+                                                        DataStorageType::DATA_STORAGE_TYPE_NFS,
+                                                        1,
+                                                        std::vector<LocationSpec>{{"tp0", "file:///tmp/key?size=1"}});
+        out_location_maps[idx][location->id()] = std::move(location);
+        out_properties[idx][PROPERTY_PREV_BLOCK_KEY] = "";
+    }
+    return EC_OK;
+}
+} // namespace raw_meta_partial_error_stub
+
 namespace remove_instance_reclaimer_state_stub {
 CacheReclaimer *reclaimer = nullptr;
 bool called = false;
@@ -1977,6 +2009,35 @@ TEST_F(CacheManagerTest, TestGetNotExistCacheMeta) {
     }
 }
 
+TEST_F(CacheManagerTest, TestGetCacheMetaDetailPreservesPartialResults) {
+    Stub stub;
+    stub.set(ADDR(MetaSearcher, BatchGetRawMeta), raw_meta_partial_error_stub::BatchGetRawMeta_stub);
+
+    BlockMask block_mask = static_cast<size_t>(0);
+    auto [ec, details] =
+        cache_manager_->GetCacheMetaDetail(request_context_.get(), "test_instance", {11, 22, 33}, {}, block_mask, 1);
+
+    ASSERT_EQ(EC_OK, ec);
+    ASSERT_EQ(3, details.size());
+
+    EXPECT_EQ(EC_OK, details[0].error_code);
+    ASSERT_EQ(1, details[0].locations.size());
+    EXPECT_EQ(CacheLocationStatus::CLS_SERVING, details[0].locations[0].status);
+
+    EXPECT_EQ(EC_TIMEOUT, details[1].error_code);
+    EXPECT_TRUE(details[1].locations.empty());
+
+    EXPECT_EQ(EC_OK, details[2].error_code);
+    ASSERT_EQ(1, details[2].locations.size());
+    EXPECT_EQ(CacheLocationStatus::CLS_NOT_FOUND, details[2].locations[0].status);
+
+    auto [all_failed_ec, all_failed_details] =
+        cache_manager_->GetCacheMetaDetail(request_context_.get(), "test_instance", {22}, {}, block_mask, 1);
+    EXPECT_EQ(EC_TIMEOUT, all_failed_ec);
+    ASSERT_EQ(1, all_failed_details.size());
+    EXPECT_EQ(EC_TIMEOUT, all_failed_details[0].error_code);
+}
+
 TEST_F(CacheManagerTest, TestRemoveCache) {
     auto expected = std::pair<ErrorCode, std::string>(EC_OK, default_storage_configs);
     ASSERT_EQ(expected,

kv_cache_manager/protocol/protobuf/meta_service.proto

@@ -426,9 +426,15 @@ message CacheLocationDetail {
 message CacheMetaDetailItem {
     int32 request_index = 1;
     int64 block_key = 2;
+    // Convenience projection of properties["BP#prev_key"]. The raw property is
+    // intentionally retained in properties as well.
     string prev_block_key = 3;
     map<string, string> properties = 4;
     repeated CacheLocationDetail locations = 5;
+    // Per-key raw metadata lookup status. A missing key is a successful lookup
+    // represented by CLS_NOT_FOUND; backend/read failures are reported here so
+    // other successful items can still be returned.
+    Status status = 6;
 }
 
 message GetCacheMetaDetailResponse {

kv_cache_manager/service/meta_service_impl.cc

@@ -665,15 +665,23 @@ void MetaServiceImpl::GetCacheMetaDetail(RequestContext *request_context,
                             request_context->error_tracer()->ToJsonString());
         KVCM_LOG_ERROR("[traceId: %s] GetCacheMetaDetail failed, ec: %d", request->trace_id().c_str(), ec_info);
     } else {
+        size_t item_error_count = 0;
         for (const auto &cache_meta_detail : cache_meta_details) {
+            item_error_count += cache_meta_detail.error_code != EC_OK;
             ProtoConvert::CacheKeyMetaDetailToProto(cache_meta_detail, response->add_items());
         }
         status->set_code(proto::meta::OK);
         request_context->set_status_code(status->code());
-        status->set_message("Cache metadata detail retrieved successfully");
-        KVCM_LOG_INFO("[traceId: %s] GetCacheMetaDetail succeeded, returned %d items",
+        if (item_error_count == 0) {
+            status->set_message("Cache metadata detail retrieved successfully");
+        } else {
+            status->set_message("Cache metadata detail retrieved with " + std::to_string(item_error_count) +
+                                " item errors");
+        }
+        KVCM_LOG_INFO("[traceId: %s] GetCacheMetaDetail succeeded, returned %d items with %zu item errors",
                       request->trace_id().c_str(),
-                      response->items_size());
+                      response->items_size(),
+                      item_error_count);
     }
     SET_SPAN_TRACER_STR_IN_HEADER(request_context);
 }

kv_cache_manager/service/util/manager_message_proto_util.h

@@ -408,6 +408,14 @@ inline void ProtoConvert::CacheLocationMetaDetailToProto(
 
 inline void ProtoConvert::CacheKeyMetaDetailToProto(const CacheKeyMetaDetail &cache_meta_detail,
                                                     proto::meta::CacheMetaDetailItem *proto_cache_meta_detail) {
+    auto *status = proto_cache_meta_detail->mutable_status();
+    if (cache_meta_detail.error_code == EC_OK) {
+        status->set_code(proto::meta::OK);
+    } else {
+        status->set_code(ToPbError<proto::meta::ErrorCode>(cache_meta_detail.error_code));
+        status->set_message("Raw metadata lookup failed with internal error code: " +
+                            std::to_string(static_cast<int32_t>(cache_meta_detail.error_code)));
+    }
     proto_cache_meta_detail->set_request_index(static_cast<int32_t>(cache_meta_detail.request_index));
     proto_cache_meta_detail->set_block_key(cache_meta_detail.block_key);
     auto prev_key_iter = cache_meta_detail.properties.find(PROPERTY_PREV_BLOCK_KEY);