[manager] implement fair eviction across instances

Author: shaohuaxi

docs/README.md

@@ -8,6 +8,7 @@
 - [高可用与选主机制](design/ha_leader_elector.md) - HA 架构、LeaderElector 状态机、CoordinationBackend、Leader 发现
 - [CacheReclaimer 异步删除与过度逐出优化](design/cache_reclaimer_async_delete.md) - 异步删除生命周期、in-flight credit、反压与无进展退避
 - [后台扫描 GC](design/cache_garbage_collector.md) - 基于 authoritative cursor 的后台全量巡检；V1 清理长期 orphan WRITING 和普通 SERVING storage-missing，并提供无副作用读取、精确值条件 CAS 与 HA 生命周期
+- [CacheReclaimer 跨 Instance 公平逐出](design/cache_reclaimer_instance_fairness.md) - 按 Instance 用量分配采样与逐出预算，并与异步 credit 协同
 
 ### 开发文档
 - [开发指南](develop/README.md) - 开发者入门指南和开发环境配置

docs/api/admin_service.md

@@ -142,7 +142,8 @@ curl -g -vvv -X POST http://localhost:6492/api/createInstanceGroup \
                 },
                 "trigger_period_seconds": 60,
                 "reclaim_step_size": 1073741824,
-                "reclaim_step_percentage": 10
+                "reclaim_step_percentage": 10,
+                "enable_instance_fairness": true
             },
             "data_storage_strategy": "CPS_PREFER_3FS",
             "meta_indexer_config": {
@@ -193,7 +194,8 @@ curl -g -vvv -X POST http://localhost:6492/api/createInstanceGroup \
                 "trigger_period_seconds": 60,
                 "reclaim_step_size": 1073741824,
                 "reclaim_step_percentage": 10,
-                "delay_before_delete_ms": 1000
+                "delay_before_delete_ms": 1000,
+                "enable_instance_fairness": true
             },
             "data_storage_strategy": "CPS_PREFER_3FS",
             "meta_indexer_config": {
@@ -249,7 +251,8 @@ curl -g -vvv -X POST http://localhost:6492/api/updateInstanceGroup \
                 "trigger_period_seconds": 60,
                 "reclaim_step_size": 1073741824,
                 "reclaim_step_percentage": 10,
-                "delay_before_delete_ms": 1000
+                "delay_before_delete_ms": 1000,
+                "enable_instance_fairness": true
             },
             "data_storage_strategy": "CPS_PREFER_3FS",
             "meta_indexer_config": {

docs/configuration.md

@@ -243,7 +243,8 @@ timeout，因此不会使用配置数组顺序作为隐式优先级。
                 "trigger_strategy": {
                     "used_percentage": 0.8 # 控制数据用量水位，当用量达到或超过quota * percentage时将触发回收（逐出）
                 },
-                "delay_before_delete_ms": 1000 # 控制从提交删除请求到实际执行删除动作的间隔，类似于租约概念
+                "delay_before_delete_ms": 1000, # 控制从提交删除请求到实际执行删除动作的间隔，类似于租约概念
+                "enable_instance_fairness": true # 是否按各 Instance 当前用量分配本轮采样/逐出预算；缺省为 true，false 完整回退旧的固定 per-instance 行为
             },
             # cache_prefer_strategy与storage candidates一起控制storage backend选择策略，可选值如下：
             # enum class CachePreferStrategy {

docs/design/cache_reclaimer_instance_fairness.md

@@ -187,6 +187,7 @@ def make_sample_instance_group(self) -> Dict:
                     "trigger_period_seconds": 60,
                     "reclaim_step_size": 512 * 1024 * 1024,
                     "reclaim_step_percentage": 5,
+                    "enable_instance_fairness": True,
                 },
                 "data_storage_strategy": 2,  # CPS_PREFER_3FS
                 "meta_indexer_config": {

integration_test/admin_service/admin_interface_cases.py

@@ -12,6 +12,7 @@ bool CacheReclaimStrategy::FromRapidValue(const rapidjson::Value &rapid_value) {
     KVCM_JSON_GET_MACRO(rapid_value, "reclaim_step_size", reclaim_step_size_);
     KVCM_JSON_GET_MACRO(rapid_value, "reclaim_step_percentage", reclaim_step_percentage_);
     KVCM_JSON_GET_MACRO(rapid_value, "delay_before_delete_ms", delay_before_delete_ms_);
+    KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "enable_instance_fairness", enable_instance_fairness_, true);
     return true;
 }
 
@@ -23,6 +24,7 @@ void CacheReclaimStrategy::ToRapidWriter(rapidjson::Writer<rapidjson::StringBuff
     Put(writer, "reclaim_step_size", reclaim_step_size_);
     Put(writer, "reclaim_step_percentage", reclaim_step_percentage_);
     Put(writer, "delay_before_delete_ms", delay_before_delete_ms_);
+    Put(writer, "enable_instance_fairness", enable_instance_fairness_);
 }
 
 bool CacheReclaimStrategy::ValidateRequiredFields(std::string &invalid_fields) const {
@@ -37,4 +39,4 @@ bool CacheReclaimStrategy::ValidateRequiredFields(std::string &invalid_fields) c
     }
     return valid;
 }
-} // namespace kv_cache_manager
+} // namespace kv_cache_manager

kv_cache_manager/config/cache_reclaim_strategy.cc

@@ -30,14 +30,16 @@ class CacheReclaimStrategy : public Jsonizable {
                          int32_t trigger_period_seconds,
                          int32_t reclaim_step_size,
                          int32_t reclaim_step_percentage,
-                         int32_t delay_before_delete_ms = 0)
+                         int32_t delay_before_delete_ms = 0,
+                         bool enable_instance_fairness = true)
         : storage_unique_name_(storage_unique_name)
         , reclaim_policy_(reclaim_policy)
         , trigger_strategy_(trigger_strategy)
         , trigger_period_seconds_(trigger_period_seconds)
         , reclaim_step_size_(reclaim_step_size)
         , reclaim_step_percentage_(reclaim_step_percentage)
-        , delay_before_delete_ms_(delay_before_delete_ms) {}
+        , delay_before_delete_ms_(delay_before_delete_ms)
+        , enable_instance_fairness_(enable_instance_fairness) {}
 
     ~CacheReclaimStrategy() override;
 
@@ -52,6 +54,7 @@ class CacheReclaimStrategy : public Jsonizable {
     int32_t reclaim_step_size() const { return reclaim_step_size_; }
     int32_t reclaim_step_percentage() const { return reclaim_step_percentage_; }
     int32_t delay_before_delete_ms() const { return delay_before_delete_ms_; }
+    bool enable_instance_fairness() const { return enable_instance_fairness_; }
 
     // Setters
     void set_storage_unique_name(const std::string &storage_unique_name) { storage_unique_name_ = storage_unique_name; }
@@ -67,6 +70,9 @@ class CacheReclaimStrategy : public Jsonizable {
     void set_delay_before_delete_ms(int32_t delay_before_delete_ms) {
         delay_before_delete_ms_ = delay_before_delete_ms;
     }
+    void set_enable_instance_fairness(bool enable_instance_fairness) {
+        enable_instance_fairness_ = enable_instance_fairness;
+    }
 
 public:
     bool FromRapidValue(const rapidjson::Value &rapid_value) override;
@@ -82,6 +88,7 @@ class CacheReclaimStrategy : public Jsonizable {
     int32_t reclaim_step_size_;
     int32_t reclaim_step_percentage_;
     int32_t delay_before_delete_ms_;
+    bool enable_instance_fairness_{true};
 };
 
-} // namespace kv_cache_manager
+} // namespace kv_cache_manager

kv_cache_manager/config/cache_reclaim_strategy.h

@@ -255,4 +255,40 @@ TEST_F(InstanceGroupTest, UnknownProtoStorageTypeFailsClosed) {
     EXPECT_EQ(DataStorageType::DATA_STORAGE_TYPE_UNKNOWN, location.type());
 }
 
+TEST_F(InstanceGroupTest, CacheReclaimFairnessProtoRoundTripPreservesExplicitFalse) {
+    CacheConfig original;
+    auto reclaim_strategy = std::make_shared<CacheReclaimStrategy>();
+    reclaim_strategy->set_enable_instance_fairness(false);
+    original.set_reclaim_strategy(reclaim_strategy);
+
+    proto::admin::CacheConfig proto_config;
+    ProtoConvert::CacheConfigToProto(original, &proto_config);
+    ASSERT_TRUE(proto_config.reclaim_strategy().has_enable_instance_fairness());
+    EXPECT_FALSE(proto_config.reclaim_strategy().enable_instance_fairness().value());
+
+    CacheConfig restored;
+    ProtoConvert::CacheConfigFromProto(&proto_config, restored);
+    ASSERT_NE(nullptr, restored.reclaim_strategy());
+    EXPECT_FALSE(restored.reclaim_strategy()->enable_instance_fairness());
+}
+
+TEST_F(InstanceGroupTest, CacheReclaimFairnessProtoDefaultsToTrueWhenMissing) {
+    proto::admin::CacheConfig proto_config;
+    proto_config.mutable_reclaim_strategy();
+
+    CacheConfig restored;
+    ProtoConvert::CacheConfigFromProto(&proto_config, restored);
+    ASSERT_NE(nullptr, restored.reclaim_strategy());
+    EXPECT_TRUE(restored.reclaim_strategy()->enable_instance_fairness());
+}
+
+TEST_F(InstanceGroupTest, CacheReclaimFairnessJsonDefaultsToTrueAndAcceptsFalse) {
+    CacheReclaimStrategy reclaim_strategy;
+    ASSERT_TRUE(reclaim_strategy.FromJsonString("{}"));
+    EXPECT_TRUE(reclaim_strategy.enable_instance_fairness());
+
+    ASSERT_TRUE(reclaim_strategy.FromJsonString(R"({"enable_instance_fairness": false})"));
+    EXPECT_FALSE(reclaim_strategy.enable_instance_fairness());
+}
+
 } // namespace kv_cache_manager