Skip to content

[manager] implement fair eviction across instances #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shaohuaxi wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[manager] implement fair eviction across instances #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
- [高可用与选主机制](design/ha_leader_elector.md) - HA 架构、LeaderElector 状态机、CoordinationBackend、Leader 发现
- [CacheReclaimer 异步删除与过度逐出优化](design/cache_reclaimer_async_delete.md) - 异步删除生命周期、in-flight credit、反压与无进展退避
- [后台扫描 GC](design/cache_garbage_collector.md) - 基于 authoritative cursor 的后台全量巡检；V1 清理长期 orphan WRITING 和普通 SERVING storage-missing，并提供无副作用读取、精确值条件 CAS 与 HA 生命周期
- [CacheReclaimer 跨 Instance 公平逐出](design/cache_reclaimer_instance_fairness.md) - 按 Instance 用量分配采样与逐出预算，并与异步 credit 协同

### 开发文档
- [开发指南](develop/README.md) - 开发者入门指南和开发环境配置
Expand Down
9 changes: 6 additions & 3 deletions docs/api/admin_service.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,8 @@ curl -g -vvv -X POST http://localhost:6492/api/createInstanceGroup \
},
"trigger_period_seconds": 60,
"reclaim_step_size": 1073741824,
"reclaim_step_percentage": 10
"reclaim_step_percentage": 10,
"enable_instance_fairness": true
},
"data_storage_strategy": "CPS_PREFER_3FS",
"meta_indexer_config": {
Expand Down Expand Up @@ -193,7 +194,8 @@ curl -g -vvv -X POST http://localhost:6492/api/createInstanceGroup \
"trigger_period_seconds": 60,
"reclaim_step_size": 1073741824,
"reclaim_step_percentage": 10,
"delay_before_delete_ms": 1000
"delay_before_delete_ms": 1000,
"enable_instance_fairness": true
},
"data_storage_strategy": "CPS_PREFER_3FS",
"meta_indexer_config": {
Expand Down Expand Up @@ -249,7 +251,8 @@ curl -g -vvv -X POST http://localhost:6492/api/updateInstanceGroup \
"trigger_period_seconds": 60,
"reclaim_step_size": 1073741824,
"reclaim_step_percentage": 10,
"delay_before_delete_ms": 1000
"delay_before_delete_ms": 1000,
"enable_instance_fairness": true
},
"data_storage_strategy": "CPS_PREFER_3FS",
"meta_indexer_config": {
Expand Down
3 changes: 2 additions & 1 deletion docs/configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,8 @@ timeout，因此不会使用配置数组顺序作为隐式优先级。
"trigger_strategy": {
"used_percentage": 0.8 # 控制数据用量水位，当用量达到或超过quota * percentage时将触发回收（逐出）
},
"delay_before_delete_ms": 1000 # 控制从提交删除请求到实际执行删除动作的间隔，类似于租约概念
"delay_before_delete_ms": 1000, # 控制从提交删除请求到实际执行删除动作的间隔，类似于租约概念
"enable_instance_fairness": true # 是否按各 Instance 当前用量分配本轮采样/逐出预算；缺省为 true，false 完整回退旧的固定 per-instance 行为
},
# cache_prefer_strategy与storage candidates一起控制storage backend选择策略，可选值如下：
# enum class CachePreferStrategy {
Expand Down
847 changes: 847 additions & 0 deletions docs/design/cache_reclaimer_instance_fairness.md
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions integration_test/admin_service/admin_interface_cases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ def make_sample_instance_group(self) -> Dict:
"trigger_period_seconds": 60,
"reclaim_step_size": 512 * 1024 * 1024,
"reclaim_step_percentage": 5,
"enable_instance_fairness": True,
},
"data_storage_strategy": 2, # CPS_PREFER_3FS
"meta_indexer_config": {
Expand Down
4 changes: 3 additions & 1 deletion kv_cache_manager/config/cache_reclaim_strategy.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ bool CacheReclaimStrategy::FromRapidValue(const rapidjson::Value &rapid_value) {
KVCM_JSON_GET_MACRO(rapid_value, "reclaim_step_size", reclaim_step_size_);
KVCM_JSON_GET_MACRO(rapid_value, "reclaim_step_percentage", reclaim_step_percentage_);
KVCM_JSON_GET_MACRO(rapid_value, "delay_before_delete_ms", delay_before_delete_ms_);
KVCM_JSON_GET_DEFAULT_MACRO(rapid_value, "enable_instance_fairness", enable_instance_fairness_, true);
return true;
}

Expand All @@ -23,6 +24,7 @@ void CacheReclaimStrategy::ToRapidWriter(rapidjson::Writer<rapidjson::StringBuff
Put(writer, "reclaim_step_size", reclaim_step_size_);
Put(writer, "reclaim_step_percentage", reclaim_step_percentage_);
Put(writer, "delay_before_delete_ms", delay_before_delete_ms_);
Put(writer, "enable_instance_fairness", enable_instance_fairness_);
}

bool CacheReclaimStrategy::ValidateRequiredFields(std::string &invalid_fields) const {
Expand All @@ -37,4 +39,4 @@ bool CacheReclaimStrategy::ValidateRequiredFields(std::string &invalid_fields) c
}
return valid;
}
} // namespace kv_cache_manager
} // namespace kv_cache_manager
13 changes: 10 additions & 3 deletions kv_cache_manager/config/cache_reclaim_strategy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,16 @@ class CacheReclaimStrategy : public Jsonizable {
int32_t trigger_period_seconds,
int32_t reclaim_step_size,
int32_t reclaim_step_percentage,
int32_t delay_before_delete_ms = 0)
int32_t delay_before_delete_ms = 0,
bool enable_instance_fairness = true)
: storage_unique_name_(storage_unique_name)
, reclaim_policy_(reclaim_policy)
, trigger_strategy_(trigger_strategy)
, trigger_period_seconds_(trigger_period_seconds)
, reclaim_step_size_(reclaim_step_size)
, reclaim_step_percentage_(reclaim_step_percentage)
, delay_before_delete_ms_(delay_before_delete_ms) {}
, delay_before_delete_ms_(delay_before_delete_ms)
, enable_instance_fairness_(enable_instance_fairness) {}

~CacheReclaimStrategy() override;

Expand All @@ -52,6 +54,7 @@ class CacheReclaimStrategy : public Jsonizable {
int32_t reclaim_step_size() const { return reclaim_step_size_; }
int32_t reclaim_step_percentage() const { return reclaim_step_percentage_; }
int32_t delay_before_delete_ms() const { return delay_before_delete_ms_; }
bool enable_instance_fairness() const { return enable_instance_fairness_; }

// Setters
void set_storage_unique_name(const std::string &storage_unique_name) { storage_unique_name_ = storage_unique_name; }
Expand All @@ -67,6 +70,9 @@ class CacheReclaimStrategy : public Jsonizable {
void set_delay_before_delete_ms(int32_t delay_before_delete_ms) {
delay_before_delete_ms_ = delay_before_delete_ms;
}
void set_enable_instance_fairness(bool enable_instance_fairness) {
enable_instance_fairness_ = enable_instance_fairness;
}

public:
bool FromRapidValue(const rapidjson::Value &rapid_value) override;
Expand All @@ -82,6 +88,7 @@ class CacheReclaimStrategy : public Jsonizable {
int32_t reclaim_step_size_;
int32_t reclaim_step_percentage_;
int32_t delay_before_delete_ms_;
bool enable_instance_fairness_{true};
};

} // namespace kv_cache_manager
} // namespace kv_cache_manager
36 changes: 36 additions & 0 deletions kv_cache_manager/config/test/instance_group_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,4 +294,40 @@ TEST_F(InstanceGroupTest, LegacyTairMempoolProtoWithoutStorageTypeRemainsDramTyp
EXPECT_EQ(kTairMemPoolMediaTypeSsd, restored_spec->media_type());
}

TEST_F(InstanceGroupTest, CacheReclaimFairnessProtoRoundTripPreservesExplicitFalse) {
CacheConfig original;
auto reclaim_strategy = std::make_shared<CacheReclaimStrategy>();
reclaim_strategy->set_enable_instance_fairness(false);
original.set_reclaim_strategy(reclaim_strategy);

proto::admin::CacheConfig proto_config;
ProtoConvert::CacheConfigToProto(original, &proto_config);
ASSERT_TRUE(proto_config.reclaim_strategy().has_enable_instance_fairness());
EXPECT_FALSE(proto_config.reclaim_strategy().enable_instance_fairness().value());

CacheConfig restored;
ProtoConvert::CacheConfigFromProto(&proto_config, restored);
ASSERT_NE(nullptr, restored.reclaim_strategy());
EXPECT_FALSE(restored.reclaim_strategy()->enable_instance_fairness());
}

TEST_F(InstanceGroupTest, CacheReclaimFairnessProtoDefaultsToTrueWhenMissing) {
proto::admin::CacheConfig proto_config;
proto_config.mutable_reclaim_strategy();

CacheConfig restored;
ProtoConvert::CacheConfigFromProto(&proto_config, restored);
ASSERT_NE(nullptr, restored.reclaim_strategy());
EXPECT_TRUE(restored.reclaim_strategy()->enable_instance_fairness());
}

TEST_F(InstanceGroupTest, CacheReclaimFairnessJsonDefaultsToTrueAndAcceptsFalse) {
CacheReclaimStrategy reclaim_strategy;
ASSERT_TRUE(reclaim_strategy.FromJsonString("{}"));
EXPECT_TRUE(reclaim_strategy.enable_instance_fairness());

ASSERT_TRUE(reclaim_strategy.FromJsonString(R"({"enable_instance_fairness": false})"));
EXPECT_FALSE(reclaim_strategy.enable_instance_fairness());
}

} // namespace kv_cache_manager
Loading
Loading