Skip to content

Commit 740a15b

Browse files
eldryotheldryoth
committed
Report persistent load balancer admin mutations
1 parent 29d02a8 commit 740a15b

7 files changed

Lines changed: 112 additions & 25 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ behavior when the change improves security or project direction.
2828
ignored on corrupt/incompatible input, and written atomically after runtime
2929
member-state, runtime weight, persistence-table, and persistence-clear
3030
changes.
31+
- Report `persistent: true` from load-balancer admin mutation responses when a
32+
pool has `proxy.load_balance.runtime_state_file` configured, and keep
33+
`persistent: false` for in-memory-only pools.
3134

3235
## 1.5.8 - 2026-06-07
3336

‎docs/config-reference.md‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -353,9 +353,10 @@ maintenance disables. `manual_resume` clears any runtime override, clears the
353353
member's passive-health failure/ejection state, and restarts slow-start ramp
354354
when slow-start is configured. `normal` clears only runtime overrides; static
355355
`drain_upstreams` and `disabled_upstreams` remain enforced until the config
356-
changes. Runtime member state is intentionally in-memory in the current `1.5.x`
357-
line, is reset by process restart or runtime rebuild, and is returned with
358-
`"persistent": false` in the mutation response. The response also includes
356+
changes. Runtime member state is in-memory unless
357+
`proxy.load_balance.runtime_state_file` is configured for that pool. Mutation
358+
responses include `"persistent": true` when the pool has a local runtime state
359+
file and `"persistent": false` otherwise. The response also includes
359360
`scope = "vhost"` or `"route"` so operators can audit which pool was changed.
360361
In `privacy-mode`, member mutation responses and structured mutation logs omit
361362
the backend address just like status output. Successful mutation metrics keep
@@ -421,8 +422,10 @@ curl -X POST \
421422

422423
Use the optional `route` query parameter or `X-Fluxheim-Lb-Route` header to
423424
target a route-local pool. The response includes `cleared_entries`,
424-
`scope = "vhost"` or `"route"`, and `"persistent": false`. The operation is
425-
local to the current runtime; it does not alter config or durable snapshots.
425+
`scope = "vhost"` or `"route"`, and a `persistent` boolean. The operation is
426+
local to the current runtime unless `proxy.load_balance.runtime_state_file` is
427+
configured for that pool, in which case the cleared table is written back to
428+
the local runtime state file; it does not alter config or durable snapshots.
426429
When metrics are compiled, successful clears are counted as
427430
`persistence_clear` in `fluxheim_load_balancer_events_total`; rejected clear
428431
requests are counted separately as `persistence_clear_invalid` or

‎release-notes/RELEASE_NOTES_1.5.9.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ Fluxheim 1.5.9 starts the restart-persistent load-balancer state line.
2222
Fluxheim loads the file best-effort at pool construction and writes it
2323
atomically after runtime member-state, runtime weight, persistence-table, and
2424
persistence-clear changes.
25+
- Admin mutation responses now report `persistent: true` when the target pool
26+
has a runtime state file configured, and `persistent: false` for in-memory
27+
pools.
2528

2629
## Stop Line
2730

‎scripts/smoke_load_balancer.sh‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,8 @@ curl -fsS -X POST \
306306
if ! grep -q '"status":"ok"' "$TMP_DIR/member-weight.json" \
307307
|| ! grep -q '"configured_weight":1' "$TMP_DIR/member-weight.json" \
308308
|| ! grep -q '"effective_weight":4' "$TMP_DIR/member-weight.json" \
309-
|| ! grep -q '"runtime_weight_override":4' "$TMP_DIR/member-weight.json"; then
309+
|| ! grep -q '"runtime_weight_override":4' "$TMP_DIR/member-weight.json" \
310+
|| ! grep -q '"persistent":true' "$TMP_DIR/member-weight.json"; then
310311
echo "load balancer member-weight endpoint did not report runtime weight override" >&2
311312
cat "$TMP_DIR/member-weight.json" >&2
312313
exit 1
@@ -484,7 +485,7 @@ curl -fsS -X POST \
484485
if ! grep -q '"status":"ok"' "$TMP_DIR/persistence-clear.json" \
485486
|| ! grep -q '"scope":"route"' "$TMP_DIR/persistence-clear.json" \
486487
|| ! grep -q '"cleared_entries":1' "$TMP_DIR/persistence-clear.json" \
487-
|| ! grep -q '"persistent":false' "$TMP_DIR/persistence-clear.json"; then
488+
|| ! grep -q '"persistent":true' "$TMP_DIR/persistence-clear.json"; then
488489
echo "load balancer persistence clear endpoint did not report cleared route entry" >&2
489490
cat "$TMP_DIR/persistence-clear.json" >&2
490491
exit 1
@@ -502,6 +503,13 @@ curl -fsS -X POST \
502503
"http://127.0.0.1:$ADMIN_PORT/_fluxheim/load-balancer/member-state?vhost=smoke&member=origin-two&state=disable" \
503504
> "$TMP_DIR/member-disable.json"
504505

506+
if ! grep -q '"status":"ok"' "$TMP_DIR/member-disable.json" \
507+
|| ! grep -q '"persistent":true' "$TMP_DIR/member-disable.json"; then
508+
echo "load balancer member-state endpoint did not report persistent runtime state" >&2
509+
cat "$TMP_DIR/member-disable.json" >&2
510+
exit 1
511+
fi
512+
505513
curl -fsS \
506514
-H "Authorization: Bearer $FLUXHEIM_ADMIN_TOKEN" \
507515
"http://127.0.0.1:$ADMIN_PORT/_fluxheim/load-balancer/status" \

‎src/admin.rs‎

Lines changed: 74 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -970,35 +970,38 @@ impl AdminApp {
970970
#[cfg(not(feature = "privacy-mode"))]
971971
log::info!(
972972
target: "fluxheim::load_balancer",
973-
"load balancer member state updated vhost={} route={} scope={} member={} state={} address={} alias={} persistent=false",
973+
"load balancer member state updated vhost={} route={} scope={} member={} state={} address={} alias={} persistent={}",
974974
result.vhost,
975975
result.route.as_deref().unwrap_or(""),
976976
scope,
977977
result.member,
978978
result.state.as_str(),
979979
result.address,
980-
result.alias.as_deref().unwrap_or("")
980+
result.alias.as_deref().unwrap_or(""),
981+
result.persistent
981982
);
982983
#[cfg(feature = "privacy-mode")]
983984
log::info!(
984985
target: "fluxheim::load_balancer",
985-
"load balancer member state updated vhost={} route={} scope={} member={} state={} alias={} persistent=false",
986+
"load balancer member state updated vhost={} route={} scope={} member={} state={} alias={} persistent={}",
986987
result.vhost,
987988
result.route.as_deref().unwrap_or(""),
988989
scope,
989990
result.member,
990991
result.state.as_str(),
991-
result.alias.as_deref().unwrap_or("")
992+
result.alias.as_deref().unwrap_or(""),
993+
result.persistent
992994
);
993995
log::info!(
994996
target: "fluxheim::audit",
995-
"load balancer member state updated vhost={} route={} scope={} member={} state={} alias={} persistent=false",
997+
"load balancer member state updated vhost={} route={} scope={} member={} state={} alias={} persistent={}",
996998
result.vhost,
997999
result.route.as_deref().unwrap_or(""),
9981000
scope,
9991001
result.member,
10001002
result.state.as_str(),
1001-
result.alias.as_deref().unwrap_or("")
1003+
result.alias.as_deref().unwrap_or(""),
1004+
result.persistent
10021005
);
10031006
record_load_balancer_event(
10041007
&result.vhost,
@@ -1019,7 +1022,7 @@ impl AdminApp {
10191022
#[cfg(not(feature = "privacy-mode"))]
10201023
body.insert("address".to_owned(), json!(result.address));
10211024
body.insert("alias".to_owned(), json!(result.alias));
1022-
body.insert("persistent".to_owned(), json!(false));
1025+
body.insert("persistent".to_owned(), json!(result.persistent));
10231026
json_response_value(StatusCode::OK, &Value::Object(body))
10241027
}
10251028
Err(error) if error.kind() == io::ErrorKind::InvalidInput => {
@@ -1090,7 +1093,7 @@ impl AdminApp {
10901093
#[cfg(not(feature = "privacy-mode"))]
10911094
log::info!(
10921095
target: "fluxheim::load_balancer",
1093-
"load balancer member weight updated vhost={} route={} scope={} member={} configured_weight={} effective_weight={} runtime_weight_override={} address={} alias={} persistent=false",
1096+
"load balancer member weight updated vhost={} route={} scope={} member={} configured_weight={} effective_weight={} runtime_weight_override={} address={} alias={} persistent={}",
10941097
result.vhost,
10951098
result.route.as_deref().unwrap_or(""),
10961099
scope,
@@ -1102,12 +1105,13 @@ impl AdminApp {
11021105
.map(|weight| weight.to_string())
11031106
.unwrap_or_else(|| "none".to_owned()),
11041107
result.address,
1105-
result.alias.as_deref().unwrap_or("")
1108+
result.alias.as_deref().unwrap_or(""),
1109+
result.persistent
11061110
);
11071111
#[cfg(feature = "privacy-mode")]
11081112
log::info!(
11091113
target: "fluxheim::load_balancer",
1110-
"load balancer member weight updated vhost={} route={} scope={} member={} configured_weight={} effective_weight={} runtime_weight_override={} alias={} persistent=false",
1114+
"load balancer member weight updated vhost={} route={} scope={} member={} configured_weight={} effective_weight={} runtime_weight_override={} alias={} persistent={}",
11111115
result.vhost,
11121116
result.route.as_deref().unwrap_or(""),
11131117
scope,
@@ -1118,11 +1122,12 @@ impl AdminApp {
11181122
.runtime_weight_override
11191123
.map(|weight| weight.to_string())
11201124
.unwrap_or_else(|| "none".to_owned()),
1121-
result.alias.as_deref().unwrap_or("")
1125+
result.alias.as_deref().unwrap_or(""),
1126+
result.persistent
11221127
);
11231128
log::info!(
11241129
target: "fluxheim::audit",
1125-
"load balancer member weight updated vhost={} route={} scope={} member={} configured_weight={} effective_weight={} runtime_weight_override={} alias={} persistent=false",
1130+
"load balancer member weight updated vhost={} route={} scope={} member={} configured_weight={} effective_weight={} runtime_weight_override={} alias={} persistent={}",
11261131
result.vhost,
11271132
result.route.as_deref().unwrap_or(""),
11281133
scope,
@@ -1133,7 +1138,8 @@ impl AdminApp {
11331138
.runtime_weight_override
11341139
.map(|weight| weight.to_string())
11351140
.unwrap_or_else(|| "none".to_owned()),
1136-
result.alias.as_deref().unwrap_or("")
1141+
result.alias.as_deref().unwrap_or(""),
1142+
result.persistent
11371143
);
11381144
record_load_balancer_event(
11391145
&result.vhost,
@@ -1165,7 +1171,7 @@ impl AdminApp {
11651171
#[cfg(not(feature = "privacy-mode"))]
11661172
body.insert("address".to_owned(), json!(result.address));
11671173
body.insert("alias".to_owned(), json!(result.alias));
1168-
body.insert("persistent".to_owned(), json!(false));
1174+
body.insert("persistent".to_owned(), json!(result.persistent));
11691175
json_response_value(StatusCode::OK, &Value::Object(body))
11701176
}
11711177
Err(error) if error.kind() == io::ErrorKind::InvalidInput => {
@@ -1217,11 +1223,12 @@ impl AdminApp {
12171223
};
12181224
log::info!(
12191225
target: "fluxheim::load_balancer",
1220-
"load balancer persistence table cleared vhost={} route={} scope={} cleared_entries={} persistent=false",
1226+
"load balancer persistence table cleared vhost={} route={} scope={} cleared_entries={} persistent={}",
12211227
result.vhost,
12221228
result.route.as_deref().unwrap_or(""),
12231229
scope,
1224-
result.cleared_entries
1230+
result.cleared_entries,
1231+
result.persistent
12251232
);
12261233
record_load_balancer_event(
12271234
&result.vhost,
@@ -1237,7 +1244,7 @@ impl AdminApp {
12371244
"route": result.route,
12381245
"scope": scope,
12391246
"cleared_entries": result.cleared_entries,
1240-
"persistent": false,
1247+
"persistent": result.persistent,
12411248
}),
12421249
)
12431250
}
@@ -3814,7 +3821,7 @@ mod tests {
38143821
use crate::config_route::RouteConfig;
38153822
use crate::proxy::{FluxProxy, ProxyHealthReporter, ProxyHealthSignal};
38163823
use crate::snapshot::SnapshotStore;
3817-
use crate::test_support::unique_temp_path;
3824+
use crate::test_support::{safe_child_path, unique_temp_path};
38183825
#[cfg(unix)]
38193826
use crate::test_support::{unique_group_writable_child, unique_world_writable_child};
38203827

@@ -3911,6 +3918,16 @@ mod tests {
39113918
}
39123919
}
39133920

3921+
#[cfg(feature = "load-balancer")]
3922+
fn load_balancer_persistent_admin_config() -> Config {
3923+
let root = unique_temp_path("admin-lb-runtime-state");
3924+
std::fs::create_dir_all(&root).unwrap();
3925+
let mut config = load_balancer_admin_config();
3926+
config.vhosts[0].proxy.load_balance.runtime_state_file =
3927+
Some(safe_child_path(&root, "lb-state.json"));
3928+
config
3929+
}
3930+
39143931
fn set_test_runtime_state(
39153932
app: &AdminApp,
39163933
runtime_snapshot: Option<String>,
@@ -4170,6 +4187,45 @@ mod tests {
41704187
assert_eq!(body["runtime_weight_override"], Value::Null);
41714188
}
41724189

4190+
#[cfg(feature = "load-balancer")]
4191+
#[test]
4192+
fn load_balancer_mutation_endpoints_report_persistent_state_file() {
4193+
#[cfg(feature = "tls-rustls-backend")]
4194+
let _ = crate::tls::install_rustls_crypto_provider();
4195+
4196+
let app = app_with_config(load_balancer_persistent_admin_config());
4197+
4198+
let response = app.handle(
4199+
"POST",
4200+
"/_fluxheim/load-balancer/member-state",
4201+
Some("vhost=one&member=app-a&state=disable"),
4202+
&auth_headers(),
4203+
);
4204+
assert_eq!(response.status, StatusCode::OK);
4205+
let body: Value = serde_json::from_slice(&response.body).unwrap();
4206+
assert_eq!(body["persistent"], true);
4207+
4208+
let response = app.handle(
4209+
"POST",
4210+
"/_fluxheim/load-balancer/member-weight",
4211+
Some("vhost=one&member=app-a&weight=5"),
4212+
&auth_headers(),
4213+
);
4214+
assert_eq!(response.status, StatusCode::OK);
4215+
let body: Value = serde_json::from_slice(&response.body).unwrap();
4216+
assert_eq!(body["persistent"], true);
4217+
4218+
let response = app.handle(
4219+
"POST",
4220+
"/_fluxheim/load-balancer/persistence/clear",
4221+
Some("vhost=one"),
4222+
&auth_headers(),
4223+
);
4224+
assert_eq!(response.status, StatusCode::OK);
4225+
let body: Value = serde_json::from_slice(&response.body).unwrap();
4226+
assert_eq!(body["persistent"], true);
4227+
}
4228+
41734229
#[cfg(feature = "load-balancer")]
41744230
#[test]
41754231
fn load_balancer_runtime_weight_parser_documents_all_reset_keywords() {

‎src/load_balancer.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ impl LoadBalancerRuntimeBackendState {
177177
pub struct LoadBalancerRuntimeBackendMutation {
178178
pub member: String,
179179
pub state: LoadBalancerRuntimeBackendState,
180+
pub persistent: bool,
180181
#[cfg(not(feature = "privacy-mode"))]
181182
pub address: String,
182183
pub alias: Option<String>,
@@ -188,6 +189,7 @@ pub struct LoadBalancerRuntimeBackendWeightMutation {
188189
pub configured_weight: usize,
189190
pub effective_weight: usize,
190191
pub runtime_weight_override: Option<usize>,
192+
pub persistent: bool,
191193
#[cfg(not(feature = "privacy-mode"))]
192194
pub address: String,
193195
pub alias: Option<String>,
@@ -934,6 +936,10 @@ impl UpstreamLoadBalancer {
934936
self.inner.health_check_frequency()
935937
}
936938

939+
pub fn runtime_state_persistent(&self) -> bool {
940+
self.runtime_state_file.is_some()
941+
}
942+
937943
#[cfg(test)]
938944
fn parallel_health_check(&self) -> bool {
939945
self.inner.parallel_health_check()
@@ -971,6 +977,7 @@ impl UpstreamLoadBalancer {
971977
Ok(LoadBalancerRuntimeBackendMutation {
972978
member: member.to_owned(),
973979
state,
980+
persistent: self.runtime_state_persistent(),
974981
#[cfg(not(feature = "privacy-mode"))]
975982
address: backend.addr.to_string(),
976983
alias: self
@@ -1007,6 +1014,7 @@ impl UpstreamLoadBalancer {
10071014
configured_weight: backend.weight,
10081015
effective_weight: self.backend_policy.effective_weight(&backend),
10091016
runtime_weight_override: self.backend_policy.runtime_backend_weight(key),
1017+
persistent: self.runtime_state_persistent(),
10101018
#[cfg(not(feature = "privacy-mode"))]
10111019
address: backend.addr.to_string(),
10121020
alias: self

‎src/proxy.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,6 +750,7 @@ pub struct LoadBalancerMemberStateResult {
750750
pub route: Option<String>,
751751
pub member: String,
752752
pub state: LoadBalancerRuntimeBackendState,
753+
pub persistent: bool,
753754
#[cfg(not(feature = "privacy-mode"))]
754755
pub address: String,
755756
pub alias: Option<String>,
@@ -773,6 +774,7 @@ pub struct LoadBalancerMemberWeightResult {
773774
pub configured_weight: usize,
774775
pub effective_weight: usize,
775776
pub runtime_weight_override: Option<usize>,
777+
pub persistent: bool,
776778
#[cfg(not(feature = "privacy-mode"))]
777779
pub address: String,
778780
pub alias: Option<String>,
@@ -791,6 +793,7 @@ pub struct LoadBalancerPersistenceClearResult {
791793
pub vhost: String,
792794
pub route: Option<String>,
793795
pub cleared_entries: usize,
796+
pub persistent: bool,
794797
}
795798

796799
impl ProxySnapshot {
@@ -1091,6 +1094,7 @@ impl ProxySnapshot {
10911094
route: route_name,
10921095
member: mutation.member,
10931096
state: mutation.state,
1097+
persistent: mutation.persistent,
10941098
#[cfg(not(feature = "privacy-mode"))]
10951099
address: mutation.address,
10961100
alias: mutation.alias,
@@ -1166,6 +1170,7 @@ impl ProxySnapshot {
11661170
configured_weight: mutation.configured_weight,
11671171
effective_weight: mutation.effective_weight,
11681172
runtime_weight_override: mutation.runtime_weight_override,
1173+
persistent: mutation.persistent,
11691174
#[cfg(not(feature = "privacy-mode"))]
11701175
address: mutation.address,
11711176
alias: mutation.alias,
@@ -1230,6 +1235,7 @@ impl ProxySnapshot {
12301235
vhost: vhost.name.clone(),
12311236
route: route_name,
12321237
cleared_entries: pool.clear_persistence(),
1238+
persistent: pool.runtime_state_persistent(),
12331239
})
12341240
}
12351241

0 commit comments

Comments
 (0)