Harden load balancer health and hash state

Author: eldryoth

CHANGELOG.md

@@ -35,12 +35,15 @@ behavior when the change improves security or project direction.
 - Replace Pingora's FNV weighted-hash selector for source, URI, header, and
   cookie hash modes with Fluxheim-owned weighted-first FNV selection over the
   current backend container.
+- Seed Fluxheim-owned FNV and consistent-hash selectors with per-boot routing
+  secrets so clients cannot precompute keys that target a chosen backend.
 - Replace Pingora's random selector dependency for power-of-two choices with a
   Fluxheim-owned weighted random first pick and unique backend fallback scan.
 - Replace Pingora's consistent-hash selector dependency with Fluxheim-owned
   rendezvous candidate ordering for consistent and bounded-load consistent
   hash modes, while preserving dynamic discovery support through the current
-  backend container.
+  backend container. This is a valid consistent-hash algorithm change and can
+  remap existing consistent-hash affinity keys once during the 1.5.7 upgrade.
 - Collapse the load-balancer factory, stats, and priority-check helpers onto a
   concrete readiness container now that Fluxheim owns all shipped selection
   algorithms, removing the remaining generic Pingora selector trait plumbing.
@@ -76,6 +79,14 @@ behavior when the change improves security or project direction.
 - Hide the remaining runtime backend value type behind the load-balancer
   backend adapter so selector and health-check modules import Fluxheim's
   boundary type instead of `pingora::lb::Backend` directly.
+- Serialize per-backend load-balancer health state updates so enable/disable
+  changes and active health observations cannot overwrite each other under
+  concurrent health checks.
+- Store refreshed backend sets before refreshed health maps and use checked
+  wake-time arithmetic in the load-balancer background loop.
+- Clarify stream upstream TLS warnings for mixed hostname and IP upstream
+  routes where only IP connections skip hostname verification without
+  `upstream_sni`.
 
 ## 1.5.6 - 2026-06-06
 

docs/config-reference.md

@@ -1070,12 +1070,15 @@ and at least one upstream must remain a normal primary.
 `maglev-header-hash`, and `maglev-cookie-hash`. Header-hash modes require
 `proxy.load_balance.hash_header = "x-session"` or another valid HTTP header
 name. Cookie-hash modes require `proxy.load_balance.hash_cookie = "session"` or
-another valid cookie name. Hash modes use weighted FNV selection; consistent
-modes use Pingora's weighted Ketama ring for lower remapping when upstream
-membership changes. Bounded-load consistent modes use the same ring and skip a
-hash target whose weighted in-flight pressure is above the configured soft
-bound when another eligible candidate is available inside `max_iterations`;
-they fall back to normal consistent selection if no bounded candidate is found.
+another valid cookie name. Hash modes use weighted FNV selection seeded with a
+per-boot secret. Consistent modes use Fluxheim-owned weighted rendezvous
+hashing, also seeded with a per-boot secret, for low remapping when upstream
+membership changes. Upgrading from the pre-1.5.7 Pingora ring-backed
+consistent selector can remap existing affinity keys once. Bounded-load
+consistent modes use the same rendezvous candidate ordering and skip a hash
+target whose weighted in-flight pressure is above the configured soft bound
+when another eligible candidate is available inside `max_iterations`; they
+fall back to normal consistent selection if no bounded candidate is found.
 `bounded_load_factor_per_mille` defaults to `1250`, meaning roughly 125% of
 current weighted average load, and is valid only with bounded-load consistent
 selectors. Maglev modes use a fixed 65,537-slot bounded lookup table for static

release-notes/RELEASE_NOTES_1.5.7.md

@@ -28,12 +28,15 @@ results as far as possible.
 - Replace Pingora's FNV weighted-hash selector for source, URI, header, and
   cookie hash modes with Fluxheim-owned weighted-first FNV selection over the
   current backend container.
+- Seed Fluxheim-owned FNV and consistent-hash selectors with per-boot routing
+  secrets so clients cannot precompute keys that target a chosen backend.
 - Replace Pingora's random selector dependency for power-of-two choices with a
   Fluxheim-owned weighted random first pick and unique backend fallback scan.
 - Replace Pingora's consistent-hash selector dependency with Fluxheim-owned
   rendezvous candidate ordering for consistent and bounded-load consistent
   hash modes. Dynamic file/DNS discovery remains supported through the current
-  backend container.
+  backend container. This is a valid consistent-hash algorithm change and can
+  remap existing consistent-hash affinity keys once during the 1.5.7 upgrade.
 - Collapse load-balancer factory, stats, and priority-check helpers onto a
   concrete readiness container now that Fluxheim owns all shipped selection
   algorithms.
@@ -68,6 +71,14 @@ results as far as possible.
 - Hide the remaining runtime backend value type behind the load-balancer
   backend adapter so selector and health-check modules use Fluxheim's boundary
   type while the final backend-type replacement remains isolated.
+- Serialize per-backend load-balancer health state updates so enable/disable
+  changes and active health observations cannot overwrite each other under
+  concurrent health checks.
+- Store refreshed backend sets before refreshed health maps and use checked
+  wake-time arithmetic in the load-balancer background loop.
+- Clarify stream upstream TLS warnings for mixed hostname and IP upstream
+  routes where only IP connections skip hostname verification without
+  `upstream_sni`.
 
 ## Boundaries
 

src/load_balancer/backend.rs

@@ -1,7 +1,9 @@
 use std::collections::{BTreeSet, HashMap};
 use std::io;
 use std::net::SocketAddr;
+use std::process;
 use std::sync::Arc;
+use std::sync::Mutex;
 use std::time::Duration;
 use std::time::Instant;
 
@@ -34,64 +36,52 @@ pub(super) trait FluxHealthCheck: Send + Sync + 'static {
     }
 }
 
-#[derive(Clone)]
 struct FluxBackendHealthInner {
     healthy: bool,
     enabled: bool,
     consecutive_counter: usize,
 }
 
-struct FluxBackendHealth(ArcSwap<FluxBackendHealthInner>);
+#[derive(Clone)]
+struct FluxBackendHealth(Arc<Mutex<FluxBackendHealthInner>>);
 
 impl Default for FluxBackendHealth {
     fn default() -> Self {
-        Self(ArcSwap::new(Arc::new(FluxBackendHealthInner {
+        Self(Arc::new(Mutex::new(FluxBackendHealthInner {
             healthy: true,
             enabled: true,
             consecutive_counter: 0,
         })))
     }
 }
 
-impl Clone for FluxBackendHealth {
-    fn clone(&self) -> Self {
-        Self(ArcSwap::new(self.0.load_full()))
+impl FluxBackendHealth {
+    fn lock(&self) -> std::sync::MutexGuard<'_, FluxBackendHealthInner> {
+        self.0.lock().unwrap_or_else(|_| process::abort())
     }
-}
 
-impl FluxBackendHealth {
     fn ready(&self) -> bool {
-        let health = self.0.load();
+        let health = self.lock();
         health.healthy && health.enabled
     }
 
     fn enable(&self, enabled: bool) {
-        let health = self.0.load();
-        if health.enabled != enabled {
-            let mut next = (**health).clone();
-            next.enabled = enabled;
-            self.0.store(Arc::new(next));
-        }
+        self.lock().enabled = enabled;
     }
 
     fn observe(&self, healthy: bool, flip_threshold: usize) -> bool {
-        let health = self.0.load();
-        let mut flipped = false;
+        let mut health = self.lock();
         if health.healthy != healthy {
-            let mut next = (**health).clone();
-            next.consecutive_counter = next.consecutive_counter.saturating_add(1);
-            if next.consecutive_counter >= flip_threshold {
-                next.healthy = healthy;
-                next.consecutive_counter = 0;
-                flipped = true;
+            health.consecutive_counter = health.consecutive_counter.saturating_add(1);
+            if health.consecutive_counter >= flip_threshold {
+                health.healthy = healthy;
+                health.consecutive_counter = 0;
+                return true;
             }
-            self.0.store(Arc::new(next));
-        } else if health.consecutive_counter > 0 {
-            let mut next = (**health).clone();
-            next.consecutive_counter = 0;
-            self.0.store(Arc::new(next));
+        } else {
+            health.consecutive_counter = 0;
         }
-        flipped
+        false
     }
 }
 
@@ -149,8 +139,8 @@ impl FluxLoadBalancerRuntime {
                 }
                 next_health.insert(key, backend_health);
             }
-            self.health.store(Arc::new(next_health));
             self.backends.store(Arc::new(new_backends));
+            self.health.store(Arc::new(next_health));
         } else {
             let health = self.health.load();
             for (key, enabled) in enablement {
@@ -187,7 +177,7 @@ impl FluxLoadBalancerRuntime {
                 if let Err(error) = self.update().await {
                     log::warn!("load-balancer discovery update failed: {error}");
                 }
-                next_update = now + self.update_frequency.unwrap_or(NEVER);
+                next_update = checked_next_wake(now, self.update_frequency.unwrap_or(NEVER));
             }
 
             if let Some(ready) = ready.take() {
@@ -196,7 +186,8 @@ impl FluxLoadBalancerRuntime {
 
             if next_health_check <= now {
                 self.run_health_check(self.parallel_health_check).await;
-                next_health_check = now + self.health_check_frequency.unwrap_or(NEVER);
+                next_health_check =
+                    checked_next_wake(now, self.health_check_frequency.unwrap_or(NEVER));
             }
 
             if self.update_frequency.is_none() && self.health_check_frequency.is_none() {
@@ -259,6 +250,11 @@ impl FluxLoadBalancerRuntime {
     }
 }
 
+fn checked_next_wake(now: Instant, delay: Duration) -> Instant {
+    now.checked_add(delay)
+        .unwrap_or_else(|| now + Duration::from_secs(3600))
+}
+
 pub(crate) trait BackendIdentity {
     fn authority(&self) -> String;
 

src/load_balancer/selection.rs

@@ -12,10 +12,11 @@ use super::state::{
 };
 use crate::flux_error::{FluxError, FluxResult};
 
+const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325;
 const MAGLEV_TABLE_SIZE: usize = 65_537;
 
 pub(super) fn fnv1a64(bytes: &[u8]) -> u64 {
-    fnv1a64_with_seed(bytes, 0xcbf2_9ce4_8422_2325)
+    fnv1a64_with_seed(bytes, FNV_OFFSET_BASIS)
 }
 
 pub(super) fn fnv1a64_with_seed(bytes: &[u8], seed: u64) -> u64 {
@@ -596,12 +597,12 @@ fn select_fnv_hash_with_backup_policy(
     }
 
     let mut seen = std::collections::HashSet::new();
-    let mut index = fnv1a64(key);
+    let mut index = fnv1a64_with_seed(key, fnv_route_secret());
     for step in 0..max_iterations.max(1) {
         let candidate_index = if step == 0 {
             weighted[index as usize % weighted.len()]
         } else {
-            index = fnv1a64(&index.to_le_bytes());
+            index = fnv1a64_with_seed(&index.to_le_bytes(), fnv_route_secret());
             index as usize % backends.len()
         };
         let candidate = &backends[candidate_index];
@@ -834,7 +835,7 @@ fn consistent_hash_candidates(
 fn consistent_backend_score(key: &[u8], backend_key: u64, weight: usize) -> u64 {
     let mut best = 0u64;
     for replica in 0..weight.max(1) {
-        let mut hash = fnv1a64_with_seed(key, 0x9e37_79b9_7f4a_7c15);
+        let mut hash = fnv1a64_with_seed(key, consistent_route_secret());
         hash = fnv1a64_with_seed(&backend_key.to_le_bytes(), hash);
         hash = fnv1a64_with_seed(&(replica as u64).to_le_bytes(), hash);
         best = best.max(hash);
@@ -870,8 +871,7 @@ impl MaglevTable {
             .iter()
             .map(|backend_key| {
                 let key = backend_key.to_le_bytes();
-                let offset =
-                    fnv1a64_with_seed(&key, 0xcbf2_9ce4_8422_2325) as usize % MAGLEV_TABLE_SIZE;
+                let offset = fnv1a64_with_seed(&key, FNV_OFFSET_BASIS) as usize % MAGLEV_TABLE_SIZE;
                 let skip = (fnv1a64_with_seed(&key, 0x8422_2325_cbf2_9ce4) as usize
                     % (MAGLEV_TABLE_SIZE - 1))
                     + 1;
@@ -961,13 +961,28 @@ pub(super) fn select_maglev(
 }
 
 fn maglev_route_secret() -> u64 {
-    static SECRET: OnceLock<u64> = OnceLock::new();
-    *SECRET.get_or_init(|| {
+    route_secret(&MAGLEV_ROUTE_SECRET, "Maglev routing hash")
+}
+
+fn consistent_route_secret() -> u64 {
+    route_secret(&CONSISTENT_ROUTE_SECRET, "consistent-hash routing")
+}
+
+fn fnv_route_secret() -> u64 {
+    route_secret(&FNV_ROUTE_SECRET, "FNV routing hash")
+}
+
+static MAGLEV_ROUTE_SECRET: OnceLock<u64> = OnceLock::new();
+static CONSISTENT_ROUTE_SECRET: OnceLock<u64> = OnceLock::new();
+static FNV_ROUTE_SECRET: OnceLock<u64> = OnceLock::new();
+
+fn route_secret(secret: &'static OnceLock<u64>, label: &'static str) -> u64 {
+    *secret.get_or_init(|| {
         let mut bytes = [0u8; 8];
         if let Err(error) = getrandom::fill(&mut bytes) {
             log::error!(
                 target: "fluxheim::security",
-                "failed to seed Maglev routing hash secret: {error}"
+                "failed to seed {label} secret: {error}"
             );
             process::abort();
         }

src/stream_tls.rs

@@ -161,7 +161,7 @@ fn warn_if_ip_upstream_tls_verification_skips_hostname(route: &StreamRouteConfig
     }
     log::warn!(
         target: "fluxheim::security",
-        "stream route '{}' enables upstream TLS certificate verification for IP-only upstreams without upstream_sni; hostname verification is skipped for those connections",
+        "stream route '{}' enables upstream TLS certificate verification for one or more IP upstreams without upstream_sni; hostname verification is skipped for those IP connections",
         route.name
     );
 }