Harden load balancer runtime state persistence

eldryoth · eldryoth · commit e892be1d55e1 · 2026-06-07T15:47:07.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,19 @@ behavior when the change improves security or project direction.
   pool has `proxy.load_balance.runtime_state_file` configured, and keep
   `persistent: false` for in-memory-only pools.
 
+### Security
+
+- Move request-path persistence state-file writes to the blocking worker pool
+  with serialized snapshot writes, while keeping admin mutation saves
+  synchronous and ordered.
+- Harden runtime state files with fd-based permission setting, temp-file
+  cleanup on failed writes, and all-or-nothing restore validation for mixed
+  policy/persistence state.
+- Document and warn when raw header or cookie persistence writes client affinity
+  identifiers to `proxy.load_balance.runtime_state_file`; prefer
+  `managed-cookie` or encrypted, access-restricted storage for session-bearing
+  identifiers.
+
 ## 1.5.8 - 2026-06-07
 
 ### Added
diff --git a/docs/config-reference.md b/docs/config-reference.md
@@ -1274,6 +1274,14 @@ Corrupt, oversized, incompatible, or stale state is ignored and rebuilt instead
 of poisoning the pool. The file is local to one Fluxheim process and does not
 share managed-cookie signing keys across nodes.
 
+When persistence is enabled, the state file includes local affinity table keys.
+`source-ip` mode writes client IP bytes, `header` mode writes the configured
+header value, and `cookie` mode writes the configured cookie value.
+`managed-cookie` mode writes opaque affinity keys generated by Fluxheim. Use
+`managed-cookie` for session affinity when possible, or place the state file on
+an encrypted, access-restricted volume when raw header or cookie identifiers are
+used.
+
 The current `1.5.x` load-balancer line does not add/remove pool members at
 runtime, apply runtime weights to hash/ring selectors, share managed-cookie
 signing keys across nodes, or synchronize load-balancer state across
diff --git a/packaging/container/load-balancer.toml b/packaging/container/load-balancer.toml
@@ -60,6 +60,9 @@ all_down_status = 503
 max_iterations = 128
 # Optional local restart-persistent runtime state. Mount the parent directory
 # as a persistent volume before enabling this in containers.
+# Header and cookie persistence modes write raw affinity identifiers to this
+# file; prefer managed-cookie mode or an encrypted volume for session-bearing
+# identifiers.
 # runtime_state_file = "/var/lib/fluxheim/load-balancer/default.json"
 
 [vhosts.proxy.load_balance.health_check]
diff --git a/release-notes/RELEASE_NOTES_1.5.9.md b/release-notes/RELEASE_NOTES_1.5.9.md
@@ -26,6 +26,22 @@ Fluxheim 1.5.9 starts the restart-persistent load-balancer state line.
   has a runtime state file configured, and `persistent: false` for in-memory
   pools.
 
+## Hardened
+
+- Persistence-table state-file writes from request selection now run on the
+  blocking worker pool instead of performing synchronous fsync work on an async
+  executor thread.
+- Runtime state file writes now set Unix permissions on the already-open file
+  descriptor and remove temporary files when a write, sync, rename, or directory
+  sync fails.
+- Runtime state restore now validates policy overrides and persistence entries
+  before committing either half, preventing partial restore of a mixed-validity
+  state file.
+- Documentation and startup warnings now call out that raw `header` and
+  `cookie` persistence modes write client affinity identifiers to
+  `proxy.load_balance.runtime_state_file`; use `managed-cookie` or encrypted,
+  access-restricted storage for session-bearing identifiers.
+
 ## Stop Line
 
 This release does not add cross-node state sync, runtime add/remove-member,
diff --git a/src/config_load_balance.rs b/src/config_load_balance.rs
@@ -145,6 +145,25 @@ impl LoadBalanceConfig {
                 "proxy.load_balance.runtime_state_file",
                 Some(path),
             )?;
+            if self.persistence.enabled
+                && matches!(
+                    self.persistence.mode,
+                    LoadBalancePersistenceMode::Header | LoadBalancePersistenceMode::Cookie
+                )
+            {
+                let persistence_mode = match self.persistence.mode {
+                    LoadBalancePersistenceMode::Header => "header",
+                    LoadBalancePersistenceMode::Cookie => "cookie",
+                    LoadBalancePersistenceMode::SourceIp
+                    | LoadBalancePersistenceMode::ManagedCookie => unreachable!(),
+                };
+                log::warn!(
+                    target: "fluxheim::security",
+                    "proxy.load_balance.runtime_state_file is configured with raw {} persistence; client affinity identifiers are written to disk at {}. Use managed-cookie mode or an encrypted, access-restricted volume for session-bearing identifiers.",
+                    persistence_mode,
+                    path.display()
+                );
+            }
         }
         self.queue.validate()?;
         Ok(())
diff --git a/src/load_balancer.rs b/src/load_balancer.rs
@@ -1,7 +1,7 @@
 use std::fmt::{Debug, Formatter};
 use std::io;
 use std::net::IpAddr;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::time::{Duration, Instant};
@@ -74,6 +74,7 @@ pub struct UpstreamLoadBalancer {
     queue_policy: LoadBalanceQueueConfig,
     queue_waiting: Arc<AtomicUsize>,
     runtime_state_file: Option<Arc<PathBuf>>,
+    runtime_state_save_lock: Arc<std::sync::Mutex<()>>,
     round_robin_cursor: Arc<AtomicUsize>,
     state_prune_counter: Arc<AtomicUsize>,
     counters: Arc<BackendConnectionCounters>,
@@ -663,12 +664,14 @@ impl UpstreamLoadBalancer {
         let selected = self.prepare_selected(selected, persistence_outcome)?;
         if let (Some(persistence), Some(key)) = (&self.persistence, persistence_key) {
             persistence.record(key, backend_key(&selected.backend));
-            self.save_runtime_state_if_configured("persistence_record");
+            self.save_runtime_state_if_configured_in_background("persistence_record");
         } else if let Some(persistence) = &self.persistence
             && let Some((key, cookie)) = persistence.new_managed_cookie()
         {
             persistence.record(&key, backend_key(&selected.backend));
-            self.save_runtime_state_if_configured("managed_cookie_persistence_record");
+            self.save_runtime_state_if_configured_in_background(
+                "managed_cookie_persistence_record",
+            );
             let mut selected = selected;
             selected.managed_affinity_cookie = Some(cookie);
             return Some(selected);
@@ -757,6 +760,7 @@ impl UpstreamLoadBalancer {
                 queue_policy: config.load_balance.queue.clone(),
                 queue_waiting: Arc::new(AtomicUsize::new(0)),
                 runtime_state_file: config.load_balance.runtime_state_file.clone().map(Arc::new),
+                runtime_state_save_lock: Arc::new(std::sync::Mutex::new(())),
                 round_robin_cursor: Arc::new(AtomicUsize::new(0)),
                 state_prune_counter: Arc::new(AtomicUsize::new(0)),
                 counters: Arc::new(BackendConnectionCounters::default()),
@@ -775,27 +779,23 @@ impl UpstreamLoadBalancer {
             return;
         }
         let backends = self.inner.backends();
-        let live_connection_keys = backends
-            .iter()
-            .map(backend_key)
-            .collect::<std::collections::HashSet<_>>();
-        let live_policy_keys = backends
+        let live_keys = backends
             .iter()
             .map(backend_key)
             .collect::<std::collections::HashSet<_>>();
-        self.counters.prune_stale(&live_connection_keys);
-        self.backend_policy.prune_stale(&live_policy_keys);
+        self.counters.prune_stale(&live_keys);
+        self.backend_policy.prune_stale(&live_keys);
         if let Some(persistence) = &self.persistence {
-            persistence.prune_stale_for_live_backends(&live_policy_keys);
+            persistence.prune_stale_for_live_backends(&live_keys);
         }
         if let Some(passive_health) = &self.passive_health {
-            passive_health.prune_stale(&live_connection_keys);
+            passive_health.prune_stale(&live_keys);
         }
         if let Some(slow_start) = &self.slow_start {
-            slow_start.prune_stale(&live_connection_keys);
+            slow_start.prune_stale(&live_keys);
         }
         if let Some(latency) = self.inner.latency_state() {
-            latency.prune_stale(&live_connection_keys);
+            latency.prune_stale(&live_keys);
         }
     }
 
@@ -1057,19 +1057,29 @@ impl UpstreamLoadBalancer {
                 "unsupported load balancer runtime state version",
             ));
         }
-        self.backend_policy
-            .restore_runtime_snapshot(&snapshot.runtime_overrides)
+        let prepared_policy = self
+            .backend_policy
+            .prepare_runtime_snapshot(&snapshot.runtime_overrides)
             .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
         let live_keys = self.live_backend_keys();
-        let persistence_entries = if let (Some(persistence), Some(snapshot)) =
+        let prepared_persistence = if let (Some(persistence), Some(snapshot)) =
             (&self.persistence, &snapshot.persistence)
         {
-            persistence
-                .restore_snapshot(snapshot, &live_keys)
-                .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?
+            Some(
+                persistence
+                    .prepare_snapshot(snapshot, &live_keys)
+                    .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?,
+            )
         } else {
-            0
+            None
         };
+        let persistence_entries = prepared_persistence
+            .as_ref()
+            .map_or(0, |snapshot| snapshot.restored_entries());
+        self.backend_policy.commit_runtime_snapshot(prepared_policy);
+        if let (Some(persistence), Some(snapshot)) = (&self.persistence, prepared_persistence) {
+            persistence.commit_snapshot(snapshot);
+        }
         Ok(LoadBalancerRuntimeStateRestore {
             persistence_entries,
         })
@@ -1108,25 +1118,51 @@ impl UpstreamLoadBalancer {
         let Some(path) = &self.runtime_state_file else {
             return;
         };
+        let _guard = self
+            .runtime_state_save_lock
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
         let snapshot = self.runtime_state_snapshot();
-        match write_runtime_state_file(path, &snapshot) {
-            Ok(()) => log::debug!(
-                target: "fluxheim::load_balancer",
-                "load balancer runtime state saved path={} reason={}",
-                path.display(),
-                reason
-            ),
-            Err(error) => log::warn!(
-                target: "fluxheim::security",
-                "load balancer runtime state save failed path={} reason={} error={}",
-                path.display(),
-                reason,
-                error
-            ),
+        write_runtime_state_snapshot(path, &snapshot, reason);
+    }
+
+    fn save_runtime_state_if_configured_in_background(&self, reason: &'static str) {
+        if self.runtime_state_file.is_none() {
+            return;
+        }
+        let balancer = self.clone();
+        if let Ok(handle) = tokio::runtime::Handle::try_current() {
+            handle.spawn_blocking(move || {
+                balancer.save_runtime_state_if_configured(reason);
+            });
+        } else {
+            self.save_runtime_state_if_configured(reason);
         }
     }
 }
 
+fn write_runtime_state_snapshot(
+    path: &Path,
+    snapshot: &LoadBalancerRuntimeStateSnapshot,
+    reason: &str,
+) {
+    match write_runtime_state_file(path, snapshot) {
+        Ok(()) => log::debug!(
+            target: "fluxheim::load_balancer",
+            "load balancer runtime state saved path={} reason={}",
+            path.display(),
+            reason
+        ),
+        Err(error) => log::warn!(
+            target: "fluxheim::security",
+            "load balancer runtime state save failed path={} reason={} error={}",
+            path.display(),
+            reason,
+            error
+        ),
+    }
+}
+
 fn backend_runtime_status_eligible(backend: &LoadBalancerBackendRuntimeStats) -> bool {
     backend.ready
         && !backend.disabled
@@ -1914,6 +1950,64 @@ mod tests {
         assert_eq!(selected.backend.addr.to_string(), "127.0.0.1:3000");
     }
 
+    #[test]
+    fn runtime_state_file_restore_is_all_or_nothing() {
+        install_test_crypto_provider();
+        let dir = unique_temp_path("lb-runtime-state-atomic-restore");
+        std::fs::create_dir_all(&dir).unwrap();
+        let state_file = safe_child_path(&dir, "lb-state.json");
+        let disabled_key = backend_key(&FluxBackend::new("127.0.0.1:3001").unwrap());
+        std::fs::write(
+            &state_file,
+            format!(
+                r#"{{
+  "version": 1,
+  "runtime_overrides": {{
+    "states": [
+      {{
+        "key": {disabled_key},
+        "state": "disabled",
+        "changed_at_unix_secs": 1
+      }}
+    ],
+    "weights": []
+  }},
+  "persistence": {{
+    "entries": [
+      {{
+        "key": [],
+        "backend_key": {disabled_key},
+        "ttl_remaining_secs": 60
+      }}
+    ]
+  }}
+}}"#
+            ),
+        )
+        .unwrap();
+        let config = ProxyConfig {
+            upstreams: vec!["127.0.0.1:3000".to_owned(), "127.0.0.1:3001".to_owned()],
+            load_balance: LoadBalanceConfig {
+                runtime_state_file: Some(state_file),
+                persistence: LoadBalancePersistenceConfig {
+                    enabled: true,
+                    ttl_secs: 60,
+                    table_max_entries: 16,
+                    ..LoadBalancePersistenceConfig::default()
+                },
+                ..LoadBalanceConfig::default()
+            },
+            ..ProxyConfig::default()
+        };
+
+        let balancer = UpstreamLoadBalancer::from_proxy_config(&config)
+            .unwrap()
+            .unwrap();
+        let stats = balancer.runtime_stats();
+        assert_eq!(stats.runtime_disabled_backend_count, 0);
+        assert_eq!(stats.persistence.entry_count, 0);
+    }
+
     #[test]
     fn header_persistence_reuses_selected_backend() {
         install_test_crypto_provider();
diff --git a/src/load_balancer/persistence.rs b/src/load_balancer/persistence.rs
@@ -55,6 +55,17 @@ pub(crate) struct LoadBalancerPersistenceSnapshot {
     pub(crate) entries: Vec<LoadBalancerPersistenceEntrySnapshot>,
 }
 
+#[derive(Debug)]
+pub(super) struct PreparedLoadBalancerPersistenceSnapshot {
+    entries: std::collections::HashMap<Vec<u8>, LoadBalancerPersistenceEntry>,
+}
+
+impl PreparedLoadBalancerPersistenceSnapshot {
+    pub(super) fn restored_entries(&self) -> usize {
+        self.entries.len()
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
 pub(crate) struct LoadBalancerPersistenceEntrySnapshot {
     pub(crate) key: Vec<u8>,
@@ -236,11 +247,23 @@ impl LoadBalancerPersistenceState {
         LoadBalancerPersistenceSnapshot { entries }
     }
 
+    #[cfg(test)]
     pub(crate) fn restore_snapshot(
         &self,
         snapshot: &LoadBalancerPersistenceSnapshot,
         live_backend_keys: &std::collections::HashSet<u64>,
     ) -> Result<usize, &'static str> {
+        let prepared = self.prepare_snapshot(snapshot, live_backend_keys)?;
+        let restored = prepared.restored_entries();
+        self.commit_snapshot(prepared);
+        Ok(restored)
+    }
+
+    pub(super) fn prepare_snapshot(
+        &self,
+        snapshot: &LoadBalancerPersistenceSnapshot,
+        live_backend_keys: &std::collections::HashSet<u64>,
+    ) -> Result<PreparedLoadBalancerPersistenceSnapshot, &'static str> {
         if snapshot.entries.len() > self.table_max_entries {
             return Err("load balancer persistence snapshot exceeds table limit");
         }
@@ -269,12 +292,14 @@ impl LoadBalancerPersistenceState {
                 return Err("load balancer persistence snapshot has duplicate keys");
             }
         }
-        let restored = next.len();
+        Ok(PreparedLoadBalancerPersistenceSnapshot { entries: next })
+    }
+
+    pub(super) fn commit_snapshot(&self, prepared: PreparedLoadBalancerPersistenceSnapshot) {
         *self
             .table
             .lock()
-            .unwrap_or_else(|poisoned| poisoned.into_inner()) = next;
-        Ok(restored)
+            .unwrap_or_else(|poisoned| poisoned.into_inner()) = prepared.entries;
     }
 
     pub(super) fn new_managed_cookie(&self) -> Option<(Vec<u8>, ManagedAffinityCookie)> {
diff --git a/src/load_balancer/policy.rs b/src/load_balancer/policy.rs
diff --git a/src/load_balancer/state_file.rs b/src/load_balancer/state_file.rs
