Skip to content

Commit 5945bcc

Browse files
committed
Harden snapshot recovery and integrity
1 parent c72280c commit 5945bcc

31 files changed

Lines changed: 1979 additions & 271 deletions

CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ behavior when the change improves security or project direction.
2020
- Support CIRCL Geo Open's combined Country and ASN MMDB schema and add an
2121
opt-in checksum-pinned real-database smoke covering static, proxy, and
2222
load-balanced country/ASN policy.
23+
- Add authenticated snapshot manifests, explicit rollback ancestry and
24+
generations, persisted self-healing state, resilient listing, store doctor,
25+
show/diff/verify commands, and protected pruning.
2326

2427
### Security
2528

@@ -35,6 +38,10 @@ behavior when the change improves security or project direction.
3538
buffer on drop with `sanitization`.
3639
- Enforce positive per-upstream and checked aggregate stream weights inside the
3740
public runtime selector boundary, independently of config validation.
41+
- Serialize snapshot mutations with a private advisory lock, publish history
42+
with create-new semantics, clean failed transactions, retain rollback state
43+
until completion, authenticate configured stores with HMAC-SHA-256, redact
44+
invalid IDs, and return clock failures without terminating the data plane.
3845
- Make OpenSSL cipher allow-lists deterministic across protocol families:
3946
omitting all TLS 1.2 or TLS 1.3 suites now disables that protocol version
4047
instead of retaining Mozilla acceptor defaults. Use the current Mozilla v5

Cargo.lock

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/fluxheim-config/src/config_admin.rs

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ pub struct AdminConfig {
3232
#[serde(default)]
3333
pub snapshot_store: Option<PathBuf>,
3434
#[serde(default)]
35+
pub snapshot_integrity_key_file: Option<PathBuf>,
36+
#[serde(default)]
3537
pub transport: AdminTransportConfig,
3638
#[serde(default)]
3739
pub ops_socket: AdminOpsSocketConfig,
@@ -54,6 +56,7 @@ pub struct AdminConfigFragment {
5456
token_env: Option<String>,
5557
token_file: Option<PathBuf>,
5658
snapshot_store: Option<PathBuf>,
59+
snapshot_integrity_key_file: Option<PathBuf>,
5760
transport: Option<AdminTransportConfigFragment>,
5861
ops_socket: Option<AdminOpsSocketConfigFragment>,
5962
health: Option<AdminHealthConfigFragment>,
@@ -71,6 +74,7 @@ impl Default for AdminConfig {
7174
token_env: None,
7275
token_file: None,
7376
snapshot_store: None,
77+
snapshot_integrity_key_file: None,
7478
transport: AdminTransportConfig::default(),
7579
ops_socket: AdminOpsSocketConfig::default(),
7680
health: AdminHealthConfig::default(),
@@ -101,6 +105,9 @@ impl AdminConfig {
101105
if let Some(snapshot_store) = fragment.snapshot_store {
102106
self.snapshot_store = Some(snapshot_store);
103107
}
108+
if let Some(key_file) = fragment.snapshot_integrity_key_file {
109+
self.snapshot_integrity_key_file = Some(key_file);
110+
}
104111
if let Some(transport) = fragment.transport {
105112
self.transport.merge(transport);
106113
}
@@ -136,6 +143,11 @@ impl AdminConfig {
136143
{
137144
*snapshot_store = base_dir.join(&snapshot_store);
138145
}
146+
if let Some(key_file) = &mut self.snapshot_integrity_key_file
147+
&& key_file.is_relative()
148+
{
149+
*key_file = base_dir.join(&key_file);
150+
}
139151
self.ops_socket.resolve_relative_paths(base_dir);
140152
}
141153

@@ -149,10 +161,32 @@ impl AdminConfig {
149161
validate_optional_env("admin.token_env", self.token_env.as_deref())?;
150162
reject_empty_admin_path("admin.token_file", self.token_file.as_deref())?;
151163
reject_empty_admin_path("admin.snapshot_store", self.snapshot_store.as_deref())?;
164+
reject_empty_admin_path(
165+
"admin.snapshot_integrity_key_file",
166+
self.snapshot_integrity_key_file.as_deref(),
167+
)?;
152168
validate_path("admin.token_file", self.token_file.as_deref())?;
153169
validate_path("admin.snapshot_store", self.snapshot_store.as_deref())?;
170+
validate_path(
171+
"admin.snapshot_integrity_key_file",
172+
self.snapshot_integrity_key_file.as_deref(),
173+
)?;
154174
validate_non_world_writable_parent("admin.token_file", self.token_file.as_deref())?;
155175
validate_non_world_writable_parent("admin.snapshot_store", self.snapshot_store.as_deref())?;
176+
validate_non_world_writable_parent(
177+
"admin.snapshot_integrity_key_file",
178+
self.snapshot_integrity_key_file.as_deref(),
179+
)?;
180+
if let (Some(store), Some(key_file)) = (
181+
self.snapshot_store.as_deref(),
182+
self.snapshot_integrity_key_file.as_deref(),
183+
) && key_file.starts_with(store)
184+
{
185+
return Err(ConfigError::UnsafePath {
186+
field: "admin.snapshot_integrity_key_file".to_owned(),
187+
path: key_file.to_path_buf(),
188+
});
189+
}
156190
self.auth_throttle.validate()?;
157191
self.self_healing.validate()?;
158192
self.client_certificate.validate()?;
@@ -222,6 +256,11 @@ impl AdminConfigFragment {
222256
{
223257
*snapshot_store = base_dir.join(&snapshot_store);
224258
}
259+
if let Some(key_file) = &mut self.snapshot_integrity_key_file
260+
&& key_file.is_relative()
261+
{
262+
*key_file = base_dir.join(&key_file);
263+
}
225264
if let Some(ops_socket) = &mut self.ops_socket {
226265
ops_socket.resolve_relative_paths(base_dir);
227266
}

crates/fluxheim-config/src/config_tests_admin.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,27 @@ fn rejects_invalid_admin_auth_throttle() {
190190
})
191191
);
192192
}
193+
194+
#[test]
195+
fn snapshot_integrity_key_must_remain_outside_snapshot_store() {
196+
let parent = secure_test_dir("config-admin-snapshot-integrity");
197+
let snapshot_store = parent.join("snapshots");
198+
let outside_key = parent.join("snapshot.key");
199+
let valid = AdminConfig {
200+
snapshot_store: Some(snapshot_store.clone()),
201+
snapshot_integrity_key_file: Some(outside_key),
202+
..AdminConfig::default()
203+
};
204+
valid.validate().unwrap();
205+
206+
let invalid = AdminConfig {
207+
snapshot_store: Some(snapshot_store.clone()),
208+
snapshot_integrity_key_file: Some(snapshot_store.join("snapshot.key")),
209+
..AdminConfig::default()
210+
};
211+
assert!(matches!(
212+
invalid.validate(),
213+
Err(ConfigError::UnsafePath { field, .. })
214+
if field == "admin.snapshot_integrity_key_file"
215+
));
216+
}

crates/fluxheim-snapshot/Cargo.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ description = "Internal Fluxheim configuration snapshot store."
1010
[dependencies]
1111
fluxheim-config = { path = "../fluxheim-config" }
1212
log = "0.4.33"
13+
ring = "0.17.14"
14+
rustix = { version = "1.1.4", features = ["fs"] }
15+
sanitization = { version = "1.2.4", features = ["alloc"] }
1316
serde = { version = "1.0.228", features = ["derive"] }
1417
toml = "1.1.2"
1518

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
use std::fs::File;
2+
use std::io::{self, Read as _};
3+
use std::path::Path;
4+
5+
use ring::{digest, hmac};
6+
use sanitization::SecretVec;
7+
use serde::{Deserialize, Serialize};
8+
9+
use crate::store::SnapshotError;
10+
11+
pub(crate) const MAX_INTEGRITY_KEY_BYTES: u64 = 4096;
12+
const MIN_INTEGRITY_KEY_BYTES: usize = 32;
13+
const SNAPSHOT_MAC_LABEL: &[u8] = b"fluxheim-snapshot-v1\0";
14+
const RECOVERY_MAC_LABEL: &[u8] = b"fluxheim-snapshot-recovery-v1\0";
15+
16+
#[derive(Debug)]
17+
pub(crate) struct SnapshotIntegrityKey {
18+
secret: SecretVec,
19+
key_id: String,
20+
}
21+
22+
#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)]
23+
#[serde(deny_unknown_fields)]
24+
pub(crate) struct SnapshotIntegrityManifest {
25+
pub algorithm: String,
26+
pub key_id: String,
27+
pub config_sha256: String,
28+
pub metadata_hmac_sha256: String,
29+
}
30+
31+
impl SnapshotIntegrityKey {
32+
pub(crate) fn load(path: &Path) -> Result<Self, SnapshotError> {
33+
if fluxheim_config::fs_trust::existing_path_or_parent_has_insecure_write_permissions(path)
34+
.unwrap_or(true)
35+
{
36+
return Err(SnapshotError::UnsafeIntegrityKey {
37+
path: path.to_path_buf(),
38+
});
39+
}
40+
let mut file = open_secret_file(path)?;
41+
let metadata = file.metadata().map_err(SnapshotError::Io)?;
42+
if !metadata.is_file() || metadata.len() > MAX_INTEGRITY_KEY_BYTES {
43+
return Err(SnapshotError::InvalidIntegrityKey);
44+
}
45+
let admitted =
46+
usize::try_from(metadata.len()).map_err(|_| SnapshotError::InvalidIntegrityKey)?;
47+
let mut secret = SecretVec::from_fn(admitted, |_| 0);
48+
secret
49+
.with_secret_mut(|bytes| file.read_exact(bytes))
50+
.map_err(SnapshotError::Io)?;
51+
let mut growth_probe = [0u8; 1];
52+
let grew = file.read(&mut growth_probe).map_err(SnapshotError::Io)? != 0;
53+
sanitization::sanitize_bytes(&mut growth_probe);
54+
if grew || secret.with_secret(|bytes| bytes.len()) < MIN_INTEGRITY_KEY_BYTES {
55+
return Err(SnapshotError::InvalidIntegrityKey);
56+
}
57+
let key_id =
58+
secret.with_secret(|bytes| hex(digest::digest(&digest::SHA256, bytes).as_ref()));
59+
Ok(Self { secret, key_id })
60+
}
61+
62+
pub(crate) fn manifest(
63+
&self,
64+
id: &str,
65+
config: &[u8],
66+
metadata: &[u8],
67+
) -> SnapshotIntegrityManifest {
68+
SnapshotIntegrityManifest {
69+
algorithm: "hmac-sha256".to_owned(),
70+
key_id: self.key_id.clone(),
71+
config_sha256: hex(digest::digest(&digest::SHA256, config).as_ref()),
72+
metadata_hmac_sha256: hex(self.sign(id, config, metadata).as_ref()),
73+
}
74+
}
75+
76+
pub(crate) fn key_id(&self) -> &str {
77+
&self.key_id
78+
}
79+
80+
pub(crate) fn sign_recovery(&self, state: &[u8]) -> String {
81+
self.secret.with_secret(|secret| {
82+
let key = hmac::Key::new(hmac::HMAC_SHA256, secret);
83+
let mut context = hmac::Context::with_key(&key);
84+
context.update(RECOVERY_MAC_LABEL);
85+
context.update(&(state.len() as u64).to_be_bytes());
86+
context.update(state);
87+
hex(context.sign().as_ref())
88+
})
89+
}
90+
91+
pub(crate) fn verify_recovery(&self, state: &[u8], signature: &str) -> bool {
92+
let Some(signature) = decode_hex_32(signature) else {
93+
return false;
94+
};
95+
self.secret.with_secret(|secret| {
96+
let key = hmac::Key::new(hmac::HMAC_SHA256, secret);
97+
let mut input = Vec::with_capacity(RECOVERY_MAC_LABEL.len() + state.len() + 8);
98+
input.extend_from_slice(RECOVERY_MAC_LABEL);
99+
input.extend_from_slice(&(state.len() as u64).to_be_bytes());
100+
input.extend_from_slice(state);
101+
hmac::verify(&key, &input, &signature).is_ok()
102+
})
103+
}
104+
105+
pub(crate) fn verify(
106+
&self,
107+
id: &str,
108+
config: &[u8],
109+
metadata: &[u8],
110+
manifest: &SnapshotIntegrityManifest,
111+
) -> Result<(), SnapshotError> {
112+
if manifest.algorithm != "hmac-sha256" || manifest.key_id != self.key_id {
113+
return Err(SnapshotError::IntegrityVerificationFailed { id: id.to_owned() });
114+
}
115+
let config_digest = hex(digest::digest(&digest::SHA256, config).as_ref());
116+
if config_digest != manifest.config_sha256 {
117+
return Err(SnapshotError::IntegrityVerificationFailed { id: id.to_owned() });
118+
}
119+
let expected = decode_hex_32(&manifest.metadata_hmac_sha256)
120+
.ok_or_else(|| SnapshotError::IntegrityVerificationFailed { id: id.to_owned() })?;
121+
self.secret
122+
.with_secret(|secret| {
123+
hmac::verify(
124+
&hmac::Key::new(hmac::HMAC_SHA256, secret),
125+
&mac_input(id, config, metadata),
126+
&expected,
127+
)
128+
})
129+
.map_err(|_| SnapshotError::IntegrityVerificationFailed { id: id.to_owned() })
130+
}
131+
132+
fn sign(&self, id: &str, config: &[u8], metadata: &[u8]) -> hmac::Tag {
133+
self.secret.with_secret(|secret| {
134+
hmac::sign(
135+
&hmac::Key::new(hmac::HMAC_SHA256, secret),
136+
&mac_input(id, config, metadata),
137+
)
138+
})
139+
}
140+
}
141+
142+
fn mac_input(id: &str, config: &[u8], metadata: &[u8]) -> Vec<u8> {
143+
let mut input = Vec::with_capacity(
144+
SNAPSHOT_MAC_LABEL.len() + id.len() + config.len() + metadata.len() + 24,
145+
);
146+
input.extend_from_slice(SNAPSHOT_MAC_LABEL);
147+
append_field(&mut input, id.as_bytes());
148+
append_field(&mut input, config);
149+
append_field(&mut input, metadata);
150+
input
151+
}
152+
153+
fn append_field(output: &mut Vec<u8>, value: &[u8]) {
154+
output.extend_from_slice(&(value.len() as u64).to_be_bytes());
155+
output.extend_from_slice(value);
156+
}
157+
158+
fn hex(bytes: &[u8]) -> String {
159+
const DIGITS: &[u8; 16] = b"0123456789abcdef";
160+
let mut output = String::with_capacity(bytes.len() * 2);
161+
for byte in bytes {
162+
output.push(char::from(DIGITS[usize::from(byte >> 4)]));
163+
output.push(char::from(DIGITS[usize::from(byte & 0x0f)]));
164+
}
165+
output
166+
}
167+
168+
fn decode_hex_32(value: &str) -> Option<[u8; 32]> {
169+
if value.len() != 64 {
170+
return None;
171+
}
172+
let mut output = [0u8; 32];
173+
for (index, pair) in value.as_bytes().chunks_exact(2).enumerate() {
174+
output[index] = hex_nibble(pair[0])?.checked_mul(16)? | hex_nibble(pair[1])?;
175+
}
176+
Some(output)
177+
}
178+
179+
fn hex_nibble(value: u8) -> Option<u8> {
180+
match value {
181+
b'0'..=b'9' => Some(value - b'0'),
182+
b'a'..=b'f' => Some(value - b'a' + 10),
183+
b'A'..=b'F' => Some(value - b'A' + 10),
184+
_ => None,
185+
}
186+
}
187+
188+
#[cfg(unix)]
189+
fn open_secret_file(path: &Path) -> Result<File, SnapshotError> {
190+
let fd = rustix::fs::open(
191+
path,
192+
rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::CLOEXEC | rustix::fs::OFlags::NOFOLLOW,
193+
rustix::fs::Mode::empty(),
194+
)
195+
.map_err(|error| SnapshotError::Io(io::Error::from_raw_os_error(error.raw_os_error())))?;
196+
Ok(fd.into())
197+
}
198+
199+
#[cfg(not(unix))]
200+
fn open_secret_file(path: &Path) -> Result<File, SnapshotError> {
201+
File::open(path).map_err(SnapshotError::Io)
202+
}

crates/fluxheim-snapshot/src/lib.rs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,28 @@
44
deny(clippy::expect_used, clippy::panic, clippy::unwrap_used)
55
)]
66

7+
mod integrity;
78
mod metadata;
9+
mod model;
10+
mod operations;
11+
mod recovery;
812
mod state;
913
mod store;
1014
mod store_fs;
15+
mod store_support;
16+
17+
#[cfg(test)]
18+
mod operations_tests;
1119

1220
pub use metadata::{MAX_SNAPSHOT_MESSAGE_BYTES, SnapshotMetadata};
21+
pub use model::{
22+
ConfigSnapshot, SnapshotEntryStatus, SnapshotError, SnapshotIntegrityStatus, SnapshotListEntry,
23+
};
24+
pub use operations::{
25+
SnapshotDiff, SnapshotDoctorReport, SnapshotPruneOptions, SnapshotPruneReport,
26+
};
1327
pub use state::{
1428
PendingValidation, SnapshotApplyMode, SnapshotHealthSignalOutcome, SnapshotRollbackReason,
1529
SnapshotRuntimeState, ValidationMetrics,
1630
};
17-
pub use store::{ConfigSnapshot, SnapshotError, SnapshotStore};
31+
pub use store::SnapshotStore;

0 commit comments

Comments
 (0)