Fail snapshot cryptography without aborting

eldryoth · eldryoth · commit 628a7655b8cd · 2026-07-11T20:12:06.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -52,6 +52,9 @@ behavior when the change improves security or project direction.
 - Route snapshot SHA-256 and HMAC-SHA-256 through Fluxheim's selected Ring,
   OpenSSL-FIPS, or AWS-LC-FIPS provider without duplicating snapshot plaintext
   into a concatenation buffer.
+- Return snapshot crypto-provider failures to the administrative caller instead
+  of aborting the process, and reject missing or replayed generation state when
+  retained snapshot metadata proves a higher generation.
 - Make OpenSSL cipher allow-lists deterministic across protocol families:
   omitting all TLS 1.2 or TLS 1.3 suites now disables that protocol version
   instead of retaining Mozilla acceptor defaults. Use the current Mozilla v5
diff --git a/crates/fluxheim-snapshot/src/generation.rs b/crates/fluxheim-snapshot/src/generation.rs
@@ -16,18 +16,18 @@ struct GenerationState {
 
 impl SnapshotStore {
     pub(crate) fn allocate_generation_unlocked(&self) -> Result<u64, SnapshotError> {
+        let observed_max = self.observed_max_generation()?;
         let current = match read_optional_regular_file_to_string_with_limit(
             &self.generation_path(),
             MAX_GENERATION_STATE_BYTES as u64,
         )? {
             Some(raw) => self.decode_generation_state(&raw)?,
-            None => self
-                .list()?
-                .into_iter()
-                .map(|snapshot| snapshot.metadata.generation)
-                .max()
-                .unwrap_or(0),
+            None if observed_max == 0 => 0,
+            None => return Err(SnapshotError::GenerationStateInvalid),
         };
+        if current < observed_max {
+            return Err(SnapshotError::GenerationStateInvalid);
+        }
         let next = current
             .checked_add(1)
             .ok_or(SnapshotError::GenerationExhausted)?;
@@ -40,24 +40,34 @@ impl SnapshotStore {
     }
 
     pub(crate) fn verify_generation_state(&self) -> Result<(), SnapshotError> {
+        let observed_max = self.observed_max_generation()?;
         let Some(raw) = read_optional_regular_file_to_string_with_limit(
             &self.generation_path(),
             MAX_GENERATION_STATE_BYTES as u64,
         )?
         else {
-            return Ok(());
+            return if observed_max == 0 {
+                Ok(())
+            } else {
+                Err(SnapshotError::GenerationStateInvalid)
+            };
         };
-        self.decode_generation_state(&raw).map(|_| ())
+        let persisted = self.decode_generation_state(&raw)?;
+        if persisted < observed_max {
+            return Err(SnapshotError::GenerationStateInvalid);
+        }
+        Ok(())
     }
 
     fn encode_generation_state(&self, generation: u64) -> Result<String, SnapshotError> {
         let payload = generation.to_be_bytes();
-        let (key_id, hmac_sha256) = self.integrity.as_deref().map_or((None, None), |key| {
-            (
+        let (key_id, hmac_sha256) = match self.integrity.as_deref() {
+            Some(key) => (
                 Some(key.key_id().to_owned()),
-                Some(key.sign_state(GENERATION_MAC_LABEL, &payload)),
-            )
-        });
+                Some(key.sign_state(GENERATION_MAC_LABEL, &payload)?),
+            ),
+            None => (None, None),
+        };
         toml::to_string(&GenerationState {
             generation,
             key_id,
@@ -100,4 +110,13 @@ impl SnapshotStore {
     pub(crate) fn generation_path(&self) -> std::path::PathBuf {
         self.root().join("generation.toml")
     }
+
+    fn observed_max_generation(&self) -> Result<u64, SnapshotError> {
+        Ok(self
+            .list()?
+            .into_iter()
+            .map(|snapshot| snapshot.metadata.generation)
+            .max()
+            .unwrap_or(0))
+    }
 }
diff --git a/crates/fluxheim-snapshot/src/integrity.rs b/crates/fluxheim-snapshot/src/integrity.rs
@@ -89,17 +89,18 @@ impl SnapshotIntegrityKey {
         id: &str,
         config: &[u8],
         metadata: &[u8],
-    ) -> SnapshotIntegrityManifest {
-        SnapshotIntegrityManifest {
+    ) -> Result<SnapshotIntegrityManifest, SnapshotError> {
+        let config_digest = self
+            .provider
+            .sha256(&[config])
+            .map_err(SnapshotError::CryptoProvider)?;
+        let signature = self.sign_result(id, config, metadata)?;
+        Ok(SnapshotIntegrityManifest {
             algorithm: "hmac-sha256".to_owned(),
             key_id: self.key_id.clone(),
-            config_sha256: self
-                .provider
-                .sha256(&[config])
-                .map(|digest| hex(&digest))
-                .unwrap_or_else(|error| crypto_provider_abort(self.provider.label(), &error)),
-            metadata_hmac_sha256: hex(&self.sign(id, config, metadata)),
-        }
+            config_sha256: hex(&config_digest),
+            metadata_hmac_sha256: hex(&signature),
+        })
     }
 
     pub(crate) fn key_id(&self) -> &str {
@@ -110,7 +111,7 @@ impl SnapshotIntegrityKey {
         &self.source_path
     }
 
-    pub(crate) fn sign_recovery(&self, state: &[u8]) -> String {
+    pub(crate) fn sign_recovery(&self, state: &[u8]) -> Result<String, SnapshotError> {
         self.sign_state(RECOVERY_MAC_LABEL, state)
     }
 
@@ -121,14 +122,17 @@ impl SnapshotIntegrityKey {
         self.verify_state(RECOVERY_MAC_LABEL, state, &signature)
     }
 
-    pub(crate) fn sign_state(&self, label: &[u8], state: &[u8]) -> String {
-        let length = (state.len() as u64).to_be_bytes();
-        self.secret.with_secret(|secret| {
-            self.provider
-                .hmac_sha256(secret, &[label, &length, state])
-                .map(|digest| hex(&digest))
-                .unwrap_or_else(|error| crypto_provider_abort(self.provider.label(), &error))
-        })
+    pub(crate) fn sign_state(&self, label: &[u8], state: &[u8]) -> Result<String, SnapshotError> {
+        let length = u64::try_from(state.len())
+            .map_err(|_| SnapshotError::CryptoProvider("state length overflow".to_owned()))?
+            .to_be_bytes();
+        self.secret
+            .with_secret(|secret| {
+                self.provider
+                    .hmac_sha256(secret, &[label, &length, state])
+                    .map(|digest| hex(&digest))
+            })
+            .map_err(SnapshotError::CryptoProvider)
     }
 
     pub(crate) fn verify_state(&self, label: &[u8], state: &[u8], signature: &[u8; 32]) -> bool {
@@ -175,22 +179,15 @@ impl SnapshotIntegrityKey {
         }
     }
 
-    fn sign(&self, id: &str, config: &[u8], metadata: &[u8]) -> [u8; 32] {
-        self.sign_result(id, config, metadata)
-            .unwrap_or_else(|error| {
-                crypto_provider_abort(self.provider.label(), &error.to_string())
-            })
-    }
-
     fn sign_result(
         &self,
         id: &str,
         config: &[u8],
         metadata: &[u8],
     ) -> Result<[u8; 32], SnapshotError> {
-        let id_length = (id.len() as u64).to_be_bytes();
-        let config_length = (config.len() as u64).to_be_bytes();
-        let metadata_length = (metadata.len() as u64).to_be_bytes();
+        let id_length = encoded_length(id.len())?;
+        let config_length = encoded_length(config.len())?;
+        let metadata_length = encoded_length(metadata.len())?;
         self.secret
             .with_secret(|secret| {
                 self.provider.hmac_sha256(
@@ -210,9 +207,10 @@ impl SnapshotIntegrityKey {
     }
 }
 
-fn crypto_provider_abort(provider: &str, error: &str) -> ! {
-    log::error!("fatal: snapshot cryptography failed through {provider}: {error}");
-    std::process::abort()
+fn encoded_length(length: usize) -> Result<[u8; 8], SnapshotError> {
+    u64::try_from(length)
+        .map(u64::to_be_bytes)
+        .map_err(|_| SnapshotError::CryptoProvider("field length overflow".to_owned()))
 }
 
 fn hex(bytes: &[u8]) -> String {
diff --git a/crates/fluxheim-snapshot/src/prune_boundary.rs b/crates/fluxheim-snapshot/src/prune_boundary.rs
@@ -82,12 +82,13 @@ impl SnapshotStore {
             records: records.iter().cloned().collect(),
         })
         .map_err(SnapshotError::Encode)?;
-        let (key_id, hmac_sha256) = self.integrity.as_deref().map_or((None, None), |key| {
-            (
+        let (key_id, hmac_sha256) = match self.integrity.as_deref() {
+            Some(key) => (
                 Some(key.key_id().to_owned()),
-                Some(key.sign_state(PRUNE_BOUNDARY_MAC_LABEL, records_toml.as_bytes())),
-            )
-        });
+                Some(key.sign_state(PRUNE_BOUNDARY_MAC_LABEL, records_toml.as_bytes())?),
+            ),
+            None => (None, None),
+        };
         let raw = toml::to_string(&PersistedPruneBoundaries {
             records_toml,
             key_id,
diff --git a/crates/fluxheim-snapshot/src/recovery.rs b/crates/fluxheim-snapshot/src/recovery.rs
@@ -43,7 +43,7 @@ impl SnapshotStore {
         let state_toml = toml::to_string_pretty(state).map_err(SnapshotError::Encode)?;
         let persisted = match self.integrity.as_deref() {
             Some(key) => PersistedRuntimeState {
-                hmac_sha256: Some(key.sign_recovery(state_toml.as_bytes())),
+                hmac_sha256: Some(key.sign_recovery(state_toml.as_bytes())?),
                 key_id: Some(key.key_id().to_owned()),
                 state_toml,
             },
diff --git a/crates/fluxheim-snapshot/src/snapshot_hardening_tests.rs b/crates/fluxheim-snapshot/src/snapshot_hardening_tests.rs
@@ -39,6 +39,35 @@ impl SnapshotCryptoProvider for TestCryptoProvider {
     }
 }
 
+#[derive(Debug)]
+struct FailingCryptoProvider {
+    fail_label: &'static [u8],
+}
+
+impl SnapshotCryptoProvider for FailingCryptoProvider {
+    fn label(&self) -> &'static str {
+        "failing-test-provider"
+    }
+
+    fn compliance_capable(&self) -> bool {
+        false
+    }
+
+    fn sha256(&self, chunks: &[&[u8]]) -> Result<[u8; 32], String> {
+        TestCryptoProvider.sha256(chunks)
+    }
+
+    fn hmac_sha256(&self, key: &[u8], chunks: &[&[u8]]) -> Result<[u8; 32], String> {
+        if chunks
+            .first()
+            .is_some_and(|label| *label == self.fail_label)
+        {
+            return Err("injected cryptographic provider failure".to_owned());
+        }
+        TestCryptoProvider.hmac_sha256(key, chunks)
+    }
+}
+
 #[cfg(unix)]
 #[test]
 fn doctor_rechecks_integrity_key_permissions() {
@@ -103,17 +132,118 @@ fn authenticated_prune_boundary_rejects_tampering() {
     assert!(!store.doctor().unwrap().healthy);
 }
 
+#[test]
+fn snapshot_generation_provider_failure_is_returned_without_abort() {
+    let dir = TestDir::new("snapshot-provider-generation-failure");
+    let store = store_with_provider(
+        &dir,
+        Arc::new(FailingCryptoProvider {
+            fail_label: b"fluxheim-snapshot-generation-v1\0",
+        }),
+    );
+
+    assert!(matches!(
+        store.snapshot_config(&Config::default(), None),
+        Err(SnapshotError::CryptoProvider(_))
+    ));
+    assert!(store.list().unwrap().is_empty());
+}
+
+#[test]
+fn snapshot_manifest_provider_failure_is_returned_without_abort() {
+    let dir = TestDir::new("snapshot-provider-manifest-failure");
+    let store = store_with_provider(
+        &dir,
+        Arc::new(FailingCryptoProvider {
+            fail_label: b"fluxheim-snapshot-v1\0",
+        }),
+    );
+
+    assert!(matches!(
+        store.snapshot_config(&Config::default(), None),
+        Err(SnapshotError::CryptoProvider(_))
+    ));
+    assert!(store.list().unwrap().is_empty());
+    assert!(store.current_id().unwrap().is_none());
+}
+
+#[test]
+fn recovery_provider_failure_is_returned_without_abort() {
+    let dir = TestDir::new("snapshot-provider-recovery-failure");
+    let store = store_with_provider(
+        &dir,
+        Arc::new(FailingCryptoProvider {
+            fail_label: b"fluxheim-snapshot-recovery-v1\0",
+        }),
+    );
+    store.snapshot_config(&Config::default(), None).unwrap();
+
+    assert!(matches!(
+        store.save_runtime_state(&crate::SnapshotRuntimeState::default()),
+        Err(SnapshotError::CryptoProvider(_))
+    ));
+    assert!(!store.root().join("self-healing.toml").exists());
+}
+
+#[test]
+fn prune_provider_failure_preserves_snapshots_without_abort() {
+    let dir = TestDir::new("snapshot-provider-prune-failure");
+    let store = store_with_provider(
+        &dir,
+        Arc::new(FailingCryptoProvider {
+            fail_label: b"fluxheim-snapshot-prune-boundary-v1\0",
+        }),
+    );
+    store.snapshot_config(&Config::default(), None).unwrap();
+    store.snapshot_config(&Config::default(), None).unwrap();
+    store.snapshot_config(&Config::default(), None).unwrap();
+
+    assert!(matches!(
+        store.prune(&SnapshotPruneOptions {
+            keep: Some(0),
+            older_than: None,
+            protected_ids: Vec::new(),
+        }),
+        Err(SnapshotError::CryptoProvider(_))
+    ));
+    assert_eq!(store.list().unwrap().len(), 3);
+}
+
+#[test]
+fn generation_state_replay_and_removal_fail_before_publication() {
+    let dir = TestDir::new("snapshot-generation-replay");
+    let (store, _key) = authenticated_store(&dir);
+    store.snapshot_config(&Config::default(), None).unwrap();
+    let generation_path = safe_child_path(store.root(), "generation.toml");
+    let generation_one = std::fs::read(&generation_path).unwrap();
+    store.snapshot_config(&Config::default(), None).unwrap();
+
+    std::fs::write(&generation_path, &generation_one).unwrap();
+    assert!(matches!(
+        store.snapshot_config(&Config::default(), None),
+        Err(SnapshotError::GenerationStateInvalid)
+    ));
+    assert_eq!(store.list().unwrap().len(), 2);
+    assert!(!store.doctor().unwrap().healthy);
+
+    std::fs::remove_file(&generation_path).unwrap();
+    assert!(matches!(
+        store.snapshot_config(&Config::default(), None),
+        Err(SnapshotError::GenerationStateInvalid)
+    ));
+    assert_eq!(store.list().unwrap().len(), 2);
+}
+
 fn authenticated_store(dir: &TestDir) -> (SnapshotStore, std::path::PathBuf) {
+    let store = store_with_provider(dir, Arc::new(TestCryptoProvider));
+    (store, dir.child("snapshot.key"))
+}
+
+fn store_with_provider(dir: &TestDir, provider: Arc<dyn SnapshotCryptoProvider>) -> SnapshotStore {
     let key = dir.child("snapshot.key");
     std::fs::write(&key, [11u8; 32]).unwrap();
     set_private_file(&key);
-    let store = SnapshotStore::with_integrity_key_file(
-        dir.child("store"),
-        &key,
-        Arc::new(TestCryptoProvider),
-    )
-    .unwrap();
-    (store, key)
+    SnapshotStore::with_integrity_key_file(dir.child("store"), &key, provider).unwrap()
 }
 
 fn set_private_file(path: &std::path::Path) {
diff --git a/crates/fluxheim-snapshot/src/store.rs b/crates/fluxheim-snapshot/src/store.rs
@@ -99,7 +99,7 @@ impl SnapshotStore {
             publish_transaction_file(&mut transaction, &config_path, raw_config.as_bytes())?;
             publish_transaction_file(&mut transaction, &metadata_path, raw_metadata.as_bytes())?;
             let integrity = if let Some(key) = self.integrity.as_deref() {
-                let manifest = key.manifest(&id, raw_config.as_bytes(), raw_metadata.as_bytes());
+                let manifest = key.manifest(&id, raw_config.as_bytes(), raw_metadata.as_bytes())?;
                 let raw_manifest =
                     toml::to_string_pretty(&manifest).map_err(SnapshotError::Encode)?;
                 publish_transaction_file(
diff --git a/docs/config-snapshots.md b/docs/config-snapshots.md
