Skip to content

Commit bbfd760

Browse files
committed
Bound asynchronous ACME account locks
1 parent a4bbb79 commit bbfd760

18 files changed

Lines changed: 559 additions & 56 deletions

.github/workflows/ci.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ jobs:
8282
- name: Run clippy
8383
run: cargo clippy --all-targets -- -D warnings
8484

85+
- name: Validate instant-acme patch provenance
86+
run: scripts/validate-instant-acme-patch.sh
87+
8588
- name: Validate panic policy on feature profiles
8689
run: |
8790
cargo clippy --no-default-features --features proxy,tls-rustls,acme --all-targets -- -D warnings

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ behavior when the change improves security or project direction.
3737
serialize bootstrap per issuer, and recover ambiguous creation with the same
3838
key before atomically promoting credentials. A pinned `instant-acme 0.8.5`
3939
API patch preserves configured contacts and EAB with caller-provided keys.
40+
- Keep blocking account files and locks off Tokio workers. Async ACME paths use
41+
nonblocking lifecycle-lock attempts on blocking workers, bounded asynchronous
42+
retry, contention diagnostics, and a 10-second lock-wait deadline.
43+
- Verify the vendored `instant-acme` source against published `0.8.5` hashes in
44+
CI and release gates after removing the single marked downstream API patch.
4045
- Validate online ACME directories structurally and require exact advertised ToS
4146
agreement, with an explicit private-directory override only for omitted terms.
4247
- Parse every advertised ACME endpoint as a bounded HTTPS URI with a real
Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
use super::*;
2+
3+
const ACCOUNT_LOCK_WAIT_TIMEOUT: Duration = Duration::from_secs(10);
4+
const ACCOUNT_LOCK_RETRY_INTERVAL: Duration = Duration::from_millis(25);
5+
6+
pub(crate) async fn load_account_credentials_async(
7+
storage: &Path,
8+
issuer_name: &str,
9+
) -> Result<Option<instant_acme::AccountCredentials>, AcmeInstantClientError> {
10+
load_account_credentials_async_with_timeout(storage, issuer_name, ACCOUNT_LOCK_WAIT_TIMEOUT)
11+
.await
12+
}
13+
14+
async fn load_account_credentials_async_with_timeout(
15+
storage: &Path,
16+
issuer_name: &str,
17+
wait_timeout: Duration,
18+
) -> Result<Option<instant_acme::AccountCredentials>, AcmeInstantClientError> {
19+
let storage = storage.to_path_buf();
20+
let issuer = issuer_name.to_owned();
21+
run_account_store_attempt(issuer_name, "credential load", wait_timeout, move || {
22+
try_load_account_credentials(&storage, &issuer)
23+
})
24+
.await
25+
}
26+
27+
pub(crate) async fn begin_account_bootstrap_async(
28+
storage: &Path,
29+
issuer_name: &str,
30+
issuer_directory: &str,
31+
) -> Result<AccountBootstrap, AcmeInstantClientError> {
32+
let storage = storage.to_path_buf();
33+
let issuer = issuer_name.to_owned();
34+
let directory = issuer_directory.to_owned();
35+
run_account_store_attempt(
36+
issuer_name,
37+
"bootstrap lock",
38+
ACCOUNT_LOCK_WAIT_TIMEOUT,
39+
move || try_begin_account_bootstrap(&storage, &issuer, &directory),
40+
)
41+
.await
42+
}
43+
44+
pub(crate) async fn promote_account_bootstrap_async(
45+
bootstrap: PendingAccountBootstrap,
46+
credentials: instant_acme::AccountCredentials,
47+
issuer_name: &str,
48+
) -> Result<(), AcmeInstantClientError> {
49+
run_account_store_task(issuer_name, "credential promotion", move || {
50+
bootstrap.promote(&credentials)
51+
})
52+
.await
53+
}
54+
55+
pub(crate) async fn begin_account_deactivation_async(
56+
storage: &Path,
57+
issuer_name: &str,
58+
) -> Result<AccountDeactivationTransaction, AcmeInstantClientError> {
59+
let storage = storage.to_path_buf();
60+
let issuer = issuer_name.to_owned();
61+
run_account_store_attempt(
62+
issuer_name,
63+
"deactivation lock",
64+
ACCOUNT_LOCK_WAIT_TIMEOUT,
65+
move || try_begin_account_deactivation(&storage, &issuer),
66+
)
67+
.await
68+
}
69+
70+
pub(crate) async fn complete_account_deactivation_async(
71+
transaction: AccountDeactivationTransaction,
72+
issuer_name: &str,
73+
) -> Result<(), AcmeInstantClientError> {
74+
run_account_store_task(issuer_name, "deactivation completion", move || {
75+
transaction.complete()
76+
})
77+
.await
78+
}
79+
80+
pub(crate) async fn rollback_account_deactivation_async(
81+
transaction: AccountDeactivationTransaction,
82+
issuer_name: &str,
83+
) -> Result<(), AcmeInstantClientError> {
84+
run_account_store_task(issuer_name, "deactivation rollback", move || {
85+
transaction.rollback()
86+
})
87+
.await
88+
}
89+
90+
async fn run_account_store_attempt<T, F>(
91+
issuer_name: &str,
92+
operation_name: &'static str,
93+
wait_timeout: Duration,
94+
operation: F,
95+
) -> Result<T, AcmeInstantClientError>
96+
where
97+
T: Send + 'static,
98+
F: Fn() -> Result<AccountStoreAttempt<T>, AcmeAccountStoreError> + Clone + Send + 'static,
99+
{
100+
let started = std::time::Instant::now();
101+
let mut contention_logged = false;
102+
loop {
103+
let attempt = tokio::task::spawn_blocking(operation.clone())
104+
.await
105+
.map_err(|error| {
106+
account_async_error(
107+
issuer_name,
108+
format!("ACME account {operation_name} task failed: {error}"),
109+
)
110+
})?
111+
.map_err(AcmeInstantClientError::AccountStore)?;
112+
match attempt {
113+
AccountStoreAttempt::Acquired(value) => return Ok(value),
114+
AccountStoreAttempt::Contended => {
115+
if !contention_logged {
116+
log::warn!(
117+
target: "fluxheim::acme",
118+
"ACME account {operation_name} waiting for issuer {issuer_name:?} lifecycle lock"
119+
);
120+
contention_logged = true;
121+
}
122+
if started.elapsed() >= wait_timeout {
123+
return Err(account_async_error(
124+
issuer_name,
125+
format!(
126+
"ACME account {operation_name} timed out after {} seconds",
127+
wait_timeout.as_secs_f64()
128+
),
129+
));
130+
}
131+
tokio::time::sleep(ACCOUNT_LOCK_RETRY_INTERVAL).await;
132+
}
133+
}
134+
}
135+
}
136+
137+
async fn run_account_store_task<T, F>(
138+
issuer_name: &str,
139+
operation_name: &'static str,
140+
operation: F,
141+
) -> Result<T, AcmeInstantClientError>
142+
where
143+
T: Send + 'static,
144+
F: FnOnce() -> Result<T, AcmeAccountStoreError> + Send + 'static,
145+
{
146+
tokio::task::spawn_blocking(operation)
147+
.await
148+
.map_err(|error| {
149+
account_async_error(
150+
issuer_name,
151+
format!("ACME account {operation_name} task failed: {error}"),
152+
)
153+
})?
154+
.map_err(AcmeInstantClientError::AccountStore)
155+
}
156+
157+
fn account_async_error(issuer: &str, message: impl Into<String>) -> AcmeInstantClientError {
158+
AcmeInstantClientError::Account {
159+
issuer: issuer.to_owned(),
160+
message: message.into(),
161+
}
162+
}
163+
164+
#[cfg(test)]
165+
pub(crate) async fn load_account_credentials_with_test_timeout(
166+
storage: &Path,
167+
issuer_name: &str,
168+
wait_timeout: Duration,
169+
) -> Result<Option<instant_acme::AccountCredentials>, AcmeInstantClientError> {
170+
load_account_credentials_async_with_timeout(storage, issuer_name, wait_timeout).await
171+
}

crates/fluxheim-acme/src/acme_account_bootstrap.rs

Lines changed: 54 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,67 @@ pub(crate) struct PendingAccountBootstrap {
2424
_lock: AcmeMutationLock,
2525
}
2626

27+
#[cfg(test)]
2728
pub(crate) fn begin_account_bootstrap(
2829
storage: &Path,
2930
issuer_name: &str,
3031
issuer_directory: &str,
3132
) -> Result<AccountBootstrap, AcmeAccountStoreError> {
3233
let credentials_path = account_credentials_path(storage, issuer_name);
33-
let directory =
34-
credentials_path
35-
.path
36-
.parent()
37-
.ok_or_else(|| AcmeAccountStoreError::UnsafePath {
38-
path: credentials_path.path.clone(),
39-
message: "credentials path has no parent directory".to_owned(),
40-
})?;
41-
ensure_safe_account_directory(directory)?;
42-
let mutation_lock = AcmeMutationLock::acquire(directory)
34+
let directory = credentials_path
35+
.path
36+
.parent()
37+
.ok_or_else(|| AcmeAccountStoreError::UnsafePath {
38+
path: credentials_path.path.clone(),
39+
message: "credentials path has no parent directory".to_owned(),
40+
})?
41+
.to_path_buf();
42+
ensure_safe_account_directory(&directory)?;
43+
let mutation_lock = AcmeMutationLock::acquire(&directory)
4344
.map_err(|error| account_store_io_error(&directory.join(".fluxheim-acme.lock"), error))?;
45+
begin_account_bootstrap_locked(
46+
credentials_path,
47+
&directory,
48+
issuer_directory,
49+
mutation_lock,
50+
)
51+
}
52+
53+
pub(crate) fn try_begin_account_bootstrap(
54+
storage: &Path,
55+
issuer_name: &str,
56+
issuer_directory: &str,
57+
) -> Result<AccountStoreAttempt<AccountBootstrap>, AcmeAccountStoreError> {
58+
let credentials_path = account_credentials_path(storage, issuer_name);
59+
let directory = credentials_path
60+
.path
61+
.parent()
62+
.ok_or_else(|| AcmeAccountStoreError::UnsafePath {
63+
path: credentials_path.path.clone(),
64+
message: "credentials path has no parent directory".to_owned(),
65+
})?
66+
.to_path_buf();
67+
ensure_safe_account_directory(&directory)?;
68+
let Some(mutation_lock) = AcmeMutationLock::try_acquire(&directory)
69+
.map_err(|error| account_store_io_error(&directory.join(".fluxheim-acme.lock"), error))?
70+
else {
71+
return Ok(AccountStoreAttempt::Contended);
72+
};
73+
begin_account_bootstrap_locked(
74+
credentials_path,
75+
&directory,
76+
issuer_directory,
77+
mutation_lock,
78+
)
79+
.map(AccountStoreAttempt::Acquired)
80+
}
81+
82+
fn begin_account_bootstrap_locked(
83+
credentials_path: AcmeAccountCredentialsPath,
84+
directory: &Path,
85+
issuer_directory: &str,
86+
mutation_lock: AcmeMutationLock,
87+
) -> Result<AccountBootstrap, AcmeAccountStoreError> {
4488
let pending_path = directory.join(ACCOUNT_BOOTSTRAP_PENDING_FILE);
4589
let credentials = load_account_credentials_locked(&credentials_path.path, directory, true)?;
4690
let pending = read_pending_key(&pending_path, issuer_directory)?;

crates/fluxheim-acme/src/acme_account_store.rs

Lines changed: 79 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@ use sanitization::SecretVec;
99
#[path = "acme_account_bootstrap.rs"]
1010
mod bootstrap;
1111
#[cfg(all(feature = "acme-client", test))]
12-
pub(crate) use bootstrap::PendingAccountBootstrap;
12+
pub(crate) use bootstrap::begin_account_bootstrap;
1313
#[cfg(feature = "acme-client")]
14-
pub(crate) use bootstrap::{AccountBootstrap, begin_account_bootstrap};
14+
pub(crate) use bootstrap::{
15+
AccountBootstrap, PendingAccountBootstrap, try_begin_account_bootstrap,
16+
};
1517

1618
#[cfg(target_os = "linux")]
1719
use super::UNIX_O_NOFOLLOW;
@@ -24,6 +26,12 @@ const ACME_ACCOUNT_DIR: &str = "accounts";
2426
const ACME_ACCOUNT_CREDENTIALS_FILE: &str = "credentials.json";
2527
const ACME_ACCOUNT_DEACTIVATION_FILE: &str = ".credentials.deactivation.pending";
2628

29+
#[cfg(feature = "acme-client")]
30+
pub(crate) enum AccountStoreAttempt<T> {
31+
Acquired(T),
32+
Contended,
33+
}
34+
2735
#[cfg(feature = "acme-client")]
2836
pub(crate) struct AccountDeactivationTransaction {
2937
directory: PathBuf,
@@ -68,6 +76,39 @@ pub fn load_account_credentials(
6876
load_account_credentials_locked(&path, directory, false)
6977
}
7078

79+
#[cfg(feature = "acme-client")]
80+
pub(crate) fn try_load_account_credentials(
81+
storage: &Path,
82+
issuer_name: &str,
83+
) -> Result<AccountStoreAttempt<Option<instant_acme::AccountCredentials>>, AcmeAccountStoreError> {
84+
let path = account_credentials_path(storage, issuer_name).path;
85+
let directory = path
86+
.parent()
87+
.ok_or_else(|| AcmeAccountStoreError::UnsafePath {
88+
path: path.clone(),
89+
message: "credentials path has no parent directory".to_owned(),
90+
})?;
91+
match fs::symlink_metadata(directory) {
92+
Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => {
93+
return Err(AcmeAccountStoreError::UnsafePath {
94+
path: directory.to_path_buf(),
95+
message: "account directory is not a real directory".to_owned(),
96+
});
97+
}
98+
Ok(_) => {}
99+
Err(error) if error.kind() == io::ErrorKind::NotFound => {
100+
return Ok(AccountStoreAttempt::Acquired(None));
101+
}
102+
Err(error) => return Err(account_store_io_error(directory, error)),
103+
}
104+
let Some(_lock) = AcmeMutationLock::try_acquire(directory)
105+
.map_err(|error| account_store_io_error(&directory.join(".fluxheim-acme.lock"), error))?
106+
else {
107+
return Ok(AccountStoreAttempt::Contended);
108+
};
109+
load_account_credentials_locked(&path, directory, false).map(AccountStoreAttempt::Acquired)
110+
}
111+
71112
fn load_account_credentials_locked(
72113
path: &Path,
73114
directory: &Path,
@@ -148,7 +189,7 @@ fn load_account_credentials_locked(
148189
})
149190
}
150191

151-
#[cfg(feature = "acme-client")]
192+
#[cfg(all(feature = "acme-client", test))]
152193
pub(crate) fn begin_account_deactivation(
153194
storage: &Path,
154195
issuer_name: &str,
@@ -159,10 +200,42 @@ pub(crate) fn begin_account_deactivation(
159200
.ok_or_else(|| AcmeAccountStoreError::UnsafePath {
160201
path: active.clone(),
161202
message: "credentials path has no parent directory".to_owned(),
162-
})?;
163-
ensure_safe_account_directory(directory)?;
164-
let lock = AcmeMutationLock::acquire(directory)
203+
})?
204+
.to_path_buf();
205+
ensure_safe_account_directory(&directory)?;
206+
let lock = AcmeMutationLock::acquire(&directory)
165207
.map_err(|error| account_store_io_error(&directory.join(".fluxheim-acme.lock"), error))?;
208+
begin_account_deactivation_locked(active, &directory, lock)
209+
}
210+
211+
#[cfg(feature = "acme-client")]
212+
pub(crate) fn try_begin_account_deactivation(
213+
storage: &Path,
214+
issuer_name: &str,
215+
) -> Result<AccountStoreAttempt<AccountDeactivationTransaction>, AcmeAccountStoreError> {
216+
let active = account_credentials_path(storage, issuer_name).path;
217+
let directory = active
218+
.parent()
219+
.ok_or_else(|| AcmeAccountStoreError::UnsafePath {
220+
path: active.clone(),
221+
message: "credentials path has no parent directory".to_owned(),
222+
})?
223+
.to_path_buf();
224+
ensure_safe_account_directory(&directory)?;
225+
let Some(lock) = AcmeMutationLock::try_acquire(&directory)
226+
.map_err(|error| account_store_io_error(&directory.join(".fluxheim-acme.lock"), error))?
227+
else {
228+
return Ok(AccountStoreAttempt::Contended);
229+
};
230+
begin_account_deactivation_locked(active, &directory, lock).map(AccountStoreAttempt::Acquired)
231+
}
232+
233+
#[cfg(feature = "acme-client")]
234+
fn begin_account_deactivation_locked(
235+
active: PathBuf,
236+
directory: &Path,
237+
lock: AcmeMutationLock,
238+
) -> Result<AccountDeactivationTransaction, AcmeAccountStoreError> {
166239
ensure_safe_account_destination(&active)?;
167240
let pending = directory.join(ACME_ACCOUNT_DEACTIVATION_FILE);
168241
match fs::symlink_metadata(&pending) {

0 commit comments

Comments
 (0)