Skip to content

Emit handler-level assertion failures in invariant logs #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aviggiano wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Emit handler-level assertion failures in invariant logs #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e80be1a
Emit handler-keyed assertion failures in invariant runs
aviggiano Feb 24, 2026
ad0dfbf
Label replayed assert-only sequences by handler
aviggiano Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/evm/evm/src/executors/invariant/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ mod replay;
pub use replay::{generate_counterexample, replay_error, replay_run};

mod result;
pub use result::InvariantFuzzTestResult;
pub use result::{ASSERTION_FAILURE_KEY_PREFIX, InvariantFuzzTestResult};

mod shrink;
pub use shrink::check_sequence;
Expand Down
48 changes: 36 additions & 12 deletions crates/evm/evm/src/executors/invariant/result.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ use revm::interpreter::InstructionResult;
use revm_inspectors::tracing::CallTraceArena;
use std::{borrow::Cow, collections::HashMap};

pub const ASSERTION_FAILURE_KEY_PREFIX: &str = "assertion_failure::";

/// The outcome of an invariant fuzz test
#[derive(Debug)]
pub struct InvariantFuzzTestResult {
Expand Down Expand Up @@ -157,6 +159,17 @@ fn invariant_inner_sequence(executor: &Executor) -> Vec<Option<BasicTxDetails>>
seq
}

fn failing_handler_key(
invariant_test: &InvariantTest,
invariant_run: &InvariantTestRun,
) -> Option<String> {
let last_input = invariant_run.inputs.last()?;
let metric_key =
invariant_test.targeted_contracts.targets.lock().fuzzed_metric_key(last_input)?;
let handler = metric_key.rsplit('.').next().unwrap_or(metric_key.as_str());
Some(format!("{ASSERTION_FAILURE_KEY_PREFIX}{handler}"))
}

/// Returns if invariant test can continue and last successful call result of the invariant test
/// function (if it can continue).
///
Expand All @@ -181,7 +194,6 @@ pub(crate) fn can_continue(
})
};

let failures = &mut invariant_test.test_data.failures;
// Assert invariants if the call did not revert and the handlers did not fail.
if !call_result.reverted && handlers_succeeded() {
if let Some(traces) = call_result.traces {
Expand All @@ -193,12 +205,20 @@ pub(crate) fn can_continue(
&invariant_test.targeted_contracts,
&invariant_run.executor,
&invariant_run.inputs,
failures,
&mut invariant_test.test_data.failures,
)?;
} else {
let should_fail_on_assert =
invariant_config.fail_on_assert && is_assertion_failure(&call_result);
let handler_failure_key = if should_fail_on_assert {
failing_handler_key(invariant_test, invariant_run)
} else {
None
};
let failures = &mut invariant_test.test_data.failures;
// Increase the amount of reverts.
failures.reverts += 1;
let should_fail_on_assert = invariant_config.fail_on_assert && is_assertion_failure(&call_result);
let mut assertion_emitted = false;
// If fail on revert is set for an invariant, or this is an assert failure and
// fail-on-assert is enabled, record invariant failure.
for (invariant, fail_on_revert) in &invariant_contract.invariant_fns {
Expand All @@ -213,22 +233,26 @@ pub(crate) fn can_continue(
&call_result,
&[],
);
failures.errors.insert(
invariant.name.clone(),
if should_fail_on_assert {
InvariantFuzzError::BrokenInvariant(case_data)
} else {
InvariantFuzzError::Revert(case_data)
},
);
let error = if should_fail_on_assert {
InvariantFuzzError::BrokenInvariant(case_data)
} else {
InvariantFuzzError::Revert(case_data)
};
if should_fail_on_assert && !assertion_emitted {
if let Some(key) = handler_failure_key.as_ref() {
failures.errors.insert(key.clone(), error.clone());
}
assertion_emitted = true;
}
failures.errors.insert(invariant.name.clone(), error);
}
}
// Remove last reverted call from inputs.
// This improves shrinking performance as irrelevant calls won't be checked again.
invariant_run.inputs.pop();
}
// Stop execution if all invariants are broken.
Ok(failures.can_continue(invariant_contract.invariant_fns.len()))
Ok(invariant_test.test_data.failures.can_continue(invariant_contract.invariant_fns.len()))
}

/// Given the executor state, asserts conditions within `afterInvariant` function.
Expand Down
61 changes: 51 additions & 10 deletions crates/forge/src/runner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ use alloy_dyn_abi::{DynSolValue, JsonAbiExt};
use alloy_json_abi::Function;
use alloy_primitives::{Address, Bytes, U256, address, map::HashMap};
use eyre::Result;
use foundry_common::{TestFunctionExt, TestFunctionKind, contracts::ContractsByAddress, sh_println};
use foundry_common::{
TestFunctionExt, TestFunctionKind, contracts::ContractsByAddress, sh_println,
};
use foundry_compilers::utils::canonicalized;
use foundry_config::{Config, FuzzCorpusConfig};
use foundry_evm::{
Expand All @@ -22,8 +24,8 @@ use foundry_evm::{
CallResult, EvmError, Executor, ITest, RawCallResult,
fuzz::FuzzedExecutor,
invariant::{
InvariantExecutor, InvariantFuzzError, check_sequence, generate_counterexample,
replay_error, replay_run,
ASSERTION_FAILURE_KEY_PREFIX, InvariantExecutor, InvariantFuzzError, check_sequence,
generate_counterexample, replay_error, replay_run,
},
},
fuzz::{
Expand Down Expand Up @@ -897,11 +899,32 @@ impl<'a> FunctionRunner<'a> {
}
}

self.result.invariant_replay_fail(
replayed_entirely,
&invariant_contract.invariant_fn.name,
call_sequence,
);
let replay_label = if invariant_config.fail_on_assert {
let assert_only_failure = check_sequence(
self.clone_executor(),
&txes,
(0..min(txes.len(), invariant_config.depth as usize)).collect(),
invariant_contract.address,
invariant_contract.invariant_fn.selector().to_vec().into(),
false,
false,
Comment on lines +909 to +910
Copy link
Copy Markdown

@chatgpt-codex-connector chatgpt-codex-connector Bot Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Keep revert checks when detecting assertion-only replay failures

This probe calls check_sequence with both fail_on_revert and fail_on_assert forced to false, but check_sequence only treats reverted handler calls as failures when one of those flags is enabled. As a result, when users run with both flags enabled and a persisted sequence fails due to a normal revert (for example a require), the probe reports assert_only_failure=true and the replay gets mislabeled as assertion failure in <handler>. That makes replay diagnostics incorrect for non-assert failures.

Useful? React with 👍 / 👎.

invariant_contract.call_after_invariant,
)
.map(|(success, _)| success)
.unwrap_or(false);
if assert_only_failure {
call_sequence
.last()
.and_then(|counterexample| counterexample.func_name.clone())
.map(|handler| format!("assertion failure in {handler}"))
.unwrap_or_else(|| invariant_contract.invariant_fn.name.clone())
} else {
invariant_contract.invariant_fn.name.clone()
}
} else {
invariant_contract.invariant_fn.name.clone()
};
self.result.invariant_replay_fail(replayed_entirely, &replay_label, call_sequence);
return self.result;
}
}
Expand All @@ -924,11 +947,26 @@ impl<'a> FunctionRunner<'a> {

let mut counterexample = None;
let success = invariant_result.errors.is_empty();
let reason = invariant_result
let main_reason = invariant_result
.errors
.get(&invariant_contract.invariant_fn.name)
.and_then(|err| err.revert_reason());
let assertion_failure = invariant_result.errors.iter().find_map(|(name, error)| {
name.strip_prefix(ASSERTION_FAILURE_KEY_PREFIX)
.map(|handler| (handler.to_string(), error))
});
let assertion_reason = assertion_failure.as_ref().map(|(handler, error)| {
let revert_reason =
error.revert_reason().unwrap_or_else(|| "assertion failure".to_string());
format!("assertion failure in {handler}: {revert_reason}")
});
let reason = main_reason.clone().or_else(|| assertion_reason.clone());
let mut other_failures = vec![];
if main_reason.is_some() {
if let Some(assertion_reason) = assertion_reason {
other_failures.push(assertion_reason);
}
}

if success {
// If invariants ran successfully, replay the last run to collect logs and
Expand All @@ -949,7 +987,10 @@ impl<'a> FunctionRunner<'a> {
}
} else {
// check if main invariant was broken and replay error
if let Some(error) = invariant_result.errors.get(&invariant_contract.invariant_fn.name)
if let Some(error) = invariant_result
.errors
.get(&invariant_contract.invariant_fn.name)
.or_else(|| assertion_failure.as_ref().map(|(_, error)| *error))
&& let InvariantFuzzError::BrokenInvariant(case_data)
| InvariantFuzzError::Revert(case_data) = error
&& let TestError::Fail(_, ref calls) = case_data.test_error
Expand Down
Loading