Skip to content

feat(invariant): decouple handler-side assertions from invariant predicates #14482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
grandizzy wants to merge 2 commits into
base: gdzzy/issue-9727
Choose a base branch
from
Draft

feat(invariant): decouple handler-side assertions from invariant predicates #14482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb8ff9f
feat(invariant): decouple handler-side assertions from invariant pred…
grandizzy Apr 28, 2026
b750247
fix: avoid private intra-doc link in HandlerAssertionFailure
grandizzy Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion crates/evm/evm/src/executors/invariant/error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use super::InvariantContract;
use crate::executors::RawCallResult;
use alloy_json_abi::Function;
use alloy_primitives::{Address, Bytes};
use alloy_primitives::{Address, Bytes, Selector};
use foundry_evm_core::{
decode::{ASSERTION_FAILED_PREFIX, EMPTY_REVERT_DATA, RevertDecoder},
evm::FoundryEvmNetwork,
Expand All @@ -10,6 +10,28 @@ use foundry_evm_fuzz::{BasicTxDetails, Reason, invariant::FuzzRunIdentifiedContr
use proptest::test_runner::TestError;
use std::{collections::HashMap, fmt};

/// Records a single handler-side assertion bug discovered during an invariant campaign.
///
/// Handler-side assertions (e.g. a `require`/`assert` inside a fuzzed handler that the campaign
/// reaches with a malformed input) are bugs in their own right, but they are *not* invariant
/// predicate violations. We record them once per `(reverter, selector)` so the campaign can keep
/// running for the rest of the budget and surface deeper bugs without polluting the invariant
/// `errors` map or stopping the run.
#[derive(Clone, Debug)]
pub struct HandlerAssertionFailure {
Copy link
Copy Markdown
Contributor

@0xalpharush 0xalpharush Apr 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could be a variant of InvariantFuzzError?

/// Address of the handler contract whose call asserted/reverted with an assertion.
pub reverter: Address,
/// 4-byte selector of the failing handler function.
pub selector: Selector,
/// Full call sequence leading up to (and including) the failing call.
pub call_sequence: Vec<BasicTxDetails>,
/// Decoded revert/assert reason.
pub revert_reason: String,
/// Always `true` for entries in this struct; mirrored for symmetry with
/// `FailedInvariantCaseData::assertion_failure`.
pub assertion_failure: bool,
}

/// Stores information about failures and reverts of the invariant tests.
#[derive(Clone, Default)]
pub struct InvariantFailures {
Expand All @@ -19,6 +41,9 @@ pub struct InvariantFailures {
pub revert_reason: Option<String>,
/// Maps a broken invariant to its specific error.
pub errors: HashMap<String, InvariantFuzzError>,
/// Handler-side assertion bugs discovered during the campaign, keyed by
/// `(reverter, selector)` so each unique handler bug is recorded once.
pub broken_handlers: HashMap<(Address, Selector), HandlerAssertionFailure>,
}

impl InvariantFailures {
Expand Down Expand Up @@ -46,6 +71,22 @@ impl InvariantFailures {
debug_assert!(invariants > 0, "invariant_fns must not be empty");
self.errors.len() < invariants
}

/// Records a handler-side assertion bug. The first occurrence for a given
/// `(reverter, selector)` wins; subsequent calls are no-ops to keep the report tidy.
pub fn record_handler_failure(
&mut self,
key: (Address, Selector),
failure: HandlerAssertionFailure,
) {
self.broken_handlers.entry(key).or_insert(failure);
}

/// Returns true if a handler-side assertion bug has already been recorded for the given
/// target/selector pair.
pub fn has_handler_failure(&self, target: Address, selector: Selector) -> bool {
self.broken_handlers.contains_key(&(target, selector))
}
}

impl fmt::Display for InvariantFailures {
Expand Down
133 changes: 117 additions & 16 deletions crates/evm/evm/src/executors/invariant/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ use std::{
};

mod error;
pub use error::{InvariantFailures, InvariantFuzzError};
pub use error::{HandlerAssertionFailure, InvariantFailures, InvariantFuzzError};
use foundry_evm_coverage::HitMaps;

mod replay;
Expand Down Expand Up @@ -203,19 +203,26 @@ fn first_broken_event<'a>(
/// This keeps the existing corpus progress metrics together with cumulative and
/// derived throughput fields so downstream benchmark tooling can consume a
/// single JSON event shape.
#[expect(clippy::too_many_arguments)]
fn build_invariant_progress_json<M: Serialize>(
timestamp_secs: u64,
invariant_name: &str,
corpus_metrics: &M,
optimization_best: Option<I256>,
throughput: InvariantThroughputMetrics,
failure_metrics: &InvariantFailureMetrics,
broken_handlers: usize,
elapsed: Duration,
) -> serde_json::Value {
let mut metrics = serde_json::to_value(corpus_metrics).unwrap_or_default();
if let Some(obj) = metrics.as_object_mut() {
obj.insert("failures".to_string(), json!(failure_metrics.failures));
obj.insert("unique_failures".to_string(), json!(failure_metrics.unique_failures.len()));
// Phase E: surface unique handler-side assertion bugs in live progress so users
// can watch them accumulate without waiting for the campaign to finish. These are
// distinct from invariant predicate violations (counted by `failures` above) and
// are routed via `InvariantFailures::broken_handlers`.
obj.insert("broken_handlers".to_string(), json!(broken_handlers));
Copy link
Copy Markdown
Contributor

@0xalpharush 0xalpharush Apr 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this belongs combined with the above failure metrics

}

let mut payload = json!({
Expand Down Expand Up @@ -513,17 +520,22 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
break 'stop;
}

// Clone the latest input so that `tx` does not borrow from `current_run.inputs`.
// This lets us pass `&tx` into `can_continue` alongside `&mut current_run`
// without conflicting borrows (`can_continue` needs the tx to attribute
// handler-side assertion failures to a specific `(target, selector)`).
let tx = current_run
.inputs
.last()
.ok_or_else(|| eyre!("no input generated to call fuzzed target."))?;
.ok_or_else(|| eyre!("no input generated to call fuzzed target."))?
.clone();

// Execute call from the randomly generated sequence without committing state.
// State is committed only if call is not a magic assume.
let mut call_result = execute_tx(&mut current_run.executor, tx)?;
let mut call_result = execute_tx(&mut current_run.executor, &tx)?;
let discarded = call_result.result.as_ref() == MAGIC_ASSUME;
if self.config.show_metrics {
invariant_test.record_metrics(tx, call_result.reverted, discarded);
invariant_test.record_metrics(&tx, call_result.reverted, discarded);
}

// Collect line coverage from last fuzzed call.
Expand Down Expand Up @@ -562,7 +574,7 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
collect_data(
&invariant_test,
&mut state_changeset,
tx,
&tx,
&call_result,
self.config.depth,
);
Expand Down Expand Up @@ -614,15 +626,64 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
&self.config,
call_result,
&state_changeset,
&tx,
)
.map_err(|e| eyre!(e.to_string()))?
} else {
// Skip invariant check but still track reverts
if call_result.reverted {
invariant_test.test_data.failures.reverts += 1;
}
if assertion_failure || (call_result.reverted && self.config.fail_on_revert)
{
if assertion_failure {
// Handler-side assertion: record once, keyed by the failing call's
// `(target, selector)`. Same routing as the `can_continue` path so
// the campaign keeps running for the full budget instead of
// attributing the assertion to the primary invariant.
let target = tx.call_details.target;
let selector = tx
.call_details
.calldata
.get(..4)
.and_then(|s| Selector::try_from(s).ok())
.unwrap_or_default();
let call_reverted = call_result.reverted;
if !invariant_test
.test_data
.failures
.has_handler_failure(target, selector)
{
let case_data = error::FailedInvariantCaseData::new(
&invariant_contract,
self.config.shrink_run_limit,
self.config.fail_on_revert,
&invariant_test.targeted_contracts,
&current_run.inputs,
call_result,
&[],
)
.with_assertion_failure(true);
let revert_reason = case_data.revert_reason.clone();
invariant_test.test_data.failures.revert_reason =
Some(revert_reason.clone());
invariant_test.test_data.failures.record_handler_failure(
(target, selector),
HandlerAssertionFailure {
reverter: target,
selector,
call_sequence: current_run.inputs.clone(),
revert_reason,
assertion_failure: true,
},
);
}
if call_reverted
&& !invariant_contract.is_optimization()
&& !self.config.has_delay()
{
current_run.inputs.pop();
}
true
} else if call_result.reverted && self.config.fail_on_revert {
let case_data = error::FailedInvariantCaseData::new(
&invariant_contract,
self.config.shrink_run_limit,
Expand All @@ -632,16 +693,12 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
call_result,
&[],
)
.with_assertion_failure(assertion_failure);
.with_assertion_failure(false);
invariant_test.test_data.failures.revert_reason =
Some(case_data.revert_reason.clone());
invariant_test.set_error(
invariant_contract.primary_invariant_fn,
if assertion_failure {
InvariantFuzzError::BrokenInvariant(case_data)
} else {
InvariantFuzzError::Revert(case_data)
},
InvariantFuzzError::Revert(case_data),
);
false
} else if call_result.reverted
Expand All @@ -661,7 +718,19 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
if !result || current_run.depth == self.config.depth - 1 {
invariant_test.set_last_run_inputs(&current_run.inputs);
}
// If test cannot continue then stop current run and exit test suite.
// Phase A: decouple "record failure" from "stop campaign" so the campaign
// can keep using its budget to surface handler-side bugs even after all
// invariant predicates are broken. Continuation is gated on
// `assert_all && !fail_on_revert`:
// - `assert_all = false` (single-invariant runs) → preserve the legacy "exit
// on first broken invariant" behavior so output is unchanged.
// - `fail_on_revert = true` → user opted into "fail fast on revert"; the
// outer-loop early-exit check (top of campaign loop) would fire on the next
// iteration anyway, so exit cleanly here instead of surfacing a confusing
// "failed to set up invariant testing environment: call reverted" error.
//
// Handler-side assertions never reach this branch — they are routed into
// `failures.broken_handlers` and return `Ok(true)`.
if !result {
// Attribute the failure event to the first invariant in declaration
// order whose entry is in `failures.errors`. Avoids the nondeterminism
Expand All @@ -672,6 +741,9 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
&invariant_test.test_data.failures,
);
failure_metrics.record_failure(name, invariant_contract.name, &reason);
if self.config.assert_all && !self.config.fail_on_revert {
break;
}
break 'stop;
}
current_run.depth += 1;
Expand Down Expand Up @@ -728,8 +800,12 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
// Display current best value, corpus metrics, and failure counts.
let best = invariant_test.test_data.optimization_best_value;
let broken = invariant_test.test_data.failures.errors.len();
// Phase E: live count of unique handler-side assertion bugs so users see
// them accumulate during the campaign (separate from invariant predicate
// breaks tracked by `broken` above).
let handler_bugs = invariant_test.test_data.failures.broken_handlers.len();
let total_invariants = invariant_contract.invariant_fns.len();
if edge_coverage_enabled || best.is_some() || broken > 0 {
if edge_coverage_enabled || best.is_some() || broken > 0 || handler_bugs > 0 {
let mut msg = String::new();
if let Some(best) = best {
msg.push_str(&format!("best: {best}"));
Expand All @@ -746,6 +822,12 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
}
msg.push_str(&format!("❌ {broken}/{total_invariants} broken"));
}
if handler_bugs > 0 {
if !msg.is_empty() {
msg.push_str(", ");
}
msg.push_str(&format!("⚠ {handler_bugs} handler bug(s)"));
}
progress.set_message(msg);
}
} else if edge_coverage_enabled
Expand All @@ -759,6 +841,7 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
invariant_test.test_data.optimization_best_value,
throughput,
&failure_metrics,
invariant_test.test_data.failures.broken_handlers.len(),
campaign_start.elapsed(),
);
let _ = sh_println!("{}", serde_json::to_string(&metrics)?);
Expand All @@ -774,6 +857,7 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
let result = invariant_test.test_data;
Ok(InvariantFuzzTestResult {
errors: result.failures.errors,
handler_errors: result.failures.broken_handlers,
cases: result.fuzz_cases,
reverts: result.failures.reverts,
last_run_inputs: result.last_run_inputs,
Expand Down Expand Up @@ -848,7 +932,19 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
invariant_contract.invariant_fns.iter().find_map(|(f, _)| failures.get_failure(f))
})
{
return Err(eyre!(error.revert_reason().unwrap_or_default()));
// Under `assert_all` the campaign is expected to keep running for the full
// budget so that handler-side bugs (and other still-live invariants) can be
// discovered. An always-failing canary invariant must not abort the entire run.
// Record the preflight failure(s) and continue; the campaign loop's
// `can_continue` will keep going as long as at least one invariant is still
// live (or, with all of them broken, until handler bugs are exhausted via the
// dedicated `broken_handlers` path).
//
// Without `assert_all` we preserve the legacy behavior of aborting on a broken
// preflight invariant.
if !self.config.assert_all {
return Err(eyre!(error.revert_reason().unwrap_or_default()));
}
}

// NOW enable call_override after the initial invariant check has passed.
Expand Down Expand Up @@ -1356,12 +1452,14 @@ mod tests {
Some(I256::try_from(42).unwrap()),
throughput,
&InvariantFailureMetrics::default(),
0,
Duration::from_secs(10),
);

assert_eq!(payload["timestamp"], json!(123));
assert_eq!(payload["invariant"], json!("invariant_balance"));
assert_eq!(payload["metrics"]["corpus_count"], json!(7));
assert_eq!(payload["metrics"]["broken_handlers"], json!(0));
assert_eq!(payload["total_txs"], json!(2));
assert_eq!(payload["total_gas"], json!(50));
assert!((payload["tx_per_sec"].as_f64().unwrap() - 0.2).abs() < 1e-12);
Expand All @@ -1381,6 +1479,7 @@ mod tests {
None,
throughput,
&InvariantFailureMetrics::default(),
0,
Duration::ZERO,
);

Expand All @@ -1403,11 +1502,13 @@ mod tests {
None,
InvariantThroughputMetrics::default(),
&failure_metrics,
7,
Duration::from_secs(1),
);

assert_eq!(payload["metrics"]["failures"], json!(3));
assert_eq!(payload["metrics"]["unique_failures"], json!(2));
assert_eq!(payload["metrics"]["broken_handlers"], json!(7));
}

#[test]
Expand Down
Loading
Loading