diff --git a/crates/config/src/invariant.rs b/crates/config/src/invariant.rs index 591af88efdee4..9220c668ee5cf 100644 --- a/crates/config/src/invariant.rs +++ b/crates/config/src/invariant.rs @@ -13,6 +13,9 @@ pub struct InvariantConfig { pub depth: u32, /// Fails the invariant fuzzing if a revert occurs pub fail_on_revert: bool, + /// Fails the invariant fuzzing if a Solidity assert failure occurs (`Panic(0x01)` or legacy + /// invalid opcode assert behavior), even if `fail_on_revert` is `false`. + pub fail_on_assert: bool, /// Allows overriding an unsafe external call when running invariant tests. eg. reentrancy /// checks pub call_override: bool, @@ -57,6 +60,7 @@ impl Default for InvariantConfig { runs: 256, depth: 500, fail_on_revert: false, + fail_on_assert: false, call_override: false, dictionary: FuzzDictionaryConfig { dictionary_weight: 80, ..Default::default() }, shrink_run_limit: 5000, diff --git a/crates/evm/evm/src/executors/invariant/error.rs b/crates/evm/evm/src/executors/invariant/error.rs index 9f48e9da82cbd..2cc63b226e122 100644 --- a/crates/evm/evm/src/executors/invariant/error.rs +++ b/crates/evm/evm/src/executors/invariant/error.rs @@ -5,6 +5,7 @@ use foundry_config::InvariantConfig; use foundry_evm_core::decode::RevertDecoder; use foundry_evm_fuzz::{BasicTxDetails, Reason, invariant::FuzzRunIdentifiedContracts}; use proptest::test_runner::TestError; +use std::collections::BTreeMap; /// Stores information about failures and reverts of the invariant tests. #[derive(Clone, Default)] @@ -15,6 +16,8 @@ pub struct InvariantFailures { pub revert_reason: Option, /// Maps a broken invariant to its specific error. pub error: Option, + /// Distinct handler-level assertion failures observed during the campaign. + pub assertion_failures: BTreeMap, } impl InvariantFailures { @@ -25,6 +28,12 @@ impl InvariantFailures { pub fn into_inner(self) -> (usize, Option) { (self.reverts, self.error) } + + pub fn record_assertion_failure(&mut self, case_data: FailedInvariantCaseData) { + let key = + case_data.failing_handler.clone().unwrap_or_else(|| "unknown_handler".to_string()); + self.assertion_failures.entry(key).or_insert(case_data); + } } #[derive(Clone, Debug)] @@ -65,6 +74,10 @@ pub struct FailedInvariantCaseData { pub shrink_run_limit: u32, /// Fail on revert, used to check sequence when shrinking. pub fail_on_revert: bool, + /// Fail on Solidity assert failures, used to check sequence when shrinking. + pub fail_on_assert: bool, + /// Handler function that triggered a fail-on-assert violation, when available. + pub failing_handler: Option, } impl FailedInvariantCaseData { @@ -97,6 +110,13 @@ impl FailedInvariantCaseData { inner_sequence: inner_sequence.to_vec(), shrink_run_limit: invariant_config.shrink_run_limit, fail_on_revert: invariant_config.fail_on_revert, + fail_on_assert: invariant_config.fail_on_assert, + failing_handler: None, } } + + pub fn with_failing_handler(mut self, failing_handler: Option) -> Self { + self.failing_handler = failing_handler; + self + } } diff --git a/crates/evm/evm/src/executors/invariant/mod.rs b/crates/evm/evm/src/executors/invariant/mod.rs index 86e8e18581764..698ff38206ede 100644 --- a/crates/evm/evm/src/executors/invariant/mod.rs +++ b/crates/evm/evm/src/executors/invariant/mod.rs @@ -35,7 +35,7 @@ use foundry_evm_traces::{CallTraceArena, SparsedTraceArena}; use indicatif::ProgressBar; use parking_lot::RwLock; use proptest::{strategy::Strategy, test_runner::TestRunner}; -use result::{assert_after_invariant, assert_invariants, can_continue}; +use result::{assert_after_invariant, assert_invariants, can_continue, is_assertion_failure}; use revm::state::Account; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -502,7 +502,46 @@ impl<'a> InvariantExecutor<'a> { // Skip invariant check but still track reverts if call_result.reverted { invariant_test.test_data.failures.reverts += 1; - if self.config.fail_on_revert { + let should_fail_on_assert = + self.config.fail_on_assert && is_assertion_failure(&call_result); + if should_fail_on_assert { + let failing_handler = + current_run.inputs.last().and_then(|last_input| { + invariant_test + .targeted_contracts + .targets + .lock() + .fuzzed_metric_key(last_input) + .map(|metric_key| { + metric_key + .rsplit('.') + .next() + .unwrap_or(metric_key.as_str()) + .to_string() + }) + }); + let case_data = error::FailedInvariantCaseData::new( + &invariant_contract, + &self.config, + &invariant_test.targeted_contracts, + ¤t_run.inputs, + call_result, + &[], + ) + .with_failing_handler(failing_handler); + invariant_test.test_data.failures.revert_reason = + Some(case_data.revert_reason.clone()); + invariant_test + .test_data + .failures + .record_assertion_failure(case_data); + if !invariant_contract.is_optimization() { + // In optimization mode, keep reverted calls to preserve + // warp/roll values for correct replay during shrinking. + current_run.inputs.pop(); + } + result::RichInvariantResults::new(true, None) + } else if self.config.fail_on_revert { let case_data = error::FailedInvariantCaseData::new( &invariant_contract, &self.config, @@ -594,10 +633,37 @@ impl<'a> InvariantExecutor<'a> { invariant_test.fuzz_state.log_stats(); let result = invariant_test.test_data; + let mut failures = result.failures; + let assertion_failures: Vec = failures.assertion_failures.keys().cloned().collect(); + if failures.error.is_none() && self.config.fail_on_assert { + if let Some((handler, case_data)) = failures.assertion_failures.iter().next() { + failures.error = Some(InvariantFuzzError::BrokenInvariant(case_data.clone())); + let base_reason = if case_data.revert_reason.is_empty() { + "assertion failure".to_string() + } else { + case_data.revert_reason.clone() + }; + let extra_handlers: Vec<_> = assertion_failures + .iter() + .filter(|name| name.as_str() != handler.as_str()) + .cloned() + .collect(); + failures.revert_reason = Some(if extra_handlers.is_empty() { + format!("assertion failure in {handler}: {base_reason}") + } else { + format!( + "assertion failure in {handler}: {base_reason}; other assertion failures in {}", + extra_handlers.join(", ") + ) + }); + } + } + Ok(InvariantFuzzTestResult { - error: result.failures.error, + error: failures.error, + assertion_failures, cases: result.fuzz_cases, - reverts: result.failures.reverts, + reverts: failures.reverts, last_run_inputs: result.last_run_inputs, gas_report_traces: result.gas_report_traces, line_coverage: result.line_coverage, diff --git a/crates/evm/evm/src/executors/invariant/result.rs b/crates/evm/evm/src/executors/invariant/result.rs index b6cd1879d29d0..4721203a40f6a 100644 --- a/crates/evm/evm/src/executors/invariant/result.rs +++ b/crates/evm/evm/src/executors/invariant/result.rs @@ -13,6 +13,7 @@ use foundry_evm_fuzz::{ BasicTxDetails, FuzzedCases, invariant::{FuzzRunIdentifiedContracts, InvariantContract}, }; +use revm::interpreter::InstructionResult; use revm_inspectors::tracing::CallTraceArena; use std::{borrow::Cow, collections::HashMap}; @@ -20,6 +21,8 @@ use std::{borrow::Cow, collections::HashMap}; #[derive(Debug)] pub struct InvariantFuzzTestResult { pub error: Option, + /// Distinct handler-level assertion failures observed during the campaign. + pub assertion_failures: Vec, /// Every successful fuzz test case pub cases: Vec, /// Number of reverted fuzz calls @@ -56,6 +59,38 @@ impl RichInvariantResults { } } +/// Returns true if this call failed due to a Solidity assert: +/// - Panic(0x01), or +/// - legacy invalid opcode assert behavior. +pub(crate) fn is_assertion_failure(call_result: &RawCallResult) -> bool { + if !call_result.reverted { + return false; + } + + is_assert_panic(call_result.result.as_ref()) + || matches!(call_result.exit_reason, Some(InstructionResult::InvalidFEOpcode)) +} + +fn is_assert_panic(data: &[u8]) -> bool { + const PANIC_SELECTOR: [u8; 4] = [0x4e, 0x48, 0x7b, 0x71]; + if data.len() < 36 || data[..4] != PANIC_SELECTOR { + return false; + } + + let panic_code = &data[4..36]; + panic_code[..31].iter().all(|byte| *byte == 0) && panic_code[31] == 0x01 +} + +fn failing_handler_name( + invariant_test: &InvariantTest, + invariant_run: &InvariantTestRun, +) -> Option { + let last_input = invariant_run.inputs.last()?; + let metric_key = + invariant_test.targeted_contracts.targets.lock().fuzzed_metric_key(last_input)?; + Some(metric_key.rsplit('.').next().unwrap_or(metric_key.as_str()).to_string()) +} + /// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the /// external `invariant_failures.failed_invariant` map and returns a generic error. /// Either returns the call result if successful, or nothing if there was an error. @@ -160,11 +195,36 @@ pub(crate) fn can_continue( } } } else { + let is_assert_failure = is_assertion_failure(&call_result); + let should_fail_on_assert = invariant_config.fail_on_assert && is_assert_failure; + let failing_handler = if should_fail_on_assert { + failing_handler_name(invariant_test, invariant_run) + } else { + None + }; // Increase the amount of reverts. let invariant_data = &mut invariant_test.test_data; invariant_data.failures.reverts += 1; - // If fail on revert is set, we must return immediately. - if invariant_config.fail_on_revert { + + // In fail-on-assert mode, keep exploring and accumulate unique assertion failures. + if should_fail_on_assert { + let case_data = FailedInvariantCaseData::new( + invariant_contract, + invariant_config, + &invariant_test.targeted_contracts, + &invariant_run.inputs, + call_result, + &[], + ) + .with_failing_handler(failing_handler); + invariant_data.failures.revert_reason = Some(case_data.revert_reason.clone()); + invariant_data.failures.record_assertion_failure(case_data); + if !is_optimization { + // Keep shrinking/replay coherent by discarding reverted calls in check mode. + invariant_run.inputs.pop(); + } + return Ok(RichInvariantResults::new(true, None)); + } else if invariant_config.fail_on_revert { let case_data = FailedInvariantCaseData::new( invariant_contract, invariant_config, @@ -175,7 +235,6 @@ pub(crate) fn can_continue( ); invariant_data.failures.revert_reason = Some(case_data.revert_reason.clone()); invariant_data.failures.error = Some(InvariantFuzzError::Revert(case_data)); - return Ok(RichInvariantResults::new(false, None)); } else if call_result.reverted && !is_optimization { // If we don't fail test on revert then remove last reverted call from inputs. @@ -211,3 +270,40 @@ pub(crate) fn assert_after_invariant( } Ok(success) } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::Bytes; + + fn panic_payload(code: u8) -> Bytes { + let mut payload = vec![0_u8; 36]; + payload[..4].copy_from_slice(&[0x4e, 0x48, 0x7b, 0x71]); + payload[35] = code; + payload.into() + } + + #[test] + fn detects_assert_panic_code() { + let call_result = + RawCallResult { reverted: true, result: panic_payload(0x01), ..Default::default() }; + assert!(is_assertion_failure(&call_result)); + } + + #[test] + fn ignores_non_assert_panic_code() { + let call_result = + RawCallResult { reverted: true, result: panic_payload(0x11), ..Default::default() }; + assert!(!is_assertion_failure(&call_result)); + } + + #[test] + fn detects_legacy_invalid_opcode_assert() { + let call_result = RawCallResult { + reverted: true, + exit_reason: Some(InstructionResult::InvalidFEOpcode), + ..Default::default() + }; + assert!(is_assertion_failure(&call_result)); + } +} diff --git a/crates/evm/evm/src/executors/invariant/shrink.rs b/crates/evm/evm/src/executors/invariant/shrink.rs index 1e56a019cc576..3844a5e151481 100644 --- a/crates/evm/evm/src/executors/invariant/shrink.rs +++ b/crates/evm/evm/src/executors/invariant/shrink.rs @@ -1,6 +1,9 @@ use crate::executors::{ EarlyExit, Executor, - invariant::{call_after_invariant_function, call_invariant_function, execute_tx}, + invariant::{ + call_after_invariant_function, call_invariant_function, execute_tx, + result::is_assertion_failure, + }, }; use alloy_primitives::{Address, Bytes, I256, U256}; use foundry_config::InvariantConfig; @@ -114,6 +117,7 @@ pub(crate) fn shrink_sequence( target_address, calldata.clone(), config.fail_on_revert, + config.fail_on_assert, invariant_contract.call_after_invariant, ) { // If candidate sequence still fails, shrink until shortest possible. @@ -146,6 +150,7 @@ pub fn check_sequence( test_address: Address, calldata: Bytes, fail_on_revert: bool, + fail_on_assert: bool, call_after_invariant: bool, ) -> eyre::Result<(bool, bool)> { // Apply the call sequence. @@ -156,7 +161,11 @@ pub fn check_sequence( // Ignore calls reverted with `MAGIC_ASSUME`. This is needed to handle failed scenarios that // are replayed with a modified version of test driver (that use new `vm.assume` // cheatcodes). - if call_result.reverted && fail_on_revert && call_result.result.as_ref() != MAGIC_ASSUME { + let should_fail_on_assert = fail_on_assert && is_assertion_failure(&call_result); + if call_result.reverted + && call_result.result.as_ref() != MAGIC_ASSUME + && (fail_on_revert || should_fail_on_assert) + { // Candidate sequence fails test. // We don't have to apply remaining calls to check sequence. return Ok((false, false)); diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index d5106a02ec84c..5281bb1cdd9ef 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -732,6 +732,9 @@ impl<'a> FunctionRunner<'a> { let runner = self.invariant_runner(); let invariant_config = &self.config.invariant; + let invariant_depth = invariant_config.depth as usize; + let fail_on_revert = invariant_config.fail_on_revert; + let fail_on_assert = invariant_config.fail_on_assert; let mut executor = self.clone_executor(); // Enable edge coverage if running with coverage guided fuzzing or with edge coverage @@ -789,10 +792,11 @@ impl<'a> FunctionRunner<'a> { if let Ok((success, replayed_entirely)) = check_sequence( self.clone_executor(), &txes, - (0..min(txes.len(), invariant_config.depth as usize)).collect(), + (0..min(txes.len(), invariant_depth)).collect(), invariant_contract.address, invariant_contract.invariant_function.selector().to_vec().into(), - invariant_config.fail_on_revert, + fail_on_revert, + fail_on_assert, invariant_contract.call_after_invariant, ) && !success { @@ -842,11 +846,32 @@ impl<'a> FunctionRunner<'a> { } } - self.result.invariant_replay_fail( - replayed_entirely, - &invariant_contract.invariant_function.name, - call_sequence, - ); + let replay_label = if fail_on_assert { + let assert_only_failure = check_sequence( + self.clone_executor(), + &txes, + (0..min(txes.len(), invariant_depth)).collect(), + invariant_contract.address, + invariant_contract.invariant_function.selector().to_vec().into(), + false, + false, + invariant_contract.call_after_invariant, + ) + .map(|(success, _)| success) + .unwrap_or(false); + if assert_only_failure { + call_sequence + .last() + .and_then(|counterexample| counterexample.func_name.clone()) + .map(|handler| format!("assertion failure in {handler}")) + .unwrap_or_else(|| invariant_contract.invariant_function.name.clone()) + } else { + invariant_contract.invariant_function.name.clone() + } + } else { + invariant_contract.invariant_function.name.clone() + }; + self.result.invariant_replay_fail(replayed_entirely, &replay_label, call_sequence); return self.result; } } @@ -868,14 +893,37 @@ impl<'a> FunctionRunner<'a> { self.result.merge_coverages(invariant_result.line_coverage); let mut counterexample = None; + let assertion_failures = invariant_result.assertion_failures.clone(); let success = invariant_result.error.is_none(); - let reason = invariant_result.error.as_ref().and_then(|err| err.revert_reason()); + let mut reason = invariant_result.error.as_ref().and_then(|err| err.revert_reason()); match invariant_result.error { // If invariants were broken, replay the error to collect logs and traces Some(error) => match error { InvariantFuzzError::BrokenInvariant(case_data) | InvariantFuzzError::Revert(case_data) => { + if case_data.fail_on_assert + && let Some(handler) = case_data.failing_handler.as_ref() + { + let assert_reason = if case_data.revert_reason.is_empty() { + "assertion failure".to_string() + } else { + case_data.revert_reason.clone() + }; + let extra_handlers: Vec<_> = assertion_failures + .iter() + .filter(|name| name.as_str() != handler.as_str()) + .cloned() + .collect(); + reason = Some(if extra_handlers.is_empty() { + format!("assertion failure in {handler}: {assert_reason}") + } else { + format!( + "assertion failure in {handler}: {assert_reason}; other assertion failures in {}", + extra_handlers.join(", ") + ) + }); + } // Replay error to create counterexample and to collect logs, traces and // coverage. match case_data.test_error { @@ -983,6 +1031,16 @@ impl<'a> FunctionRunner<'a> { } } + if !assertion_failures.is_empty() + && !reason.as_ref().is_some_and(|r| r.contains("assertion failure in")) + { + let summary = format!("assertion failures in {}", assertion_failures.join(", ")); + reason = Some(match reason { + Some(existing) => format!("{existing}; {summary}"), + None => summary, + }); + } + self.result.invariant_result( invariant_result.gas_report_traces, success, diff --git a/crates/forge/tests/cli/config.rs b/crates/forge/tests/cli/config.rs index b172ea70c24e0..2d5adeec21eb7 100644 --- a/crates/forge/tests/cli/config.rs +++ b/crates/forge/tests/cli/config.rs @@ -198,6 +198,7 @@ show_logs = false runs = 256 depth = 500 fail_on_revert = false +fail_on_assert = false call_override = false dictionary_weight = 80 include_storage = true @@ -1280,6 +1281,7 @@ forgetest_init!(test_default_config, |prj, cmd| { "runs": 256, "depth": 500, "fail_on_revert": false, + "fail_on_assert": false, "call_override": false, "dictionary_weight": 80, "include_storage": true, diff --git a/crates/forge/tests/cli/test_cmd/invariant/common.rs b/crates/forge/tests/cli/test_cmd/invariant/common.rs index eb28ca8c15ca0..385a67589f8d3 100644 --- a/crates/forge/tests/cli/test_cmd/invariant/common.rs +++ b/crates/forge/tests/cli/test_cmd/invariant/common.rs @@ -713,6 +713,229 @@ Tip: Run `forge test --rerun` to retry only the 1 failed test "#]]); }); +forgetest_init!(invariant_fail_on_assert_panic, |prj, cmd| { + prj.update_config(|config| { + config.invariant.runs = 1; + config.invariant.depth = 10; + config.invariant.fail_on_revert = false; + config.invariant.fail_on_assert = true; + }); + + prj.add_test( + "InvariantFailOnAssertPanic.t.sol", + r#" +import "forge-std/Test.sol"; + +contract AssertHandler { + uint256 public calls; + + function alwaysAssert() external { + calls++; + assert(false); + } +} + +contract InvariantFailOnAssertPanic is Test { + AssertHandler handler; + + function setUp() public { + handler = new AssertHandler(); + targetContract(address(handler)); + } + + function invariant_fail_on_assert_panic() public view {} +} +"#, + ); + + assert_invariant(cmd.args(["test"])).failure().stdout_eq(str![[r#" +... +Ran 1 test for test/InvariantFailOnAssertPanic.t.sol:InvariantFailOnAssertPanic +[FAIL: panic: assertion failed (0x01)] +... + invariant_fail_on_assert_panic() ([RUNS]) +... +"#]]); +}); + +forgetest_init!(invariant_ignore_assert_panic_when_flag_off, |prj, cmd| { + prj.update_config(|config| { + config.invariant.runs = 1; + config.invariant.depth = 10; + config.invariant.fail_on_revert = false; + config.invariant.fail_on_assert = false; + }); + + prj.add_test( + "InvariantIgnoreAssertWhenFlagOff.t.sol", + r#" +import "forge-std/Test.sol"; + +contract AssertHandler { + uint256 public calls; + + function alwaysAssert() external { + calls++; + assert(false); + } +} + +contract InvariantIgnoreAssertWhenFlagOff is Test { + AssertHandler handler; + + function setUp() public { + handler = new AssertHandler(); + targetContract(address(handler)); + } + + function invariant_assert_discarded() public view {} +} +"#, + ); + + assert_invariant(cmd.args(["test"])).success().stdout_eq(str![[r#" +... +[PASS] invariant_assert_discarded() ([RUNS]) +... +"#]]); +}); + +forgetest_init!(invariant_fail_on_assert_ignores_non_assert_panic, |prj, cmd| { + prj.update_config(|config| { + config.invariant.runs = 1; + config.invariant.depth = 10; + config.invariant.fail_on_revert = false; + config.invariant.fail_on_assert = true; + }); + + prj.add_test( + "InvariantIgnoreNonAssertPanic.t.sol", + r#" +import "forge-std/Test.sol"; + +contract OverflowHandler { + uint256 public calls; + + function alwaysOverflow() external { + calls++; + uint256 x = type(uint256).max; + x = x + 1; + } +} + +contract InvariantIgnoreNonAssertPanic is Test { + OverflowHandler handler; + + function setUp() public { + handler = new OverflowHandler(); + targetContract(address(handler)); + } + + function invariant_non_assert_panic_discarded() public view {} +} +"#, + ); + + assert_invariant(cmd.args(["test"])).success().stdout_eq(str![[r#" +... +[PASS] invariant_non_assert_panic_discarded() ([RUNS]) +... +"#]]); +}); + +forgetest_init!(invariant_fail_on_assert_ignores_require_revert, |prj, cmd| { + prj.update_config(|config| { + config.invariant.runs = 1; + config.invariant.depth = 10; + config.invariant.fail_on_revert = false; + config.invariant.fail_on_assert = true; + }); + + prj.add_test( + "InvariantIgnoreRequireRevert.t.sol", + r#" +import "forge-std/Test.sol"; + +contract RequireHandler { + uint256 public calls; + + function alwaysRequire() external { + calls++; + require(false, "require failed"); + } +} + +contract InvariantIgnoreRequireRevert is Test { + RequireHandler handler; + + function setUp() public { + handler = new RequireHandler(); + targetContract(address(handler)); + } + + function invariant_require_revert_discarded() public view {} +} +"#, + ); + + assert_invariant(cmd.args(["test"])).success().stdout_eq(str![[r#" +... +[PASS] invariant_require_revert_discarded() ([RUNS]) +... +"#]]); +}); + +forgetest_init!(invariant_replay_fail_on_assert, |prj, cmd| { + prj.update_config(|config| { + config.fuzz.seed = Some(U256::from(119u32)); + config.invariant.fail_on_revert = false; + config.invariant.fail_on_assert = true; + config.invariant.runs = 1; + config.invariant.depth = 200; + }); + + prj.add_test( + "InvariantReplayFailOnAssert.t.sol", + r#" +import "forge-std/Test.sol"; + +contract ReplayAssertHandler { + uint256 public calls; + + function alwaysAssert() external { + calls++; + assert(false); + } +} + +contract ReplayFailOnAssertTest is Test { + ReplayAssertHandler handler; + + function setUp() public { + handler = new ReplayAssertHandler(); + targetContract(address(handler)); + } + + function invariant_replay_fail_on_assert() public view {} +} +"#, + ); + + cmd.args(["test"]).assert_failure().stdout_eq(str![[r#" +... +Ran 1 test for test/InvariantReplayFailOnAssert.t.sol:ReplayFailOnAssertTest +[FAIL: panic: assertion failed (0x01)] +... +"#]]); + + cmd.assert_failure().stdout_eq(str![[r#" +... +Ran 1 test for test/InvariantReplayFailOnAssert.t.sol:ReplayFailOnAssertTest +[FAIL: invariant_replay_fail_on_assert persisted failure revert] +... +"#]]); +}); + // Here we test that the fuzz engine can include a contract created during the fuzz // in its fuzz dictionary and eventually break the invariant. // Specifically, can Judas, a created contract from Jesus, break Jesus contract