Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions crates/config/src/invariant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ pub struct InvariantConfig {
pub depth: u32,
/// Fails the invariant fuzzing if a revert occurs
pub fail_on_revert: bool,
/// Fails the invariant fuzzing if a Solidity assert failure occurs (`Panic(0x01)` or legacy
/// invalid opcode assert behavior), even if `fail_on_revert` is `false`.
pub fail_on_assert: bool,
/// Allows overriding an unsafe external call when running invariant tests. eg. reentrancy
/// checks
pub call_override: bool,
Expand Down Expand Up @@ -57,6 +60,7 @@ impl Default for InvariantConfig {
runs: 256,
depth: 500,
fail_on_revert: false,
fail_on_assert: false,
call_override: false,
dictionary: FuzzDictionaryConfig { dictionary_weight: 80, ..Default::default() },
shrink_run_limit: 5000,
Expand Down
20 changes: 20 additions & 0 deletions crates/evm/evm/src/executors/invariant/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use foundry_config::InvariantConfig;
use foundry_evm_core::decode::RevertDecoder;
use foundry_evm_fuzz::{BasicTxDetails, Reason, invariant::FuzzRunIdentifiedContracts};
use proptest::test_runner::TestError;
use std::collections::BTreeMap;

/// Stores information about failures and reverts of the invariant tests.
#[derive(Clone, Default)]
Expand All @@ -15,6 +16,8 @@ pub struct InvariantFailures {
pub revert_reason: Option<String>,
/// Maps a broken invariant to its specific error.
pub error: Option<InvariantFuzzError>,
/// Distinct handler-level assertion failures observed during the campaign.
pub assertion_failures: BTreeMap<String, FailedInvariantCaseData>,
}

impl InvariantFailures {
Expand All @@ -25,6 +28,12 @@ impl InvariantFailures {
pub fn into_inner(self) -> (usize, Option<InvariantFuzzError>) {
(self.reverts, self.error)
}

pub fn record_assertion_failure(&mut self, case_data: FailedInvariantCaseData) {
let key =
case_data.failing_handler.clone().unwrap_or_else(|| "unknown_handler".to_string());
self.assertion_failures.entry(key).or_insert(case_data);
}
}

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -65,6 +74,10 @@ pub struct FailedInvariantCaseData {
pub shrink_run_limit: u32,
/// Fail on revert, used to check sequence when shrinking.
pub fail_on_revert: bool,
/// Fail on Solidity assert failures, used to check sequence when shrinking.
pub fail_on_assert: bool,
/// Handler function that triggered a fail-on-assert violation, when available.
pub failing_handler: Option<String>,
}

impl FailedInvariantCaseData {
Expand Down Expand Up @@ -97,6 +110,13 @@ impl FailedInvariantCaseData {
inner_sequence: inner_sequence.to_vec(),
shrink_run_limit: invariant_config.shrink_run_limit,
fail_on_revert: invariant_config.fail_on_revert,
fail_on_assert: invariant_config.fail_on_assert,
failing_handler: None,
}
}

pub fn with_failing_handler(mut self, failing_handler: Option<String>) -> Self {
self.failing_handler = failing_handler;
self
}
}
74 changes: 70 additions & 4 deletions crates/evm/evm/src/executors/invariant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ use foundry_evm_traces::{CallTraceArena, SparsedTraceArena};
use indicatif::ProgressBar;
use parking_lot::RwLock;
use proptest::{strategy::Strategy, test_runner::TestRunner};
use result::{assert_after_invariant, assert_invariants, can_continue};
use result::{assert_after_invariant, assert_invariants, can_continue, is_assertion_failure};
use revm::state::Account;
use serde::{Deserialize, Serialize};
use serde_json::json;
Expand Down Expand Up @@ -502,7 +502,46 @@ impl<'a> InvariantExecutor<'a> {
// Skip invariant check but still track reverts
if call_result.reverted {
invariant_test.test_data.failures.reverts += 1;
if self.config.fail_on_revert {
let should_fail_on_assert =
self.config.fail_on_assert && is_assertion_failure(&call_result);
if should_fail_on_assert {
let failing_handler =
current_run.inputs.last().and_then(|last_input| {
invariant_test
.targeted_contracts
.targets
.lock()
.fuzzed_metric_key(last_input)
.map(|metric_key| {
metric_key
.rsplit('.')
.next()
.unwrap_or(metric_key.as_str())
.to_string()
})
});
let case_data = error::FailedInvariantCaseData::new(
&invariant_contract,
&self.config,
&invariant_test.targeted_contracts,
&current_run.inputs,
call_result,
&[],
)
.with_failing_handler(failing_handler);
invariant_test.test_data.failures.revert_reason =
Some(case_data.revert_reason.clone());
invariant_test
.test_data
.failures
.record_assertion_failure(case_data);
if !invariant_contract.is_optimization() {
// In optimization mode, keep reverted calls to preserve
// warp/roll values for correct replay during shrinking.
current_run.inputs.pop();
}
result::RichInvariantResults::new(true, None)
} else if self.config.fail_on_revert {
let case_data = error::FailedInvariantCaseData::new(
&invariant_contract,
&self.config,
Expand Down Expand Up @@ -594,10 +633,37 @@ impl<'a> InvariantExecutor<'a> {
invariant_test.fuzz_state.log_stats();

let result = invariant_test.test_data;
let mut failures = result.failures;
let assertion_failures: Vec<String> = failures.assertion_failures.keys().cloned().collect();
if failures.error.is_none() && self.config.fail_on_assert {
if let Some((handler, case_data)) = failures.assertion_failures.iter().next() {
failures.error = Some(InvariantFuzzError::BrokenInvariant(case_data.clone()));
let base_reason = if case_data.revert_reason.is_empty() {
"assertion failure".to_string()
} else {
case_data.revert_reason.clone()
};
let extra_handlers: Vec<_> = assertion_failures
.iter()
.filter(|name| name.as_str() != handler.as_str())
.cloned()
.collect();
failures.revert_reason = Some(if extra_handlers.is_empty() {
format!("assertion failure in {handler}: {base_reason}")
} else {
format!(
"assertion failure in {handler}: {base_reason}; other assertion failures in {}",
extra_handlers.join(", ")
)
});
}
}

Ok(InvariantFuzzTestResult {
error: result.failures.error,
error: failures.error,
assertion_failures,
cases: result.fuzz_cases,
reverts: result.failures.reverts,
reverts: failures.reverts,
last_run_inputs: result.last_run_inputs,
gas_report_traces: result.gas_report_traces,
line_coverage: result.line_coverage,
Expand Down
102 changes: 99 additions & 3 deletions crates/evm/evm/src/executors/invariant/result.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,16 @@ use foundry_evm_fuzz::{
BasicTxDetails, FuzzedCases,
invariant::{FuzzRunIdentifiedContracts, InvariantContract},
};
use revm::interpreter::InstructionResult;
use revm_inspectors::tracing::CallTraceArena;
use std::{borrow::Cow, collections::HashMap};

/// The outcome of an invariant fuzz test
#[derive(Debug)]
pub struct InvariantFuzzTestResult {
pub error: Option<InvariantFuzzError>,
/// Distinct handler-level assertion failures observed during the campaign.
pub assertion_failures: Vec<String>,
/// Every successful fuzz test case
pub cases: Vec<FuzzedCases>,
/// Number of reverted fuzz calls
Expand Down Expand Up @@ -56,6 +59,38 @@ impl RichInvariantResults {
}
}

/// Returns true if this call failed due to a Solidity assert:
/// - Panic(0x01), or
/// - legacy invalid opcode assert behavior.
pub(crate) fn is_assertion_failure(call_result: &RawCallResult) -> bool {
if !call_result.reverted {
return false;
}

is_assert_panic(call_result.result.as_ref())
|| matches!(call_result.exit_reason, Some(InstructionResult::InvalidFEOpcode))
}

fn is_assert_panic(data: &[u8]) -> bool {
const PANIC_SELECTOR: [u8; 4] = [0x4e, 0x48, 0x7b, 0x71];
if data.len() < 36 || data[..4] != PANIC_SELECTOR {
return false;
}

let panic_code = &data[4..36];
panic_code[..31].iter().all(|byte| *byte == 0) && panic_code[31] == 0x01
}

fn failing_handler_name(
invariant_test: &InvariantTest,
invariant_run: &InvariantTestRun,
) -> Option<String> {
let last_input = invariant_run.inputs.last()?;
let metric_key =
invariant_test.targeted_contracts.targets.lock().fuzzed_metric_key(last_input)?;
Some(metric_key.rsplit('.').next().unwrap_or(metric_key.as_str()).to_string())
}

/// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the
/// external `invariant_failures.failed_invariant` map and returns a generic error.
/// Either returns the call result if successful, or nothing if there was an error.
Expand Down Expand Up @@ -160,11 +195,36 @@ pub(crate) fn can_continue(
}
}
} else {
let is_assert_failure = is_assertion_failure(&call_result);
let should_fail_on_assert = invariant_config.fail_on_assert && is_assert_failure;
let failing_handler = if should_fail_on_assert {
failing_handler_name(invariant_test, invariant_run)
} else {
None
};
// Increase the amount of reverts.
let invariant_data = &mut invariant_test.test_data;
invariant_data.failures.reverts += 1;
// If fail on revert is set, we must return immediately.
if invariant_config.fail_on_revert {

// In fail-on-assert mode, keep exploring and accumulate unique assertion failures.
if should_fail_on_assert {
let case_data = FailedInvariantCaseData::new(
invariant_contract,
invariant_config,
&invariant_test.targeted_contracts,
&invariant_run.inputs,
call_result,
&[],
)
.with_failing_handler(failing_handler);
invariant_data.failures.revert_reason = Some(case_data.revert_reason.clone());
invariant_data.failures.record_assertion_failure(case_data);
if !is_optimization {
// Keep shrinking/replay coherent by discarding reverted calls in check mode.
invariant_run.inputs.pop();
}
return Ok(RichInvariantResults::new(true, None));
} else if invariant_config.fail_on_revert {
let case_data = FailedInvariantCaseData::new(
invariant_contract,
invariant_config,
Expand All @@ -175,7 +235,6 @@ pub(crate) fn can_continue(
);
invariant_data.failures.revert_reason = Some(case_data.revert_reason.clone());
invariant_data.failures.error = Some(InvariantFuzzError::Revert(case_data));

return Ok(RichInvariantResults::new(false, None));
} else if call_result.reverted && !is_optimization {
// If we don't fail test on revert then remove last reverted call from inputs.
Expand Down Expand Up @@ -211,3 +270,40 @@ pub(crate) fn assert_after_invariant(
}
Ok(success)
}

#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::Bytes;

fn panic_payload(code: u8) -> Bytes {
let mut payload = vec![0_u8; 36];
payload[..4].copy_from_slice(&[0x4e, 0x48, 0x7b, 0x71]);
payload[35] = code;
payload.into()
}

#[test]
fn detects_assert_panic_code() {
let call_result =
RawCallResult { reverted: true, result: panic_payload(0x01), ..Default::default() };
assert!(is_assertion_failure(&call_result));
}

#[test]
fn ignores_non_assert_panic_code() {
let call_result =
RawCallResult { reverted: true, result: panic_payload(0x11), ..Default::default() };
assert!(!is_assertion_failure(&call_result));
}

#[test]
fn detects_legacy_invalid_opcode_assert() {
let call_result = RawCallResult {
reverted: true,
exit_reason: Some(InstructionResult::InvalidFEOpcode),
..Default::default()
};
assert!(is_assertion_failure(&call_result));
}
}
13 changes: 11 additions & 2 deletions crates/evm/evm/src/executors/invariant/shrink.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
use crate::executors::{
EarlyExit, Executor,
invariant::{call_after_invariant_function, call_invariant_function, execute_tx},
invariant::{
call_after_invariant_function, call_invariant_function, execute_tx,
result::is_assertion_failure,
},
};
use alloy_primitives::{Address, Bytes, I256, U256};
use foundry_config::InvariantConfig;
Expand Down Expand Up @@ -114,6 +117,7 @@ pub(crate) fn shrink_sequence(
target_address,
calldata.clone(),
config.fail_on_revert,
config.fail_on_assert,
invariant_contract.call_after_invariant,
) {
// If candidate sequence still fails, shrink until shortest possible.
Expand Down Expand Up @@ -146,6 +150,7 @@ pub fn check_sequence(
test_address: Address,
calldata: Bytes,
fail_on_revert: bool,
fail_on_assert: bool,
call_after_invariant: bool,
) -> eyre::Result<(bool, bool)> {
// Apply the call sequence.
Expand All @@ -156,7 +161,11 @@ pub fn check_sequence(
// Ignore calls reverted with `MAGIC_ASSUME`. This is needed to handle failed scenarios that
// are replayed with a modified version of test driver (that use new `vm.assume`
// cheatcodes).
if call_result.reverted && fail_on_revert && call_result.result.as_ref() != MAGIC_ASSUME {
let should_fail_on_assert = fail_on_assert && is_assertion_failure(&call_result);
if call_result.reverted
&& call_result.result.as_ref() != MAGIC_ASSUME
&& (fail_on_revert || should_fail_on_assert)
{
// Candidate sequence fails test.
// We don't have to apply remaining calls to check sequence.
return Ok((false, false));
Expand Down
Loading