Tests and Nits

Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019dbf48-3fb0-7762-a01f-b5e966339e73

Author: grandizzy

crates/config/src/invariant.rs

@@ -49,6 +49,8 @@ pub struct InvariantConfig {
     ///
     /// Example: `check_interval = 10` means assert after calls 10, 20, 30, ... and the last call.
     pub check_interval: u32,
+    /// Continue invariant run until all invariants declared in current test suite breaks.
+    pub continuous_run: bool,
 }
 
 impl Default for InvariantConfig {
@@ -70,6 +72,7 @@ impl Default for InvariantConfig {
             max_time_delay: None,
             max_block_delay: None,
             check_interval: 1,
+            continuous_run: false,
         }
     }
 }

crates/evm/evm/src/executors/invariant/error.rs

@@ -2,7 +2,6 @@ use super::InvariantContract;
 use crate::executors::RawCallResult;
 use alloy_json_abi::Function;
 use alloy_primitives::{Address, Bytes};
-use foundry_config::InvariantConfig;
 use foundry_evm_core::{
     decode::{ASSERTION_FAILED_PREFIX, EMPTY_REVERT_DATA, RevertDecoder},
     evm::FoundryEvmNetwork,

crates/evm/evm/src/executors/invariant/mod.rs

@@ -6,7 +6,7 @@ use crate::{
     inspectors::Fuzzer,
 };
 use alloy_json_abi::Function;
-use alloy_primitives::{Address, Bytes, FixedBytes, I256, Selector, U256, map::{AddressMap, HashMap}};
+use alloy_primitives::{Address, Bytes, FixedBytes, I256, Selector, U256, map::AddressMap};
 use alloy_sol_types::{SolCall, sol};
 use eyre::{ContextCompat, Result, eyre};
 use foundry_common::{
@@ -35,7 +35,7 @@ use foundry_evm_traces::{CallTraceArena, SparsedTraceArena};
 use indicatif::ProgressBar;
 use parking_lot::RwLock;
 use proptest::{strategy::Strategy, test_runner::TestRunner};
-use result::{assert_after_invariant, assert_invariants, can_continue, did_fail_on_assert, invariant_preflight_check};
+use result::{assert_after_invariant, can_continue, did_fail_on_assert, invariant_preflight_check};
 use revm::{context::Block, state::Account};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
@@ -220,7 +220,7 @@ fn build_invariant_progress_json<M: Serialize>(
 }
 
 /// Contains data collected during invariant test runs.
-struct InvariantTestData<FEN: FoundryEvmNetwork> {
+struct InvariantTestData {
     // Consumed gas and calldata of every successful fuzz call.
     fuzz_cases: Vec<FuzzedCases>,
     // Data related to reverts or failed assertions of the test.
@@ -229,8 +229,6 @@ struct InvariantTestData<FEN: FoundryEvmNetwork> {
     last_run_inputs: Vec<BasicTxDetails>,
     // Additional traces for gas report.
     gas_report_traces: Vec<Vec<CallTraceArena>>,
-    // Last call results of the invariant test.
-    last_call_results: Option<RawCallResult<FEN>>,
     // Line coverage information collected from all fuzzed calls.
     line_coverage: Option<HitMaps>,
     // Metrics for each fuzzed selector.
@@ -249,30 +247,28 @@ struct InvariantTestData<FEN: FoundryEvmNetwork> {
 }
 
 /// Contains invariant test data.
-struct InvariantTest<FEN: FoundryEvmNetwork> {
+struct InvariantTest {
     // Fuzz state of invariant test.
     fuzz_state: EvmFuzzState,
     // Contracts fuzzed by the invariant test.
     targeted_contracts: FuzzRunIdentifiedContracts,
     // Data collected during invariant runs.
-    test_data: InvariantTestData<FEN>,
+    test_data: InvariantTestData,
 }
 
-impl<FEN: FoundryEvmNetwork> InvariantTest<FEN> {
+impl InvariantTest {
     /// Instantiates an invariant test.
     fn new(
         fuzz_state: EvmFuzzState,
         targeted_contracts: FuzzRunIdentifiedContracts,
         failures: InvariantFailures,
-        last_call_results: Option<RawCallResult<FEN>>,
         branch_runner: TestRunner,
     ) -> Self {
         let test_data = InvariantTestData {
             fuzz_cases: vec![],
             failures,
             last_run_inputs: vec![],
             gas_report_traces: vec![],
-            last_call_results,
             line_coverage: None,
             metrics: Map::default(),
             branch_runner,
@@ -297,11 +293,6 @@ impl<FEN: FoundryEvmNetwork> InvariantTest<FEN> {
         self.test_data.failures.record_failure(invariant, error);
     }
 
-    /// Set last invariant test call results.
-    fn set_last_call_results(&mut self, call_result: Option<RawCallResult<FEN>>) {
-        self.test_data.last_call_results = call_result;
-    }
-
     /// Set last invariant run call sequence.
     fn set_last_run_inputs(&mut self, inputs: &Vec<BasicTxDetails>) {
         self.test_data.last_run_inputs.clone_from(inputs);
@@ -332,7 +323,7 @@ impl<FEN: FoundryEvmNetwork> InvariantTest<FEN> {
 
     /// End invariant test run by collecting results, cleaning collected artifacts and reverting
     /// created fuzz state.
-    fn end_run(&mut self, run: InvariantTestRun<FEN>, gas_samples: usize) {
+    fn end_run<FEN: FoundryEvmNetwork>(&mut self, run: InvariantTestRun<FEN>, gas_samples: usize) {
         // We clear all the targeted contracts created during this run.
         self.targeted_contracts.clear_created_contracts(run.created_contracts);
 
@@ -600,7 +591,7 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
                                 || is_last_call
                         };
 
-                    let can_continue_result = if should_check_invariant {
+                    let result = if should_check_invariant {
                         can_continue(
                             &invariant_contract,
                             &mut invariant_test,
@@ -649,11 +640,11 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
                         }
                     };
 
-                    if !can_continue_result || current_run.depth == self.config.depth - 1 {
+                    if !result || current_run.depth == self.config.depth - 1 {
                         invariant_test.set_last_run_inputs(&current_run.inputs);
                     }
                     // If test cannot continue then stop current run and exit test suite.
-                    if !can_continue_result {
+                    if !result {
                         let reason = invariant_test
                             .test_data
                             .failures
@@ -787,7 +778,7 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
         invariant_contract: &InvariantContract<'_>,
         fuzz_fixtures: &FuzzFixtures,
         fuzz_state: EvmFuzzState,
-    ) -> Result<(InvariantTest<FEN>, WorkerCorpus)> {
+    ) -> Result<(InvariantTest, WorkerCorpus)> {
         // Finds out the chosen deployed contracts and/or senders.
         self.select_contract_artifacts(invariant_contract.address)?;
         let (targeted_senders, targeted_contracts) =
@@ -880,7 +871,6 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
             fuzz_state,
             targeted_contracts,
             failures,
-            None,
             self.runner.clone(),
         );
 
@@ -1228,7 +1218,7 @@ impl<'a, FEN: FoundryEvmNetwork> InvariantExecutor<'a, FEN> {
 /// before inserting it into the dictionary. Otherwise, we flood the dictionary with
 /// randomly generated addresses.
 fn collect_data<FEN: FoundryEvmNetwork>(
-    invariant_test: &InvariantTest<FEN>,
+    invariant_test: &InvariantTest,
     state_changeset: &mut AddressMap<Account>,
     tx: &BasicTxDetails,
     call_result: &RawCallResult<FEN>,

crates/evm/evm/src/executors/invariant/result.rs

@@ -51,20 +51,6 @@ pub struct InvariantFuzzTestResult {
     pub optimization_best_sequence: Vec<BasicTxDetails>,
 }
 
-/// Enriched results of an invariant run check.
-///
-/// Contains the success condition and call results of the last run
-pub(crate) struct RichInvariantResults<FEN: FoundryEvmNetwork> {
-    pub(crate) can_continue: bool,
-    pub(crate) call_result: Option<RawCallResult<FEN>>,
-}
-
-impl<FEN: FoundryEvmNetwork> RichInvariantResults<FEN> {
-    pub(crate) const fn new(can_continue: bool, call_result: Option<RawCallResult<FEN>>) -> Self {
-        Self { can_continue, call_result }
-    }
-}
-
 /// Given the executor state, asserts that no invariant has been broken. Otherwise, it fills the
 /// external `invariant_failures.failed_invariant` map and returns a generic error.
 /// Either returns the call result if successful, or nothing if there was an error.
@@ -223,13 +209,12 @@ fn invariant_inner_sequence<FEN: FoundryEvmNetwork>(executor: &Executor<FEN>) ->
 /// For check mode, asserts the invariant and fails if broken.
 pub(crate) fn can_continue<FEN: FoundryEvmNetwork>(
     invariant_contract: &InvariantContract<'_>,
-    invariant_test: &mut InvariantTest<FEN>,
+    invariant_test: &mut InvariantTest,
     invariant_run: &mut InvariantTestRun<FEN>,
     invariant_config: &InvariantConfig,
     call_result: RawCallResult<FEN>,
     state_changeset: &StateChangeset,
-) -> Result<RichInvariantResults<FEN>> {
-    let mut call_results = None;
+) -> Result<bool> {
     let is_optimization = invariant_contract.is_optimization();
 
     let handlers_succeeded = || {
@@ -243,8 +228,6 @@ pub(crate) fn can_continue<FEN: FoundryEvmNetwork>(
         })
     };
 
-    let failures = &mut invariant_test.test_data.failures;
-
     if !call_result.reverted && handlers_succeeded() {
         if let Some(traces) = call_result.traces {
             invariant_run.run_traces.push(traces);
@@ -269,26 +252,25 @@ pub(crate) fn can_continue<FEN: FoundryEvmNetwork>(
                     invariant_run.optimization_prefix_len = invariant_run.inputs.len();
                 }
             }
-            call_results = Some(inv_result);
         } else {
             // Check mode: assert invariants and fail if broken.
-            call_results = assert_invariants(
+            let call_results = assert_invariants(
                 invariant_contract,
                 invariant_config,
                 &invariant_test.targeted_contracts,
                 &invariant_run.executor,
                 &invariant_run.inputs,
-                failures,
+                &mut invariant_test.test_data.failures,
             )?;
             if call_results.is_none() {
-                return Ok(RichInvariantResults::new(false, None));
+                return Ok(false);
             }
         }
     } else {
         let is_assert_failure = did_fail_on_assert(&call_result, state_changeset);
 
         if call_result.reverted {
-            failures.reverts += 1;
+            invariant_test.test_data.failures.reverts += 1;
         }
 
         if is_assert_failure || (call_result.reverted && invariant_config.fail_on_revert) {
@@ -302,8 +284,9 @@ pub(crate) fn can_continue<FEN: FoundryEvmNetwork>(
                 &[],
             )
             .with_assertion_failure(is_assert_failure);
-            failures.revert_reason = Some(case_data.revert_reason.clone());
-            failures.record_failure(
+            invariant_test.test_data.failures.revert_reason =
+                Some(case_data.revert_reason.clone());
+            invariant_test.test_data.failures.record_failure(
                 invariant_contract.invariant_fn,
                 if is_assert_failure {
                     InvariantFuzzError::BrokenInvariant(case_data)
@@ -312,7 +295,7 @@ pub(crate) fn can_continue<FEN: FoundryEvmNetwork>(
                 },
             );
 
-            return Ok(RichInvariantResults::new(false, None));
+            return Ok(false);
         } else if call_result.reverted && !is_optimization && !invariant_config.has_delay() {
             // If we don't fail test on revert then remove the reverted call from inputs.
             // Delay-enabled campaigns keep reverted calls so shrinking can preserve their
@@ -321,17 +304,14 @@ pub(crate) fn can_continue<FEN: FoundryEvmNetwork>(
         }
     }
 
-    Ok(RichInvariantResults::new(
-        failures.can_continue(invariant_contract.invariant_fns.len()),
-        call_results,
-    ))
+    Ok(invariant_test.test_data.failures.can_continue(invariant_contract.invariant_fns.len()))
 }
 
 /// Given the executor state, asserts conditions within `afterInvariant` function.
 /// If call fails then the invariant test is considered failed.
 pub(crate) fn assert_after_invariant<FEN: FoundryEvmNetwork>(
     invariant_contract: &InvariantContract<'_>,
-    invariant_test: &mut InvariantTest<FEN>,
+    invariant_test: &mut InvariantTest,
     invariant_run: &InvariantTestRun<FEN>,
     invariant_config: &InvariantConfig,
 ) -> Result<bool> {

crates/evm/evm/src/executors/invariant/shrink.rs

@@ -388,7 +388,7 @@ pub(crate) fn shrink_sequence_value<FEN: FoundryEvmNetwork>(
     reset_shrink_progress(config, progress);
 
     let target_address = invariant_contract.address;
-    let calldata: Bytes = invariant_contract.invariant_function.selector().to_vec().into();
+    let calldata: Bytes = invariant_contract.invariant_fn.selector().to_vec().into();
 
     // Special case: check if target value is achieved with 0 calls.
     if check_sequence_value(executor.clone(), calls, vec![], target_address, calldata.clone())?

crates/forge/src/result.rs

@@ -421,6 +421,9 @@ pub struct TestResult {
     /// still be successful (i.e self.success == true) when it's expected to fail.
     pub reason: Option<String>,
 
+    /// This field will be populated if there are additional invariant broken besides the main one.
+    pub other_failures: Vec<String>,
+
     /// Minimal reproduction test case for failing test
     pub counterexample: Option<CounterExample>,
 
@@ -527,6 +530,12 @@ impl fmt::Display for TestResult {
                 } else {
                     s.push(']');
                 }
+                if !self.other_failures.is_empty() {
+                    writeln!(s).unwrap();
+                    for failure in &self.other_failures {
+                        writeln!(s, "{failure}").unwrap();
+                    }
+                }
                 s.red().wrap().fmt(f)
             }
         }
@@ -730,6 +739,7 @@ impl TestResult {
         gas_report_traces: Vec<Vec<CallTraceArena>>,
         success: bool,
         reason: Option<String>,
+        other_failures: Vec<String>,
         counterexample: Option<CounterExample>,
         cases: Vec<FuzzedCases>,
         reverts: usize,
@@ -752,6 +762,7 @@ impl TestResult {
             TestStatus::Failure
         };
         self.reason = reason;
+        self.other_failures = other_failures;
         self.counterexample = counterexample;
         self.gas_report_traces = gas_report_traces;
     }

crates/forge/src/runner.rs

@@ -781,11 +781,19 @@ impl<'a, FEN: FoundryEvmNetwork> FunctionRunner<'a, FEN> {
             identified_contracts,
             &self.cr.mcr.known_contracts,
         );
+        // Filter out additional invariants to test if we already have a persisted failure.
         let invariant_contract = InvariantContract::new(
             self.address,
             self.cr.name,
             func,
-            invariants,
+            invariants
+                .into_iter()
+                .filter(|(invariant_fn, _)| {
+                    *invariant_fn == func
+                        || (invariant_config.continuous_run
+                            && !canonicalized(failure_dir.join(invariant_fn.name.clone())).exists())
+                })
+                .collect(),
             call_after_invariant,
             &self.cr.contract.abi,
         );
@@ -922,6 +930,7 @@ impl<'a, FEN: FoundryEvmNetwork> FunctionRunner<'a, FEN> {
             .errors
             .get(&invariant_contract.invariant_fn.name)
             .and_then(|err| err.revert_reason());
+        let mut other_failures = vec![];
 
         if success {
             if let Some(best_value) = invariant_result.optimization_best_value {
@@ -1031,11 +1040,20 @@ impl<'a, FEN: FoundryEvmNetwork> FunctionRunner<'a, FEN> {
                     continue;
                 }
 
-                if let Some(error) = invariant_result.errors.get(&invariant.name)
+                // Generate counterexamples for broken invariant, if there is no failure persisted
+                // already.
+                let persisted_failure = canonicalized(failure_dir.join(invariant.name.clone()));
+                if !persisted_failure.exists()
+                    && let Some(error) = invariant_result.errors.get(&invariant.name)
                     && let InvariantFuzzError::BrokenInvariant(case_data)
                     | InvariantFuzzError::Revert(case_data) = error
                     && let TestError::Fail(_, ref calls) = case_data.test_error
                 {
+                    other_failures.push(format!(
+                        "{}: {}",
+                        invariant.name,
+                        error.revert_reason().unwrap_or_default()
+                    ));
                     match generate_counterexample(
                         self.clone_executor(),
                         &self.cr.mcr.known_contracts,
@@ -1046,7 +1064,7 @@ impl<'a, FEN: FoundryEvmNetwork> FunctionRunner<'a, FEN> {
                         Ok(call_sequence) => {
                             record_invariant_failure(
                                 failure_dir.as_path(),
-                                canonicalized(failure_dir.join(invariant.name.clone())).as_path(),
+                                persisted_failure.as_path(),
                                 &call_sequence,
                                 &current_settings,
                                 false,
@@ -1064,6 +1082,7 @@ impl<'a, FEN: FoundryEvmNetwork> FunctionRunner<'a, FEN> {
             invariant_result.gas_report_traces,
             success,
             reason,
+            other_failures,
             counterexample,
             invariant_result.cases,
             invariant_result.reverts,