allow foundry unit test generation to handle assertion failure with events

Author: gustavo-grieco

lib/Echidna/Output/Foundry.hs

@@ -6,7 +6,7 @@ module Echidna.Output.Foundry (foundryTest) where
 
 import Data.Aeson (Value(..), object, (.=))
 import Data.Functor ((<&>))
-import Data.List (elemIndex, nub)
+import Data.List (elemIndex, isPrefixOf, nub)
 import Data.Maybe (fromMaybe, mapMaybe)
 import Data.Text (Text, unpack)
 import Data.Text.Lazy (fromStrict)
@@ -33,33 +33,67 @@ foundryTest :: Maybe Text -> Addr -> EchidnaTest -> TL.Text
 foundryTest mContractName psender test =
   case test.testType of
     AssertionTest{} ->
-      let testData = createTestData mContractName Nothing test
+      let testData = createTestData mContractName Nothing Nothing test
       in fromStrict $ substituteValue template (toMustache testData)
     PropertyTest name _ ->
-      let testData = createTestData mContractName (Just (name, psender)) test
+      let testData = createTestData mContractName (Just (name, psender)) Nothing test
+      in fromStrict $ substituteValue template (toMustache testData)
+    CallTest name _ | "AssertionFailed" `isPrefixOf` unpack name ->
+      -- Echidna detects assertion failures via events named AssertionFailed
+      -- with any argument types (see checkAssertionEvent in Echidna.Test).
+      -- We check all overloads defined in crytic's fuzzlib (LibLog.sol):
+      --   AssertionFailed()
+      --   AssertionFailed(string)
+      --   AssertionFailed(string,string)
+      --   AssertionFailed(string,bytes)
+      --   AssertionFailed(string,uint256)
+      --   AssertionFailed(string,int256)
+      --   AssertionFailed(string,address)
+      --   AssertionFailed(string,bool)
+      --   AssertionFailed(string,bytes32)
+      let eventAssert = Just $
+            "        // Check that an AssertionFailed event was emitted\n"
+            ++ "        Vm.Log[] memory entries = vm.getRecordedLogs();\n"
+            ++ "        bool found = false;\n"
+            ++ "        for (uint i = 0; i < entries.length; i++) {\n"
+            ++ "            if (entries[i].topics.length > 0 && _isAssertionFailed(entries[i].topics[0])) {\n"
+            ++ "                found = true;\n"
+            ++ "                break;\n"
+            ++ "            }\n"
+            ++ "        }\n"
+            ++ "        assertTrue(found, \"Expected AssertionFailed event\");"
+          testData = createTestData mContractName Nothing eventAssert test
       in fromStrict $ substituteValue template (toMustache testData)
     _ -> ""
 
 -- | Create an Aeson Value from test data for the Mustache template.
 -- When a property name and psender are provided, a final assertion is added
 -- to call the property from psender and check it returns false.
-createTestData :: Maybe Text -> Maybe (Text, Addr) -> EchidnaTest -> Value
-createTestData mContractName mProperty test =
+-- When an event assertion is provided, vm.recordLogs() is added at the start
+-- and the event check is added at the end.
+createTestData :: Maybe Text -> Maybe (Text, Addr) -> Maybe String -> EchidnaTest -> Value
+createTestData mContractName mProperty mEventAssert test =
   let
     senders = nub $ map (.src) test.reproducer
     actors = zipWith actorObject senders [1..]
     repro = mapMaybe (foundryTx senders) test.reproducer
     cName = fromMaybe "YourContract" mContractName
-    propAssertion = mProperty <&> \(name, addr) ->
-      "        vm.stopPrank();\n        vm.prank(" ++ formatAddr addr ++ ");\n"
-      ++ "        assertFalse(Target." ++ unpack name ++ "());"
+    propAssertion = case mProperty of
+      Just (name, addr) -> Just $
+        "        vm.stopPrank();\n        vm.prank(" ++ formatAddr addr ++ ");\n"
+        ++ "        assertFalse(Target." ++ unpack name ++ "());"
+      Nothing -> mEventAssert
+    preamble = case mEventAssert of
+      Just _ -> Just ("        vm.recordLogs();" :: String)
+      Nothing -> Nothing
   in
   object
     [ "testName"     .= ("FoundryTest" :: Text)
     , "contractName" .= cName
     , "actors"       .= actors
     , "reproducer"   .= repro
     , "propertyAssertion" .= propAssertion
+    , "preamble"     .= preamble
     ]
 
 -- | Create a JSON object for an actor.

lib/Echidna/Output/assets/foundry.mustache

@@ -17,6 +17,9 @@ contract {{testName}} is Test {
   }
 
   function test_replay() public {
+{{#preamble}}
+{{{.}}}
+{{/preamble}}
 {{#reproducer}}
     {{{prelude}}}
     {{{call}}}
@@ -35,4 +38,20 @@ contract {{testName}} is Test {
       vm.warp(block.timestamp + timeInSeconds);
       vm.roll(block.number + numBlocks);
   }
+
+{{#preamble}}
+  /// @dev Checks if a topic matches any known AssertionFailed event signature.
+  /// Covers all overloads from crytic/fuzzlib (LibLog.sol).
+  function _isAssertionFailed(bytes32 t) internal pure returns (bool) {
+      return t == keccak256("AssertionFailed()")
+          || t == keccak256("AssertionFailed(string)")
+          || t == keccak256("AssertionFailed(string,string)")
+          || t == keccak256("AssertionFailed(string,bytes)")
+          || t == keccak256("AssertionFailed(string,uint256)")
+          || t == keccak256("AssertionFailed(string,int256)")
+          || t == keccak256("AssertionFailed(string,address)")
+          || t == keccak256("AssertionFailed(string,bool)")
+          || t == keccak256("AssertionFailed(string,bytes32)");
+  }
+{{/preamble}}
 }

src/test/Tests/FoundryTestGen.hs

@@ -31,6 +31,7 @@ foundryTestGenTests = testGroup "Foundry test generation"
   , testCase "fallback function syntax" testFallbackSyntax
   , testCase "null bytes in arguments" testNullBytes
   , testCase "property test generates assertFalse" testPropertyTestGen
+  , testCase "event assertion generates recordLogs" testEventAssertionTestGen
   , testGroup "Concrete execution (fuzzing)"
       [ testForgeStd "solves assertTrue"
           "foundry/FoundryAsserts.sol"
@@ -396,6 +397,42 @@ solcSupportsForgeStd = unsafePerformIO $ do
       (a, _:b) -> a : splitOn c b
       (a, [])  -> [a]
 
+-- | Test that event-based assertion failures (CallTest "AssertionFailed(..)")
+-- generate Foundry reproducers with vm.recordLogs() and event checks.
+testEventAssertionTestGen :: IO ()
+testEventAssertionTestGen = do
+  let
+    reproducerTx = Tx
+      { call = SolCall ("inc", [])
+      , src = 0x10000
+      , dst = 0
+      , value = 0
+      , gas = 0
+      , gasprice = 0
+      , delay = (0, 0)
+      }
+    test = mkMinimalTest
+      { testType = CallTest "AssertionFailed(..)" (\_ _ -> BoolValue True)
+      , reproducer = [reproducerTx, reproducerTx, reproducerTx, reproducerTx]
+      }
+    generated = TL.unpack $ foundryTest (Just "EventAssertion") defaultPsender test
+  assertBool ("should not be empty, got: " ++ generated)
+    (not $ null generated)
+  assertBool ("should contain vm.recordLogs(), got: " ++ generated)
+    ("vm.recordLogs()" `isInfixOf` generated)
+  assertBool ("should contain vm.getRecordedLogs(), got: " ++ generated)
+    ("vm.getRecordedLogs()" `isInfixOf` generated)
+  assertBool ("should contain _isAssertionFailed helper call, got: " ++ generated)
+    ("_isAssertionFailed(" `isInfixOf` generated)
+  assertBool ("should contain _isAssertionFailed function definition, got: " ++ generated)
+    ("function _isAssertionFailed" `isInfixOf` generated)
+  assertBool ("should check all fuzzlib overloads, got: " ++ generated)
+    ("AssertionFailed(string,uint256)" `isInfixOf` generated)
+  assertBool ("should contain assertTrue, got: " ++ generated)
+    ("assertTrue(found" `isInfixOf` generated)
+  assertBool ("should contain inc() call, got: " ++ generated)
+    ("Target.inc()" `isInfixOf` generated)
+
 mkMinimalTest :: EchidnaTest
 mkMinimalTest = EchidnaTest
   -- Foundry tests are only generated for solved/large tests.

tests/solidity/foundry/EventAssertion.sol

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: MIT
+pragma solidity ^0.8.0;
+
+// Test that event-based assertion failures generate Foundry reproducers.
+// In assertion mode, emitting AssertionFailed() is detected as a failure.
+contract EventAssertion {
+    event AssertionFailed();
+
+    uint256 public counter;
+
+    function inc() external {
+        counter++;
+        if (counter > 3) {
+            emit AssertionFailed();
+        }
+    }
+
+    function dummy() external {}
+}

tests/solidity/foundry/EventAssertion.yaml

@@ -0,0 +1,2 @@
+testMode: assertion
+seed: 1234