Suggestions for: feat(consume): add ExceptionMapper support to the consume-engine simulator (#1464)

* refactor(consume): cache exception mappers in session-scoped fixture

* refactor(consume): make string match more robust by applying `lower()`

* feat(consume): allow disabling of strict exception checking per fork

* refactor(consume/hive): create `strict_exception_matching` fixture

---------

Co-authored-by: Mario Vega <[email protected]>

Author: danceratopz

Diff for: src/pytest_plugins/consume/hive_simulators/conftest.py

@@ -48,7 +48,7 @@ def pytest_addoption(parser):
         dest="disable_strict_exception_matching",
         default="",
         help=(
-            "Comma-separated list of the names of clients which should NOT use strict "
+            "Comma-separated list of client names and/or forks which should NOT use strict "
             "exception matching."
         ),
     )
@@ -208,20 +208,31 @@ def buffered_genesis(client_genesis: dict) -> io.BufferedReader:
     return io.BufferedReader(cast(io.RawIOBase, io.BytesIO(genesis_bytes)))
 
 
+@pytest.fixture(scope="session")
+def client_exception_mapper_cache():
+    """Cache for exception mappers by client type."""
+    return {}
+
+
 @pytest.fixture(scope="function")
 def client_exception_mapper(
-    client_type: ClientType,
+    client_type: ClientType, client_exception_mapper_cache
 ) -> ExceptionMapper | None:
-    """Return the exception mapper for the client type."""
-    for client in EXCEPTION_MAPPERS:
-        if client in client_type.name:
-            return EXCEPTION_MAPPERS[client]
-    return None
+    """Return the exception mapper for the client type, with caching."""
+    if client_type.name not in client_exception_mapper_cache:
+        for client in EXCEPTION_MAPPERS:
+            if client in client_type.name:
+                client_exception_mapper_cache[client_type.name] = EXCEPTION_MAPPERS[client]
+                break
+        else:
+            client_exception_mapper_cache[client_type.name] = None
+
+    return client_exception_mapper_cache[client_type.name]
 
 
 @pytest.fixture(scope="session")
 def disable_strict_exception_matching(request: pytest.FixtureRequest) -> List[str]:
-    """Return the list of clients that should NOT use strict exception matching."""
+    """Return the list of clients or forks that should NOT use strict exception matching."""
     config_string = request.config.getoption("disable_strict_exception_matching")
     return config_string.split(",") if config_string else []
 
@@ -232,7 +243,30 @@ def client_strict_exception_matching(
     disable_strict_exception_matching: List[str],
 ) -> bool:
     """Return True if the client type should use strict exception matching."""
-    return not any(client in client_type.name for client in disable_strict_exception_matching)
+    return not any(
+        client.lower() in client_type.name.lower() for client in disable_strict_exception_matching
+    )
+
+
+@pytest.fixture(scope="function")
+def fork_strict_exception_matching(
+    fixture: BlockchainFixtureCommon,
+    disable_strict_exception_matching: List[str],
+) -> bool:
+    """Return True if the fork should use strict exception matching."""
+    # NOTE: `in` makes it easier for transition forks ("Prague" in "CancunToPragueAtTime15k")
+    return not any(
+        fork.lower() in fixture.fork.lower() for fork in disable_strict_exception_matching
+    )
+
+
+@pytest.fixture(scope="function")
+def strict_exception_matching(
+    client_strict_exception_matching: bool,
+    fork_strict_exception_matching: bool,
+) -> bool:
+    """Return True if the test should use strict exception matching."""
+    return client_strict_exception_matching and fork_strict_exception_matching
 
 
 @pytest.fixture(scope="function")

Diff for: src/pytest_plugins/consume/hive_simulators/engine/test_via_engine.py

@@ -22,7 +22,7 @@ def test_blockchain_via_engine(
     eth_rpc: EthRPC,
     engine_rpc: EngineRPC,
     fixture: BlockchainEngineFixture,
-    client_strict_exception_matching: bool,
+    strict_exception_matching: bool,
 ):
     """
     1. Check the client genesis block hash matches `fixture.genesis.block_hash`.
@@ -100,7 +100,7 @@ def test_blockchain_via_engine(
                                     f'returned exception: "{payload_response.validation_error}" '
                                     f'(mapper: "{payload_response.validation_error.mapper_name}")'
                                 )
-                                if client_strict_exception_matching:
+                                if strict_exception_matching:
                                     logger.fail(message)
                                     raise Exception(message)
                                 else:
@@ -115,11 +115,12 @@ def test_blockchain_via_engine(
                                         f'got: "{payload_response.validation_error}" '
                                         f'expected: "{payload.validation_error}"'
                                     )
-                                    if client_strict_exception_matching:
+                                    if strict_exception_matching:
                                         logger.fail(message)
                                         raise Exception(message)
                                     else:
                                         logger.warning(message)
+
                     except JSONRPCError as e:
                         logger.info(f"JSONRPC error encountered: {e.code} - {e.message}")
                         if payload.error_code is None: