fix(fixtures,specs): Fix Engine X Env->Genesis Conversion (#1914)

* fix(specs,fixtures): Pre-alloc groups on tests that modify the pre-allocation

* fix(specs): state tests and blockchain tests with default env have same pre-alloc group

* fix(tests): Remove some unnecessary custom envs

* refactor(specs): Genesis header creation

* fix(fixtures): Use defaults in FixtureHeader for genesis generation

Author: marioevz

src/ethereum_test_fixtures/blockchain.py

@@ -119,25 +119,27 @@ class FixtureHeader(CamelModel):
     We combine the `Environment` and `Result` contents to create this model.
     """
 
-    parent_hash: Hash
+    parent_hash: Hash = Hash(0)
     ommers_hash: Hash = Field(Hash(EmptyOmmersRoot), alias="uncleHash")
     fee_recipient: Address = Field(
         ..., alias="coinbase", validation_alias=AliasChoices("coinbase", "miner")
     )
     state_root: Hash
     transactions_trie: Hash = Field(
-        validation_alias=AliasChoices("transactionsTrie", "transactionsRoot")
+        Hash(EmptyTrieRoot), validation_alias=AliasChoices("transactionsTrie", "transactionsRoot")
     )
     receipts_root: Hash = Field(
-        ..., alias="receiptTrie", validation_alias=AliasChoices("receiptTrie", "receiptsRoot")
+        Hash(EmptyTrieRoot),
+        alias="receiptTrie",
+        validation_alias=AliasChoices("receiptTrie", "receiptsRoot"),
     )
     logs_bloom: Bloom = Field(
-        ..., alias="bloom", validation_alias=AliasChoices("bloom", "logsBloom")
+        Bloom(0), alias="bloom", validation_alias=AliasChoices("bloom", "logsBloom")
     )
     difficulty: ZeroPaddedHexNumber = ZeroPaddedHexNumber(0)
     number: ZeroPaddedHexNumber
     gas_limit: ZeroPaddedHexNumber
-    gas_used: ZeroPaddedHexNumber
+    gas_used: ZeroPaddedHexNumber = ZeroPaddedHexNumber(0)
     timestamp: ZeroPaddedHexNumber
     extra_data: Bytes
     prev_randao: Hash = Field(Hash(0), alias="mixHash")
@@ -219,32 +221,16 @@ def block_hash(self) -> Hash:
     @classmethod
     def genesis(cls, fork: Fork, env: Environment, state_root: Hash) -> "FixtureHeader":
         """Get the genesis header for the given fork."""
-        return FixtureHeader(
-            parent_hash=0,
-            ommers_hash=EmptyOmmersRoot,
-            fee_recipient=0,
-            state_root=state_root,
-            transactions_trie=EmptyTrieRoot,
-            receipts_root=EmptyTrieRoot,
-            logs_bloom=0,
-            difficulty=0x20000 if env.difficulty is None else env.difficulty,
-            number=0,
-            gas_limit=env.gas_limit,
-            gas_used=0,
-            timestamp=0,
-            extra_data=b"\x00",
-            prev_randao=0,
-            nonce=0,
-            base_fee_per_gas=env.base_fee_per_gas,
-            blob_gas_used=env.blob_gas_used,
-            excess_blob_gas=env.excess_blob_gas,
-            withdrawals_root=(
-                Withdrawal.list_root(env.withdrawals) if env.withdrawals is not None else None
-            ),
-            parent_beacon_block_root=env.parent_beacon_block_root,
-            requests_hash=Requests() if fork.header_requests_required(0, 0) else None,
-            fork=fork,
-        )
+        environment_values = env.model_dump(exclude_none=True, exclude={"withdrawals"})
+        if env.withdrawals is not None:
+            environment_values["withdrawals_root"] = Withdrawal.list_root(env.withdrawals)
+        environment_values["extra_data"] = env.extra_data
+        extras = {
+            "state_root": state_root,
+            "requests_hash": Requests() if fork.header_requests_required(0, 0) else None,
+            "fork": fork,
+        }
+        return FixtureHeader(**environment_values, **extras)
 
 
 class FixtureExecutionPayload(CamelModel):

src/ethereum_test_fixtures/pre_alloc_groups.py

@@ -23,28 +23,30 @@ class PreAllocGroup(CamelModel):
 
     model_config = {"populate_by_name": True}  # Allow both field names and aliases
 
-    test_count: int = Field(0, description="Number of tests in this group")
-    pre_account_count: int = Field(0, description="Number of accounts in the pre-allocation")
-    test_ids: List[str] = Field(default_factory=list, alias="testIds")
+    test_ids: List[str] = Field(default_factory=list)
     environment: Environment = Field(..., description="Grouping environment for this test group")
     fork: Fork = Field(..., alias="network")
     pre: Alloc
 
-    def model_post_init(self, __context):
-        """Post-init hook to ensure pre is not None."""
-        super().model_post_init(__context)
+    @computed_field(description="Number of accounts in the pre-allocation")  # type: ignore[prop-decorator]
+    @property
+    def pre_account_count(self) -> int:
+        """Return the amount of accounts the pre-allocation group holds."""
+        return len(self.pre.root)
 
-        self.pre = Alloc.merge(
-            Alloc.model_validate(self.fork.pre_allocation_blockchain()),
-            self.pre,
-        )
+    @computed_field(description="Number of tests in this group")  # type: ignore[prop-decorator]
+    @property
+    def test_count(self) -> int:
+        """Return the amount of tests that use this pre-allocation group."""
+        return len(self.test_ids)
 
-    @computed_field  # type: ignore[misc]
+    @computed_field  # type: ignore[prop-decorator]
+    @property
     def genesis(self) -> FixtureHeader:
         """Get the genesis header for this group."""
         return FixtureHeader.genesis(
             self.fork,
-            self.environment.set_fork_requirements(self.fork),
+            self.environment,
             self.pre.state_root(),
         )
 
@@ -58,8 +60,6 @@ def to_file(self, file: Path) -> None:
                     for account in previous_pre_alloc_group.pre:
                         if account not in self.pre:
                             self.pre[account] = previous_pre_alloc_group.pre[account]
-                    self.pre_account_count += previous_pre_alloc_group.pre_account_count
-                    self.test_count += previous_pre_alloc_group.test_count
                     self.test_ids.extend(previous_pre_alloc_group.test_ids)
 
             with open(file, "w") as f:

src/ethereum_test_specs/base.py

@@ -237,18 +237,19 @@ def update_pre_alloc_groups(
             )
             group.fork = fork
             group.test_ids.append(str(test_id))
-            group.test_count = len(group.test_ids)
-            group.pre_account_count = len(group.pre.root)
             pre_alloc_groups[pre_alloc_hash] = group
         else:
             # Create new group - use Environment instead of expensive genesis generation
+            genesis_env = self.get_genesis_environment(fork)
+            pre_alloc = Alloc.merge(
+                Alloc.model_validate(fork.pre_allocation_blockchain()),
+                self.pre,
+            )
             group = PreAllocGroup(
-                test_count=1,
-                pre_account_count=len(self.pre.root),
                 test_ids=[str(test_id)],
                 fork=fork,
-                environment=self.get_genesis_environment(fork),
-                pre=self.pre,
+                environment=genesis_env,
+                pre=pre_alloc,
             )
             pre_alloc_groups[pre_alloc_hash] = group
         return pre_alloc_groups

src/ethereum_test_specs/blockchain.py

@@ -385,6 +385,18 @@ def verify_block_exception(self, transition_tool_exceptions_reliable: bool):
         )
 
 
+GENESIS_ENVIRONMENT_DEFAULTS: Dict[str, Any] = {
+    "fee_recipient": 0,
+    "number": 0,
+    "timestamp": 0,
+    "extra_data": b"\x00",
+    "prev_randao": 0,
+}
+"""
+Default values for the genesis environment that are used to create all genesis headers.
+"""
+
+
 class BlockchainTest(BaseTest):
     """Filler type that tests multiple blocks (valid or invalid) in a chain."""
 
@@ -432,25 +444,31 @@ def discard_fixture_format_by_marks(
             return fixture_format != BlockchainEngineFixture
         return False
 
-    @staticmethod
+    def get_genesis_environment(self, fork: Fork) -> Environment:
+        """Get the genesis environment for pre-allocation groups."""
+        modified_values = self.genesis_environment.set_fork_requirements(fork).model_dump(
+            exclude_unset=True
+        )
+        return Environment(**(GENESIS_ENVIRONMENT_DEFAULTS | modified_values))
+
     def make_genesis(
-        genesis_environment: Environment,
-        pre: Alloc,
-        fork: Fork,
+        self, *, fork: Fork, apply_pre_allocation_blockchain: bool
     ) -> Tuple[Alloc, FixtureBlock]:
         """Create a genesis block from the blockchain test definition."""
-        env = genesis_environment.set_fork_requirements(fork)
+        env = self.get_genesis_environment(fork)
         assert env.withdrawals is None or len(env.withdrawals) == 0, (
             "withdrawals must be empty at genesis"
         )
         assert env.parent_beacon_block_root is None or env.parent_beacon_block_root == Hash(0), (
             "parent_beacon_block_root must be empty at genesis"
         )
 
-        pre_alloc = Alloc.merge(
-            Alloc.model_validate(fork.pre_allocation_blockchain()),
-            pre,
-        )
+        pre_alloc = self.pre
+        if apply_pre_allocation_blockchain:
+            pre_alloc = Alloc.merge(
+                Alloc.model_validate(fork.pre_allocation_blockchain()),
+                pre_alloc,
+            )
         if empty_accounts := pre_alloc.empty_accounts():
             raise Exception(f"Empty accounts in pre state: {empty_accounts}")
         state_root = pre_alloc.state_root()
@@ -635,7 +653,7 @@ def make_fixture(
         """Create a fixture from the blockchain test definition."""
         fixture_blocks: List[FixtureBlock | InvalidFixtureBlock] = []
 
-        pre, genesis = BlockchainTest.make_genesis(self.genesis_environment, self.pre, fork)
+        pre, genesis = self.make_genesis(fork=fork, apply_pre_allocation_blockchain=True)
 
         alloc = pre
         env = environment_from_parent_header(genesis.header)
@@ -692,7 +710,10 @@ def make_hive_fixture(
         """Create a hive fixture from the blocktest definition."""
         fixture_payloads: List[FixtureEngineNewPayload] = []
 
-        pre, genesis = BlockchainTest.make_genesis(self.genesis_environment, self.pre, fork)
+        pre, genesis = self.make_genesis(
+            fork=fork,
+            apply_pre_allocation_blockchain=fixture_format != BlockchainEngineXFixture,
+        )
         alloc = pre
         env = environment_from_parent_header(genesis.header)
         head_hash = genesis.header.block_hash
@@ -787,10 +808,6 @@ def make_hive_fixture(
             )
             return BlockchainEngineFixture(**fixture_data)
 
-    def get_genesis_environment(self, fork: Fork) -> Environment:
-        """Get the genesis environment for pre-allocation groups."""
-        return self.genesis_environment
-
     def generate(
         self,
         t8n: TransitionTool,
@@ -799,9 +816,7 @@ def generate(
     ) -> BaseFixture:
         """Generate the BlockchainTest fixture."""
         t8n.reset_traces()
-        if fixture_format == BlockchainEngineFixture:
-            return self.make_hive_fixture(t8n, fork, fixture_format)
-        elif fixture_format == BlockchainEngineXFixture:
+        if fixture_format in [BlockchainEngineFixture, BlockchainEngineXFixture]:
             return self.make_hive_fixture(t8n, fork, fixture_format)
         elif fixture_format == BlockchainFixture:
             return self.make_fixture(t8n, fork)

src/ethereum_test_specs/state.py

@@ -242,7 +242,7 @@ def make_state_test_fixture(
 
     def get_genesis_environment(self, fork: Fork) -> Environment:
         """Get the genesis environment for pre-allocation groups."""
-        return self._generate_blockchain_genesis_environment(fork=fork)
+        return self.generate_blockchain_test(fork=fork).get_genesis_environment(fork=fork)
 
     def generate(
         self,

src/ethereum_test_specs/tests/test_fixtures.py

@@ -470,7 +470,7 @@ def genesis_environment(self):  # noqa: D102
         return Environment(
             gas_limit=100_000_000_000_000_000,
             base_fee_per_gas=1000,
-            fee_recipient="0xba5e000000000000000000000000000000000000",
+            fee_recipient="0x0000000000000000000000000000000000000000",
         )
 
     @pytest.fixture
@@ -867,7 +867,7 @@ def test_fill_blockchain_invalid_txs(
     genesis_environment = Environment(
         gas_limit=100_000_000_000_000_000,
         base_fee_per_gas=1000,
-        fee_recipient="0xba5e000000000000000000000000000000000000",
+        fee_recipient="0x0000000000000000000000000000000000000000",
     )
 
     fixture_format: FixtureFormat = (