fix(consume): fix the test re-run hive commands displayed in hive ui (#1494)

* fix(consume): fix the test re-run commands displayed in hive ui

* docs: update changelog

* fix(consume): only show the client relevant to test in hive command

* chore(consume): add `ClientFile` model; serialize to YAML

* fix(consume): add forgotten client file to hive command-line

* chore(consume): remove debugging log message

* fix(consume): add forgotten pytest fixture to function sig

---------

Co-authored-by: Mario Vega <[email protected]>

Author: danceratopz

docs/CHANGELOG.md

@@ -26,6 +26,7 @@ Users can expect that all tests currently living in [ethereum/tests](https://git
 
 #### `consume`
 
+- 🐞 Fix the Hive commands used to reproduce test executions that are displayed in test descriptions in the Hive UI ([#1494](https://github.com/ethereum/execution-spec-tests/pull/1494)).
 - 🐞 Fix consume direct fails for geth blockchain tests ([#1502](https://github.com/ethereum/execution-spec-tests/pull/1502)).
 
 ### 📋 Misc

src/pytest_plugins/consume/hive_simulators/conftest.py

@@ -3,6 +3,9 @@
 import io
 import json
 import logging
+import textwrap
+import urllib
+import warnings
 from pathlib import Path
 from typing import Dict, Generator, List, Literal, cast
 
@@ -22,7 +25,7 @@
 from ethereum_test_rpc import EthRPC
 from pytest_plugins.consume.consume import FixturesSource
 from pytest_plugins.consume.hive_simulators.ruleset import ruleset  # TODO: generate dynamically
-from pytest_plugins.pytest_hive.hive_info import ClientInfo
+from pytest_plugins.pytest_hive.hive_info import ClientFile, HiveInfo
 
 from .exceptions import EXCEPTION_MAPPERS
 from .timing import TimingData
@@ -61,91 +64,161 @@ def eth_rpc(client: Client) -> EthRPC:
 
 
 @pytest.fixture(scope="function")
-def hive_client_config_file_parameter(
-    client_type: ClientType, client_file: List[ClientInfo]
-) -> List[str]:
-    """Return the hive client config file that is currently being used to configure tests."""
-    for client in client_file:
-        if client_type.name.startswith(client.client):
-            return ["--client-file", f"<('{client.model_dump_json(exclude_none=True)}')"]
-    return []
+def hive_clients_yaml_target_filename() -> str:
+    """Return the name of the target clients YAML file."""
+    return "clients_eest.yaml"
 
 
 @pytest.fixture(scope="function")
-def hive_consume_command(
-    test_suite_name: str,
+def hive_clients_yaml_generator_command(
     client_type: ClientType,
+    client_file: ClientFile,
+    hive_clients_yaml_target_filename: str,
+    hive_info: HiveInfo,
+) -> str:
+    """Generate a shell command that creates a clients YAML file for the current client."""
+    try:
+        if not client_file:
+            raise ValueError("No client information available - try updating hive")
+        client_config = [c for c in client_file.root if c.client in client_type.name]
+        if not client_config:
+            raise ValueError(f"Client '{client_type.name}' not found in client file")
+        try:
+            yaml_content = ClientFile(root=[client_config[0]]).yaml().replace(" ", "&nbsp;")
+            return f'echo "\\\n{yaml_content}" > {hive_clients_yaml_target_filename}'
+        except Exception as e:
+            raise ValueError(f"Failed to generate YAML: {str(e)}") from e
+    except ValueError as e:
+        error_message = str(e)
+        warnings.warn(
+            f"{error_message}. The Hive clients YAML generator command will not be available.",
+            stacklevel=2,
+        )
+
+        issue_title = f"Client {client_type.name} configuration issue"
+        issue_body = f"Error: {error_message}\nHive version: {hive_info.commit}\n"
+        issue_url = f"https://github.com/ethereum/execution-spec-tests/issues/new?title={urllib.parse.quote(issue_title)}&body={urllib.parse.quote(issue_body)}"
+
+        return (
+            f"Error: {error_message}\n"
+            f'Please <a href="{issue_url}">create an issue</a> to report this problem.'
+        )
+
+
+@pytest.fixture(scope="function")
+def filtered_hive_options(hive_info: HiveInfo) -> List[str]:
+    """Filter Hive command options to remove unwanted options."""
+    logger.info("Hive info: %s", hive_info.command)
+
+    unwanted_options = [
+        "--client",  # gets overwritten: we specify a single client; the one from the test case
+        "--client-file",  # gets overwritten: we'll write our own client file
+        "--results-root",  # use default value instead (or you have to pass it to ./hiveview)
+        "--sim.limit",  # gets overwritten: we only run the current test case id
+        "--sim.parallelism",  # skip; we'll only be running a single test
+    ]
+
+    command_parts = []
+    skip_next = False
+    for part in hive_info.command:
+        if skip_next:
+            skip_next = False
+            continue
+
+        if part in unwanted_options:
+            skip_next = True
+            continue
+
+        if any(part.startswith(f"{option}=") for option in unwanted_options):
+            continue
+
+        command_parts.append(part)
+
+    return command_parts
+
+
+@pytest.fixture(scope="function")
+def hive_client_config_file_parameter(hive_clients_yaml_target_filename: str) -> str:
+    """Return the hive client config file parameter."""
+    return f"--client-file {hive_clients_yaml_target_filename}"
+
+
+@pytest.fixture(scope="function")
+def hive_consume_command(
     test_case: TestCaseIndexFile | TestCaseStream,
-    hive_client_config_file_parameter: List[str],
-) -> List[str]:
+    hive_client_config_file_parameter: str,
+    filtered_hive_options: List[str],
+    client_type: ClientType,
+) -> str:
     """Command to run the test within hive."""
-    command = ["./hive", "--sim", f"ethereum/{test_suite_name}"]
-    if hive_client_config_file_parameter:
-        command += hive_client_config_file_parameter
-    command += ["--client", client_type.name, "--sim.limit", f'"id:{test_case.id}"']
-    return command
+    command_parts = filtered_hive_options.copy()
+    command_parts.append(f"{hive_client_config_file_parameter}")
+    command_parts.append(f"--client={client_type.name}")
+    command_parts.append(f'--sim.limit="id:{test_case.id}"')
+
+    return " ".join(command_parts)
 
 
 @pytest.fixture(scope="function")
 def hive_dev_command(
     client_type: ClientType,
-    hive_client_config_file_parameter: List[str],
-) -> List[str]:
+    hive_client_config_file_parameter: str,
+) -> str:
     """Return the command used to instantiate hive alongside the `consume` command."""
-    hive_dev = ["./hive", "--dev"]
-    if hive_client_config_file_parameter:
-        hive_dev += hive_client_config_file_parameter
-    hive_dev += ["--client", client_type.name]
-    return hive_dev
+    return f"./hive --dev {hive_client_config_file_parameter} --client {client_type.name}"
 
 
 @pytest.fixture(scope="function")
 def eest_consume_command(
     test_suite_name: str,
     test_case: TestCaseIndexFile | TestCaseStream,
     fixture_source_flags: List[str],
-) -> List[str]:
+) -> str:
     """Commands to run the test within EEST using a hive dev back-end."""
+    flags = " ".join(fixture_source_flags)
     return (
-        ["consume", test_suite_name.split("-")[-1], "-v"]
-        + fixture_source_flags
-        + [
-            "-k",
-            f'"{test_case.id}"',
-        ]
+        f"uv run consume {test_suite_name.split('-')[-1]} "
+        f'{flags} --sim.limit="id:{test_case.id}" -v -s'
     )
 
 
 @pytest.fixture(scope="function")
 def test_case_description(
     fixture: BaseFixture,
     test_case: TestCaseIndexFile | TestCaseStream,
-    hive_consume_command: List[str],
-    hive_dev_command: List[str],
-    eest_consume_command: List[str],
+    hive_clients_yaml_generator_command: str,
+    hive_consume_command: str,
+    hive_dev_command: str,
+    eest_consume_command: str,
 ) -> str:
-    """
-    Create the description of the current blockchain fixture test case.
-    Includes reproducible commands to re-run the test case against the target client.
-    """
-    description = f"Test id: {test_case.id}"
-    if "url" in fixture.info:
-        description += f"\n\nTest source: {fixture.info['url']}"
-    if "description" not in fixture.info:
-        description += "\n\nNo description field provided in the fixture's 'info' section."
+    """Create the description of the current blockchain fixture test case."""
+    test_url = fixture.info.get("url", "")
+
+    if "description" not in fixture.info or fixture.info["description"] is None:
+        test_docstring = "No documentation available."
     else:
-        description += f"\n\n{fixture.info['description']}"
-    description += (
-        f"\n\nCommand to reproduce entirely in hive:"
-        f"\n<code>{' '.join(hive_consume_command)}</code>"
-    )
-    eest_commands = "\n".join(
-        f"{i + 1}. <code>{' '.join(cmd)}</code>"
-        for i, cmd in enumerate([hive_dev_command, eest_consume_command])
-    )
-    description += (
-        f"\n\nCommands to reproduce within EEST using a hive dev back-end:\n{eest_commands}"
-    )
+        # this prefix was included in the fixture description field for fixtures <= v4.3.0
+        test_docstring = fixture.info["description"].replace("Test function documentation:\n", "")  # type: ignore
+
+    description = textwrap.dedent(f"""
+        <b>Test Details</b>
+        <code>{test_case.id}</code>
+        {f'<a href="{test_url}">[source]</a>' if test_url else ""}
+
+        {test_docstring}
+
+        <b>Run This Test Locally:</b>
+        To run this test in <a href="https://github.com/ethereum/hive">hive</a></i>:
+        <code>{hive_clients_yaml_generator_command}
+            {hive_consume_command}</code>
+
+        <b>Advanced: Run the test against a hive developer backend using EEST's <code>consume</code> command</b>
+        Create the client YAML file, as above, then:
+        1. Start hive in dev mode: <code>{hive_dev_command}</code>
+        2. In the EEST repository root: <code>{eest_consume_command}</code>
+    """)  # noqa: E501
+
+    description = description.strip()
     description = description.replace("\n", "<br/>")
     return description
 

src/pytest_plugins/pytest_hive/hive_info.py

@@ -1,25 +1,54 @@
 """Hive instance information structures."""
 
-from typing import List
+from typing import Dict, List, Optional
 
-from pydantic import BaseModel, Field
+import yaml
+from pydantic import BaseModel, Field, RootModel
 
 from ethereum_test_base_types import CamelModel
 
 
-class ClientInfo(BaseModel):
-    """Client information."""
+class YAMLModel(BaseModel):
+    """A helper class for YAML serialization of pydantic models."""
+
+    def yaml(self, **kwargs):
+        """Return the YAML representation of the model."""
+        return yaml.dump(self.model_dump(), **kwargs)
+
+    @classmethod
+    def parse_yaml(cls, yaml_string):
+        """Parse a YAML string into a model instance."""
+        data = yaml.safe_load(yaml_string)
+        return cls(**data)
+
+
+class ClientConfig(YAMLModel):
+    """
+    Client configuration for YAML serialization.
+
+    Represents a single client entry in the clients.yaml file.
+    """
 
     client: str
-    nametag: str | None = None
-    dockerfile: str | None = None
-    build_args: dict[str, str] | None = None
+    nametag: Optional[str] = None
+    dockerfile: Optional[str] = None
+    build_args: Optional[Dict[str, str]] = Field(default_factory=lambda: {})
+
+
+class ClientFile(RootModel, YAMLModel):
+    """
+    Represents the entire clients.yaml file structure.
+
+    The clients.yaml file is a list of client configurations.
+    """
+
+    root: List[ClientConfig]
 
 
 class HiveInfo(CamelModel):
     """Hive instance information."""
 
     command: List[str]
-    client_file: List[ClientInfo] = Field(default_factory=list)
+    client_file: ClientFile = Field(default_factory=lambda: ClientFile(root=[]))
     commit: str
     date: str

src/pytest_plugins/pytest_hive/pytest_hive.py

@@ -34,7 +34,6 @@
 import warnings
 from dataclasses import asdict
 from pathlib import Path
-from typing import List
 
 import pytest
 from filelock import FileLock
@@ -43,7 +42,7 @@
 from hive.testing import HiveTest, HiveTestResult, HiveTestSuite
 
 from ..logging import get_logger
-from .hive_info import ClientInfo, HiveInfo
+from .hive_info import ClientFile, HiveInfo
 
 logger = get_logger(__name__)
 
@@ -122,7 +121,7 @@ def pytest_report_header(config, start_path):
             f"hive commit: {hive_info.commit}",
             f"hive date: {hive_info.date}",
         ]
-        for client in hive_info.client_file:
+        for client in hive_info.client_file.root:
             header_lines += [
                 f"hive client ({client.client}): {client.model_dump_json(exclude_none=True)}",
             ]
@@ -160,10 +159,10 @@ def hive_info(simulator: Simulation):
 
 
 @pytest.fixture(scope="session")
-def client_file(hive_info: HiveInfo | None) -> List[ClientInfo]:
+def client_file(hive_info: HiveInfo | None) -> ClientFile:
     """Return the client file used when launching hive."""
     if hive_info is None:
-        return []
+        return ClientFile(root=[])
     return hive_info.client_file
 
 

src/pytest_plugins/shared/execute_fill.py

@@ -158,11 +158,7 @@ def test_case_description(request: pytest.FixtureRequest) -> str:
     if hasattr(request.node, "cls"):
         test_class_doc = f"Test class documentation:\n{request.cls.__doc__}" if request.cls else ""
     if hasattr(request.node, "function"):
-        test_function_doc = (
-            f"Test function documentation:\n{request.function.__doc__}"
-            if request.function.__doc__
-            else ""
-        )
+        test_function_doc = f"{request.function.__doc__}" if request.function.__doc__ else ""
     if not test_class_doc and not test_function_doc:
         return description_unavailable
     combined_docstring = f"{test_class_doc}\n\n{test_function_doc}".strip()

whitelist.txt

@@ -883,3 +883,4 @@ coeffs
 uncomp
 isogeny
 codomain
+nametag