feat(consume): eofparse output is now correctly parsed into expected json format (ethereum#1232)

Author: felix314159

src/ethereum_clis/clis/geth.py

@@ -250,27 +250,71 @@ class GethFixtureConsumer(
 ):
     """Geth's implementation of the fixture consumer."""
 
-    def consume_blockchain_test(
+    # ------------------------------------ eof test -----------------------------------------------
+
+    def extract_int_from_subprocess_string(
+        self, s: str, substring_of_interest_prefix: str
+    ) -> int | None:
+        """Hacky method to extract relevant substring from string returned by subprocess execution."""  # noqa: E501
+        # get everything after this substring occurrence, then remove everything not in [0,9]
+        relevant_substring = re.sub(r"\D", "", s.split('total executed"=', 1)[1])
+        if relevant_substring.isdigit():
+            return int(relevant_substring)
+        else:
+            return None
+
+    def eofparse_subprocess_output_to_dict(
+        self, process_result: str, fixture_name: str | None, fork_name: str
+    ) -> Dict:
+        """Take subprocess output from geth's eofparse, a fixture_name and a fork name and return relevant data as dict."""  # noqa: E501
+        #      remove whitespaces if more than 1 whitespace occurs at a time
+        process_result_cleaned: str = re.sub(r"\s+", " ", process_result).strip()
+
+        #       determine amount of total tests
+        total_tests_amount: int | None = self.extract_int_from_subprocess_string(
+            process_result_cleaned, 'total executed"='
+        )
+
+        #       determine amount of passed tests
+        passed_tests_amount: int | None = self.extract_int_from_subprocess_string(
+            process_result_cleaned, "tests passed="
+        )
+
+        #       determine if all tests of this test vector passed
+        all_tests_passed: bool = False
+        if total_tests_amount is not None and passed_tests_amount is not None:
+            if total_tests_amount == passed_tests_amount:
+                all_tests_passed = True
+
+        #       return relevant data as dict
+        return {
+            "name": fixture_name,
+            "pass": all_tests_passed,
+            "stateRoot": None,
+            "fork": fork_name,
+        }
+
+    def consume_eof_test_file(
         self,
         fixture_path: Path,
         fixture_name: Optional[str] = None,
         debug_output_path: Optional[Path] = None,
-    ):
+    ) -> List[Dict[str, Any]]:
         """
-        Consume a single blockchain test.
+        Consume an entire EOF validation test file.
 
-        The `evm blocktest` command takes the `--run` argument which can be used to select a
-        specific fixture from the fixture file when executing.
+
+        The `evm eofparse` will always validate all the eof byte codes contained in a file without
+        the possibility of selecting a single test, so this function is cached in order to only
+        call the command once and `consume_eof_test` can simply select the result that was
+        requested.
         """
-        subcommand = "blocktest"
+        subcommand = "eofparse"
         global_options = []
-        subcommand_options = []
+        subcommand_options = ["--test"]
         if debug_output_path:
             global_options += ["--verbosity", "100"]
-            subcommand_options += ["--trace"]
-
-        if fixture_name:
-            subcommand_options += ["--run", re.escape(fixture_name)]
+            # --trace does not exist for eofparse
 
         command = (
             [str(self.binary)]
@@ -290,19 +334,69 @@ def consume_blockchain_test(
                 f"Unexpected exit code:\n{' '.join(command)}\n\n Error:\n{result.stderr}"
             )
 
+        result_json = [
+            self.eofparse_subprocess_output_to_dict(result.stderr, fixture_name, "Osaka")
+        ]
+        if not isinstance(result_json, list):
+            raise Exception(f"Unexpected result from evm eofparse: {result_json}")
+        return result_json
+
     def consume_eof_test(
         self,
         fixture_path: Path,
         fixture_name: Optional[str] = None,
         debug_output_path: Optional[Path] = None,
     ):
-        """Consume an EOF validation test (valid yes/no) without running any EOF code."""
-        subcommand = "eofparse"
+        """
+        Consume a single eof test.
+
+        Uses the cached result from `consume_eof_test_file` in order to not call the command
+        every time an select a single result from there.
+        """
+        file_results = self.consume_eof_test_file(
+            fixture_path=fixture_path,
+            fixture_name=fixture_name,
+            debug_output_path=debug_output_path,
+        )
+        if fixture_name:
+            test_result = [
+                test_result for test_result in file_results if test_result["name"] == fixture_name
+            ]
+            assert len(test_result) < 2, f"Multiple test results for {fixture_name}"
+            assert len(test_result) == 1, f"Test result for {fixture_name} missing"
+            assert test_result[0]["pass"], f"EOF test failed: {test_result[0]['error']}"
+        else:
+            if any(not test_result["pass"] for test_result in file_results):
+                exception_text = "EOF test failed: \n" + "\n".join(
+                    f"{test_result['name']}: " + test_result["error"]
+                    for test_result in file_results
+                    if not test_result["pass"]
+                )
+                raise Exception(exception_text)
+
+    # ---------------------------------- blockchain test ------------------------------------------
+
+    def consume_blockchain_test(
+        self,
+        fixture_path: Path,
+        fixture_name: Optional[str] = None,
+        debug_output_path: Optional[Path] = None,
+    ):
+        """
+        Consume a single blockchain test.
+
+        The `evm blocktest` command takes the `--run` argument which can be used to select a
+        specific fixture from the fixture file when executing.
+        """
+        subcommand = "blocktest"
         global_options = []
-        subcommand_options = ["--test"]
+        subcommand_options = []
         if debug_output_path:
             global_options += ["--verbosity", "100"]
-            # --trace does not exist for eofparse
+            subcommand_options += ["--trace"]
+
+        if fixture_name:
+            subcommand_options += ["--run", re.escape(fixture_name)]
 
         command = (
             [str(self.binary)]
@@ -322,6 +416,8 @@ def consume_eof_test(
                 f"Unexpected exit code:\n{' '.join(command)}\n\n Error:\n{result.stderr}"
             )
 
+    # ------------------------------------ state test ---------------------------------------------
+
     @cache  # noqa
     def consume_state_test_file(
         self,
@@ -361,6 +457,7 @@ def consume_state_test_file(
             )
 
         result_json = json.loads(result.stdout)
+        print("Output from json.loads(result.stdout):", result_json)
         if not isinstance(result_json, list):
             raise Exception(f"Unexpected result from evm statetest: {result_json}")
         return result_json