nethtest: add --engineTest, --stateTest, --jsonout, --workers flags

nethtest

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec dotnet run --no-build -c Release --project /Users/spencer/ethereum/clients/nethermind/src/Nethermind/Nethermind.Test.Runner/Nethermind.Test.Runner.csproj -- "$@"

src/Nethermind/Ethereum.Test.Base/BlockchainTest.cs

@@ -11,6 +11,7 @@ namespace Ethereum.Test.Base
 {
     public class BlockchainTest : EthereumTest
     {
+        public string? ForkName { get; set; }
         public IReleaseSpec? Network { get; set; }
         public IReleaseSpec? NetworkAfterTransition { get; set; }
         public ForkActivation? TransitionForkActivation { get; set; }

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -161,6 +161,9 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
         try
         {
             BlockHeader parentHeader;
+            string lastPayloadStatus = "";
+            string? lastValidationError = null;
+            string? asyncBlockError = null;
             // Genesis processing
             using (stateProvider.BeginScope(null))
             {
@@ -212,16 +215,21 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
 
             if (test.Blocks is not null)
             {
-                // blockchain test
-                parentHeader = SuggestBlocks(test, failOnInvalidRlp, blockValidator, blockTree, parentHeader);
+                // blockchain test — capture async block processing errors via event
+                blockchainProcessor.BlockRemoved += (_, args) =>
+                {
+                    if (args.ProcessingResult != ProcessingResult.Success)
+                        asyncBlockError = args.Message ?? args.Exception?.Message;
+                };
+                (parentHeader, lastValidationError) = SuggestBlocks(test, failOnInvalidRlp, blockValidator, blockTree, parentHeader);
             }
             else if (test.EngineNewPayloads is not null)
             {
                 // engine test — route through JsonRpcService for realistic deserialization
                 IJsonRpcService rpcService = container.Resolve<IJsonRpcService>();
                 JsonRpcUrl engineUrl = new(Uri.UriSchemeHttp, "localhost", 8551, RpcEndpoint.Http, true, ["engine"]);
                 JsonRpcContext rpcContext = new(RpcEndpoint.Http, url: engineUrl);
-                await RunNewPayloads(test.EngineNewPayloads, rpcService, rpcContext, parentHeader.Hash!);
+                (lastPayloadStatus, lastValidationError) = await RunNewPayloads(test.EngineNewPayloads, rpcService, rpcContext, parentHeader.Hash!);
             }
             else
             {
@@ -231,6 +239,7 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             // NOTE: Tracer removal must happen AFTER StopAsync to ensure all blocks are traced
             // Blocks are queued asynchronously, so we need to wait for processing to complete
             await blockchainProcessor.StopAsync(true);
+            lastValidationError ??= asyncBlockError;
             stopwatch?.Stop();
 
             IBlockCachePreWarmer? preWarmer = container.Resolve<MainProcessingContext>().LifetimeScope.ResolveOptional<IBlockCachePreWarmer>();
@@ -243,7 +252,7 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             Assert.That(headBlock, Is.Not.Null);
             if (headBlock is null)
             {
-                return new EthereumTestResult(test.Name, null, false);
+                return new EthereumTestResult(test.Name, test.ForkName, false) { Error = "head block is null" };
             }
 
             List<string> differences;
@@ -263,7 +272,16 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
             }
 
             Assert.That(differences, Is.Empty, "differences");
-            return new EthereumTestResult(test.Name, null, testPassed);
+            var result = new EthereumTestResult(test.Name, test.ForkName, testPassed);
+            if (headBlock?.Hash is not null)
+                result.LastBlockHash = headBlock.Hash;
+            if (!string.IsNullOrEmpty(lastPayloadStatus))
+                result.LastPayloadStatus = lastPayloadStatus;
+            if (lastValidationError is not null && result.Pass)
+                result.Error = lastValidationError;
+            if (!testPassed)
+                result.Error = string.Join("; ", differences);
+            return result;
         }
         catch (Exception)
         {
@@ -272,8 +290,9 @@ protected async Task<EthereumTestResult> RunTest(BlockchainTest test, Stopwatch?
         }
     }
 
-    private static BlockHeader SuggestBlocks(BlockchainTest test, bool failOnInvalidRlp, IBlockValidator blockValidator, IBlockTree blockTree, BlockHeader parentHeader)
+    private static (BlockHeader header, string? lastBlockError) SuggestBlocks(BlockchainTest test, bool failOnInvalidRlp, IBlockValidator blockValidator, IBlockTree blockTree, BlockHeader parentHeader)
     {
+        string? lastBlockError = null;
         List<(Block Block, string ExpectedException)> correctRlp = DecodeRlps(test, failOnInvalidRlp);
         for (int i = 0; i < correctRlp.Count; i++)
         {
@@ -287,55 +306,53 @@ private static BlockHeader SuggestBlocks(BlockchainTest test, bool failOnInvalid
 
             bool expectsException = correctRlp[i].ExpectedException is not null;
             // Validate block structure first (mimics SyncServer validation)
-            if (blockValidator.ValidateSuggestedBlock(correctRlp[i].Block, parentHeader, out string? validationError))
+            bool blockValid = blockValidator.ValidateSuggestedBlock(correctRlp[i].Block, parentHeader, out string? validationError);
+            if (blockValid)
             {
-                Assert.That(!expectsException, $"Expected block {correctRlp[i].Block.Hash} to fail with '{correctRlp[i].ExpectedException}', but it passed validation");
                 try
                 {
-                    // All validations passed, suggest the block
                     blockTree.SuggestBlock(correctRlp[i].Block);
-
                 }
                 catch (InvalidBlockException e)
                 {
-                    // Exception thrown during block processing
-                    Assert.That(expectsException, $"Unexpected invalid block {correctRlp[i].Block.Hash}: {validationError}, Exception: {e}");
-                    // else: Expected to fail and did fail via exception → this is correct behavior
+                    Assert.That(expectsException, $"Unexpected invalid block {correctRlp[i].Block.Hash}: {e.Message}");
+                    lastBlockError = e.Message;
                 }
                 catch (Exception e)
                 {
                     Assert.Fail($"Unexpected exception during processing: {e}");
                 }
                 finally
                 {
-                    // Dispose AccountChanges to prevent memory leaks in tests
                     correctRlp[i].Block.DisposeAccountChanges();
                 }
             }
             else
             {
-                // Validation FAILED
+                // Header validation failed
                 Assert.That(expectsException, $"Unexpected invalid block {correctRlp[i].Block.Hash}: {validationError}");
-                // else: Expected to fail and did fail → this is correct behavior
+                lastBlockError = validationError;
             }
 
             parentHeader = correctRlp[i].Block.Header;
         }
-        return parentHeader;
+        return (parentHeader, lastBlockError);
     }
 
     private static readonly Dictionary<int, int> s_newPayloadParamCounts = Enumerable
         .Range(1, EngineApiVersions.NewPayload.Latest)
         .ToDictionary(v => v, v => (typeof(IEngineRpcModule).GetMethod($"engine_newPayloadV{v}")
             ?? throw new NotSupportedException($"engine_newPayloadV{v} not found on IEngineRpcModule")).GetParameters().Length);
 
-    private async static Task RunNewPayloads(TestEngineNewPayloadsJson[]? newPayloads, IJsonRpcService rpcService, JsonRpcContext rpcContext, Hash256 initialHeadHash)
+    private async static Task<(string status, string? validationError)> RunNewPayloads(TestEngineNewPayloadsJson[]? newPayloads, IJsonRpcService rpcService, JsonRpcContext rpcContext, Hash256 initialHeadHash)
     {
-        if (newPayloads is null || newPayloads.Length == 0) return;
+        if (newPayloads is null || newPayloads.Length == 0) return ("", null);
 
         int initialFcuVersion = int.Parse(newPayloads[0].ForkChoiceUpdatedVersion ?? EngineApiVersions.Fcu.Latest.ToString());
         AssertRpcSuccess(await SendFcu(rpcService, rpcContext, initialFcuVersion, initialHeadHash.ToString()));
 
+        string lastStatus = "";
+        string? lastValidationError = null;
         foreach (TestEngineNewPayloadsJson enginePayload in newPayloads)
         {
             int newPayloadVersion = int.Parse(enginePayload.NewPayloadVersion ?? EngineApiVersions.NewPayload.Latest.ToString());
@@ -351,23 +368,31 @@ private async static Task RunNewPayloads(TestEngineNewPayloadsJson[]? newPayload
             // RPC-level errors (e.g. wrong payload version) are valid for negative tests
             if (npResponse is JsonRpcErrorResponse errorResponse)
             {
-                Assert.That(validationError, Is.Not.Null,
-                    $"engine_newPayloadV{newPayloadVersion} RPC error: {errorResponse.Error?.Code} {errorResponse.Error?.Message}");
+                if (validationError is null)
+                    throw new Exception(
+                        $"engine_newPayloadV{newPayloadVersion} unexpected RPC error: {errorResponse.Error?.Code} {errorResponse.Error?.Message}");
                 continue;
             }
 
             PayloadStatusV1 payloadStatus = (PayloadStatusV1)((JsonRpcSuccessResponse)npResponse).Result!;
+            lastStatus = payloadStatus.Status;
+            if (payloadStatus.ValidationError is not null)
+                lastValidationError = payloadStatus.ValidationError;
             string expectedStatus = validationError is null ? PayloadStatus.Valid : PayloadStatus.Invalid;
-            Assert.That(payloadStatus.Status, Is.EqualTo(expectedStatus),
-                $"engine_newPayloadV{newPayloadVersion} returned {payloadStatus.Status}, expected {expectedStatus}. " +
-                $"ValidationError: {payloadStatus.ValidationError}");
+            if (payloadStatus.Status != expectedStatus)
+                throw new Exception(
+                    $"engine_newPayloadV{newPayloadVersion}: expected {expectedStatus} status" +
+                    (validationError is not null ? $" for validation error \"{validationError}\"" : "") +
+                    $", got {payloadStatus.Status}" +
+                    (payloadStatus.ValidationError is not null ? $" (err: {payloadStatus.ValidationError})" : ""));
 
             if (payloadStatus.Status == PayloadStatus.Valid)
             {
                 string blockHash = enginePayload.Params[0].GetProperty("blockHash").GetString()!;
                 AssertRpcSuccess(await SendFcu(rpcService, rpcContext, fcuVersion, blockHash));
             }
         }
+        return (lastStatus, lastValidationError);
     }
 
     private static async Task<JsonRpcResponse> SendRpc(IJsonRpcService rpcService, JsonRpcContext context, string method, string paramsJson)
@@ -383,8 +408,13 @@ private static Task<JsonRpcResponse> SendFcu(IJsonRpcService rpcService, JsonRpc
 
     private static void AssertRpcSuccess(JsonRpcResponse response)
     {
-        Assert.That(response, Is.InstanceOf<JsonRpcSuccessResponse>(),
-            response is JsonRpcErrorResponse err ? $"RPC error: {err.Error?.Code} {err.Error?.Message}" : "unexpected response type");
+        if (response is not JsonRpcSuccessResponse)
+        {
+            string message = response is JsonRpcErrorResponse err
+                ? $"RPC error: {err.Error?.Code} {err.Error?.Message}"
+                : "unexpected response type";
+            throw new Exception(message);
+        }
     }
 
     private static List<(Block Block, string ExpectedException)> DecodeRlps(BlockchainTest test, bool failOnInvalidRlp)
@@ -406,18 +436,16 @@ private static void AssertRpcSuccess(JsonRpcResponse response)
                     {
                         Assert.That(suggestedBlock.Uncles[uncleIndex].Hash, Is.EqualTo(new Hash256(testBlockJson.UncleHeaders![uncleIndex].Hash)));
                     }
-
-                    correctRlp.Add((suggestedBlock, testBlockJson.ExpectException));
                 }
+
+                correctRlp.Add((suggestedBlock, testBlockJson.ExpectException));
             }
             catch (Exception e)
             {
                 if (testBlockJson.ExpectException is null)
                 {
                     string invalidRlpMessage = $"Invalid RLP ({i}) {e}";
                     Assert.That(!failOnInvalidRlp, invalidRlpMessage);
-                    // ForgedTests don't have ExpectedException and at the same time have invalid rlps
-                    // Don't fail here. If test executed incorrectly will fail at last check
                     _logger.Warn(invalidRlpMessage);
                 }
                 else

src/Nethermind/Ethereum.Test.Base/EthereumTestResult.cs

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Text.Json.Serialization;
 using Nethermind.Core.Crypto;
 
 namespace Ethereum.Test.Base
@@ -20,20 +21,33 @@ public EthereumTestResult(string? name, string? fork, string loadFailure)
             Fork = fork ?? "unknown";
             Pass = false;
             LoadFailure = loadFailure;
+            Error = loadFailure;
         }
 
         public EthereumTestResult(string? name, string? loadFailure)
             : this(name, null, loadFailure)
         {
         }
 
+        [JsonIgnore]
         public string? LoadFailure { get; set; }
         public string Name { get; set; }
         public bool Pass { get; set; }
         public string Fork { get; set; }
 
+        public string Error { get; set; } = "";
+
+        [JsonIgnore]
         public double TimeInMs { get; set; }
 
-        public Hash256 StateRoot { get; set; } = Keccak.EmptyTreeHash;
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public Hash256? StateRoot { get; set; }
+
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public Hash256? LastBlockHash { get; set; }
+
+        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public string? LastPayloadStatus { get; set; }
+
     }
 }

src/Nethermind/Ethereum.Test.Base/FileTestsSource.cs

@@ -4,7 +4,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Text;
 
 namespace Ethereum.Test.Base
 {
@@ -26,12 +25,15 @@ public IEnumerable<EthereumTest> LoadTests(TestType testType)
                     return [];
                 }
 
-                string json = File.ReadAllText(_fileName, Encoding.Default);
+                // Read as UTF-8 bytes directly — avoids the intermediate string allocation
+                // from File.ReadAllText. System.Text.Json can deserialize from byte spans
+                // without encoding conversion overhead.
+                byte[] jsonBytes = File.ReadAllBytes(_fileName);
 
                 return testType switch
                 {
-                    TestType.State => JsonToEthereumTest.ConvertStateTest(json),
-                    _ => JsonToEthereumTest.ConvertToBlockchainTests(json)
+                    TestType.State => JsonToEthereumTest.ConvertStateTest(jsonBytes),
+                    _ => JsonToEthereumTest.ConvertToBlockchainTests(jsonBytes)
                 };
             }
             catch (Exception e)

src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs

@@ -188,10 +188,18 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer)
             }
 
             List<string> differences = RunAssertions(test, stateProvider);
+            // Capture tx error for exception mapping (even when test passes)
+            string txError = "";
+            if (txResult is not null && txResult.Value != TransactionResult.Ok)
+                txError = txResult.Value.ErrorDescription;
+            else if (blockValidationError is not null)
+                txError = blockValidationError;
+
             EthereumTestResult testResult = new(test.Name, test.ForkName, differences.Count == 0)
             {
                 TimeInMs = stopwatch.Elapsed.TotalMilliseconds,
-                StateRoot = stateProvider.StateRoot
+                StateRoot = stateProvider.StateRoot,
+                Error = differences.Count > 0 ? string.Join("; ", differences) : txError,
             };
 
             if (differences.Count > 0)