nethtest: add --stateTest, --engineTest, --jsonout, --workers flags

Author: spencer-tb

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -351,16 +351,20 @@ private async static Task RunNewPayloads(TestEngineNewPayloadsJson[]? newPayload
             // RPC-level errors (e.g. wrong payload version) are valid for negative tests
             if (npResponse is JsonRpcErrorResponse errorResponse)
             {
-                Assert.That(validationError, Is.Not.Null,
-                    $"engine_newPayloadV{newPayloadVersion} RPC error: {errorResponse.Error?.Code} {errorResponse.Error?.Message}");
+                if (validationError is null)
+                    throw new Exception(
+                        $"engine_newPayloadV{newPayloadVersion} unexpected RPC error: {errorResponse.Error?.Code} {errorResponse.Error?.Message}");
                 continue;
             }
 
             PayloadStatusV1 payloadStatus = (PayloadStatusV1)((JsonRpcSuccessResponse)npResponse).Result!;
             string expectedStatus = validationError is null ? PayloadStatus.Valid : PayloadStatus.Invalid;
-            Assert.That(payloadStatus.Status, Is.EqualTo(expectedStatus),
-                $"engine_newPayloadV{newPayloadVersion} returned {payloadStatus.Status}, expected {expectedStatus}. " +
-                $"ValidationError: {payloadStatus.ValidationError}");
+            if (payloadStatus.Status != expectedStatus)
+                throw new Exception(
+                    $"engine_newPayloadV{newPayloadVersion}: expected {expectedStatus} status" +
+                    (validationError is not null ? $" for validation error \"{validationError}\"" : "") +
+                    $", got {payloadStatus.Status}" +
+                    (payloadStatus.ValidationError is not null ? $" (err: {payloadStatus.ValidationError})" : ""));
 
             if (payloadStatus.Status == PayloadStatus.Valid)
             {
@@ -383,8 +387,13 @@ private static Task<JsonRpcResponse> SendFcu(IJsonRpcService rpcService, JsonRpc
 
     private static void AssertRpcSuccess(JsonRpcResponse response)
     {
-        Assert.That(response, Is.InstanceOf<JsonRpcSuccessResponse>(),
-            response is JsonRpcErrorResponse err ? $"RPC error: {err.Error?.Code} {err.Error?.Message}" : "unexpected response type");
+        if (response is not JsonRpcSuccessResponse)
+        {
+            string message = response is JsonRpcErrorResponse err
+                ? $"RPC error: {err.Error?.Code} {err.Error?.Message}"
+                : "unexpected response type";
+            throw new Exception(message);
+        }
     }
 
     private static List<(Block Block, string ExpectedException)> DecodeRlps(BlockchainTest test, bool failOnInvalidRlp)

src/Nethermind/Ethereum.Test.Base/EthereumTestResult.cs

@@ -35,5 +35,11 @@ public EthereumTestResult(string? name, string? loadFailure)
         public double TimeInMs { get; set; }
 
         public Hash256 StateRoot { get; set; } = Keccak.EmptyTreeHash;
+
+        /// <summary>
+        /// The actual validation error string returned by the engine for each payload.
+        /// Populated only for engine tests. Allows consume direct to perform exception mapping.
+        /// </summary>
+        public string? EngineValidationError { get; set; }
     }
 }

src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs

@@ -6,6 +6,7 @@
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Ethereum.Test.Base;
+using Nethermind.Serialization.Json;
 
 namespace Nethermind.Test.Runner;
 
@@ -15,11 +16,14 @@ public class BlockchainTestsRunner(
     ulong chainId,
     bool trace = false,
     bool traceMemory = false,
-    bool traceNoStack = false)
+    bool traceNoStack = false,
+    bool jsonOutput = false,
+    bool suppressOutput = false)
     : BlockchainTestBase, IBlockchainTestRunner
 {
     private readonly ConsoleColor _defaultColor = Console.ForegroundColor;
     private readonly ITestSourceLoader _testsSource = testsSource ?? throw new ArgumentNullException(nameof(testsSource));
+    private static readonly IJsonSerializer _serializer = new EthereumJsonSerializer();
 
     public async Task<IEnumerable<EthereumTestResult>> RunTestsAsync()
     {
@@ -29,7 +33,7 @@ public async Task<IEnumerable<EthereumTestResult>> RunTestsAsync()
         {
             if (loadedTest as FailedToLoadTest is not null)
             {
-                WriteRed(loadedTest.LoadFailure);
+                if (!jsonOutput && !suppressOutput) WriteRed(loadedTest.LoadFailure);
                 testResults.Add(new EthereumTestResult(loadedTest.Name, loadedTest.LoadFailure));
                 continue;
             }
@@ -43,10 +47,11 @@ public async Task<IEnumerable<EthereumTestResult>> RunTestsAsync()
 
             if (filter is not null && test.Name is not null && !Regex.Match(test.Name, $"^({filter})").Success)
                 continue;
-            Console.Write($"{test,-120} ");
+
+            if (!jsonOutput && !suppressOutput) Console.Write($"{test,-120} ");
             if (test.LoadFailure is not null)
             {
-                WriteRed(test.LoadFailure);
+                if (!jsonOutput && !suppressOutput) WriteRed(test.LoadFailure);
                 testResults.Add(new EthereumTestResult(test.Name, test.LoadFailure));
             }
             else
@@ -55,13 +60,21 @@ public async Task<IEnumerable<EthereumTestResult>> RunTestsAsync()
 
                 EthereumTestResult result = await RunTest(test, tracer: tracer);
                 testResults.Add(result);
-                if (result.Pass)
-                    WriteGreen("PASS");
-                else
-                    WriteRed("FAIL");
+                if (!jsonOutput && !suppressOutput)
+                {
+                    if (result.Pass)
+                        WriteGreen("PASS");
+                    else
+                        WriteRed("FAIL");
+                }
             }
         }
 
+        if (jsonOutput && !suppressOutput)
+        {
+            Console.Out.Write(_serializer.Serialize(testResults, true));
+        }
+
         return testResults;
     }