Ability to run EOF exec spec tests in CI

build.gradle

@@ -171,7 +171,7 @@ configure(allprojects - project(':platform')) {
     def ethExecSpecTestsRepo = ivy {
       url 'https://github.com'
       patternLayout {
-        artifact '/[organisation]/[module]/releases/download/v[revision]/[classifier].[ext]'
+        artifact '/[organisation]/[module]/releases/download/[revision]/[artifact]_[classifier].[ext]'
       }
       metadataSources {
         artifact()

ethereum/referencetests/build.gradle

@@ -13,6 +13,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+configurations {
+  referenceTestAnnotationProcessor.extendsFrom testAnnotationProcessor
+  executionRefTestImplementation.extendsFrom referenceTestImplementation
+  eofRefTestImplementation.extendsFrom referenceTestImplementation
+}
 
 def blockchainReferenceTests = tasks.register("blockchainReferenceTests")  {
   final referenceTestsPath = 'src/reference-test/external-resources/BlockchainTests'
@@ -63,10 +68,58 @@ def eipStateReferenceTests = tasks.register("eipStateReferenceTests")  {
     )
 }
 
+def eofSpecTests = tasks.register("eofSpecTests") {
+  final referenceTestsPath = "$buildDir/execution-spec-tests/eof"
+  final generatedTestsPath = "$buildDir/generated/sources/reference-test/$name/java"
+  def tarPath = configurations.eofRefTestImplementation.files.find {
+    it.absolutePath.contains('execution-spec-tests')
+  }
+
+  copy {
+    from tarTree(tarPath)
+    into "$referenceTestsPath"
+  }
+
+  inputs.files fileTree(referenceTestsPath), fileTree(generatedTestsPath)
+  outputs.files generatedTestsPath
+
+  // generate blockchain_tests:
+  generateTestFiles(
+    fileTree(referenceTestsPath + "/fixtures/blockchain_tests"),
+    file("src/reference-test/templates/BlockchainReferenceTest.java.template"),
+    "eof/fixtures",
+    "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/blockchain/eof",
+    "EofSpecBlockchainTest",
+    "org.hyperledger.besu.ethereum.vm.blockchain.eof",
+    )
+
+  // generate state_tests:
+  generateTestFiles(
+    fileTree(referenceTestsPath + "/fixtures/state_tests"),
+    file("src/reference-test/templates/GeneralStateReferenceTest.java.template"),
+    "eof/fixtures",
+    "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/state/eof",
+    "EofSpecStateTest",
+    "org.hyperledger.besu.ethereum.vm.state.eof",
+    )
+
+  // generate eof_tests:
+  generateTestFiles(
+    fileTree(referenceTestsPath + "/fixtures/eof_tests"),
+    file("src/reference-test/templates/EOFReferenceTest.java.template"),
+    "eof/fixtures",
+    "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/container/eof",
+    "EofSpecContainerTest",
+    "org.hyperledger.besu.ethereum.vm.container.eof"
+    )
+}
+
 def executionSpecTests = tasks.register("executionSpecTests") {
-  final referenceTestsPath = "$buildDir/execution-spec-tests/"
+  final referenceTestsPath = "$buildDir/execution-spec-tests/stable"
   final generatedTestsPath = "$buildDir/generated/sources/reference-test/$name/java"
-  def tarPath = configurations.tarConfig.files.find{ it.name.startsWith('execution-spec-tests')}
+  def tarPath = configurations.executionRefTestImplementation.files.find {
+    it.absolutePath.contains('execution-spec-tests')
+  }
 
   copy {
     from tarTree(tarPath)
@@ -79,7 +132,7 @@ def executionSpecTests = tasks.register("executionSpecTests") {
   generateTestFiles(
     fileTree(referenceTestsPath + "/fixtures/blockchain_tests"),
     file("src/reference-test/templates/BlockchainReferenceTest.java.template"),
-    "fixtures",
+    "stable/fixtures",
     "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/executionspec",
     "ExecutionSpecBlockchainTest",
     "org.hyperledger.besu.ethereum.vm.executionspec",
@@ -89,7 +142,7 @@ def executionSpecTests = tasks.register("executionSpecTests") {
   generateTestFiles(
     fileTree(referenceTestsPath + "/fixtures/state_tests"),
     file("src/reference-test/templates/GeneralStateReferenceTest.java.template"),
-    "fixtures",
+    "stable/fixtures",
     "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/executionspec",
     "ExecutionSpecStateTest",
     "org.hyperledger.besu.ethereum.vm.executionspec",
@@ -128,22 +181,6 @@ def generalstateRegressionReferenceTests = tasks.register("generalstateRegressio
     )
 }
 
-def eofReferenceTests = tasks.register("eofReferenceTests") {
-  final referenceTestsPath = "src/reference-test/external-resources/EOFTests"
-  final generatedTestsPath = "$buildDir/generated/sources/reference-test/$name/java"
-  inputs.files fileTree(referenceTestsPath),
-    fileTree(generatedTestsPath)
-  outputs.files generatedTestsPath
-  generateTestFiles(
-    fileTree(referenceTestsPath),
-    file("src/reference-test/templates/EOFReferenceTest.java.template"),
-    "EOFTests",
-    "$generatedTestsPath/org/hyperledger/besu/ethereum/vm/eof",
-    "EOFReferenceTest",
-    "org.hyperledger.besu.ethereum.vm.eof"
-    )
-}
-
 sourceSets {
   referenceTest {
     java {
@@ -154,9 +191,9 @@ sourceSets {
         eipBlockchainReferenceTests,
         eipStateReferenceTests,
         executionSpecTests,
+        eofSpecTests,
         generalstateReferenceTests,
-        generalstateRegressionReferenceTests,
-        eofReferenceTests
+        generalstateRegressionReferenceTests
     }
     resources {
       srcDirs 'src/reference-test/resources',
@@ -197,8 +234,6 @@ dependencies {
   referenceTestImplementation project(path: ':plugin-api')
   referenceTestImplementation project(path: ':testutil')
   referenceTestImplementation project(path: ':util')
-  // the following will be resolved via custom ivy repository declared in root build.gradle
-  referenceTestImplementation 'ethereum:execution-spec-tests:4.1.0:[email protected]'
   referenceTestImplementation 'com.fasterxml.jackson.core:jackson-databind'
   referenceTestImplementation 'com.google.guava:guava'
   referenceTestImplementation 'io.consensys.tuweni:tuweni-bytes'
@@ -207,6 +242,28 @@ dependencies {
   referenceTestImplementation 'org.junit.jupiter:junit-jupiter-api'
   referenceTestImplementation 'org.junit.jupiter:junit-jupiter-params'
 
+  // the following will be resolved via custom ivy repository declared in root build.gradle
+  executionRefTestImplementation dependencies.create('ethereum:execution-spec-tests') {
+    version {
+      strictly 'v4.1.0'
+    }
+    artifact {
+      name = 'fixtures'
+      classifier = 'stable'
+      type = 'tar.gz'
+    }
+  }
+  eofRefTestImplementation dependencies.create('ethereum:execution-spec-tests') {
+    version {
+      strictly '[email protected]'
+    }
+    artifact {
+      name = 'fixtures'
+      classifier = 'eip7692'
+      type = 'tar.gz'
+    }
+  }
+
   referenceTestRuntimeOnly 'org.junit.jupiter:junit-jupiter'
 }
 
@@ -295,13 +352,3 @@ def generateTestFiles(
     testFile.newWriter().withWriter { w -> w << testFileContents }
   }
 }
-
-configurations {
-  referenceTestAnnotationProcessor.extendsFrom testAnnotationProcessor
-  // we need this because referenceTestImplementation defaults to 'canBeResolved=false'.
-  tarConfig.extendsFrom referenceTestImplementation
-  tarConfig {
-    canBeResolved = true
-    canBeConsumed = false
-  }
-}

ethereum/referencetests/src/reference-test/templates/EOFReferenceTest.java.template

@@ -1,4 +1,4 @@
-package org.hyperledger.besu.ethereum.vm.eof;
+package %%PACKAGE_NAME%%;
 
 import static org.hyperledger.besu.ethereum.eof.EOFReferenceTestTools.executeTest;
 import static org.hyperledger.besu.ethereum.eof.EOFReferenceTestTools.generateTestParametersForConfig;

gradle/verification-metadata.xml

@@ -1351,6 +1351,16 @@
             <sha256 value="55541b830302589576cf36526dcf54839d68e2501dfe0fa6f8cb9851e5b352d4" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="ethereum" name="execution-spec-tests" version="[email protected]">
+         <artifact name="[email protected]">
+            <sha256 value="cc33835ac51105ec5cabc568ed5618ef9bbf14719fdeb87879dc9f35f800b55b" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="ethereum" name="execution-spec-tests" version="v4.1.0">
+         <artifact name="fixtures-v4.1.0-stable.tar.gz">
+            <sha256 value="275ecc7b0cb57d1b4fcca3943ea66dc7f61f5e54774427c5a13894da12a0a9e5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="info.picocli" name="picocli" version="4.7.6">
          <artifact name="picocli-4.7.6.jar">
             <sha256 value="ed441183f309b93f104ca9e071e314a4062a893184e18a3c7ad72ec9cba12ba0" origin="Generated by Gradle"/>

testutil/src/main/java/org/hyperledger/besu/testutil/JsonTestParameters.java

@@ -304,10 +304,9 @@ private static <T> boolean matchAny(final T toTest, final List<Predicate<T>> tes
   }
 
   private Collection<File> getFilteredFiles(final String[] paths) {
-    final ClassLoader classLoader = JsonTestParameters.class.getClassLoader();
     final List<File> files = new ArrayList<>();
     for (final String path : paths) {
-      final URL url = classLoader.getResource(path);
+      final URL url = ClassLoader.getSystemClassLoader().getResource(path);
       checkState(url != null, "Cannot find test directory %s", path);
       final Path dir;
       try {