execution: extract spec tests out of make test-all

Author: taratorio

.github/workflows/ci-gate.yml

@@ -47,6 +47,10 @@ jobs:
     uses: ./.github/workflows/test-all-erigon-race.yml
     secrets: inherit
 
+  spec-tests:
+    uses: ./.github/workflows/test-spec.yml
+    secrets: inherit
+
   hive:
     uses: ./.github/workflows/test-hive.yml
     secrets: inherit
@@ -95,6 +99,7 @@ jobs:
       - lint
       - tests
       - race-tests
+      - spec-tests
       - hive
       - hive-eest
       - caplin

.github/workflows/test-all-erigon.yml

@@ -17,9 +17,9 @@ jobs:
           - macos-15
           - windows-2025
         include:
-          # eest_blockchain tests take ~1190s on the GitHub-hosted windows-2025
-          # runner (right at the 20m limit). Give Windows extra headroom to avoid
-          # marginal timeouts while keeping Linux/macOS at the tighter 20m.
+          # Windows GitHub-hosted runners run noticeably slower than Linux for
+          # this suite; give them extra headroom to avoid marginal timeouts
+          # while keeping Linux/macOS at the tighter 20m.
           - os: windows-2025
             test_timeout: 30m
             skip_execution_tests: 'true'

.github/workflows/test-spec.yml

@@ -0,0 +1,81 @@
+name: Spec tests
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  spec-tests:
+    name: spec-${{ matrix.suite }}-${{ matrix.fixtures }}
+    # Skip enginextests-devnet until the devnet tarball ships
+    # blockchain_tests_engine_x/. The non-`_x` `blockchain_tests_engine/`
+    # variant uses a different JSON schema (inline `pre`, no `preHash`)
+    # that the cmd/evm enginextest runner cannot consume. Re-enable by
+    # removing this guard once the fixture lands.
+    if: ${{ !(matrix.suite == 'enginextests' && matrix.fixtures == 'devnet') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { suite: statetests,   fixtures: stable, workers: 12, max-allowed-failures: 37 }
+          - { suite: statetests,   fixtures: devnet, workers: 12, max-allowed-failures: 5253 }
+          - { suite: blocktests,   fixtures: stable, workers: 12, max-allowed-failures: 0 }
+          - { suite: blocktests,   fixtures: devnet, workers: 12, max-allowed-failures: 6206 }
+          - { suite: enginextests, fixtures: stable, workers: 8,  max-allowed-failures: 0 }
+          - { suite: enginextests, fixtures: devnet, workers: 8,  max-allowed-failures: 0 }
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      RAMDISK_SIZE_MB: '8192'
+
+    steps:
+      - name: Declare runner
+        run: echo "Runner $RUNNER_NAME"
+
+      - name: Checkout custom actions
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+
+      - uses: ./.github/actions/setup-erigon
+        id: erigon
+        with:
+          submodules: false
+          lfs: false
+          cleanup-space: true
+          ramdisk: true
+          build-cache-extra-key: spec-${{ matrix.suite }}-${{ matrix.fixtures }}
+
+      - name: Cache test-fixtures tarballs
+        uses: actions/cache@v4
+        with:
+          # Same key as test-all-erigon.yml so the two workflows share the
+          # cache: tarballs are downloaded once per OS+manifest-hash and
+          # restored here. Only the .tar.gz files are cached; extraction
+          # happens fresh each run via tools/test-fixtures.sh.
+          path: test-fixtures-cache/*.tar.gz
+          key: test-fixtures-${{ runner.os }}-${{ hashFiles('test-fixtures.json') }}
+          restore-keys: |
+            test-fixtures-${{ runner.os }}-
+
+      - name: Run spec-${{ matrix.suite }}-${{ matrix.fixtures }}
+        env:
+          SPEC_MAX_FAILURES: ${{ matrix.max-allowed-failures }}
+          SPEC_WORKERS: ${{ matrix.workers }}
+        run: make spec-${{ matrix.suite }}-${{ matrix.fixtures }}
+
+      - uses: ./.github/actions/cleanup-erigon
+        if: always()
+
+      # In the merge queue, cancel the run on first failure so the gate
+      # doesn't stall waiting for still-running siblings. PR runs keep
+      # going so authors see the full failure picture.
+      - name: Cancel workflow run on failure
+        if: failure() && github.event_name == 'merge_group'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh run cancel ${{ github.run_id }} || true

Makefile

@@ -248,6 +248,24 @@ test-fixtures:
 	rm -rf test-fixtures-cache/cl_mainnet/tests/mainnet/eip7805
 	rm -rf test-fixtures-cache/cl_mainnet/tests/mainnet/gloas
 
+## test-fixtures-eest:                 download & extract EEST fixture tarballs only (subset of test-fixtures)
+.PHONY: test-fixtures-eest
+test-fixtures-eest:
+	tools/test-fixtures.sh test-fixtures.json test-fixtures-cache eest_stable eest_devnet
+
+# Spec tests: run cmd/evm runners (statetest, blocktest, enginextest) against
+# EEST fixtures. CI defines failure budgets in .github/workflows/test-spec.yml's
+# matrix.include:; local runs use the per-shard fallback defaults baked into
+# tools/run-spec-test.sh. Override either with SPEC_MAX_FAILURES / SPEC_WORKERS.
+SPEC_SHARDS := \
+	statetests-stable statetests-devnet \
+	blocktests-stable blocktests-devnet \
+	enginextests-stable enginextests-devnet
+
+.PHONY: $(addprefix spec-,$(SPEC_SHARDS))
+$(addprefix spec-,$(SPEC_SHARDS)): spec-%: test-fixtures-eest evm
+	@bash tools/run-spec-test.sh $(subst -, ,$*)
+
 ## test-bench:                         check the benchmarks compile and run
 test-bench: override GO_FLAGS += -run=^$$ -bench=. -benchtime=1x -short -timeout=5m
 test-bench:

cmd/evm/enginexrunner.go

@@ -57,6 +57,7 @@ var engineXTestCommand = cli.Command{
 			Usage:    "Directory containing engine-x pre-alloc JSON files",
 			Required: true,
 		},
+		&JSONOutputFlag,
 		&RunFlag,
 		&VerbosityFlag,
 		&WorkersFlag,

db/kv/memdb/memory_database.go

@@ -40,8 +40,8 @@ func NewChainDB(tb testing.TB, tmpDir string) kv.RwDB {
 
 func NewTestDB(tb testing.TB, label kv.Label) kv.RwDB {
 	tb.Helper()
-	// we can't use tb.TempDir() here becuase things like TestExecutionSpecBlockchain
-	// produces test names that cause 'file name too long' errors
+	// we can't use tb.TempDir() here because some tests produce names long
+	// enough to cause 'file name too long' errors when reused as paths
 	dirname, err := os.MkdirTemp("", "testdb-"+string(label)+"-*")
 	if err != nil {
 		tb.Fatal(err)

execution/execmodule/execmoduletester/exec_module_tester.go

@@ -416,8 +416,8 @@ func New(tb testing.TB, opts ...Option) *ExecModuleTester {
 		panic(err)
 	}
 	if tb != nil {
-		// we can't use tb.TempDir() here becuase things like TestExecutionSpecBlockchain
-		// produces test names that cause 'file name too long' errors
+		// we can't use tb.TempDir() here because some tests produce names long
+		// enough to cause 'file name too long' errors when reused as paths
 		tb.Cleanup(func() {
 			dir.RemoveAll(tmpdir)
 		})