Merge branch 'master' into support-binary

Author: courtneyeh

.codespell/.codespellrc

@@ -0,0 +1,5 @@
+[codespell]
+skip = .git,package-lock.json,LOG.old.*
+count =
+quiet-level = 3
+ignore-words = ./.codespell/wordlist.txt

.codespell/wordlist.txt

@@ -0,0 +1,6 @@
+afterall
+errorprone
+vertx
+dout
+interruptors
+bu

.github/workflows/codespell.yml

@@ -0,0 +1,23 @@
+# A Github action that using codespell to check spell.
+# .codespell/.codespellrc is a config file.
+# .codespell/wordlist.txt is a list of words that will ignore word checks.
+# More details please check the following link:
+# https://github.com/codespell-project/codespell
+
+name: Codespell
+
+on: pull_request
+
+jobs:
+  codespell:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Install prerequisites
+        run: pip install codespell
+
+      - name: Spell check
+        run: codespell --config=./.codespell/.codespellrc

CHANGELOG.md

@@ -10,12 +10,9 @@ the [releases page](https://github.com/Consensys/teku/releases).
 ## Unreleased Changes
 
 ### Breaking Changes
-- When a lock file is unable to be cleaned up, the (BN or VC) will now exit code 2 in preference to being 'up' but not able to perform duties. This will not self recover and will need intervention from the node operator.
 
 ### Additions and Improvements
-- Improve block rewards calculation performance for `/eth/v3/validator/blocks/{slot}` block production beacon node API.
-- Updated Javalin to v.6 (used by rest-api and keymanager-api).
-- Docker image tags now default to jdk21 images unless a jdk-specific tag is used.
+- Introduced [Validator Slashing Prevention feature](https://docs.teku.consensys.io/how-to/prevent-slashing/detect-slashing).
+- If the EL supports the `engine_getClientVersionV1` Engine API method, the default graffiti (when no graffiti has been configured by the validator) will include EL as well as CL version information. For more details, please see https://github.com/ethereum/execution-apis/pull/517.
 
 ### Bug Fixes
-- Fixed an issue where stale lock files weren't able to be cleaned up and would effectively park the service (BN or VC) with no user errors or any indication that the service was in a bad state.

README.md

@@ -7,7 +7,8 @@
  [![Twitter Follow](https://img.shields.io/twitter/follow/Teku_Consensys)](https://twitter.com/Teku_Consensys)
  [![GitPOAP Badge](https://public-api.gitpoap.io/v1/repo/ConsenSys/teku/badge)](https://www.gitpoap.io/gh/ConsenSys/teku)
 
-Teku is an open-source Ethereum consensus client written in Java and containing a full beacon node and validator client implementation.  
+Teku is an open-source Ethereum consensus client written in Java and containing a full beacon node and validator client implementation.
+
 See the [Changelog](https://github.com/Consensys/teku/releases) for details of the latest releases and upcoming breaking changes.
 
 ## Useful links

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java

@@ -13,12 +13,20 @@
 
 package tech.pegasys.teku.test.acceptance;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
 import java.nio.file.Path;
 import java.util.Collections;
+import java.util.Map;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
 import tech.pegasys.teku.test.acceptance.dsl.GenesisGenerator;
+import tech.pegasys.teku.test.acceptance.dsl.SimpleHttpClient;
 import tech.pegasys.teku.test.acceptance.dsl.TekuBeaconNode;
 import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfig;
 import tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfigBuilder;
@@ -27,6 +35,7 @@
 import tech.pegasys.teku.test.acceptance.dsl.tools.deposits.ValidatorKeystores;
 
 public class ValidatorClientServiceAcceptanceTest extends AcceptanceTestBase {
+  private static final Logger LOG = LogManager.getLogger();
 
   @Test
   void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnBeaconNode() throws Exception {
@@ -56,8 +65,7 @@ void bn_shouldFailIfValidatorKeyLocked(@TempDir final Path tempDir) throws Excep
                 .withInitialState(genesis)
                 .build());
 
-    beaconNode.startWithFailure(
-        "Unable to initialize validator keys, please manually correct errors and try again.");
+    beaconNode.startWithFailure("FATAL - Failed to load keystore", 20);
   }
 
   @Test
@@ -82,8 +90,7 @@ void vc_shouldFailIfValidatorKeyLocked(@TempDir final Path tempDir) throws Excep
                 .withWritableKeystorePath(initialKeystores, tempDir)
                 .build());
 
-    validatorNode.startWithFailure(
-        "Unable to initialize validator keys, please manually correct errors and try again.");
+    validatorNode.startWithFailure("FATAL - Failed to load keystore", 20);
   }
 
   @Test
@@ -104,8 +111,7 @@ void bn_shouldFailIfCannotLockKeys(@TempDir final Path tempDir) throws Exception
                 .withReadOnlyKeystorePath(initialKeystores, tempDir)
                 .build());
 
-    beaconNode.startWithFailure(
-        "Unable to initialize validator keys, please manually correct errors and try again.");
+    beaconNode.startWithFailure("FATAL - Please check the logs for details.");
   }
 
   @Test
@@ -127,8 +133,7 @@ void bn_shouldFailIfCannotLockKeysUnhandledException() throws Exception {
                 .withValidatorKeystoreLockingEnabled(true)
                 .build());
 
-    beaconNode.startWithFailure(
-        "Unable to initialize validator keys, please manually correct errors and try again.");
+    beaconNode.startWithFailure("FATAL - Failed to load keystore, error Access Denied", 20);
   }
 
   @Test
@@ -183,4 +188,54 @@ void shouldNotFailWithNoValidatorKeysWhenExitOptionDisabledOnValidatorClient() t
     validatorClient.stop();
     beaconNode.stop();
   }
+
+  @Test
+  void shouldStartValidatorApiWithoutSslAndAccessData() throws Exception {
+    final TekuBeaconNode beaconNode =
+        createTekuBeaconNode(
+            TekuNodeConfigBuilder.createBeaconNode()
+                .withExitWhenNoValidatorKeysEnabled(false)
+                .withValidatorApiNoSsl(true)
+                .withSpecifiedBearerToken("admin")
+                .withInteropValidators(0, 8)
+                .build());
+    beaconNode.start();
+    beaconNode.waitForLogMessageContaining("UNSAFE");
+    try {
+      final SimpleHttpClient client = new SimpleHttpClient();
+      final String result =
+          client.get(
+              beaconNode.getValidatorApiUrl(),
+              "/eth/v1/keystores",
+              Map.of("Authorization", "Bearer admin"));
+      assertThat(result).contains("validating_pubkey");
+    } catch (AssertionError ex) {
+      fail("Failed to read response from keystores");
+    }
+  }
+
+  @Test
+  void shouldStartValidatorApiWithoutSslAndRequireCorrectBearer() throws Exception {
+    boolean caught = false;
+    final TekuBeaconNode beaconNode =
+        createTekuBeaconNode(
+            TekuNodeConfigBuilder.createBeaconNode()
+                .withExitWhenNoValidatorKeysEnabled(false)
+                .withValidatorApiNoSsl(true)
+                .withSpecifiedBearerToken("admin")
+                .withInteropValidators(0, 8)
+                .build());
+    beaconNode.start();
+    beaconNode.waitForLogMessageContaining("UNSAFE");
+    try {
+      final SimpleHttpClient client = new SimpleHttpClient();
+      // without a bearer token this will fail
+      client.get(beaconNode.getValidatorApiUrl(), "/eth/v1/keystores");
+    } catch (AssertionError ex) {
+      caught = true;
+      LOG.debug(ex.getMessage());
+      assertThat(ex.getMessage()).contains("401");
+    }
+    assertTrue(caught);
+  }
 }

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/validatorslashing/MultiPeersStandAloneVcAcceptanceTest.java

@@ -28,7 +28,7 @@
  * - Node 2: Stand-alone VC with a separate BN <br>
  * The slashing event is sent to the first node via the POST attester/proposer slashing REST API. It
  * is then sent <br>
- * to the second BN which sends it to it's VC vie the attester/proposer slashing SSE channel
+ * to the second BN which sends it to it's VC via the attester/proposer slashing SSE channel
  */
 public class MultiPeersStandAloneVcAcceptanceTest extends ValidatorSlashingDetectionAcceptanceTest {
 

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/validatorslashing/MultiPeersStandAloneVcBlocksAcceptanceTest.java

@@ -28,7 +28,7 @@
  * - Node 2: Stand-alone VC with a separate BN <br>
  * The slashing event is sent to the first node via the POST attester/proposer slashing REST API. It
  * is then sent <br>
- * to the second BN withing a block which sends it to it's VC via the attester/proposer slashing SSE
+ * to the second BN within a block which sends it to it's VC via the attester/proposer slashing SSE
  * channel
  */
 public class MultiPeersStandAloneVcBlocksAcceptanceTest

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/validatorslashing/ValidatorSlashingDetectionAcceptanceTest.java

@@ -45,7 +45,8 @@ public class ValidatorSlashingDetectionAcceptanceTest extends AcceptanceTestBase
   final SystemTimeProvider timeProvider = new SystemTimeProvider();
   final String network = "swift";
   final String slashingActionLog =
-      "Validator(s) with public key(s) %s got slashed. Shutting down...";
+      "Validator slashing detection is enabled and validator(s) with public key(s) %s detected as slashed. "
+          + "Shutting down...";
   final int shutdownWaitingSeconds = 60;
 
   enum SlashingEventType {

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuBeaconNode.java

@@ -113,9 +113,13 @@ private TekuBeaconNode(
     super(network, TEKU_DOCKER_IMAGE_NAME, version, LOG);
     this.config = tekuNodeConfig;
     this.spec = SpecFactory.create(config.getNetworkName(), config.getSpecConfigModifier());
+    if (config.getConfigMap().containsKey("validator-api-enabled")) {
+      container.addExposedPort(VALIDATOR_API_PORT);
+    }
+
+    container.addExposedPorts(METRICS_PORT, REST_API_PORT);
     container
         .withWorkingDirectory(WORKING_DIRECTORY)
-        .withExposedPorts(REST_API_PORT, METRICS_PORT)
         .waitingFor(
             new HttpWaitStrategy()
                 .forPort(REST_API_PORT)

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java

@@ -31,6 +31,7 @@
 import tech.pegasys.teku.api.response.v1.EventType;
 
 public abstract class TekuNode extends Node {
+  public static final int VALIDATOR_API_PORT = 9052;
   private static final Logger LOG = LogManager.getLogger();
   protected boolean started = false;
 
@@ -68,12 +69,18 @@ public void start() throws Exception {
     container.start();
   }
 
+  // be aware that these regex's slow down quickly the bigger they are
+  // by default 10 second timeout, needs to be a short, simple match.
   public void startWithFailure(final String expectedError) throws Exception {
+    startWithFailure(expectedError, 10);
+  }
+
+  public void startWithFailure(final String expectedError, final int timeout) throws Exception {
     setUpStart();
     container.waitingFor(
         new LogMessageWaitStrategy()
-            .withRegEx(".*" + expectedError + ".*")
-            .withStartupTimeout(Duration.ofSeconds(10)));
+            .withRegEx(".*?" + expectedError + ".*")
+            .withStartupTimeout(Duration.ofSeconds(timeout)));
     container.start();
   }
 
@@ -140,4 +147,15 @@ private String getEventUrl(List<EventType> events) {
   protected URI getRestApiUrl() {
     return URI.create("http://127.0.0.1:" + container.getMappedPort(REST_API_PORT));
   }
+
+  public URI getValidatorApiUrl() {
+    final boolean isUseSsl =
+        (boolean) getConfig().getConfigMap().getOrDefault("Xvalidator-api-ssl-enabled", true);
+    final String prefix = isUseSsl ? "https" : "http";
+
+    final URI uri =
+        URI.create(prefix + "://127.0.0.1:" + container.getMappedPort(VALIDATOR_API_PORT));
+    LOG.debug("Validator URL: {}", uri);
+    return uri;
+  }
 }

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNodeConfigBuilder.java

@@ -25,9 +25,9 @@
 import static tech.pegasys.teku.test.acceptance.dsl.Node.SENTRY_NODE_CONFIG_FILE_PATH;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.WORKING_DIRECTORY;
 import static tech.pegasys.teku.test.acceptance.dsl.Node.copyToTmpFile;
+import static tech.pegasys.teku.test.acceptance.dsl.TekuNode.VALIDATOR_API_PORT;
 import static tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfig.DEFAULT_VALIDATOR_COUNT;
 import static tech.pegasys.teku.test.acceptance.dsl.TekuNodeConfig.INITIAL_STATE_FILE;
-import static tech.pegasys.teku.test.acceptance.dsl.TekuValidatorNode.VALIDATOR_API_PORT;
 
 import com.google.common.io.Resources;
 import io.libp2p.core.PeerId;
@@ -300,6 +300,27 @@ public TekuNodeConfigBuilder withValidatorApiEnabled() {
     return this;
   }
 
+  public TekuNodeConfigBuilder withValidatorApiNoSsl(final boolean ignoreSslInterface) {
+    LOG.debug("Validator api enabled, no SSL, ignoreInterface={}", ignoreSslInterface);
+    configMap.put("validator-api-enabled", true);
+    configMap.put("Xvalidator-api-ssl-enabled", false);
+    configMap.put("validator-api-port", VALIDATOR_API_PORT);
+    configMap.put("validator-api-host-allowlist", "*");
+    configMap.put("Xvalidator-api-unsafe-hosts-enabled", ignoreSslInterface);
+    return this;
+  }
+
+  public TekuNodeConfigBuilder withSpecifiedBearerToken(final String password) throws IOException {
+    final String bearerPath = "/bearer.txt";
+    LOG.debug("Setting bearer password, and mapping to file {}}", bearerPath);
+    configMap.put("validator-api-bearer-file", bearerPath);
+    final File bearerFile = File.createTempFile("bearer", ".txt");
+    Files.writeString(bearerFile.toPath(), password, UTF_8);
+    bearerFile.deleteOnExit();
+    configFileMap.put(bearerFile, bearerPath);
+    return this;
+  }
+
   public TekuNodeConfigBuilder withWritableKeystorePath(
       ValidatorKeystores keystores, Path tempDir) {
     LOG.debug("Xinterop-enabled=false");
@@ -390,8 +411,8 @@ public TekuNodeConfigBuilder withExternalMetricsClient(
   }
 
   public TekuNodeConfigBuilder withStopVcWhenValidatorSlashedEnabled() {
-    LOG.debug("Xshut-down-when-validator-slashed-enabled={}", true);
-    configMap.put("Xshut-down-when-validator-slashed-enabled", true);
+    LOG.debug("shut-down-when-validator-slashed-enabled={}", true);
+    configMap.put("shut-down-when-validator-slashed-enabled", true);
     return this;
   }
 

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuValidatorNode.java

@@ -17,7 +17,6 @@
 import static tech.pegasys.teku.test.acceptance.dsl.metrics.MetricConditions.withNameEqualsTo;
 import static tech.pegasys.teku.test.acceptance.dsl.metrics.MetricConditions.withValueGreaterThan;
 
-import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import org.apache.logging.log4j.LogManager;
@@ -29,7 +28,6 @@
 public class TekuValidatorNode extends TekuNode {
 
   private static final Logger LOG = LogManager.getLogger();
-  public static final int VALIDATOR_API_PORT = 9052;
   protected static final String VALIDATOR_PATH = DATA_PATH + "validator/";
 
   private final TekuNodeConfig config;
@@ -101,10 +99,6 @@ public void waitForBlockPublishedTo(final TekuBeaconNode node) {
         withValueGreaterThan(0));
   }
 
-  private URI getValidatorApiUrl() {
-    return URI.create("https://127.0.0.1:" + container.getMappedPort(VALIDATOR_API_PORT));
-  }
-
   private String getApiPassword() {
     return container.copyFileFromContainer(
         VALIDATOR_PATH + "key-manager/validator-api-bearer",

beacon/pow/build.gradle

@@ -1,5 +1,6 @@
 dependencies {
   implementation project(':ethereum:pow:api')
+  implementation project(':ethereum:pow:merkletree')
   implementation project(':ethereum:spec')
   implementation project(':infrastructure:async')
   implementation project(':infrastructure:bls')

beacon/pow/src/main/java/tech/pegasys/teku/beacon/pow/DepositSnapshotFileLoader.java

@@ -31,6 +31,7 @@
 import org.apache.tuweni.bytes.Bytes;
 import tech.pegasys.teku.ethereum.pow.api.DepositTreeSnapshot;
 import tech.pegasys.teku.ethereum.pow.api.schema.LoadDepositSnapshotResult;
+import tech.pegasys.teku.ethereum.pow.merkletree.DepositTree;
 import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
 import tech.pegasys.teku.infrastructure.http.UrlSanitizer;
 import tech.pegasys.teku.infrastructure.io.resource.ResourceLoader;
@@ -68,11 +69,19 @@ public LoadDepositSnapshotResult loadDepositSnapshot() {
       try {
         STATUS_LOG.loadingDepositSnapshotResource(sanitizedUrl);
         final DepositTreeSnapshot depositTreeSnapshot = loadFromUrl(depositSnapshotResourceUrl);
+        // Validate
+        DepositTree.fromSnapshot(depositTreeSnapshot);
         STATUS_LOG.onDepositSnapshot(
             depositTreeSnapshot.getDepositCount(), depositTreeSnapshot.getExecutionBlockHash());
         return LoadDepositSnapshotResult.create(Optional.of(depositTreeSnapshot));
       } catch (final Exception e) {
-        LOG.warn("Failed to load deposit tree snapshot from " + sanitizedUrl, e);
+        if (e instanceof IllegalArgumentException) {
+          LOG.warn(
+              "Deposit tree snapshot loaded from " + sanitizedUrl + " is not a correct snapshot",
+              e);
+        } else {
+          LOG.warn("Failed to load deposit tree snapshot from " + sanitizedUrl, e);
+        }
 
         if (isRequired) {
           throw new InvalidConfigurationException(