Headers backward

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -482,7 +482,11 @@ public ThreadBesuNodeRunner provideThreadBesuNodeRunner() {
   public static class BesuControllerModule {
     @Provides
     @Singleton
-    public SynchronizerConfiguration provideSynchronizationConfiguration() {
+    public SynchronizerConfiguration provideSynchronizationConfiguration(final BesuNode node) {
+      // Use the synchronizer configuration set on the node, otherwise use default
+      if (node.getSynchronizerConfiguration() != null) {
+        return node.getSynchronizerConfiguration();
+      }
       final SynchronizerConfiguration synchronizerConfiguration =
           SynchronizerConfiguration.builder().build();
       return synchronizerConfiguration;

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/bft/BftSyncAcceptanceTest.java

@@ -16,6 +16,8 @@
 
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.snapsync.ImmutableSnapSyncConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncConfiguration;
 import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
 
 import java.util.stream.Stream;
@@ -51,10 +53,24 @@ public void shouldSyncValidatorNode(
     final BesuNode validator3 = nodeFactory.createBonsaiNodeFixedPort(besu, "validator3");
     final BesuNode validator4 = nodeFactory.createBonsaiNodeFixedPort(besu, "validator4");
 
-    // Configure validators with specified sync mode
+    // Enable snap server on validators 1-3 so they can serve world state data
+    // This is required for SNAP/CHECKPOINT sync modes to work
+    final SnapSyncConfiguration snapServerEnabledConfig =
+        ImmutableSnapSyncConfiguration.builder().isSnapServerEnabled(true).build();
+    final SynchronizerConfiguration fullSyncWithSnapServer =
+        SynchronizerConfiguration.builder()
+            .syncMode(SyncMode.FULL)
+            .syncMinimumPeerCount(1)
+            .snapSyncConfiguration(snapServerEnabledConfig)
+            .build();
+
+    validator1.setSynchronizerConfiguration(fullSyncWithSnapServer);
+    validator2.setSynchronizerConfiguration(fullSyncWithSnapServer);
+    validator3.setSynchronizerConfiguration(fullSyncWithSnapServer);
+
+    // Configure validator4 with the test's specified sync mode (FULL/SNAP/CHECKPOINT)
     final SynchronizerConfiguration syncConfig =
         SynchronizerConfiguration.builder().syncMode(syncMode).syncMinimumPeerCount(1).build();
-
     validator4.setSynchronizerConfiguration(syncConfig);
 
     // Start first three validators

build.gradle

@@ -236,7 +236,7 @@ configure(allprojects - project(':platform')) {
     java {
       // This path needs to be relative to each project
       target 'src/**/*.java'
-      targetExclude '**/src/reference-test/**', '**/src/main/generated/**', '**/src/test/generated/**', '**/src/jmh/generated/**'
+      targetExclude '**/src/reference-test/**', '**/src/main/generated/**', '**/src/test/generated/**', '**/src/jmh/generated/**', '**/build/generated/**'
       removeUnusedImports()
       googleJavaFormat('1.25.2')
       importOrder 'org.hyperledger', 'java', ''

consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/blockcreation/BftMiningCoordinator.java

@@ -160,7 +160,7 @@ public void subscribe() {
     }
     syncState.subscribeSyncStatus(
         syncStatus -> {
-          if (syncState.syncTarget().isPresent()) {
+          if (syncState.syncTarget().isPresent() || !syncState.isInitialSyncPhaseDone()) {
             // We're syncing so stop doing other stuff
             LOG.info("Stopping BFT mining coordinator while we are syncing");
             stop();

consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/BFTPivotSelectorFromPeers.java

@@ -29,6 +29,7 @@
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 
 import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -85,7 +86,7 @@ protected boolean weAreAValidator(final ValidatorProvider validatorProvider) {
   }
 
   @Override
-  public Optional<FastSyncState> selectNewPivotBlock() {
+  public CompletableFuture<FastSyncState> selectNewPivotBlock() {
 
     final BftContext bftContext = protocolContext.getConsensusContext(BftContext.class);
     final ValidatorProvider validatorProvider = bftContext.getValidatorProvider();
@@ -102,7 +103,11 @@ public Optional<FastSyncState> selectNewPivotBlock() {
         throw new NoSyncRequiredException();
       }
 
-      return bestPeer.flatMap(this::fromBestPeer);
+      return bestPeer
+          .map(this::fromBestPeer)
+          .orElse(
+              CompletableFuture.failedFuture(
+                  new RuntimeException("No peers with sufficient height")));
     } else {
       // Treat us being the only validator as a special case. We are the only node that can produce
       // blocks so we won't wait to sync with a non-validator node that may or may not exist
@@ -154,6 +159,7 @@ public Optional<FastSyncState> selectNewPivotBlock() {
       }
     }
 
-    return Optional.empty();
+    return CompletableFuture.failedFuture(
+        new RuntimeException("Not enough peers to select pivot block"));
   }
 }

consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/sync/QbftPivotSelectorTest.java

@@ -14,7 +14,6 @@
  */
 package org.hyperledger.besu.consensus.qbft.sync;
 
-import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.fail;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
@@ -30,13 +29,11 @@
 import org.hyperledger.besu.ethereum.eth.manager.EthPeers;
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
-import org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState;
 import org.hyperledger.besu.ethereum.eth.sync.fastsync.NoSyncRequiredException;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -79,8 +76,12 @@ public void returnEmptySyncStateIfValidatorWithOtherValidatorsButNoPeers() {
     BFTPivotSelectorFromPeers pivotSelector =
         new BFTPivotSelectorFromPeers(
             ethContext, syncConfig, syncState, protocolContext, nodeKey, blockHeader);
-    Optional<FastSyncState> pivotState = pivotSelector.selectNewPivotBlock();
-    assertThat(pivotState.isEmpty()).isTrue();
+    try {
+      pivotSelector.selectNewPivotBlock().get();
+      fail("Expected CompletableFuture to be failed but it completed successfully");
+    } catch (Exception e) {
+      // Expected - the future should fail when no peers are available
+    }
   }
 
   @Test
@@ -99,7 +100,7 @@ public void returnNoSyncRequiredIfOnlyValidatorAndNoPeers() {
             ethContext, syncConfig, syncState, protocolContext, nodeKey, blockHeader);
 
     try {
-      Optional<FastSyncState> pivotState = pivotSelector.selectNewPivotBlock();
+      pivotSelector.selectNewPivotBlock();
       fail("Expected NoSyncRequiredException but none thrown");
     } catch (NoSyncRequiredException e) {
       // Ignore - just make sure we get here
@@ -114,8 +115,12 @@ public void returnEmptySyncStateIfNonValidatorWithNoBestPeer() {
         new BFTPivotSelectorFromPeers(
             ethContext, syncConfig, syncState, protocolContext, nodeKey, blockHeader);
 
-    Optional<FastSyncState> pivotState = pivotSelector.selectNewPivotBlock();
-    assertThat(pivotState.isEmpty()).isTrue();
+    try {
+      pivotSelector.selectNewPivotBlock().get();
+      fail("Expected CompletableFuture to be failed but it completed successfully");
+    } catch (Exception e) {
+      // Expected - the future should fail when no best peer is available
+    }
   }
 
   @Test
@@ -127,7 +132,11 @@ public void returnEmptySyncStateIfValidatorAndNotAtGenesisAndOtherValidators() {
         new BFTPivotSelectorFromPeers(
             ethContext, syncConfig, syncState, protocolContext, nodeKey, blockHeader);
 
-    Optional<FastSyncState> pivotState = pivotSelector.selectNewPivotBlock();
-    assertThat(pivotState.isEmpty()).isTrue();
+    try {
+      pivotSelector.selectNewPivotBlock().get();
+      fail("Expected CompletableFuture to be failed but it completed successfully");
+    } catch (Exception e) {
+      // Expected - the future should fail when no best peer is available
+    }
   }
 }

ethereum/api/build.gradle

@@ -135,8 +135,10 @@ tasks.register('generateTestBlockchain') {
   dependsOn(configurations.testResourceGeneration)
   dependsOn(processTestResources)
   doLast {
-    if(!dataPath.exists()) {
-      mkdir(dataPath)
+    if(!file(blocksBin).exists()) {
+      if(!dataPath.exists()) {
+        mkdir(dataPath)
+      }
 
       javaexec {
         main = 'org.hyperledger.besu.Besu'

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/AbstractEngineForkchoiceUpdated.java

@@ -126,7 +126,7 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
         mergeCoordinator.getOrSyncHeadByHash(
             forkChoice.getHeadBlockHash(), forkChoice.getFinalizedBlockHash());
 
-    if (maybeNewHead.isEmpty()) {
+    if (maybeNewHead.isEmpty() || mergeContext.get().isSyncing()) {
       return syncingResponse(requestId, forkChoice);
     }
 
@@ -191,10 +191,6 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
       return new JsonRpcSuccessResponse(requestId, parameterValidationResult);
     }
 
-    if (mergeContext.get().isSyncing()) {
-      return syncingResponse(requestId, forkChoice);
-    }
-
     maybePayloadAttributes.ifPresentOrElse(
         this::logPayload, () -> LOG.debug("Payload attributes are null"));
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/Blockchain.java

@@ -137,6 +137,15 @@ default boolean blockIsOnCanonicalChain(final Hash blockHeaderHash) {
    */
   Optional<BlockHeader> getBlockHeader(long blockNumber);
 
+  /**
+   * Returns a list of block headers starting from the given block number.
+   *
+   * @param startBlockNumber The number of the first block header.
+   * @param numberOfHeaders The number of blocks to retrieve.
+   * @return The block headers.
+   */
+  List<BlockHeader> getBlockHeaders(long startBlockNumber, int numberOfHeaders);
+
   /**
    * Return true if the block corresponding the hash is present.
    *

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java

@@ -30,6 +30,8 @@
 import org.hyperledger.besu.ethereum.core.Difficulty;
 import org.hyperledger.besu.ethereum.core.LogWithMetadata;
 import org.hyperledger.besu.ethereum.core.SyncBlock;
+import org.hyperledger.besu.ethereum.core.SyncBlockBody;
+import org.hyperledger.besu.ethereum.core.SyncBlockWithReceipts;
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.core.TransactionReceipt;
 import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList;
@@ -88,7 +90,7 @@ public class DefaultBlockchain implements MutableBlockchain {
 
   private Counter gasUsedCounter = NoOpMetricsSystem.NO_OP_COUNTER;
   private Counter numberOfTransactionsCounter = NoOpMetricsSystem.NO_OP_COUNTER;
-  // difficultyForSyncing is thread safe, as it is only used in the one thread of the import step
+
   private Difficulty difficultyForSyncing = Difficulty.ZERO;
 
   private DefaultBlockchain(
@@ -371,6 +373,35 @@ public Optional<BlockHeader> getBlockHeader(final long blockNumber) {
     return blockchainStorage.getBlockHash(blockNumber).flatMap(this::getBlockHeader);
   }
 
+  @Override
+  public List<BlockHeader> getBlockHeaders(final long firstBlock, final int numberOfHeaders) {
+    List<BlockHeader> headers = new ArrayList<>(numberOfHeaders);
+
+    final Optional<BlockHeader> optionalBlockHeaderStart =
+        getBlockHeader(firstBlock + numberOfHeaders - 1);
+    if (optionalBlockHeaderStart.isEmpty()) {
+      LOG.error("Missing header at block number {}", firstBlock + numberOfHeaders - 1);
+      throw new IllegalStateException(
+          "Missing header at block " + (firstBlock + numberOfHeaders - 1));
+    } else {
+      final BlockHeader startBlockHeader = optionalBlockHeaderStart.get();
+      headers.add(startBlockHeader);
+      Hash nextHash = startBlockHeader.getParentHash();
+      for (int i = 0; i < numberOfHeaders - 1; i++) {
+        Optional<BlockHeader> optionalBlockHeader = getBlockHeader(nextHash);
+        if (optionalBlockHeader.isEmpty()) {
+          LOG.error("Missing header for block {}", nextHash);
+          throw new IllegalStateException("Missing header for block " + nextHash);
+        } else {
+          BlockHeader blockHeader = optionalBlockHeader.get();
+          headers.addFirst(blockHeader);
+          nextHash = blockHeader.getParentHash();
+        }
+      }
+    }
+    return headers;
+  }
+
   @Override
   public Optional<BlockHeader> getBlockHeader(final Hash blockHeaderHash) {
     return blockHeadersCache
@@ -523,6 +554,18 @@ public synchronized void storeBlock(
     appendBlockHelper(new BlockWithReceipts(block, receipts), true, true);
   }
 
+  @Override
+  public void storeBlockHeaders(final List<BlockHeader> blockHeaders) {
+    final BlockchainStorage.Updater updater = blockchainStorage.updater();
+    blockHeaders.forEach(
+        header -> {
+          final Hash hash = header.getHash();
+          updater.putBlockHeader(hash, header);
+          updater.putBlockHash(header.getNumber(), hash);
+        });
+    updater.commit();
+  }
+
   @Override
   public void unsafeStoreHeader(final BlockHeader blockHeader, final Difficulty totalDifficulty) {
     // as this is used only to store premerge block headers, we don't cache the header in this case
@@ -650,6 +693,31 @@ public synchronized void unsafeImportBlock(
     updater.commit();
   }
 
+  @Override
+  public void unsafeImportSyncBodiesAndReceipts(
+      final List<SyncBlockWithReceipts> blocksAndReceipts, final boolean indexTransactions) {
+    final BlockchainStorage.Updater updater = blockchainStorage.updater();
+    for (final SyncBlockWithReceipts blockAndReceipts : blocksAndReceipts) {
+      final SyncBlock block = blockAndReceipts.getBlock();
+      final BlockHeader header = block.getHeader();
+      final Hash blockHash = header.getHash();
+      final SyncBlockBody body = block.getBody();
+      updater.putBlockHash(header.getNumber(), blockHash);
+      updater.putSyncBlockBody(blockHash, body);
+      updater.putTransactionReceipts(blockHash, blockAndReceipts.getReceipts());
+      this.totalDifficulty = calculateTotalDifficultyForSyncing(header);
+      updater.putTotalDifficulty(blockHash, totalDifficulty);
+      this.chainHeader = header;
+      if (indexTransactions) {
+        final List<Hash> listOfTxHashes =
+            body.getEncodedTransactions().stream().map(Hash::hash).toList();
+        indexTransactionsForBlock(updater, blockHash, listOfTxHashes);
+      }
+    }
+    updater.setChainHead(chainHeader.getBlockHash());
+    updater.commit();
+  }
+
   @Override
   public synchronized void unsafeSetChainHead(
       final BlockHeader blockHeader, final Difficulty totalDifficulty) {
@@ -674,6 +742,14 @@ public Difficulty calculateTotalDifficulty(final BlockHeader blockHeader) {
     return blockHeader.getDifficulty().add(parentTotalDifficulty);
   }
 
+  /**
+   * Calculates the total difficulty (TD) during the sync. This method calculates the TD relying on
+   * the TD of the previous block being available. This method has to be called IN ORDER and is NOT
+   * THREAD SAFE.
+   *
+   * @param blockHeader The block header of the block for which the total difficulty is needed.
+   * @return The total difficulty
+   */
   private Difficulty calculateTotalDifficultyForSyncing(final BlockHeader blockHeader) {
     if (blockHeader.getNumber() == BlockHeader.GENESIS_BLOCK_NUMBER) {
       difficultyForSyncing = blockHeader.getDifficulty();
@@ -682,7 +758,13 @@ private Difficulty calculateTotalDifficultyForSyncing(final BlockHeader blockHea
           blockchainStorage
               .getTotalDifficulty(blockHeader.getParentHash())
               .orElseThrow(
-                  () -> new IllegalStateException("Blockchain is missing total difficulty data."));
+                  () ->
+                      new IllegalStateException(
+                          "Blockchain is missing total difficulty data for block "
+                              + (blockHeader.getNumber() - 1)
+                              + ", block hash "
+                              + blockHeader.getParentHash()
+                              + "."));
       difficultyForSyncing = parentTotalDifficulty.add(blockHeader.getDifficulty());
     } else {
       difficultyForSyncing = difficultyForSyncing.add(blockHeader.getDifficulty());
@@ -736,6 +818,13 @@ private BlockAddedEvent updateCanonicalChainData(
       if (newBlock.getHeader().getParentHash().equals(chainHead) || chainHead == null) {
         return handleNewHead(updater, newBlock, receipts, transactionIndexing);
       } else {
+        LOG.error(
+            "New block {} parent hash {} does not match current head block {} hash {} (chainHead= {}).",
+            newBlock.getHeader().getNumber(),
+            newBlock.getHeader().getParentHash(),
+            chainHeader.getNumber(),
+            chainHeader.getHash(),
+            chainHead);
         throw new RuntimeException("Blocks during sync should always be in order");
       }
     } catch (final NoSuchElementException e) {