Merge branch 'main' into feat/bal_devnet_test_v5.6.1

Author: daniellehrner

CHANGELOG.md

@@ -51,6 +51,7 @@ are provided with different values, using input as per the execution-apis spec i
 - Support [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189): snap/2 - block access list exchange
 - Limit pooled tx requests by size and remove pre-eth/68 transaction announcement support [#9990](https://github.com/besu-eth/besu/pull/9990)
 - Reduce tx p2p broadcast bandwidth and memory used [#9937](https://github.com/besu-eth/besu/pull/9937) 
+- Improve syncing time of the experimental Bonsai Archive storage by migrating after a Bonsai full sync [#9979](https://github.com/besu-eth/besu/pull/9997)
 
 ### Plugin API
 - Plugin API: Allow the registration of multiple PluginTransactionPoolValidatorFactory [#9964](https://github.com/hyperledger/besu/pull/9964)

app/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -92,11 +92,15 @@
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.cache.CodeCache;
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.storage.flat.BonsaiArchiveFlatDbStrategy;
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiArchiver;
+import org.hyperledger.besu.ethereum.trie.pathbased.bonsaiarchive.BonsaiFlatDbToArchiveMigrator;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.storage.PathBasedWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.trie.pathbased.common.storage.flat.CodeHashCodeStorageStrategy;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.trielog.TrieLogManager;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
+import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
 import org.hyperledger.besu.ethereum.worldstate.PathBasedExtraStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive.WorldStateHealer;
@@ -121,6 +125,8 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalLong;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
@@ -888,6 +894,10 @@ public BesuController build() {
       }
     }
 
+    final List<Closeable> closeables = new ArrayList<>();
+    closeables.add(protocolContext.getWorldStateArchive());
+    closeables.add(storageProvider);
+
     if (DataStorageFormat.X_BONSAI_ARCHIVE.equals(
         dataStorageConfiguration.getDataStorageFormat())) {
       final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
@@ -898,12 +908,38 @@ public BesuController build() {
               blockchain,
               scheduler,
               ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager());
-      blockchain.observeBlockAdded(archiver);
-    }
 
-    final List<Closeable> closeables = new ArrayList<>();
-    closeables.add(protocolContext.getWorldStateArchive());
-    closeables.add(storageProvider);
+      if (worldStateStorageCoordinator.isMatchingFlatMode(FlatDbMode.FULL)
+          || worldStateStorageCoordinator.isMatchingFlatMode(FlatDbMode.PARTIAL)) {
+        final BonsaiFlatDbToArchiveMigrator archiveMigrator =
+            createArchiveMigrator(worldStateStorageCoordinator, worldStateArchive, blockchain);
+        closeables.add(archiveMigrator);
+
+        final AtomicBoolean migrationStarted = new AtomicBoolean(false);
+        final AtomicLong syncSubscriptionId = new AtomicLong();
+        syncSubscriptionId.set(
+            synchronizer.subscribeInSync(
+                (inSync) -> {
+                  if (inSync && migrationStarted.compareAndSet(false, true)) {
+                    synchronizer.unsubscribeInSync(syncSubscriptionId.get());
+                    LOG.info("Node is in sync, starting Bonsai archive migration");
+                    archiveMigrator
+                        .migrate()
+                        .thenRun(() -> blockchain.observeBlockAdded(archiver))
+                        .exceptionally(
+                            error -> {
+                              LOG.error(
+                                  "Archive migration failed, archiver will remain disabled until restart",
+                                  error);
+                              return null;
+                            });
+                  }
+                },
+                0));
+      } else {
+        blockchain.observeBlockAdded(archiver);
+      }
+    }
 
     return new BesuController(
         protocolSchedule,
@@ -1011,6 +1047,27 @@ private BonsaiArchiver createBonsaiArchiver(
     return archiver;
   }
 
+  private BonsaiFlatDbToArchiveMigrator createArchiveMigrator(
+      final WorldStateStorageCoordinator worldStateStorageCoordinator,
+      final WorldStateArchive worldStateArchive,
+      final Blockchain blockchain) {
+    final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
+        worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);
+    final TrieLogManager trieLogManager =
+        ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager();
+    final ScheduledExecutorService migrationExecutor =
+        MonitoredExecutors.newScheduledThreadPool("archive-migrator", 1, metricsSystem);
+    final BonsaiArchiveFlatDbStrategy archiveStrategy =
+        new BonsaiArchiveFlatDbStrategy(metricsSystem, new CodeHashCodeStorageStrategy());
+    return new BonsaiFlatDbToArchiveMigrator(
+        worldStateKeyValueStorage,
+        trieLogManager,
+        blockchain,
+        migrationExecutor,
+        metricsSystem,
+        archiveStrategy);
+  }
+
   /**
    * Create synchronizer synchronizer.
    *

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/pathbased/bonsai/BonsaiArchiveWorldStateProvider.java

@@ -24,6 +24,7 @@
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.provider.WorldStateQueryParams;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.worldview.PathBasedWorldState;
+import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
 import org.hyperledger.besu.ethereum.worldstate.PathBasedExtraStorageConfiguration;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 import org.hyperledger.besu.plugin.ServiceManager;
@@ -60,6 +61,12 @@ public BonsaiArchiveWorldStateProvider(
 
   @Override
   public Optional<MutableWorldState> getWorldState(final WorldStateQueryParams queryParams) {
+    // If not in archive mode then the migration is not yet complete, so fallback to
+    // the regular BonsaiWorldStateProvider
+    if (!worldStateKeyValueStorage.getFlatDbMode().equals(FlatDbMode.ARCHIVE)) {
+      return super.getWorldState(queryParams);
+    }
+
     if (queryParams.shouldWorldStateUpdateHead()) {
       return getFullWorldState(queryParams);
     } else {

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/pathbased/bonsai/storage/BonsaiWorldStateKeyValueStorage.java

@@ -47,9 +47,13 @@
 
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class BonsaiWorldStateKeyValueStorage extends PathBasedWorldStateKeyValueStorage
     implements WorldStateKeyValueStorage {
+  private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class);
+
   protected final BonsaiFlatDbStrategyProvider flatDbStrategyProvider;
 
   public BonsaiWorldStateKeyValueStorage(
@@ -167,6 +171,21 @@ public void upgradeToFullFlatDbMode() {
     flatDbStrategyProvider.upgradeToFullFlatDbMode(composedWorldStateStorage);
   }
 
+  public void upgradeToArchiveFlatDbMode() {
+    flatDbStrategyProvider.upgradeToArchiveFlatDbMode(composedWorldStateStorage);
+    // Invalidate cached world state snapshots that were created under the previous strategy.
+    // Snapshots share the flatDbStrategyProvider, so after the switch they would use the new
+    // ARCHIVE read path against stale snapshot data that lacks complete archive-keyed entries.
+    subscribers.forEach(
+        subscriber -> {
+          try {
+            subscriber.onClearFlatDatabaseStorage();
+          } catch (final Exception e) {
+            LOG.error("Error notifying subscriber of flat database storage upgrade", e);
+          }
+        });
+  }
+
   public void downgradeToPartialFlatDbMode() {
     flatDbStrategyProvider.downgradeToPartialFlatDbMode(composedWorldStateStorage);
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/pathbased/bonsai/storage/flat/BonsaiArchiveFlatDbStrategy.java

@@ -281,20 +281,33 @@ protected Stream<Pair<Bytes32, Bytes>> storageToPairStream(
   }
 
   /*
-   * Puts the account data for the given account hash and block context.
+   * Puts the account data for the given account hash.
    */
   @Override
   public void putFlatAccount(
       final SegmentedKeyValueStorage storage,
       final SegmentedKeyValueStorageTransaction transaction,
       final Hash accountHash,
       final Bytes accountValue) {
+    putFlatAccount(
+        getStateArchiveContextForWrite(storage).get(), transaction, accountHash, accountValue);
+  }
 
-    // key suffixed with block context, or MIN_BLOCK_SUFFIX if we have no context:
+  /**
+   * Puts the account data for the given account hash and block context.
+   *
+   * @param context the block context supplying the block number suffix for the archive key
+   * @param transaction the transaction to write into
+   * @param accountHash the hash of the account address
+   * @param accountValue the RLP-encoded account value
+   */
+  public void putFlatAccount(
+      final BonsaiContext context,
+      final SegmentedKeyValueStorageTransaction transaction,
+      final Hash accountHash,
+      final Bytes accountValue) {
     byte[] keySuffixed =
-        calculateArchiveKeyWithMinSuffix(
-            getStateArchiveContextForWrite(storage).get(), accountHash.getBytes().toArrayUnsafe());
-
+        calculateArchiveKeyWithMinSuffix(context, accountHash.getBytes().toArrayUnsafe());
     transaction.put(ACCOUNT_INFO_STATE_ARCHIVE, keySuffixed, accountValue.toArrayUnsafe());
   }
 
@@ -303,12 +316,22 @@ public void removeFlatAccount(
       final SegmentedKeyValueStorage storage,
       final SegmentedKeyValueStorageTransaction transaction,
       final Hash accountHash) {
+    removeFlatAccount(getStateArchiveContextForWrite(storage).get(), transaction, accountHash);
+  }
 
-    // insert a key suffixed with block context, with 'deleted account' value
+  /**
+   * Removes account data for the given account hash and block context.
+   *
+   * @param context the block context supplying the block number suffix for the archive key
+   * @param transaction the transaction to write into
+   * @param accountHash the hash of the account address
+   */
+  public void removeFlatAccount(
+      final BonsaiContext context,
+      final SegmentedKeyValueStorageTransaction transaction,
+      final Hash accountHash) {
     byte[] keySuffixed =
-        calculateArchiveKeyWithMinSuffix(
-            getStateArchiveContextForWrite(storage).get(), accountHash.getBytes().toArrayUnsafe());
-
+        calculateArchiveKeyWithMinSuffix(context, accountHash.getBytes().toArrayUnsafe());
     transaction.put(ACCOUNT_INFO_STATE_ARCHIVE, keySuffixed, DELETED_ACCOUNT_VALUE);
   }
 
@@ -390,13 +413,33 @@ public void putFlatAccountStorageValueByStorageSlotHash(
       final Hash accountHash,
       final Hash slotHash,
       final Bytes storageValue) {
+    putFlatAccountStorageValueByStorageSlotHash(
+        getStateArchiveContextForWrite(storage).get(),
+        transaction,
+        accountHash,
+        slotHash,
+        storageValue);
+  }
 
+  /**
+   * Puts the storage value for the given account hash and storage slot key for a given context.
+   *
+   * @param context the block context supplying the block number suffix for the archive key
+   * @param transaction the transaction to write into
+   * @param accountHash the hash of the account address
+   * @param slotHash the hash of the storage slot key
+   * @param storageValue the storage value
+   */
+  public void putFlatAccountStorageValueByStorageSlotHash(
+      final BonsaiContext context,
+      final SegmentedKeyValueStorageTransaction transaction,
+      final Hash accountHash,
+      final Hash slotHash,
+      final Bytes storageValue) {
     // get natural key from account hash and slot key
     byte[] naturalKey = calculateNaturalSlotKey(accountHash, slotHash);
     // keyNearest, use MIN_BLOCK_SUFFIX in the absence of a block context:
-    byte[] keyNearest =
-        calculateArchiveKeyWithMinSuffix(getStateArchiveContextForWrite(storage).get(), naturalKey);
-
+    byte[] keyNearest = calculateArchiveKeyWithMinSuffix(context, naturalKey);
     transaction.put(ACCOUNT_STORAGE_ARCHIVE, keyNearest, storageValue.toArrayUnsafe());
   }
 
@@ -409,13 +452,27 @@ public void removeFlatAccountStorageValueByStorageSlotHash(
       final SegmentedKeyValueStorageTransaction transaction,
       final Hash accountHash,
       final Hash slotHash) {
+    removeFlatAccountStorageValueByStorageSlotHash(
+        getStateArchiveContextForWrite(storage).get(), transaction, accountHash, slotHash);
+  }
 
+  /**
+   * Removes the storage value for the given account hash and storage slot key for a given context.
+   *
+   * @param context the block context supplying the block number suffix for the archive key
+   * @param transaction the transaction to write into
+   * @param accountHash the hash of the account address
+   * @param slotHash the hash of the storage slot key
+   */
+  public void removeFlatAccountStorageValueByStorageSlotHash(
+      final BonsaiContext context,
+      final SegmentedKeyValueStorageTransaction transaction,
+      final Hash accountHash,
+      final Hash slotHash) {
     // get natural key from account hash and slot key
     byte[] naturalKey = calculateNaturalSlotKey(accountHash, slotHash);
     // insert a key suffixed with block context, with 'deleted account' value
-    byte[] keySuffixed =
-        calculateArchiveKeyWithMinSuffix(getStateArchiveContextForWrite(storage).get(), naturalKey);
-
+    byte[] keySuffixed = calculateArchiveKeyWithMinSuffix(context, naturalKey);
     transaction.put(ACCOUNT_STORAGE_ARCHIVE, keySuffixed, DELETED_STORAGE_VALUE);
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/trie/pathbased/bonsai/storage/flat/BonsaiFlatDbStrategyProvider.java

@@ -22,7 +22,6 @@
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
 import org.hyperledger.besu.plugin.services.MetricsSystem;
-import org.hyperledger.besu.plugin.services.storage.DataStorageFormat;
 import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorage;
 import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
 
@@ -45,11 +44,7 @@ protected FlatDbMode getRequestedFlatDbMode(
             .getPathBasedExtraStorageConfiguration()
             .getUnstable()
             .getFullFlatDbEnabled()
-        ? (dataStorageConfiguration
-                .getDataStorageFormat()
-                .equals(DataStorageFormat.X_BONSAI_ARCHIVE)
-            ? FlatDbMode.ARCHIVE
-            : FlatDbMode.FULL)
+        ? FlatDbMode.FULL
         : FlatDbMode.PARTIAL;
   }
 
@@ -61,16 +56,19 @@ protected FlatDbMode alternativeFlatDbModeForExistingDatabase() {
   public void upgradeToFullFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
     final SegmentedKeyValueStorageTransaction transaction =
         composedWorldStateStorage.startTransaction();
-    if (dataStorageConfiguration.getDataStorageFormat() == DataStorageFormat.BONSAI) {
-      LOG.info("setting FlatDbStrategy to FULL");
-      transaction.put(
-          TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
-    } else if (dataStorageConfiguration.getDataStorageFormat()
-        == DataStorageFormat.X_BONSAI_ARCHIVE) {
-      LOG.info("setting FlatDbStrategy to ARCHIVE");
-      transaction.put(
-          TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.ARCHIVE.getVersion().toArrayUnsafe());
-    }
+    LOG.info("setting FlatDbStrategy to FULL");
+    transaction.put(
+        TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.FULL.getVersion().toArrayUnsafe());
+    transaction.commit();
+    loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy
+  }
+
+  public void upgradeToArchiveFlatDbMode(final SegmentedKeyValueStorage composedWorldStateStorage) {
+    final SegmentedKeyValueStorageTransaction transaction =
+        composedWorldStateStorage.startTransaction();
+    LOG.info("setting FlatDbStrategy to ARCHIVE");
+    transaction.put(
+        TRIE_BRANCH_STORAGE, FLAT_DB_MODE, FlatDbMode.ARCHIVE.getVersion().toArrayUnsafe());
     transaction.commit();
     loadFlatDbStrategy(composedWorldStateStorage); // force reload of flat db reader strategy
   }