Merge branch 'main' into issue-10115

Signed-off-by: Sagar Khandagre <sagar.khandagre998@gmail.com>

Author: sagarkhandagre998

CHANGELOG.md

@@ -41,6 +41,7 @@ are provided with different values, using input as per the execution-apis spec i
 - Upgrade besu-native libraries version to 1.5.0. This fixes the issue of besu-native/secp256r1 exporting OpenSSL symbols in JVM space. [besu-native #308](https://github.com/besu-eth/besu-native/pull/308)
 
 ### Additions and Improvements
+- Add `transactionReceipts` subscription type to `eth_subscribe` that pushes all transaction receipts when a new block is added, with optional `transactionHashes` filter to receive receipts for specific transactions only [#10190](https://github.com/besu-eth/besu/pull/10190)
 - `--net-restrict` now supports IPv6 CIDR notation (e.g. `fd00::/64`) in addition to IPv4, enabling subnet-based peer filtering in IPv6 and dual-stack deployments [#10028](https://github.com/besu-eth/besu/pull/10028)
 - Stop EngineQosTimer as part of shutdown [#9903](https://github.com/hyperledger/besu/pull/9903)
 - Add `--max-blobs-per-transaction` CLI option to configure the maximum number of blobs per transaction [#9912](https://github.com/hyperledger/besu/pull/9912)
@@ -51,6 +52,7 @@ are provided with different values, using input as per the execution-apis spec i
 - Support [EIP-8189](https://eips.ethereum.org/EIPS/eip-8189): snap/2 - block access list exchange
 - Limit pooled tx requests by size and remove pre-eth/68 transaction announcement support [#9990](https://github.com/besu-eth/besu/pull/9990)
 - Reduce tx p2p broadcast bandwidth and memory used [#9937](https://github.com/besu-eth/besu/pull/9937) 
+- Improve syncing time of the experimental Bonsai Archive storage by migrating after a Bonsai full sync [#9979](https://github.com/besu-eth/besu/pull/9997)
 
 ### Plugin API
 - Plugin API: Allow the registration of multiple PluginTransactionPoolValidatorFactory [#9964](https://github.com/hyperledger/besu/pull/9964)
@@ -127,6 +129,7 @@ are provided with different values, using input as per the execution-apis spec i
 - Add ability to pass a custom tracer to block simulation [#9708](https://github.com/hyperledger/besu/pull/9708)
 - Add support for `4byteTracer` in `debug_trace*` methods to collect function selectors from internal calls via PR [#9642](https://github.com/hyperledger/besu/pull/9642). Thanks to [@JukLee0ira](https://github.com/JukLee0ira).
 - Update assertj to v3.27.7 [#9710](https://github.com/hyperledger/besu/pull/9710)
+- Update rocksdbjni lib version from 9.7.3 to 10.6.2 [#9767](https://github.com/hyperledger/besu/pull/9767)
 - Update vertx to 4.5.24 [#9645](https://github.com/hyperledger/besu/pull/9645)
 - Add byte-level metrics for P2P message exchange [#9666](https://github.com/hyperledger/besu/pull/9666)
 - Add IPv6 dual-stack support for DiscV5 peer discovery (enabled via `--Xv5-discovery-enabled`): new `--p2p-host-ipv6`, `--p2p-interface-ipv6`, and `--p2p-port-ipv6` CLI options enable a second UDP discovery socket; `--p2p-ipv6-outbound-enabled` controls whether IPv6 is preferred for outbound connections when a peer advertises both address families [#9763](https://github.com/hyperledger/besu/pull/9763); RLPx now also binds a second TCP socket on the IPv6 interface so IPv6-only peers can establish connections [#9873](https://github.com/hyperledger/besu/pull/9873)

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java

@@ -62,7 +62,6 @@
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.slf4j.Logger;
@@ -210,11 +209,4 @@ protected void waitForFile(final Path path) {
               }
             });
   }
-
-  @Test
-  public void dryRunDetector() {
-    assertThat(true)
-        .withFailMessage("This test is here so gradle --dry-run executes this class")
-        .isTrue();
-  }
 }

acceptance-tests/tests/src/acceptanceTest/java/org/hyperledger/besu/tests/acceptance/jsonrpc/AbstractJsonRpcTest.java

@@ -40,7 +40,6 @@
 import okhttp3.Request;
 import okhttp3.RequestBody;
 import okhttp3.Response;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -148,11 +147,4 @@ public static Stream<Arguments> testCasesFromPath(final String testCasesPath)
 
     return Arrays.stream(testCasesList).sorted().map(File::toURI).map(Arguments::of);
   }
-
-  @Test
-  void dryRunDetector() {
-    assertThat(true)
-        .withFailMessage("This test is here so gradle --dry-run executes this class")
-        .isTrue();
-  }
 }

acceptance-tests/tests/src/acceptanceTest/resources/jsonrpc/engine/paris/test-cases/08_unknown_payload.json

@@ -11,8 +11,8 @@
     "jsonrpc": "2.0",
     "id": 67,
     "error": {
-      "code": -32001,
-      "message": "Payload does not exist / is not available"
+      "code": -38001,
+      "message": "Unknown payload"
     }
   },
   "statusCode": 200

app/src/main/java/org/hyperledger/besu/RunnerBuilder.java

@@ -61,6 +61,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.pending.PendingTransactionDroppedSubscriptionService;
 import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.pending.PendingTransactionSubscriptionService;
 import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.syncing.SyncingSubscriptionService;
+import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.subscription.transactionreceipts.TransactionReceiptsSubscriptionService;
 import org.hyperledger.besu.ethereum.api.query.BlockchainQueries;
 import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
@@ -934,6 +935,9 @@ public Runner build() {
           context.getBlockchain(), blockchainQueries, subscriptionManager);
 
       createSyncingSubscriptionService(synchronizer, subscriptionManager);
+
+      createTransactionReceiptsSubscriptionService(
+          context.getBlockchain(), blockchainQueries, subscriptionManager, protocolSchedule);
     }
 
     Optional<EngineJsonRpcService> engineJsonRpcService = Optional.empty();
@@ -1435,6 +1439,18 @@ private void createNewBlockHeadersSubscriptionService(
     blockchain.observeBlockAdded(newBlockHeadersSubscriptionService);
   }
 
+  private void createTransactionReceiptsSubscriptionService(
+      final Blockchain blockchain,
+      final BlockchainQueries blockchainQueries,
+      final SubscriptionManager subscriptionManager,
+      final ProtocolSchedule protocolSchedule) {
+    final TransactionReceiptsSubscriptionService transactionReceiptsSubscriptionService =
+        new TransactionReceiptsSubscriptionService(
+            subscriptionManager, blockchainQueries, protocolSchedule);
+
+    blockchain.observeBlockAdded(transactionReceiptsSubscriptionService);
+  }
+
   private WebSocketService createWebsocketService(
       final Vertx vertx,
       final WebSocketConfiguration configuration,

app/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1917,7 +1917,10 @@ private void configure() throws Exception {
 
     jsonRpcConfiguration =
         jsonRpcHttpOptions.jsonRpcConfiguration(
-            hostsAllowlist, p2PDiscoveryConfig.p2pHost(), unstableRPCOptions.getHttpTimeoutSec());
+            hostsAllowlist,
+            p2PDiscoveryConfig.p2pHost(),
+            unstableRPCOptions.getHttpTimeoutSec(),
+            unstableRPCOptions.getHttpStreamingTimeoutSec());
     logger.info("RPC HTTP JSON-RPC config: {}", jsonRpcConfiguration);
     if (isEngineApiEnabled()) {
       engineJsonRpcConfiguration = createEngineJsonRpcConfiguration();
@@ -2086,7 +2089,8 @@ private JsonRpcConfiguration createEngineJsonRpcConfiguration() {
         jsonRpcHttpOptions.jsonRpcConfiguration(
             engineRPCConfig.engineHostsAllowlist(),
             p2PDiscoveryConfig.p2pHost(),
-            unstableRPCOptions.getHttpTimeoutSec());
+            unstableRPCOptions.getHttpTimeoutSec(),
+            unstableRPCOptions.getHttpStreamingTimeoutSec());
     engineConfig.setPort(engineRPCConfig.engineRpcPort());
     engineConfig.setRpcApis(Arrays.asList("ENGINE", "ETH"));
     engineConfig.setEnabled(isEngineApiEnabled());

app/src/main/java/org/hyperledger/besu/cli/options/JsonRpcHttpOptions.java

@@ -302,17 +302,22 @@ public JsonRpcConfiguration jsonRpcConfiguration() {
    * @param hostsAllowlist List of hosts allowed
    * @param defaultHostAddress Default host address
    * @param timeoutSec timeout in seconds
+   * @param streamingTimeoutSec timeout in seconds for streaming methods
    * @return A JsonRpcConfiguration instance
    */
   public JsonRpcConfiguration jsonRpcConfiguration(
-      final List<String> hostsAllowlist, final String defaultHostAddress, final Long timeoutSec) {
+      final List<String> hostsAllowlist,
+      final String defaultHostAddress,
+      final Long timeoutSec,
+      final Long streamingTimeoutSec) {
 
     final JsonRpcConfiguration jsonRpcConfiguration = this.jsonRpcConfiguration();
 
     jsonRpcConfiguration.setHost(
         Strings.isNullOrEmpty(rpcHttpHost) ? defaultHostAddress : rpcHttpHost);
     jsonRpcConfiguration.setHostsAllowlist(hostsAllowlist);
     jsonRpcConfiguration.setHttpTimeoutSec(timeoutSec);
+    jsonRpcConfiguration.setHttpStreamingTimeoutSec(streamingTimeoutSec);
     return jsonRpcConfiguration;
   }
 

app/src/main/java/org/hyperledger/besu/cli/options/RPCOptions.java

@@ -15,6 +15,7 @@
 package org.hyperledger.besu.cli.options;
 
 import org.hyperledger.besu.ethereum.api.handlers.TimeoutOptions;
+import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcConfiguration;
 
 import picocli.CommandLine;
 
@@ -27,6 +28,14 @@ public class RPCOptions {
       description = "HTTP timeout in seconds (default: ${DEFAULT-VALUE})")
   private final Long httpTimeoutSec = TimeoutOptions.defaultOptions().getTimeoutSeconds();
 
+  @CommandLine.Option(
+      hidden = true,
+      names = {"--Xhttp-streaming-timeout-seconds"},
+      description =
+          "HTTP timeout in seconds for streaming methods like debug_traceBlock (default: ${DEFAULT-VALUE})")
+  private final Long httpStreamingTimeoutSec =
+      JsonRpcConfiguration.DEFAULT_HTTP_STREAMING_TIMEOUT_SEC;
+
   @CommandLine.Option(
       hidden = true,
       names = {"--Xws-timeout-seconds"},
@@ -54,6 +63,15 @@ public Long getHttpTimeoutSec() {
     return httpTimeoutSec;
   }
 
+  /**
+   * Gets http streaming timeout sec.
+   *
+   * @return the http streaming timeout sec
+   */
+  public Long getHttpStreamingTimeoutSec() {
+    return httpStreamingTimeoutSec;
+  }
+
   /**
    * Gets WebSocket timeout sec.
    *

app/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -92,11 +92,15 @@
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.cache.BonsaiCachedMerkleTrieLoader;
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.cache.CodeCache;
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.storage.BonsaiWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.storage.flat.BonsaiArchiveFlatDbStrategy;
 import org.hyperledger.besu.ethereum.trie.pathbased.bonsai.worldview.BonsaiArchiver;
+import org.hyperledger.besu.ethereum.trie.pathbased.bonsaiarchive.BonsaiFlatDbToArchiveMigrator;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.storage.PathBasedWorldStateKeyValueStorage;
+import org.hyperledger.besu.ethereum.trie.pathbased.common.storage.flat.CodeHashCodeStorageStrategy;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.trielog.TrieLogManager;
 import org.hyperledger.besu.ethereum.trie.pathbased.common.trielog.TrieLogPruner;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
+import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
 import org.hyperledger.besu.ethereum.worldstate.PathBasedExtraStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive.WorldStateHealer;
@@ -121,6 +125,8 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalLong;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
@@ -888,6 +894,10 @@ public BesuController build() {
       }
     }
 
+    final List<Closeable> closeables = new ArrayList<>();
+    closeables.add(protocolContext.getWorldStateArchive());
+    closeables.add(storageProvider);
+
     if (DataStorageFormat.X_BONSAI_ARCHIVE.equals(
         dataStorageConfiguration.getDataStorageFormat())) {
       final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
@@ -898,12 +908,38 @@ public BesuController build() {
               blockchain,
               scheduler,
               ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager());
-      blockchain.observeBlockAdded(archiver);
-    }
 
-    final List<Closeable> closeables = new ArrayList<>();
-    closeables.add(protocolContext.getWorldStateArchive());
-    closeables.add(storageProvider);
+      if (worldStateStorageCoordinator.isMatchingFlatMode(FlatDbMode.FULL)
+          || worldStateStorageCoordinator.isMatchingFlatMode(FlatDbMode.PARTIAL)) {
+        final BonsaiFlatDbToArchiveMigrator archiveMigrator =
+            createArchiveMigrator(worldStateStorageCoordinator, worldStateArchive, blockchain);
+        closeables.add(archiveMigrator);
+
+        final AtomicBoolean migrationStarted = new AtomicBoolean(false);
+        final AtomicLong syncSubscriptionId = new AtomicLong();
+        syncSubscriptionId.set(
+            synchronizer.subscribeInSync(
+                (inSync) -> {
+                  if (inSync && migrationStarted.compareAndSet(false, true)) {
+                    synchronizer.unsubscribeInSync(syncSubscriptionId.get());
+                    LOG.info("Node is in sync, starting Bonsai archive migration");
+                    archiveMigrator
+                        .migrate()
+                        .thenRun(() -> blockchain.observeBlockAdded(archiver))
+                        .exceptionally(
+                            error -> {
+                              LOG.error(
+                                  "Archive migration failed, archiver will remain disabled until restart",
+                                  error);
+                              return null;
+                            });
+                  }
+                },
+                0));
+      } else {
+        blockchain.observeBlockAdded(archiver);
+      }
+    }
 
     return new BesuController(
         protocolSchedule,
@@ -1011,6 +1047,27 @@ private BonsaiArchiver createBonsaiArchiver(
     return archiver;
   }
 
+  private BonsaiFlatDbToArchiveMigrator createArchiveMigrator(
+      final WorldStateStorageCoordinator worldStateStorageCoordinator,
+      final WorldStateArchive worldStateArchive,
+      final Blockchain blockchain) {
+    final BonsaiWorldStateKeyValueStorage worldStateKeyValueStorage =
+        worldStateStorageCoordinator.getStrategy(BonsaiWorldStateKeyValueStorage.class);
+    final TrieLogManager trieLogManager =
+        ((BonsaiWorldStateProvider) worldStateArchive).getTrieLogManager();
+    final ScheduledExecutorService migrationExecutor =
+        MonitoredExecutors.newScheduledThreadPool("archive-migrator", 1, metricsSystem);
+    final BonsaiArchiveFlatDbStrategy archiveStrategy =
+        new BonsaiArchiveFlatDbStrategy(metricsSystem, new CodeHashCodeStorageStrategy());
+    return new BonsaiFlatDbToArchiveMigrator(
+        worldStateKeyValueStorage,
+        trieLogManager,
+        blockchain,
+        migrationExecutor,
+        metricsSystem,
+        archiveStrategy);
+  }
+
   /**
    * Create synchronizer synchronizer.
    *

app/src/test/java/org/hyperledger/besu/ForkIdsNetworkConfigTest.java

@@ -49,7 +49,6 @@
 
 import com.google.common.collect.Streams;
 import org.apache.tuweni.bytes.Bytes;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -207,11 +206,4 @@ public Stream<Long> streamMilestoneBlocks() {
           MilestoneStreamingProtocolSchedule::streamMilestoneBlocks);
     }
   }
-
-  @Test
-  void dryRunDetector() {
-    assertThat(true)
-        .withFailMessage("This test is here so gradle --dry-run executes this class")
-        .isTrue();
-  }
 }