Merge branch 'main' of github.com:hyperledger/besu into HeadersBackward

Author: pinges

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## Unreleased
 
+### Breaking Changes
+
+### Upcoming Breaking Changes
+- ETC Classic and Mordor network support in Besu is deprecated [#9437](https://github.com/hyperledger/besu/pull/9437)
+- Holesky network is deprecated [#9437](https://github.com/hyperledger/besu/pull/9437)
+- Sunsetting features - for more context on the reasoning behind the deprecation of these features, including alternative options, read [this blog post](https://www.lfdecentralizedtrust.org/blog/sunsetting-tessera-and-simplifying-hyperledger-besu)
+    - ETC (Ethereum Classic) network support
+    - Proof of Work consensus (PoW)
+    - Clique Block Production (mining) - you will still be able to sync existing Clique networks, but not be a validator or create new Clique networks.
+    - Fast Sync
+  
+### Additions and Improvements
+
+##  25.12.0-RC1
+
 ### Breaking Changes
 - Remove these deprecated CLI options [#9385](https://github.com/hyperledger/besu/pull/9385)
   - Remove`--Xbonsai-parallel-tx-processing-enabled` deprecated since 25.7.0. Use `--bonsai-parallel-tx-processing-enabled` instead.
@@ -32,11 +47,17 @@
 - Performance: Optimise `engine_getPayload*` methods and `engine_getBlobsV2` [#9445](https://github.com/hyperledger/besu/pull/9445)
 - Add Linea named networks for `linea_mainnet` and `linea_sepolia` [#9436](https://github.com/hyperledger/besu/pull/9436), [#9518](https://github.com/hyperledger/besu/pull/9518)
 - Add `eth_subscribe` and `eth_unsubscribe` support to IPC service [#9504](https://github.com/hyperledger/besu/pull/9504)
+- Add experimental `callTracer` tracer option to `debug_trace*` methods. Enabled using `--Xenable-extra-debug-tracers=true` option. Issue [#8326][issue_8326] implemented via PR [#8960][PR_8960] and [#9072][PR_9072].
 
 ### Bug fixes
+- Fix non-deterministic sub-protocol registration during IBFT2 to QBFT consensus migration [#9516](https://github.com/hyperledger/besu/pull/9516)
 - Fix loss of colored output in terminal when using `--color-enabled=true` option [#8908](https://github.com/hyperledger/besu/issues/8908)
 - Fix an issue where Besu does not support `0x80` as transaction type when decoding eth/69 receipts [#9520](https://github.com/hyperledger/besu/issues/9520)
 
+[issue_8326]: https://github.com/hyperledger/besu/issues/8326
+[PR_9072]: https://github.com/hyperledger/besu/pull/9072
+[PR_8960]: https://github.com/hyperledger/besu/pull/8960
+
 ## 25.11.0
 
 ### Breaking Changes

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/RunVersionTest.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright contributors to Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.plugin.BesuPlugin;
+import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase;
+import org.hyperledger.besu.tests.acceptance.dsl.WaitUtils;
+import org.hyperledger.besu.tests.acceptance.dsl.node.BesuNode;
+import org.hyperledger.besu.util.BesuVersionUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ServiceLoader;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+
+public class RunVersionTest extends AcceptanceTestBase {
+
+  @Test
+  public void testShowsVersionAndExits() throws IOException {
+    final BesuNode node =
+        besu.createPluginsNode(
+            "run --version",
+            List.of("testPlugins"),
+            besuNodeConfigurationBuilder -> besuNodeConfigurationBuilder.run("--version"));
+    cluster.startConsoleCapture();
+    cluster.runNodeStart(node);
+
+    final var serviceLoader = ServiceLoader.load(BesuPlugin.class);
+    final var plugins = serviceLoader.stream().map(ServiceLoader.Provider::get).toList();
+
+    final var versionOutput =
+        BesuVersionUtils.version()
+            + System.lineSeparator()
+            + plugins.stream()
+                .map(
+                    bp ->
+                        bp.getName().orElseThrow()
+                            + "/test-plugins/"
+                            + BesuVersionUtils.shortVersion())
+                .collect(Collectors.joining(System.lineSeparator()));
+
+    WaitUtils.waitFor(5000, () -> node.verify(exitedSuccessfully));
+
+    final String consoleContents = cluster.getConsoleContents();
+    assertThat(consoleContents).startsWith(versionOutput);
+  }
+}

app/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -257,6 +257,10 @@
 import io.vertx.core.VertxOptions;
 import io.vertx.core.json.DecodeException;
 import io.vertx.core.json.jackson.DatabindCodec;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.impl.Log4jContextFactory;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.units.bigints.UInt256;
 import org.slf4j.Logger;
@@ -853,13 +857,40 @@ private IExecutionStrategy createExecuteTask(final IExecutionStrategy nextStep)
 
   private IExecutionStrategy createPluginRegistrationTask(final IExecutionStrategy nextStep) {
     return parseResult -> {
+      if (parseResult.isUsageHelpRequested() || parseResult.isVersionHelpRequested()) {
+        // suppressing the info log to avoid that plugin registrations logs are printed
+        // before the help or the version information
+        suppressInfoLog();
+      }
       besuPluginContext.initialize(PluginsConfigurationOptions.fromCommandLine(commandLine));
       besuPluginContext.registerPlugins();
       commandLine.setExecutionStrategy(nextStep);
       return commandLine.execute(parseResult.originalArgs().toArray(new String[0]));
     };
   }
 
+  @SuppressWarnings("BannedMethod")
+  private void suppressInfoLog() {
+    // this is specific for Log4j2, in case we switch to another logging framework,
+    // this need to be adapted for it
+
+    // silence already created loggers
+    LoggerContext.getContext(false).getLoggers().forEach(logger -> logger.setLevel(Level.WARN));
+
+    // silence future loggers by configuration
+    if (LogManager.getFactory() instanceof Log4jContextFactory log4jContextFactory) {
+      final var selector = log4jContextFactory.getSelector();
+      selector
+          .getLoggerContexts()
+          .forEach(
+              ctx ->
+                  ctx.getConfiguration()
+                      .getLoggers()
+                      .values()
+                      .forEach(loggerConfig -> loggerConfig.setLevel(Level.WARN)));
+    }
+  }
+
   private IExecutionStrategy createDefaultValueProviderTask(final IExecutionStrategy nextStep) {
     return new ConfigDefaultValueProviderStrategy(nextStep, environment);
   }

app/src/main/java/org/hyperledger/besu/cli/util/VersionProvider.java

@@ -39,7 +39,8 @@ public String[] getVersion() {
     // the PluginVersionsProvider has registered plugins and their versions by this time.
     return Stream.concat(
             Stream.of(BesuVersionUtils.version()),
-            pluginVersionsProvider.getPluginVersions().stream())
+            pluginVersionsProvider.getPluginVersions().entrySet().stream()
+                .map(e -> e.getKey() + "/" + e.getValue()))
         .toArray(String[]::new);
   }
 }

app/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java

@@ -49,6 +49,8 @@
 import org.hyperledger.besu.ethereum.mainnet.BalConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration;
+import org.hyperledger.besu.ethereum.p2p.network.ProtocolManager;
+import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol;
 import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
@@ -61,10 +63,12 @@
 import java.nio.file.Path;
 import java.time.Clock;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Optional;
+import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.BiFunction;
 import java.util.stream.Collectors;
@@ -225,9 +229,39 @@ protected JsonRpcMethods createAdditionalJsonRpcMethodFactory(
   protected SubProtocolConfiguration createSubProtocolConfiguration(
       final EthProtocolManager ethProtocolManager,
       final Optional<SnapProtocolManager> maybeSnapProtocolManager) {
-    return besuControllerBuilderSchedule
-        .get(besuControllerBuilderSchedule.keySet().stream().skip(1).findFirst().orElseThrow())
-        .createSubProtocolConfiguration(ethProtocolManager, maybeSnapProtocolManager);
+    // During consensus migration, we need BOTH wire protocols registered.
+    // This allows nodes to communicate before AND after the migration block.
+    // Previously used .skip(1) which was non-deterministic with HashMap ordering.
+    //
+    // We merge all sub-protocol configurations, which registers both:
+    // - IBF/1 (IBFT2) for pre-migration communication
+    // - istanbul/100 (QBFT) for post-migration communication
+    final SubProtocolConfiguration mergedConfig = new SubProtocolConfiguration();
+    final Set<String> addedProtocolNames = new HashSet<>();
+
+    // Add sub-protocols from each consensus builder (sorted by block number for determinism)
+    besuControllerBuilderSchedule.entrySet().stream()
+        .sorted(Map.Entry.comparingByKey())
+        .forEach(
+            entry -> {
+              final SubProtocolConfiguration builderConfig =
+                  entry
+                      .getValue()
+                      .createSubProtocolConfiguration(ethProtocolManager, maybeSnapProtocolManager);
+              final List<SubProtocol> subProtocols = builderConfig.getSubProtocols();
+              final List<ProtocolManager> protocolManagers = builderConfig.getProtocolManagers();
+              for (int i = 0; i < subProtocols.size(); i++) {
+                final SubProtocol subProtocol = subProtocols.get(i);
+                final ProtocolManager protocolManager = protocolManagers.get(i);
+                // Only add if not already present (avoid duplicates like EthProtocol)
+                if (!addedProtocolNames.contains(subProtocol.getName())) {
+                  mergedConfig.withSubProtocol(subProtocol, protocolManager);
+                  addedProtocolNames.add(subProtocol.getName());
+                }
+              }
+            });
+
+    return mergedConfig;
   }
 
   @Override

app/src/main/java/org/hyperledger/besu/services/BesuPluginContextImpl.java

@@ -30,10 +30,10 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
@@ -84,7 +84,7 @@ private enum Lifecycle {
 
   private final List<BesuPlugin> registeredPlugins = new ArrayList<>();
 
-  private final List<String> pluginVersions = new ArrayList<>();
+  private final Map<String, String> pluginVersions = new LinkedHashMap<>();
   private PluginConfiguration config;
 
   /** Instantiates a new Besu plugin context. */
@@ -206,7 +206,7 @@ private void registerPlugins(final List<BesuPlugin> pluginsToRegister) {
   private boolean registerPlugin(final BesuPlugin plugin) {
     try {
       plugin.register(this);
-      pluginVersions.add(plugin.getVersion());
+      pluginVersions.put(plugin.getName().orElse("<Unnamed Plugin>"), plugin.getVersion());
       LOG.info("Registered plugin of type {}.", plugin.getClass().getName());
     } catch (final Exception e) {
       if (config.isContinueOnPluginError()) {
@@ -378,8 +378,8 @@ private Optional<ClassLoader> pluginDirectoryLoader(final Path pluginsDir) {
   }
 
   @Override
-  public Collection<String> getPluginVersions() {
-    return Collections.unmodifiableList(pluginVersions);
+  public Map<String, String> getPluginVersions() {
+    return Collections.unmodifiableMap(pluginVersions);
   }
 
   /**

app/src/test/java/org/hyperledger/besu/cli/util/BesuCommandCustomFactoryTest.java

@@ -20,8 +20,7 @@
 import org.hyperledger.besu.plugin.services.PluginVersionsProvider;
 import org.hyperledger.besu.util.BesuVersionUtils;
 
-import java.util.Arrays;
-
+import com.google.common.collect.ImmutableMap;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -35,7 +34,8 @@ public class BesuCommandCustomFactoryTest {
 
   @BeforeEach
   public void initMocks() {
-    when(pluginVersionsProvider.getPluginVersions()).thenReturn(Arrays.asList("v1", "v2"));
+    when(pluginVersionsProvider.getPluginVersions())
+        .thenReturn(ImmutableMap.of("plugin1", "v1", "plugin2", "v2"));
   }
 
   @Test
@@ -44,6 +44,6 @@ public void testCreateVersionProviderInstance() throws Exception {
         new BesuCommandCustomFactory(pluginVersionsProvider);
     final VersionProvider versionProvider = besuCommandCustomFactory.create(VersionProvider.class);
     assertThat(versionProvider.getVersion())
-        .containsExactly(BesuVersionUtils.version(), "v1", "v2");
+        .containsExactly(BesuVersionUtils.version(), "plugin1/v1", "plugin2/v2");
   }
 }

app/src/test/java/org/hyperledger/besu/cli/util/VersionProviderTest.java

@@ -14,7 +14,7 @@
  */
 package org.hyperledger.besu.cli.util;
 
-import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
 
@@ -35,15 +35,17 @@ public class VersionProviderTest {
 
   @Test
   public void validateEmptyListGenerateBesuInfoVersionOnly() {
-    when(pluginVersionsProvider.getPluginVersions()).thenReturn(emptyList());
+    when(pluginVersionsProvider.getPluginVersions()).thenReturn(emptyMap());
     final VersionProvider versionProvider = new VersionProvider(pluginVersionsProvider);
     assertThat(versionProvider.getVersion()).containsOnly(BesuVersionUtils.version());
   }
 
   @Test
   public void validateVersionListGenerateValidValues() {
-    when(pluginVersionsProvider.getPluginVersions()).thenReturn(Collections.singletonList("test"));
+    when(pluginVersionsProvider.getPluginVersions())
+        .thenReturn(Collections.singletonMap("plugin", "test"));
     final VersionProvider versionProvider = new VersionProvider(pluginVersionsProvider);
-    assertThat(versionProvider.getVersion()).containsExactly(BesuVersionUtils.version(), "test");
+    assertThat(versionProvider.getVersion())
+        .containsExactly(BesuVersionUtils.version(), "plugin/test");
   }
 }

app/src/test/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilderTest.java

@@ -37,6 +37,9 @@
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
+import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration;
+import org.hyperledger.besu.ethereum.p2p.network.ProtocolManager;
+import org.hyperledger.besu.ethereum.p2p.rlpx.wire.SubProtocol;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
 
 import java.math.BigInteger;
@@ -201,4 +204,35 @@ public void createsMigratingContext() {
     assertThat(contextSchedule.getFork(10).getValue()).isSameAs(context2);
     assertThat(contextSchedule.getFork(11).getValue()).isSameAs(context2);
   }
+
+  @Test
+  public void createsSubProtocolConfigurationWithAllSubProtocols() {
+    final Map<Long, BesuControllerBuilder> consensusSchedule =
+        Map.of(0L, besuControllerBuilder1, 50L, besuControllerBuilder2);
+
+    final SubProtocol subProtocol1 = mock(SubProtocol.class);
+    final SubProtocol subProtocol2 = mock(SubProtocol.class);
+    when(subProtocol1.getName()).thenReturn("IBF");
+    when(subProtocol2.getName()).thenReturn("istanbul");
+
+    final SubProtocolConfiguration config1 = new SubProtocolConfiguration();
+    config1.withSubProtocol(subProtocol1, mock(ProtocolManager.class));
+
+    final SubProtocolConfiguration config2 = new SubProtocolConfiguration();
+    config2.withSubProtocol(subProtocol2, mock(ProtocolManager.class));
+
+    when(besuControllerBuilder1.createSubProtocolConfiguration(any(), any())).thenReturn(config1);
+    when(besuControllerBuilder2.createSubProtocolConfiguration(any(), any())).thenReturn(config2);
+
+    final ConsensusScheduleBesuControllerBuilder builder =
+        new ConsensusScheduleBesuControllerBuilder(consensusSchedule);
+
+    final SubProtocolConfiguration result =
+        builder.createSubProtocolConfiguration(mock(EthProtocolManager.class), Optional.empty());
+
+    assertThat(result.getSubProtocols()).hasSize(2);
+    assertThat(result.getSubProtocols())
+        .extracting(SubProtocol::getName)
+        .containsExactlyInAnyOrder("IBF", "istanbul");
+  }
 }