Feature/preimage store rebased

app/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -2765,6 +2765,9 @@ public void setIgnorableStorageSegments() {
         && !dataStorageConfiguration.getHistoryExpiryPruneEnabled()) {
       rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE);
     }
+    if (!dataStorageOptions.toDomainObject().getHashPreImageStorageEnabled()) {
+      rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.HASH_PREIMAGE_STORE);
+    }
   }
 
   private void validatePostMergeCheckpointBlockRequirements() {

app/src/main/java/org/hyperledger/besu/cli/options/storage/DataStorageOptions.java

@@ -14,6 +14,7 @@
  */
 package org.hyperledger.besu.cli.options.storage;
 
+import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_HASH_PREIMAGE_STORAGE_ENABLED;
 import static org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration.DEFAULT_RECEIPT_COMPACTION_ENABLED;
 
 import org.hyperledger.besu.cli.options.CLIOptions;
@@ -35,6 +36,9 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
 
   private static final String DATA_STORAGE_FORMAT = "--data-storage-format";
 
+  /** Feature to store hash preimages as part of worldstate */
+  public static final String HASH_STORE_PREIMAGE_ENABLED = "--hash-preimage-storage-enabled";
+
   // Use Bonsai DB
   @Option(
       names = {DATA_STORAGE_FORMAT},
@@ -54,6 +58,12 @@ public class DataStorageOptions implements CLIOptions<DataStorageConfiguration>
           "Convenience option to enable online history pruning and configure BlobDB garbage collection settings (default: ${DEFAULT-VALUE}). \"--history-expiry-prune\" is deprecated and will be removed in a future release.")
   private Boolean historyExpiryPrune = false;
 
+  @CommandLine.Option(
+      names = {HASH_STORE_PREIMAGE_ENABLED},
+      description = "Whether to enable preimage storage. (default: ${DEFAULT-VALUE}).",
+      arity = "1")
+  private Boolean hashStorePreimagesEnabled = DEFAULT_HASH_PREIMAGE_STORAGE_ENABLED;
+
   /**
    * Options specific to path-based storage modes. Holds the necessary parameters to configure
    * path-based storage, such as the Bonsai mode or Verkle in the future.
@@ -97,6 +107,7 @@ public static DataStorageOptions fromConfig(final DataStorageConfiguration domai
         PathBasedExtraStorageOptions.fromConfig(
             domainObject.getPathBasedExtraStorageConfiguration());
     dataStorageOptions.historyExpiryPrune = domainObject.getHistoryExpiryPruneEnabled();
+    dataStorageOptions.hashStorePreimagesEnabled = domainObject.getHashPreImageStorageEnabled();
     return dataStorageOptions;
   }
 
@@ -107,6 +118,7 @@ public DataStorageConfiguration toDomainObject() {
             .dataStorageFormat(dataStorageFormat)
             .receiptCompactionEnabled(receiptCompactionEnabled)
             .historyExpiryPruneEnabled(historyExpiryPrune)
+            .hashPreImageStorageEnabled(hashStorePreimagesEnabled)
             .pathBasedExtraStorageConfiguration(pathBasedExtraStorageOptions.toDomainObject());
     return builder.build();
   }

app/src/main/java/org/hyperledger/besu/cli/options/storage/PathBasedExtraStorageOptions.java

@@ -65,6 +65,7 @@ public class PathBasedExtraStorageOptions
   public static final String PARALLEL_STATE_ROOT_COMPUTATION_ENABLED =
       "--bonsai-parallel-state-root-computation-enabled";
 
+  @SuppressWarnings("ExperimentalCliOptionMustBeCorrectlyDisplayed")
   @Option(
       names = {LIMIT_TRIE_LOGS_ENABLED},
       fallbackValue = "true",

app/src/main/java/org/hyperledger/besu/cli/subcommands/storage/PreimageSubCommand.java

@@ -0,0 +1,284 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.cli.subcommands.storage;
+
+import org.hyperledger.besu.cli.util.VersionProvider;
+import org.hyperledger.besu.controller.BesuController;
+import org.hyperledger.besu.datatypes.Hash;
+import org.hyperledger.besu.ethereum.storage.StorageProvider;
+import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
+import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
+import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
+import org.hyperledger.besu.ethereum.worldstate.ImmutablePathBasedExtraStorageConfiguration;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Optional;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.tuweni.bytes.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+/** The preimage subcommand. */
+@CommandLine.Command(
+    name = "preimage",
+    description = "import/export hash preimage data",
+    mixinStandardHelpOptions = true,
+    versionProvider = VersionProvider.class,
+    subcommands = {PreimageSubCommand.Export.class, PreimageSubCommand.Import.class})
+public class PreimageSubCommand implements Runnable {
+
+  /** Default constructor. */
+  public PreimageSubCommand() {}
+
+  private static final Logger LOG = LoggerFactory.getLogger(PreimageSubCommand.class);
+
+  @SuppressWarnings("UnusedVariable")
+  @CommandLine.ParentCommand
+  private static StorageSubCommand parentCommand;
+
+  @SuppressWarnings("unused")
+  @CommandLine.Spec
+  private CommandLine.Model.CommandSpec spec; // Picocli injects reference to command spec
+
+  @Override
+  public void run() {
+    final PrintWriter out = spec.commandLine().getOut();
+    spec.commandLine().usage(out);
+  }
+
+  private static BesuController createBesuController() {
+    final DataStorageConfiguration config = parentCommand.besuCommand.getDataStorageConfiguration();
+    // disable limit trie logs to avoid preloading during subcommand execution
+    return parentCommand
+        .besuCommand
+        .setupControllerBuilder()
+        .dataStorageConfiguration(
+            ImmutableDataStorageConfiguration.copyOf(config)
+                .withPathBasedExtraStorageConfiguration(
+                    ImmutablePathBasedExtraStorageConfiguration.copyOf(
+                            config.getPathBasedExtraStorageConfiguration())
+                        .withLimitTrieLogsEnabled(false)))
+        .build();
+  }
+
+  private static Path orDataDir() {
+    return Paths.get(
+        PreimageSubCommand.parentCommand
+            .besuCommand
+            .dataDir()
+            .resolve("preimages.dat")
+            .toAbsolutePath()
+            .toString());
+  }
+
+  @CommandLine.Command(
+      name = "export",
+      description = "This command exports the preimages from persistent storage to a flat file",
+      mixinStandardHelpOptions = true,
+      versionProvider = VersionProvider.class)
+  static class Export implements Runnable {
+
+    @VisibleForTesting
+    @CommandLine.Option(
+        names = "--preimage-file-path",
+        description = "The flat file to which to export preimage data",
+        arity = "1..1")
+    Path preimageFilePath = null;
+
+    @VisibleForTesting
+    @CommandLine.Option(
+        names = "--hex",
+        description = "export data in hex (human readable) format",
+        arity = "1..1")
+    boolean hex = false;
+
+    StorageProvider storageProvider = null;
+
+    StorageProvider getStorageProvider() {
+      if (storageProvider == null) {
+        storageProvider = createBesuController().getStorageProvider();
+      }
+      return storageProvider;
+    }
+
+    @Override
+    public void run() {
+
+      final StorageProvider storageProvider = getStorageProvider();
+      var preimagesPath =
+          Optional.ofNullable(preimageFilePath).orElseGet(PreimageSubCommand::orDataDir);
+
+      var preimageStorage =
+          storageProvider.getStorageBySegmentIdentifier(
+              KeyValueSegmentIdentifier.HASH_PREIMAGE_STORE);
+
+      try {
+        if (hex) {
+          // HEX MODE: Write each preimage as a hex string line
+          try (var writer = Files.newBufferedWriter(preimagesPath, StandardCharsets.UTF_8)) {
+            preimageStorage.stream()
+                .forEach(
+                    pair -> {
+                      byte[] value = pair.getValue();
+                      String hexString = Bytes.wrap(value).toHexString();
+                      try {
+                        writer.write(hexString);
+                        writer.newLine();
+                      } catch (IOException e) {
+                        throw new UncheckedIOException("Failed to write hex preimage", e);
+                      }
+                    });
+          }
+        } else {
+          // BINARY MODE: Write [1-byte length][N-byte value]
+          try (var out = Files.newOutputStream(preimagesPath)) {
+            preimageStorage.stream()
+                .forEach(
+                    pair -> {
+                      byte[] value = pair.getValue();
+                      int length = value.length;
+                      if (length != 20 && length != 32) {
+                        throw new IllegalArgumentException(
+                            "Unsupported preimage length: " + length);
+                      }
+                      try {
+                        out.write(length);
+                        out.write(value);
+                      } catch (IOException e) {
+                        throw new UncheckedIOException("Failed to write binary preimage", e);
+                      }
+                    });
+          }
+        }
+      } catch (IOException e) {
+        throw new UncheckedIOException("Failed to open preimages file: " + preimagesPath, e);
+      }
+    }
+  }
+
+  @CommandLine.Command(
+      name = "import",
+      description = "This command imports preimage data from a flat file to persistent storage",
+      mixinStandardHelpOptions = true,
+      versionProvider = VersionProvider.class)
+  static class Import implements Runnable {
+
+    @VisibleForTesting
+    @CommandLine.Option(
+        names = "--preimage-file-path",
+        description = "The flat file from which to load preimage data",
+        arity = "1..1")
+    Path preimageFilePath = null;
+
+    @VisibleForTesting
+    @CommandLine.Option(
+        names = "--hex",
+        description = "import data from a hex formatted, line delimited input file",
+        arity = "1..1")
+    boolean hex = false;
+
+    StorageProvider storageProvider = null;
+
+    StorageProvider getStorageProvider() {
+      if (storageProvider == null) {
+        storageProvider = createBesuController().getStorageProvider();
+      }
+      return storageProvider;
+    }
+
+    @Override
+    public void run() {
+      final StorageProvider storageProvider = getStorageProvider();
+      var preimageStorage =
+          storageProvider.getStorageBySegmentIdentifier(
+              KeyValueSegmentIdentifier.HASH_PREIMAGE_STORE);
+
+      var preimagesPath =
+          Optional.ofNullable(preimageFilePath).orElseGet(PreimageSubCommand::orDataDir);
+
+      int batchSize = 1_000_000;
+      int counter = 0;
+      var preimageTx = preimageStorage.startTransaction();
+
+      try {
+        if (hex) {
+          // HEX MODE: read line-by-line
+          try (var lines = Files.lines(preimagesPath)) {
+            var filteredLines =
+                lines
+                    .map(String::trim)
+                    .filter(s -> !s.isEmpty())
+                    .toList(); // force evaluation while stream is open
+
+            for (String line : filteredLines) {
+              if (line.length() > 66) {
+                LOG.error("Discarding input with length > 32 byte hex value: {}", line);
+                continue;
+              }
+
+              Bytes byteVal = Bytes.fromHexString(line);
+              preimageTx.put(Hash.hash(byteVal).getBytes().toArrayUnsafe(), byteVal.toArrayUnsafe());
+
+              if (++counter % batchSize == 0) {
+                preimageTx.commit();
+                preimageTx = preimageStorage.startTransaction();
+              }
+            }
+          }
+        } else {
+          // BINARY MODE: read [1-byte length][N-byte value] format
+          try (var in = Files.newInputStream(preimagesPath)) {
+            while (true) {
+              int lenByte = in.read();
+              if (lenByte == -1) break; // EOF
+
+              if (lenByte != 20 && lenByte != 32) {
+                throw new IOException("Invalid preimage length prefix: " + lenByte);
+              }
+
+              byte[] value = in.readNBytes(lenByte);
+              if (value.length != lenByte) {
+                throw new IOException("Incomplete record: expected " + lenByte + " bytes");
+              }
+
+              Bytes byteVal = Bytes.wrap(value);
+              preimageTx.put(Hash.hash(byteVal).getBytes().toArrayUnsafe(), byteVal.toArrayUnsafe());
+
+              if (++counter % batchSize == 0) {
+                preimageTx.commit();
+                preimageTx = preimageStorage.startTransaction();
+              }
+            }
+          }
+        }
+
+        if (counter % batchSize != 0) {
+          preimageTx.commit();
+        }
+
+      } catch (IOException e) {
+        throw new UncheckedIOException("Failed to load preimages from file: " + preimagesPath, e);
+      }
+    }
+  }
+}

app/src/main/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommand.java

@@ -50,6 +50,7 @@
       StorageSubCommand.RevertVariablesStorage.class,
       RocksDbSubCommand.class,
       TrieLogSubCommand.class,
+      PreimageSubCommand.class,
       RevertMetadataSubCommand.class,
       PrunePreMergeBlockDataSubCommand.class
     })

app/src/main/java/org/hyperledger/besu/services/BesuConfigurationImpl.java

@@ -153,7 +153,7 @@ public DataStoreConfigurationImpl(final DataStorageConfiguration dataStorageConf
     }
 
     @Override
-    public DataStorageFormat getDatabaseFormat() {
+    public DataStorageFormat getDataStorageFormat() {
       return dataStorageConfiguration.getDataStorageFormat();
     }
 
@@ -166,5 +166,10 @@ public boolean getReceiptCompactionEnabled() {
     public boolean isHistoryExpiryPruneEnabled() {
       return dataStorageConfiguration.getHistoryExpiryPruneEnabled();
     }
+
+    @Override
+    public boolean getHashPreImageStorageEnabled() {
+      return dataStorageConfiguration.getHashPreImageStorageEnabled();
+    }
   }
 }