Cache index and filter blocks for archive column families

Archive CFs (ACCOUNT_INFO_STATE_ARCHIVE, ACCOUNT_STORAGE_ARCHIVE,
ACCOUNT_INFO_STATE_FREEZER, ACCOUNT_STORAGE_FREEZER) accumulate large
numbers of SST files during migration. With cache_index_and_filter_blocks=false
(the prior hardcoded default), each SST file's index and filter blocks are
held in unbounded native memory outside the block cache, causing RSS to
grow ~1.5 GB/hr and reach 15+ GB above control nodes by the time the
full 24M-block migration completes.

Add isCacheIndexAndFilterBlocks() to SegmentIdentifier (default false,
preserving existing behaviour for all other segments) and set it true on
the four archive CFs so their index/filter blocks are evictable via the
bounded block cache.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Signed-off-by: Jason Frame <jason.frame@consensys.net>

Author: jframe

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java

@@ -44,24 +44,28 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
       EnumSet.of(X_BONSAI_ARCHIVE),
       true,
       false,
+      true,
       true),
   ACCOUNT_STORAGE_ARCHIVE(
       "ACCOUNT_STORAGE_ARCHIVE".getBytes(StandardCharsets.UTF_8),
       EnumSet.of(X_BONSAI_ARCHIVE),
       true,
       false,
+      true,
       true),
   ACCOUNT_INFO_STATE_FREEZER(
       "ACCOUNT_INFO_STATE_FREEZER".getBytes(StandardCharsets.UTF_8),
       EnumSet.of(X_BONSAI_ARCHIVE),
       true,
       false,
+      true,
       true),
   ACCOUNT_STORAGE_FREEZER(
       "ACCOUNT_STORAGE_FREEZER".getBytes(StandardCharsets.UTF_8),
       EnumSet.of(X_BONSAI_ARCHIVE),
       true,
       false,
+      true,
       true),
   VARIABLES(new byte[] {11}), // formerly GOQUORUM_PRIVATE_WORLD_STATE
 
@@ -81,6 +85,7 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
   private final boolean containsStaticData;
   private final boolean eligibleToHighSpecFlag;
   private final boolean staticDataGarbageCollectionEnabled;
+  private final boolean cacheIndexAndFilterBlocks;
 
   KeyValueSegmentIdentifier(final byte[] id) {
     this(id, EnumSet.allOf(DataStorageFormat.class));
@@ -96,11 +101,22 @@ public enum KeyValueSegmentIdentifier implements SegmentIdentifier {
       final boolean containsStaticData,
       final boolean eligibleToHighSpecFlag,
       final boolean staticDataGarbageCollectionEnabled) {
+    this(id, formats, containsStaticData, eligibleToHighSpecFlag, staticDataGarbageCollectionEnabled, false);
+  }
+
+  KeyValueSegmentIdentifier(
+      final byte[] id,
+      final EnumSet<DataStorageFormat> formats,
+      final boolean containsStaticData,
+      final boolean eligibleToHighSpecFlag,
+      final boolean staticDataGarbageCollectionEnabled,
+      final boolean cacheIndexAndFilterBlocks) {
     this.id = id;
     this.formats = formats;
     this.containsStaticData = containsStaticData;
     this.eligibleToHighSpecFlag = eligibleToHighSpecFlag;
     this.staticDataGarbageCollectionEnabled = staticDataGarbageCollectionEnabled;
+    this.cacheIndexAndFilterBlocks = cacheIndexAndFilterBlocks;
   }
 
   @Override
@@ -128,6 +144,11 @@ public boolean isStaticDataGarbageCollectionEnabled() {
     return staticDataGarbageCollectionEnabled;
   }
 
+  @Override
+  public boolean isCacheIndexAndFilterBlocks() {
+    return cacheIndexAndFilterBlocks;
+  }
+
   @Override
   public boolean includeInDatabaseFormat(final DataStorageFormat format) {
     return formats.contains(format);

plugin-api/build.gradle

@@ -71,7 +71,7 @@ Calculated : ${currentHash}
 tasks.register('checkAPIChanges', FileStateChecker) {
   description = "Checks that the API for the Plugin-API project does not change without deliberate thought"
   files = sourceSets.main.allJava.files
-  knownHash = 'fDkUshUOXLQDBHsugUujwwJkJtYVYnqipuScb8CIof0='
+  knownHash = 'cUSVd6mW2LySwDlG8U7C1tBmFsN7dugAqqRCcyi4BJo='
 }
 check.dependsOn('checkAPIChanges')
 

plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java

@@ -74,4 +74,16 @@ default boolean includeInDatabaseFormat(final DataStorageFormat format) {
   default boolean isStaticDataGarbageCollectionEnabled() {
     return false;
   }
+
+  /**
+   * Whether index and filter blocks for this segment should be stored in the block cache. When
+   * false (the default), index and filter blocks are held in unbounded native memory per SST file.
+   * Segments that accumulate large numbers of SST files (e.g. archive column families written
+   * during migration) should return true to bound their memory via the block cache.
+   *
+   * @return true if index and filter blocks should be cached in the block cache
+   */
+  default boolean isCacheIndexAndFilterBlocks() {
+    return false;
+  }
 }

plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/segmented/RocksDBColumnarKeyValueStorage.java

@@ -292,7 +292,7 @@ private BlockBasedTableConfig createBlockBasedTableConfig(
         .setBlockCache(cache)
         .setFilterPolicy(new BloomFilter(10, false))
         .setPartitionFilters(true)
-        .setCacheIndexAndFilterBlocks(false)
+        .setCacheIndexAndFilterBlocks(segment.isCacheIndexAndFilterBlocks())
         .setBlockSize(ROCKSDB_BLOCK_SIZE);
   }