Raise process FD limit and use RocksDB default max_open_files

Remove the hardcoded OPEN_FILE_LIMIT=750 and state_snapshot's
max_open_files=1024 overrides. With multiple RocksDB databases open
simultaneously (comments archive + account history + snapshot manifest),
the combined table cache reservation of 1500+ FDs exceeds the process
soft limit on many systems, causing "Too many open files" crashes
during replay.

Instead, raise the process soft FD limit toward the hard limit at
startup (capped at 65536) and let RocksDB use its recommended default
of max_open_files=-1 (keep all SST files open). This is simpler, more
performant (no table cache eviction), and aligned with RocksDB's own
design recommendations.

Fixes #851

Author: Gandalf-the-Grey

libraries/chain/CMakeLists.txt

@@ -92,6 +92,7 @@ set( SOURCES
              external_storage/rocksdb_comment_archive.cpp
              external_storage/rocksdb_snapshot.cpp
              external_storage/types.cpp
+             external_storage/fd_budget.cpp
 
              ${HEADERS}
            )

libraries/chain/external_storage/fd_budget.cpp

@@ -0,0 +1,71 @@
+#include <hive/chain/external_storage/fd_budget.hpp>
+
+#include <sys/resource.h>
+#include <algorithm>
+#include <cerrno>
+#include <cstdint>
+#include <mutex>
+
+#include <fc/log/logger.hpp>
+
+namespace hive { namespace chain {
+
+namespace {
+
+constexpr rlim_t MAX_FD_CAP = 65536;
+constexpr rlim_t FD_WARN_THRESHOLD = 8192;
+
+std::once_flag raise_flag;
+
+void do_raise_fd_limit()
+{
+  struct rlimit rl;
+  if( getrlimit( RLIMIT_NOFILE, &rl ) != 0 )
+  {
+    wlog( "Failed to read file descriptor limit (errno=${e}). "
+          "RocksDB will use max_open_files=-1 with current system defaults.",
+          ("e", errno) );
+    return;
+  }
+
+  rlim_t old_soft = rl.rlim_cur;
+  rlim_t hard = rl.rlim_max;
+  rlim_t target = std::min( hard, MAX_FD_CAP );
+
+  if( old_soft < target )
+  {
+    rl.rlim_cur = target;
+    if( setrlimit( RLIMIT_NOFILE, &rl ) != 0 )
+    {
+      wlog( "Failed to raise file descriptor soft limit from ${old} to ${target} (errno=${e}). "
+            "Keeping current soft limit. Consider raising ulimit -n manually.",
+            ("old", static_cast<uint64_t>( old_soft ))
+            ("target", static_cast<uint64_t>( target ))
+            ("e", errno) );
+      rl.rlim_cur = old_soft;
+    }
+  }
+
+  ilog( "File descriptor limits: soft ${old} -> ${cur}, hard ${hard}",
+        ("old", static_cast<uint64_t>( old_soft ))
+        ("cur", static_cast<uint64_t>( rl.rlim_cur ))
+        ("hard", static_cast<uint64_t>( hard )) );
+
+  if( rl.rlim_cur < FD_WARN_THRESHOLD )
+  {
+    wlog( "File descriptor limit is low (${cur}). "
+          "This may cause RocksDB errors under heavy load. "
+          "Consider raising the limit: ulimit -n ${recommended} or adjust system configuration.",
+          ("cur", static_cast<uint64_t>( rl.rlim_cur ))
+          ("recommended", static_cast<uint64_t>( FD_WARN_THRESHOLD )) );
+  }
+}
+
+} // anonymous namespace
+
+void raise_fd_limit()
+{
+  std::call_once( raise_flag, do_raise_fd_limit );
+}
+
+} } // hive::chain

libraries/chain/external_storage/rocksdb_storage_provider.cpp

@@ -2,6 +2,7 @@
 #include <hive/chain/external_storage/rocksdb_storage_provider.hpp>
 
 #include <hive/chain/external_storage/comment_rocksdb_objects.hpp>
+#include <hive/chain/external_storage/fd_budget.hpp>
 #include <hive/chain/external_storage/utilities.hpp>
 #include <hive/chain/external_storage/types.hpp>
 
@@ -45,7 +46,7 @@ void rocksdb_storage_provider::openDb( uint32_t expected_lib )
   /// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
   options.IncreaseParallelism();
   options.OptimizeLevelStyleCompaction();
-  options.max_open_files = OPEN_FILE_LIMIT;
+  raise_fd_limit();
 
   auto status = DB::Open( DBOptions( options ), strPath, columnDefs, &_columnHandles, &_db );
   ilog( "Database is ${status}.", ("status", status.ok() ? "opened" : "not opened") );
@@ -123,7 +124,7 @@ std::tuple<bool, bool> rocksdb_storage_provider::createDbSchema( const bfs::path
   /// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
   options.IncreaseParallelism();
   options.OptimizeLevelStyleCompaction();
-  options.max_open_files = OPEN_FILE_LIMIT;
+  raise_fd_limit();
 
   std::tuple<bool, bool> _result{ false, false };
 

libraries/chain/include/hive/chain/external_storage/fd_budget.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+namespace hive { namespace chain {
+
+/// Raises the process soft file descriptor limit toward the hard limit.
+/// Called before opening RocksDB databases so they can use max_open_files = -1
+/// (RocksDB's recommended default) without hitting OS limits.
+/// Thread-safe and idempotent (uses std::call_once internally).
+void raise_fd_limit();
+
+} } // hive::chain

libraries/chain/include/hive/chain/external_storage/types.hpp

@@ -21,8 +21,6 @@ using ::rocksdb::Comparator;
 #define STORE_MAJOR_VERSION          1
 #define STORE_MINOR_VERSION          1
 
-#define OPEN_FILE_LIMIT 750
-
 #define checkStatus(s) FC_ASSERT((s).ok(), "Data access failed: ${m}", ("m", (s).ToString()))
 
 /** Because localtion_id_pair stores block_number paired with operation_id_vop_pair, which stores operation id on 63 bits,

libraries/plugins/state_snapshot/state_snapshot_plugin.cpp

@@ -15,6 +15,7 @@
 #include <hive/chain/util/decoded_types_data_storage.hpp>
 #include <hive/chain/util/state_checker_tools.hpp>
 
+#include <hive/chain/external_storage/fd_budget.hpp>
 #include <hive/chain/external_storage/state_snapshot_provider.hpp>
 #include <hive/chain/external_storage/types.hpp>
 
@@ -292,8 +293,7 @@ class snapshot_processor_data : public BaseClass
       {
       ::rocksdb::Options opts;
       opts.comparator = &_size_t_comparator;
-
-      opts.max_open_files = 1024;
+      hive::chain::raise_fd_limit();
 
       return opts;
       }
@@ -1022,9 +1022,9 @@ void state_snapshot_plugin::impl::store_snapshot_manifest(const bfs::path& actua
   if(bfs::exists(manifestDbPath) == false)
     bfs::create_directories(manifestDbPath);
 
+  hive::chain::raise_fd_limit();
   ::rocksdb::Options dbOptions;
   dbOptions.create_if_missing = true;
-  dbOptions.max_open_files = 1024;
 
   rocksdb_cleanup_helper db = rocksdb_cleanup_helper::open(dbOptions, manifestDbPath);
   ::rocksdb::ColumnFamilyHandle* manifestCF = db.create_column_family("INDEX_MANIFEST");
@@ -1174,10 +1174,10 @@ state_snapshot_plugin::impl::load_snapshot_manifest(const bfs::path& actualStora
   bfs::path manifestDbPath(actualStoragePath);
   manifestDbPath /= "snapshot-manifest";
 
+  hive::chain::raise_fd_limit();
   ::rocksdb::Options dbOptions;
   dbOptions.create_if_missing = false;
-  dbOptions.max_open_files = 1024;
-  
+
   ::rocksdb::ColumnFamilyDescriptor cfDescriptor;
   cfDescriptor.name = "INDEX_MANIFEST";