db: move db setup from silkworm.cpp into Node and db (#2368)

chain_data_init (old run_db_checklist):

* refactored to accept only required settings
* refactored to not modify node_settings, and return ChainConfig
* moved from cmd/common to db
* called from Node, then chaindata_env is created inside Node.

Author: battlmonstr

cmd/CMakeLists.txt

@@ -45,8 +45,6 @@ if(NOT SILKWORM_CORE_ONLY)
   # [=] "all-in-one" Silkworm component
   set(SILKWORM_CMD_SRC
       silkworm.cpp
-      common/db_checklist.cpp
-      common/db_checklist.hpp
       common/db_max_readers_option.cpp
       common/db_max_readers_option.hpp
       common/node_options.cpp

cmd/silkworm.cpp

@@ -36,7 +36,6 @@
 #include <silkworm/node/node.hpp>
 
 #include "common/common.hpp"
-#include "common/db_checklist.hpp"
 #include "common/node_options.hpp"
 #include "common/rpcdaemon_options.hpp"
 #include "common/sentry_options.hpp"
@@ -167,6 +166,7 @@ void parse_silkworm_command_line(CLI::App& cli, int argc, char* argv[], node::Se
     }
 
     node_settings.data_directory = std::make_unique<DataDirectory>(data_dir_path, /*create=*/true);
+    node_settings.chaindata_env_config.path = node_settings.data_directory->chaindata().path().string();
 
     // Parse prune mode
     sw_db::PruneDistance olderHistory, olderReceipts, olderSenders, olderTxIndex, olderCallTraces;
@@ -232,25 +232,13 @@ int main(int argc, char* argv[]) {
 
         sw_log::Info("Silkworm", build_info_as_log_args(silkworm_get_buildinfo()));
 
-        // Output mdbx build info
-        auto mdbx_ver{mdbx::get_version()};
-        auto mdbx_bld{mdbx::get_build()};
-        sw_log::Debug("libmdbx",
-                      {"version", mdbx_ver.git.describe, "build", mdbx_bld.target, "compiler", mdbx_bld.compiler});
-
-        // Prepare database for takeoff
-        cmd::common::run_db_checklist(settings.node_settings);
-
-        mdbx::env_managed chaindata_env = db::open_env(settings.node_settings.chaindata_env_config);
-
         silkworm::rpc::ClientContextPool context_pool{
             settings.server_settings.context_pool_settings,
         };
 
         silkworm::node::Node execution_node{
             context_pool,
             settings,
-            chaindata_env,  // NOLINT(cppcoreguidelines-slicing)
         };
 
         // Trap OS signals

cmd/common/db_checklist.cpp silkworm/db/chain_data_init.cppcmd/common/db_checklist.cpp renamed to silkworm/db/chain_data_init.cpp

@@ -14,7 +14,7 @@
    limitations under the License.
 */
 
-#include "db_checklist.hpp"
+#include "chain_data_init.hpp"
 
 #include <regex>
 
@@ -24,22 +24,28 @@
 #include <silkworm/db/stages.hpp>
 #include <silkworm/infra/common/log.hpp>
 
-namespace silkworm::cmd::common {
-
-void run_db_checklist(NodeSettings& node_settings, bool init_if_empty) {
-    node_settings.data_directory->deploy();                                  // Ensures all subdirs are present
-    bool chaindata_exclusive{node_settings.chaindata_env_config.exclusive};  // Save setting
-    {
-        auto& config = node_settings.chaindata_env_config;
-        config.path = node_settings.data_directory->chaindata().path().string();
-        config.create =
-            !std::filesystem::exists(db::get_datafile_path(node_settings.data_directory->chaindata().path()));
-        config.exclusive = true;  // Will be cleared after this phase
-    }
+namespace silkworm::db {
+
+ChainConfig chain_data_init(const ChainDataInitSettings& node_settings) {
+    // Output mdbx build info
+    log::Debug(
+        "libmdbx",
+        {
+            "version",
+            mdbx::get_version().git.describe,
+            "build",
+            mdbx::get_build().target,
+            "compiler",
+            mdbx::get_build().compiler,
+        });
+
+    auto chaindata_env_config = node_settings.chaindata_env_config;
+    chaindata_env_config.create = !std::filesystem::exists(db::get_datafile_path(chaindata_env_config.path));
+    chaindata_env_config.exclusive = true;
 
     // Open chaindata environment and check tables are consistent
-    log::Info("Opening database", {"path", node_settings.data_directory->chaindata().path().string()});
-    auto chaindata_env{db::open_env(node_settings.chaindata_env_config)};
+    log::Info("Opening database", {"path", chaindata_env_config.path});
+    mdbx::env_managed chaindata_env = db::open_env(chaindata_env_config);
     db::RWTxnManaged tx(chaindata_env);
 
     // Ensures all tables are present
@@ -51,9 +57,10 @@ void run_db_checklist(NodeSettings& node_settings, bool init_if_empty) {
     const auto header_download_progress{db::stages::read_stage_progress(tx, db::stages::kHeadersKey)};
 
     // Check db is initialized with chain config
+    std::optional<ChainConfig> chain_config;
     {
-        node_settings.chain_config = db::read_chain_config(tx);
-        if (!node_settings.chain_config.has_value() && init_if_empty) {
+        chain_config = db::read_chain_config(tx);
+        if (!chain_config.has_value() && node_settings.init_if_empty) {
             auto source_data{read_genesis_data(node_settings.network_id)};
             auto genesis_json = nlohmann::json::parse(source_data, nullptr, /* allow_exceptions = */ false);
             if (genesis_json.is_discarded()) {
@@ -63,26 +70,26 @@ void run_db_checklist(NodeSettings& node_settings, bool init_if_empty) {
             log::Message("Priming database", {"network id", std::to_string(node_settings.network_id)});
             db::initialize_genesis(tx, genesis_json, /*allow_exceptions=*/true);
             tx.commit_and_renew();
-            node_settings.chain_config = db::read_chain_config(tx);
+            chain_config = db::read_chain_config(tx);
         }
 
-        if (!node_settings.chain_config.has_value()) {
+        if (!chain_config.has_value()) {
             throw std::runtime_error("Unable to retrieve chain configuration");
         }
 
-        const ChainId chain_id{node_settings.chain_config->chain_id};
+        const ChainId chain_id = chain_config->chain_id;
         if (chain_id != node_settings.network_id) {
             throw std::runtime_error("Incompatible network id. Command line expects " +
                                      std::to_string(node_settings.network_id) + "; Database has " +
                                      std::to_string(chain_id));
         }
 
         const auto known_chain{kKnownChainConfigs.find(chain_id)};
-        if (known_chain && **known_chain != *(node_settings.chain_config)) {
+        if (known_chain && **known_chain != *chain_config) {
             // If loaded config is known we must ensure is up-to-date with hardcoded one
             // Loop all respective JSON members to find discrepancies
             auto known_chain_config_json{(*known_chain)->to_json()};
-            auto active_chain_config_json{node_settings.chain_config->to_json()};
+            auto active_chain_config_json = chain_config->to_json();
             bool new_members_added{false};
             bool old_members_changed(false);
             for (auto& [known_key, known_value] : known_chain_config_json.items()) {
@@ -152,17 +159,17 @@ void run_db_checklist(NodeSettings& node_settings, bool init_if_empty) {
             if (new_members_added || old_members_changed) {
                 db::update_chain_config(tx, **known_chain);
                 tx.commit_and_renew();
-                node_settings.chain_config = **known_chain;
+                chain_config = **known_chain;
             }
         }
 
         // Load genesis_hash
-        node_settings.chain_config->genesis_hash = db::read_canonical_header_hash(tx, 0);
-        if (!node_settings.chain_config->genesis_hash.has_value())
+        chain_config->genesis_hash = db::read_canonical_header_hash(tx, 0);
+        if (!chain_config->genesis_hash.has_value())
             throw std::runtime_error("Could not load genesis hash");
 
         log::Info("Starting Silkworm", {"chain", (known_chain ? std::to_string(chain_id) : "unknown/custom"),
-                                        "config", node_settings.chain_config->to_json().dump()});
+                                        "config", chain_config->to_json().dump()});
     }
 
     // Detect prune-mode and verify is compatible
@@ -176,15 +183,14 @@ void run_db_checklist(NodeSettings& node_settings, bool init_if_empty) {
                                          db_prune_mode.to_string() + " got " + node_settings.prune_mode.to_string());
             }
             db::write_prune_mode(*tx, node_settings.prune_mode);
-            node_settings.prune_mode = db::PruneMode(db::read_prune_mode(*tx));
         }
         log::Info("Effective pruning", {"mode", node_settings.prune_mode.to_string()});
     }
 
     tx.commit_and_stop();
     chaindata_env.close();
-    node_settings.chaindata_env_config.exclusive = chaindata_exclusive;
-    node_settings.chaindata_env_config.create = false;  // Has already been created
+
+    return std::move(*chain_config);
 }
 
-}  // namespace silkworm::cmd::common
+}  // namespace silkworm::db

cmd/common/db_checklist.hpp silkworm/db/chain_data_init.hppcmd/common/db_checklist.hpp renamed to silkworm/db/chain_data_init.hpp

@@ -16,11 +16,20 @@
 
 #pragma once
 
-#include <silkworm/node/common/node_settings.hpp>
+#include <silkworm/core/chain/config.hpp>
+#include <silkworm/db/mdbx/mdbx.hpp>
+#include <silkworm/db/prune_mode.hpp>
 
-namespace silkworm::cmd::common {
+namespace silkworm::db {
+
+struct ChainDataInitSettings {
+    db::EnvConfig chaindata_env_config;
+    db::PruneMode prune_mode;
+    ChainId network_id{0};
+    bool init_if_empty{true};
+};
 
 //! \brief Ensure database is ready to take off and consistent with command line arguments
-void run_db_checklist(NodeSettings& node_settings, bool init_if_empty = true);
+ChainConfig chain_data_init(const ChainDataInitSettings& node_settings);
 
-}  // namespace silkworm::cmd::common
+}  // namespace silkworm::db

silkworm/node/node.cpp

@@ -20,6 +20,7 @@
 
 #include <boost/asio/io_context.hpp>
 
+#include <silkworm/db/chain_data_init.hpp>
 #include <silkworm/db/chain_head.hpp>
 #include <silkworm/db/snapshot_sync.hpp>
 #include <silkworm/execution/api/active_direct_service.hpp>
@@ -50,8 +51,7 @@ class NodeImpl final {
   public:
     NodeImpl(
         rpc::ClientContextPool& context_pool,
-        Settings& settings,
-        mdbx::env chaindata_env);
+        Settings& settings);
 
     NodeImpl(const NodeImpl&) = delete;
     NodeImpl& operator=(const NodeImpl&) = delete;
@@ -70,9 +70,10 @@ class NodeImpl final {
     Task<void> run_chain_sync();
 
     Settings& settings_;
-    ChainConfig& chain_config_;
 
-    mdbx::env chaindata_env_;
+    mdbx::env_managed chaindata_env_;
+
+    ChainConfig& chain_config_;
 
     //! The execution layer server engine
     boost::asio::io_context execution_context_;
@@ -96,7 +97,18 @@ class NodeImpl final {
     std::unique_ptr<snapshots::bittorrent::BitTorrentClient> bittorrent_client_;
 };
 
-static rpc::ServerSettings make_execution_server_settings(std::optional<std::string> exec_api_address) {
+static mdbx::env_managed init_chain_data_db(NodeSettings& node_settings) {
+    node_settings.data_directory->deploy();
+    node_settings.chain_config = db::chain_data_init(db::ChainDataInitSettings{
+        .chaindata_env_config = node_settings.chaindata_env_config,
+        .prune_mode = node_settings.prune_mode,
+        .network_id = node_settings.network_id,
+        .init_if_empty = true,
+    });
+    return db::open_env(node_settings.chaindata_env_config);
+}
+
+static rpc::ServerSettings make_execution_server_settings(const std::optional<std::string>& exec_api_address) {
     return rpc::ServerSettings{
         .address_uri = exec_api_address.value_or("localhost:9092"),
         .context_pool_settings = {.num_contexts = 1},  // just one execution context
@@ -150,11 +162,10 @@ static sentry::SessionSentryClient::StatusDataProvider make_sentry_eth_status_da
 
 NodeImpl::NodeImpl(
     rpc::ClientContextPool& context_pool,
-    Settings& settings,
-    mdbx::env chaindata_env)
+    Settings& settings)
     : settings_{settings},
+      chaindata_env_{init_chain_data_db(settings.node_settings)},
       chain_config_{*settings_.node_settings.chain_config},
-      chaindata_env_{std::move(chaindata_env)},
       execution_engine_{
           execution_context_.get_executor(),
           settings_.node_settings,
@@ -165,7 +176,13 @@ NodeImpl::NodeImpl(
       execution_service_{std::make_shared<execution::api::ActiveDirectService>(execution_engine_, execution_context_)},
       execution_server_{make_execution_server_settings(settings_.node_settings.exec_api_address), execution_service_},
       execution_direct_client_{execution_service_},
-      snapshot_sync_{settings.snapshot_settings, chain_config_.chain_id, chaindata_env_, settings_.node_settings.data_directory->temp().path(), execution_engine_.stage_scheduler()},
+      snapshot_sync_{
+          settings.snapshot_settings,
+          chain_config_.chain_id,
+          chaindata_env_,  // NOLINT(cppcoreguidelines-slicing)
+          settings_.node_settings.data_directory->temp().path(),
+          execution_engine_.stage_scheduler(),
+      },
       sentry_{
           sentry::SentryClientFactory::make_sentry(
               std::move(settings.sentry_settings),
@@ -175,7 +192,7 @@ NodeImpl::NodeImpl(
               make_sentry_eth_status_data_provider(db::ROAccess{chaindata_env_}, chain_config_))},
       chain_sync_{
           context_pool.any_executor(),
-          chaindata_env_,
+          chaindata_env_,  // NOLINT(cppcoreguidelines-slicing)
           execution_direct_client_,
           std::get<0>(sentry_),
           chain_config_,
@@ -253,12 +270,10 @@ Task<void> NodeImpl::run_chain_sync() {
 
 Node::Node(
     rpc::ClientContextPool& context_pool,
-    Settings& settings,
-    mdbx::env chaindata_env)
+    Settings& settings)
     : p_impl_(std::make_unique<NodeImpl>(
           context_pool,
-          settings,
-          std::move(chaindata_env))) {}
+          settings)) {}
 
 // Must be here (not in header) because NodeImpl size is necessary for std::unique_ptr in PIMPL idiom
 Node::~Node() = default;

silkworm/node/node.hpp

@@ -34,8 +34,7 @@ class Node {
   public:
     Node(
         rpc::ClientContextPool& context_pool,
-        Settings& settings,
-        mdbx::env chaindata_env);
+        Settings& settings);
     ~Node();
 
     Node(const Node&) = delete;