datastore: .ef v2 format

Author: battlmonstr

silkworm/capi/cli/execute.cpp

@@ -18,6 +18,7 @@
 
 #include <silkworm/capi/silkworm.h>
 #include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/blocks/step_block_num_converter.hpp>
 #include <silkworm/db/datastore/kvdb/mdbx.hpp>
 #include <silkworm/db/datastore/snapshots/snapshot_repository.hpp>
 #include <silkworm/infra/cli/common.hpp>
@@ -319,7 +320,7 @@ int build_indexes(SilkwormHandle handle, const BuildIndexesSettings& settings, c
         if (!snapshot_path.has_value())
             throw std::runtime_error("Invalid snapshot path");
 
-        segment::SegmentFileReader& segment = segments.emplace_back(*snapshot_path);
+        segment::SegmentFileReader& segment = segments.emplace_back(*snapshot_path, db::blocks::kStepToBlockNumConverter);
 
         auto mmf = new SilkwormMemoryMappedFile{
             .file_path = make_path(*snapshot_path),

silkworm/capi/silkworm_test.cpp

@@ -17,6 +17,7 @@
 #include <silkworm/db/datastore/kvdb/mdbx.hpp>
 #include <silkworm/db/datastore/snapshots/schema.hpp>
 #include <silkworm/db/datastore/snapshots/segment/segment_reader.hpp>
+#include <silkworm/db/state/step_txn_id_converter.hpp>
 #include <silkworm/db/test_util/temp_snapshots.hpp>
 #include <silkworm/db/test_util/test_database_context.hpp>
 #include <silkworm/infra/common/directories.hpp>
@@ -813,6 +814,8 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_execute_blocks_perpetual multiple bloc
 }
 
 TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_blocks_snapshot_bundle", "[capi]") {
+    static constexpr datastore::StepToTimestampConverter kStepConverter = db::blocks::kStepToBlockNumConverter;
+
     snapshot_test::SampleHeaderSnapshotFile header_segment_file{tmp_dir.path()};
     auto& header_segment_path = header_segment_file.path();
     snapshot_test::SampleBodySnapshotFile body_segment_file{tmp_dir.path()};
@@ -823,20 +826,20 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_blocks_snapshot_bundle", "[capi]")
     auto header_index_builder = snapshots::HeaderIndex::make(header_segment_path);
     header_index_builder.set_base_data_id(header_segment_file.block_num_range().start);
     REQUIRE_NOTHROW(header_index_builder.build());
-    snapshots::segment::SegmentFileReader header_segment{header_segment_path};
+    snapshots::segment::SegmentFileReader header_segment{header_segment_path, kStepConverter};
     snapshots::rec_split::AccessorIndex idx_header_hash{header_segment_path.related_path_ext(db::blocks::kIdxExtension)};
 
     auto body_index_builder = snapshots::BodyIndex::make(body_segment_path);
     body_index_builder.set_base_data_id(body_segment_file.block_num_range().start);
     REQUIRE_NOTHROW(body_index_builder.build());
-    snapshots::segment::SegmentFileReader body_segment{body_segment_path};
+    snapshots::segment::SegmentFileReader body_segment{body_segment_path, kStepConverter};
     snapshots::rec_split::AccessorIndex idx_body_number{body_segment_path.related_path_ext(db::blocks::kIdxExtension)};
 
     auto tx_index_builder = snapshots::TransactionIndex::make(body_segment_path, txn_segment_path);
     tx_index_builder.build();
     auto tx_index_hash_to_block_builder = snapshots::TransactionToBlockIndex::make(body_segment_path, txn_segment_path, txn_segment_file.block_num_range().start);
     tx_index_hash_to_block_builder.build();
-    snapshots::segment::SegmentFileReader txn_segment{txn_segment_path};
+    snapshots::segment::SegmentFileReader txn_segment{txn_segment_path, kStepConverter};
     snapshots::rec_split::AccessorIndex idx_txn_hash{txn_segment_path.related_path_ext(db::blocks::kIdxExtension)};
     snapshots::rec_split::AccessorIndex idx_txn_hash_2_block{tx_index_hash_to_block_builder.path()};
 
@@ -968,7 +971,7 @@ TEST_CASE_METHOD(CApiTest, "CAPI silkworm_add_state_snapshot", "[capi]") {
     constexpr uint32_t kZeroSalt{0};
 
     const snapshot_test::SampleAccountsDomainSegmentFile kv_segment_file{tmp_dir.path()};
-    segment::KVSegmentFileReader kv_segment{kv_segment_file.path(), seg::CompressionKind::kAll};
+    segment::KVSegmentFileReader kv_segment{kv_segment_file.path(), db::state::kStepToTxnIdConverter, seg::CompressionKind::kAll};
     const auto kv_segment_path_string{kv_segment_file.path().path().string()};
 
     const snapshot_test::SampleAccountsDomainExistenceIndexFile existence_index_file{tmp_dir.path()};

silkworm/capi/snapshots.cpp

@@ -41,11 +41,13 @@ static SnapshotPath parse_snapshot_path(const char* file_path) {
 static void build_inverted_index_bundle_data(
     const SilkwormInvertedIndexSnapshot& snapshot,
     const Schema::EntityDef& entity_def,
+    const datastore::StepToTimestampConverter& step_converter,
     SnapshotBundleEntityData& data) {
     data.kv_segments.emplace(
         Schema::kInvIdxKVSegmentName,
         segment::KVSegmentFileReader{
             parse_snapshot_path(snapshot.segment.file_path),
+            step_converter,
             entity_def.kv_segment(Schema::kInvIdxKVSegmentName).compression_kind(),
             make_region(snapshot.segment),
         });
@@ -59,21 +61,24 @@ static void build_inverted_index_bundle_data(
 
 static snapshots::SnapshotBundleEntityData build_inverted_index_bundle_data(
     const SilkwormInvertedIndexSnapshot& snapshot,
-    const Schema::EntityDef& entity_def) {
+    const Schema::EntityDef& entity_def,
+    const datastore::StepToTimestampConverter& step_converter) {
     SnapshotBundleEntityData data;
-    build_inverted_index_bundle_data(snapshot, entity_def, data);
+    build_inverted_index_bundle_data(snapshot, entity_def, step_converter, data);
     return data;
 }
 
 static snapshots::SnapshotBundleEntityData build_domain_bundle_data(
     const SilkwormDomainSnapshot& snapshot,
     const Schema::EntityDef& entity_def,
+    const datastore::StepToTimestampConverter& step_converter,
     uint32_t index_salt) {
     SnapshotBundleEntityData data;
     data.kv_segments.emplace(
         Schema::kDomainKVSegmentName,
         segment::KVSegmentFileReader{
             parse_snapshot_path(snapshot.segment.file_path),
+            step_converter,
             entity_def.kv_segment(Schema::kDomainKVSegmentName).compression_kind(),
             make_region(snapshot.segment),
         });
@@ -102,12 +107,14 @@ static snapshots::SnapshotBundleEntityData build_domain_bundle_data(
 
 static snapshots::SnapshotBundleEntityData build_history_bundle_data(
     const SilkwormHistorySnapshot& snapshot,
-    const Schema::EntityDef& entity_def) {
+    const Schema::EntityDef& entity_def,
+    const datastore::StepToTimestampConverter& step_converter) {
     SnapshotBundleEntityData data;
     data.segments.emplace(
         Schema::kHistorySegmentName,
         segment::SegmentFileReader{
             parse_snapshot_path(snapshot.segment.file_path),
+            step_converter,
             make_region(snapshot.segment),
         });
     data.accessor_indexes.emplace(
@@ -117,7 +124,7 @@ static snapshots::SnapshotBundleEntityData build_history_bundle_data(
             make_region(snapshot.accessor_index),
         });
 
-    build_inverted_index_bundle_data(snapshot.inverted_index, entity_def, data);
+    build_inverted_index_bundle_data(snapshot.inverted_index, entity_def, step_converter, data);
     return data;
 }
 
@@ -126,22 +133,24 @@ static snapshots::SnapshotBundle build_state_snapshot_bundle_latest(
     const Schema::RepositoryDef& schema,
     uint32_t salt) {
     SnapshotBundleData bundle_data;
+    datastore::StepToTimestampConverter step_converter = schema.make_step_converter();
+
     bundle_data.entities.emplace(
         db::state::kDomainNameAccounts,
-        build_domain_bundle_data(bundle->accounts, schema.domain(db::state::kDomainNameAccounts), salt));
+        build_domain_bundle_data(bundle->accounts, schema.domain(db::state::kDomainNameAccounts), step_converter, salt));
     bundle_data.entities.emplace(
         db::state::kDomainNameStorage,
-        build_domain_bundle_data(bundle->storage, schema.domain(db::state::kDomainNameStorage), salt));
+        build_domain_bundle_data(bundle->storage, schema.domain(db::state::kDomainNameStorage), step_converter, salt));
     bundle_data.entities.emplace(
         db::state::kDomainNameCode,
-        build_domain_bundle_data(bundle->code, schema.domain(db::state::kDomainNameCode), salt));
+        build_domain_bundle_data(bundle->code, schema.domain(db::state::kDomainNameCode), step_converter, salt));
     // TODO(canepat): enable after fixing .kvi configuration with IndexList-like implementation
     // bundle_data.entities.emplace(
     //     db::state::kDomainNameCommitment,
-    //     build_domain_bundle_data(bundle->commitment, schema.domain(db::state::kDomainNameCommitment), salt));
+    //     build_domain_bundle_data(bundle->commitment, schema.domain(db::state::kDomainNameCommitment), step_converter, salt));
     bundle_data.entities.emplace(
         db::state::kDomainNameReceipts,
-        build_domain_bundle_data(bundle->receipts, schema.domain(db::state::kDomainNameReceipts), salt));
+        build_domain_bundle_data(bundle->receipts, schema.domain(db::state::kDomainNameReceipts), step_converter, salt));
 
     return SnapshotBundle{
         parse_snapshot_path(bundle->accounts.segment.file_path).step_range(),
@@ -153,32 +162,33 @@ static snapshots::SnapshotBundle build_state_snapshot_bundle_historical(
     const SilkwormStateSnapshotBundleHistorical* bundle,
     const Schema::RepositoryDef& schema) {
     SnapshotBundleData bundle_data;
+    datastore::StepToTimestampConverter step_converter = schema.make_step_converter();
 
     bundle_data.entities.emplace(
         db::state::kDomainNameAccounts,
-        build_history_bundle_data(bundle->accounts, schema.history(db::state::kDomainNameAccounts)));
+        build_history_bundle_data(bundle->accounts, schema.history(db::state::kDomainNameAccounts), step_converter));
     bundle_data.entities.emplace(
         db::state::kDomainNameStorage,
-        build_history_bundle_data(bundle->storage, schema.history(db::state::kDomainNameStorage)));
+        build_history_bundle_data(bundle->storage, schema.history(db::state::kDomainNameStorage), step_converter));
     bundle_data.entities.emplace(
         db::state::kDomainNameCode,
-        build_history_bundle_data(bundle->code, schema.history(db::state::kDomainNameCode)));
+        build_history_bundle_data(bundle->code, schema.history(db::state::kDomainNameCode), step_converter));
     bundle_data.entities.emplace(
         db::state::kDomainNameReceipts,
-        build_history_bundle_data(bundle->receipts, schema.history(db::state::kDomainNameReceipts)));
+        build_history_bundle_data(bundle->receipts, schema.history(db::state::kDomainNameReceipts), step_converter));
 
     bundle_data.entities.emplace(
         db::state::kInvIdxNameLogAddress,
-        build_inverted_index_bundle_data(bundle->log_addresses, schema.inverted_index(db::state::kInvIdxNameLogAddress)));
+        build_inverted_index_bundle_data(bundle->log_addresses, schema.inverted_index(db::state::kInvIdxNameLogAddress), step_converter));
     bundle_data.entities.emplace(
         db::state::kInvIdxNameLogTopics,
-        build_inverted_index_bundle_data(bundle->log_topics, schema.inverted_index(db::state::kInvIdxNameLogTopics)));
+        build_inverted_index_bundle_data(bundle->log_topics, schema.inverted_index(db::state::kInvIdxNameLogTopics), step_converter));
     bundle_data.entities.emplace(
         db::state::kInvIdxNameTracesFrom,
-        build_inverted_index_bundle_data(bundle->traces_from, schema.inverted_index(db::state::kInvIdxNameTracesFrom)));
+        build_inverted_index_bundle_data(bundle->traces_from, schema.inverted_index(db::state::kInvIdxNameTracesFrom), step_converter));
     bundle_data.entities.emplace(
         db::state::kInvIdxNameTracesTo,
-        build_inverted_index_bundle_data(bundle->traces_to, schema.inverted_index(db::state::kInvIdxNameTracesTo)));
+        build_inverted_index_bundle_data(bundle->traces_to, schema.inverted_index(db::state::kInvIdxNameTracesTo), step_converter));
 
     return SnapshotBundle{
         parse_snapshot_path(bundle->accounts.segment.file_path).step_range(),
@@ -278,13 +288,17 @@ SILKWORM_EXPORT int silkworm_build_recsplit_indexes(SilkwormHandle handle, struc
     return SILKWORM_OK;
 }
 
-static snapshots::SnapshotBundle build_blocks_snapshot_bundle(const SilkwormBlocksSnapshotBundle* bundle) {
+static snapshots::SnapshotBundle build_blocks_snapshot_bundle(
+    const SilkwormBlocksSnapshotBundle* bundle,
+    const Schema::RepositoryDef& schema) {
     snapshots::SnapshotBundleEntityData data;
+    datastore::StepToTimestampConverter step_converter = schema.make_step_converter();
 
     data.segments.emplace(
         db::blocks::kHeaderSegmentName,
         snapshots::segment::SegmentFileReader{
             parse_snapshot_path(bundle->headers.segment.file_path),
+            step_converter,
             make_region(bundle->headers.segment),
         });
     data.accessor_indexes.emplace(
@@ -298,6 +312,7 @@ static snapshots::SnapshotBundle build_blocks_snapshot_bundle(const SilkwormBloc
         db::blocks::kBodySegmentName,
         snapshots::segment::SegmentFileReader{
             parse_snapshot_path(bundle->bodies.segment.file_path),
+            step_converter,
             make_region(bundle->bodies.segment),
         });
     data.accessor_indexes.emplace(
@@ -311,6 +326,7 @@ static snapshots::SnapshotBundle build_blocks_snapshot_bundle(const SilkwormBloc
         db::blocks::kTxnSegmentName,
         snapshots::segment::SegmentFileReader{
             parse_snapshot_path(bundle->transactions.segment.file_path),
+            step_converter,
             make_region(bundle->transactions.segment),
         });
     data.accessor_indexes.emplace(
@@ -348,7 +364,7 @@ SILKWORM_EXPORT int silkworm_add_blocks_snapshot_bundle(
 
         auto& repository = *handle->blocks_repository;
 
-        repository.add_snapshot_bundle(build_blocks_snapshot_bundle(bundle));
+        repository.add_snapshot_bundle(build_blocks_snapshot_bundle(bundle, repository.schema()));
         return SILKWORM_OK;
     } catch (const InvalidSnapshotPathException&) {
         return SILKWORM_INVALID_PATH;

silkworm/db/blocks/bodies/body_txs_amount_query_test.cpp

@@ -5,6 +5,7 @@
 
 #include <catch2/catch_test_macros.hpp>
 
+#include <silkworm/db/blocks/step_block_num_converter.hpp>
 #include <silkworm/db/test_util/temp_snapshots.hpp>
 #include <silkworm/infra/common/directories.hpp>
 #include <silkworm/infra/common/log.hpp>
@@ -15,7 +16,7 @@ namespace silkworm::snapshots {
 TEST_CASE("BodyTxsAmountSegmentQuery") {
     TemporaryDirectory tmp_dir;
     test_util::SampleBodySnapshotFile snapshot_file{tmp_dir.path()};
-    segment::SegmentFileReader snapshot{snapshot_file.path()};
+    segment::SegmentFileReader snapshot{snapshot_file.path(), db::blocks::kStepToBlockNumConverter};
 
     BodyTxsAmountSegmentQuery query{snapshot};
     auto result = query.exec();

silkworm/db/blocks/headers/header_segment.cpp

@@ -28,8 +28,11 @@ void decode_word_into_header(ByteView word, BlockHeader& header) {
     success_or_throw(decode_result, "decode_word_into_header: rlp::decode error");
 }
 
-void check_sanity_of_header_with_metadata(const BlockHeader& header, datastore::StepRange step_range) {
-    auto block_num_range = db::blocks::kStepToBlockNumConverter.timestamp_range_from_step_range(step_range);
+void check_sanity_of_header_with_metadata(
+    const BlockHeader& header,
+    datastore::StepRange step_range,
+    const datastore::StepToTimestampConverter& step_converter) {
+    auto block_num_range = step_converter.timestamp_range_from_step_range(step_range);
     BlockNum block_from = block_num_range.start;
     BlockNum block_to = block_num_range.end;
     ensure((header.number >= block_from) && (header.number < block_to), [&]() {

silkworm/db/blocks/headers/header_segment.hpp

@@ -13,7 +13,10 @@ namespace silkworm::snapshots {
 
 void encode_word_from_header(Bytes& word, const BlockHeader& header);
 void decode_word_into_header(ByteView word, BlockHeader& header);
-void check_sanity_of_header_with_metadata(const BlockHeader& header, datastore::StepRange step_range);
+void check_sanity_of_header_with_metadata(
+    const BlockHeader& header,
+    datastore::StepRange step_range,
+    const datastore::StepToTimestampConverter& step_converter);
 
 struct HeaderSegmentWordEncoder : public Encoder {
     BlockHeader value;
@@ -39,8 +42,8 @@ struct HeaderSegmentWordDecoder : public Decoder {
         decode_word_into_header(word, value);
     }
 
-    void check_sanity_with_metadata(const SnapshotPath& path) override {
-        check_sanity_of_header_with_metadata(value, path.step_range());
+    void check_sanity_with_metadata(const SnapshotPath& path, const datastore::StepToTimestampConverter& step_converter) override {
+        check_sanity_of_header_with_metadata(value, path.step_range(), step_converter);
     }
 };
 

silkworm/db/blocks/schema_config.cpp

@@ -11,6 +11,7 @@ namespace silkworm::db::blocks {
 snapshots::Schema::RepositoryDef make_blocks_repository_schema() {
     snapshots::Schema::RepositoryDef repository_schema;
     repository_schema.index_salt_file_name("salt-blocks.txt");
+    repository_schema.step_size(kStepSizeForBlockSnapshots);
     snapshots::Schema::EntityDef& schema = repository_schema.default_entity();
 
     schema.segment(kHeaderSegmentName)
@@ -48,12 +49,12 @@ snapshots::SnapshotRepository make_blocks_repository(
     std::filesystem::path dir_path,
     bool open,
     std::optional<uint32_t> index_salt) {
+    auto schema = make_blocks_repository_schema();
     return snapshots::SnapshotRepository{
         kBlocksRepositoryName,
         std::move(dir_path),
         open,
-        make_blocks_repository_schema(),
-        kStepToBlockNumConverter,
+        schema,
         index_salt,
         make_blocks_index_builders_factory(),
         std::nullopt,  // no domain caches

silkworm/db/blocks/transactions/txn_index.cpp

@@ -6,6 +6,7 @@
 #include <silkworm/db/blocks/bodies/body_txs_amount_query.hpp>
 #include <silkworm/db/datastore/snapshots/segment/segment_reader.hpp>
 
+#include "silkworm/db/blocks/step_block_num_converter.hpp"
 #include "txn_segment_word_codec.hpp"
 
 namespace silkworm::snapshots {
@@ -17,7 +18,7 @@ Bytes TransactionKeyFactory::make(ByteView key_data, uint64_t i) {
 std::pair<uint64_t, uint64_t> TransactionIndex::compute_txs_amount(
     SnapshotPath bodies_segment_path,
     std::optional<MemoryMappedRegion> bodies_segment_region) {
-    segment::SegmentFileReader body_segment{std::move(bodies_segment_path), bodies_segment_region};
+    segment::SegmentFileReader body_segment{std::move(bodies_segment_path), db::blocks::kStepToBlockNumConverter, bodies_segment_region};
     auto result = BodyTxsAmountSegmentQuery{body_segment}.exec();
     return {result.first_tx_id, result.count};
 }