Skip to content

Commit ae0f69c

Browse files
canepatcanepat
authored
db: streams hiding pagination in kv::api::Transaction range Data API (#2864)
db: streams hiding pagination in kv::api::Transaction range Data API db: implement direct kv::api::LocalTransaction using ranges/views db: move paginated implementation to kv::api::RemoteTransaction db: support skipping empty values in range queries for direct implementation db: fix empty end key in range queries db: move ranges' vector from iterator to view in MergeManyView rpc: fix block parameter lifecycle in parity_listStorageKeys rpc: rename PaginatedStream as Stream
1 parent 640908f commit ae0f69c

31 files changed

+537
-416
lines changed

‎.github/workflows/run_integration_tests.sh‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ rm -rf ./mainnet/results/
1515
# eth_getLogs: waiting erigon fix on wrong FirstLogIndex in ReceiptsDomain
1616
# debug_traceBlockByNumber[24-28]: response different wrt erigon
1717
python3 ./run_tests.py --continue --blockchain mainnet --jwt "$2" --display-only-fail --json-diff --port 51515 --transport_type http -x \
18-
debug_accountRange,\
19-
debug_storageRangeAt,\
2018
debug_traceBlockByNumber/test_24,\
2119
debug_traceBlockByNumber/test_25,\
2220
debug_traceBlockByNumber/test_26,\
@@ -36,7 +34,6 @@ eth_getLogs/test_17,\
3634
eth_getLogs/test_18,\
3735
eth_getLogs/test_19,\
3836
eth_getLogs/test_20,\
39-
parity_listStorageKeys,\
4037
trace_replayBlockTransactions/test_29,\
4138
trace_transaction/test_44,\
4239
trace_transaction/test_47

‎silkworm/db/datastore/common/ranges/merge_many_view.hpp‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,10 @@ class MergeManyView : public std::ranges::view_interface<MergeManyView<Range, Ra
4343
using pointer = std::remove_reference_t<reference>*;
4444

4545
Iterator() = default;
46-
Iterator(
47-
Ranges& ranges,
48-
const Comp* comp, Proj proj)
49-
: ranges_{vector_from_range(ranges)},
50-
comp_{comp},
46+
Iterator(std::vector<Range>& ranges, const Comp* comp, Proj proj)
47+
: comp_{comp},
5148
proj_{std::move(proj)} {
52-
for (Range& range : ranges_) {
49+
for (Range& range : ranges) {
5350
iterators_.emplace_back(std::ranges::begin(range));
5451
sentinels_.emplace_back(std::ranges::end(range));
5552
}
@@ -156,7 +153,6 @@ class MergeManyView : public std::ranges::view_interface<MergeManyView<Range, Ra
156153
return std::ranges::iter_move(iterators_[i]);
157154
}
158155

159-
std::vector<Range> ranges_;
160156
std::vector<RangeIterator> iterators_;
161157
std::vector<RangeSentinel> sentinels_;
162158
const Comp* comp_{nullptr};
@@ -170,7 +166,7 @@ class MergeManyView : public std::ranges::view_interface<MergeManyView<Range, Ra
170166
MergeManyView(
171167
Ranges ranges,
172168
Comp comp, Proj proj)
173-
: ranges_{std::move(ranges)},
169+
: ranges_{vector_from_range(std::move(ranges))},
174170
comp_{std::move(comp)},
175171
proj_{std::move(proj)} {}
176172
MergeManyView() = default;
@@ -182,7 +178,7 @@ class MergeManyView : public std::ranges::view_interface<MergeManyView<Range, Ra
182178
std::default_sentinel_t end() const { return std::default_sentinel; }
183179

184180
private:
185-
Ranges ranges_;
181+
std::vector<Range> ranges_;
186182
Comp comp_;
187183
Proj proj_;
188184
};

‎silkworm/db/datastore/common/ranges/merge_many_view_test.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ namespace silkworm::views {
1212

1313
static_assert(std::ranges::input_range<MergeManyView<std::vector<std::vector<int>>>>);
1414
static_assert(std::ranges::view<MergeManyView<std::vector<std::vector<int>>>>);
15+
static_assert(std::ranges::view<std::ranges::take_view<MergeManyView<std::vector<std::vector<int>>>>>);
1516

1617
template <std::ranges::input_range TRange>
1718
std::vector<TRange> ranges(TRange r1, TRange r2) {

‎silkworm/db/datastore/domain_range_as_of_query.hpp‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,27 +38,44 @@ struct DomainRangeAsOfQuery {
3838
using Key = decltype(TKeyEncoder1::value);
3939
using ResultItem = typename DomainRangeLatestQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2>::ResultItem;
4040

41-
auto exec(const Key& key_start, const Key& key_end, std::optional<Timestamp> timestamp, bool ascending) {
41+
auto exec(const Key& key_start, const Key& key_end, std::optional<Timestamp> timestamp, bool ascending, bool skip_empty_values) {
4242
return silkworm::views::if_view(
4343
!timestamp.has_value(),
4444
query2_.exec(key_start, key_end, ascending),
45-
this->exec(key_start, key_end, timestamp.value_or(0), ascending));
45+
this->exec(key_start, key_end, timestamp.value_or(0), ascending, skip_empty_values));
4646
}
4747

48-
auto exec(const Key& key_start, const Key& key_end, Timestamp timestamp, bool ascending) {
48+
auto exec(const Key& key_start, const Key& key_end, Timestamp timestamp, bool ascending, bool skip_empty_values) {
4949
SILKWORM_ASSERT(ascending); // descending is not implemented
5050

51+
auto skip_empty_value_predicate = [skip_empty_values](std::pair<Bytes, Bytes>& kv_pair) {
52+
if (!skip_empty_values) return true;
53+
return !kv_pair.second.empty();
54+
};
55+
5156
return silkworm::views::merge_unique(
52-
query1_.exec(key_start, key_end, timestamp, ascending),
53-
query2_.exec(key_start, key_end, ascending),
54-
silkworm::views::MergeCompareFunc{},
55-
PairGetFirst<typename ResultItem::first_type, typename ResultItem::second_type>{},
56-
PairGetFirst<typename ResultItem::first_type, typename ResultItem::second_type>{});
57+
query1_.exec(key_start, key_end, timestamp, ascending),
58+
query2_.exec(key_start, key_end, ascending),
59+
silkworm::views::MergeCompareFunc{},
60+
PairGetFirst<typename ResultItem::first_type, typename ResultItem::second_type>{},
61+
PairGetFirst<typename ResultItem::first_type, typename ResultItem::second_type>{}) |
62+
std::views::filter(std::move(skip_empty_value_predicate));
5763
}
5864

5965
private:
6066
HistoryRangeByKeysQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2> query1_;
6167
DomainRangeLatestQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2> query2_;
6268
};
6369

70+
template <
71+
kvdb::EncoderConcept TKeyEncoder1, snapshots::EncoderConcept TKeyEncoder2,
72+
kvdb::DecoderConcept TKeyDecoder1, snapshots::DecoderConcept TKeyDecoder2,
73+
kvdb::DecoderConcept TValueDecoder1, snapshots::DecoderConcept TValueDecoder2>
74+
using DomainRangeAsOfQueryResult = decltype(std::declval<DomainRangeAsOfQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2>>().exec(
75+
std::declval<const typename DomainRangeAsOfQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2>::Key&>(),
76+
std::declval<const typename DomainRangeAsOfQuery<TKeyEncoder1, TKeyEncoder2, TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2>::Key&>(),
77+
std::declval<std::optional<Timestamp>>(),
78+
std::declval<bool>(),
79+
std::declval<bool>()));
80+
6481
} // namespace silkworm::datastore

‎silkworm/db/datastore/history_range_in_period_query.hpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,11 @@ struct HistoryRangeInPeriodQuery {
5858
snapshots::HistoryRangeInPeriodQuery<TKeyDecoder2, TValueDecoder2> query2_;
5959
};
6060

61+
template <
62+
kvdb::DecoderConcept TKeyDecoder1, snapshots::DecoderConcept TKeyDecoder2,
63+
kvdb::DecoderConcept TValueDecoder1, snapshots::DecoderConcept TValueDecoder2>
64+
using HistoryRangeInPeriodQueryResult = decltype(std::declval<HistoryRangeInPeriodQuery<TKeyDecoder1, TKeyDecoder2, TValueDecoder1, TValueDecoder2>>().exec(
65+
std::declval<TimestampRange>(),
66+
std::declval<bool>()));
67+
6168
} // namespace silkworm::datastore

‎silkworm/db/datastore/inverted_index_range_by_key_query.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,4 +54,10 @@ struct InvertedIndexRangeByKeyQuery {
5454
snapshots::InvertedIndexRangeByKeyQuery<TKeyEncoder2> query2_;
5555
};
5656

57+
template <kvdb::EncoderConcept TKeyEncoder1, snapshots::EncoderConcept TKeyEncoder2>
58+
using InvertedIndexRangeByKeyQueryResult = decltype(std::declval<InvertedIndexRangeByKeyQuery<TKeyEncoder1, TKeyEncoder2>>().exec(
59+
std::declval<typename InvertedIndexRangeByKeyQuery<TKeyEncoder1, TKeyEncoder2>::Key>(),
60+
std::declval<TimestampRange>(),
61+
std::declval<bool>()));
62+
5763
} // namespace silkworm::datastore

‎silkworm/db/datastore/kvdb/domain_range_latest_query.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ struct DomainRangeLatestQuery {
7171
}
7272

7373
auto before_key_end_predicate = [key_end = std::move(key_end)](const std::pair<ByteView, ByteView>& kv_pair) {
74-
return kv_pair.first < key_end;
74+
return key_end.empty() || kv_pair.first < key_end;
7575
};
7676

7777
return std::ranges::subrange{std::move(begin_it), CursorKVIteratorRaw{}} |

‎silkworm/db/datastore/kvdb/history_range_by_keys_query.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ struct HistoryRangeByKeysQuery {
126126
}
127127

128128
auto before_key_end_predicate = [key_end = std::move(key_end)](const std::shared_ptr<ROCursor>& cursor) {
129+
if (key_end.empty()) return true;
129130
auto result = cursor->current();
130131
SILKWORM_ASSERT(result);
131132
return from_slice(result.key) < ByteView{key_end};

‎silkworm/db/datastore/snapshots/domain_range_latest_query.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ struct DomainRangeLatestSegmentQuery {
3939
auto begin_it = entity_.btree_index.seek(key_start, entity_.kv_segment).value_or(btree::BTreeIndex::Cursor{});
4040

4141
return std::ranges::subrange{std::move(begin_it), std::default_sentinel} |
42-
std::views::take_while([key_end = std::move(key_end)](const auto& kv_pair) { return ByteView{kv_pair.first} < key_end; });
42+
std::views::take_while([key_end = std::move(key_end)](const auto& kv_pair) { return key_end.empty() || ByteView{kv_pair.first} < key_end; });
4343
}
4444

4545
auto exec(Bytes key_start, Bytes key_end, bool ascending) {
@@ -112,6 +112,7 @@ struct DomainRangeLatestQuery {
112112
PairGetFirst<DomainRangeLatestSegmentQuery::ResultItem::first_type, DomainRangeLatestSegmentQuery::ResultItem::second_type>{});
113113

114114
return silkworm::ranges::owning_view(std::move(results)) |
115+
std::views::filter([](const auto& kv_pair) { return !ByteView{kv_pair.second}.empty(); }) |
115116
std::views::transform(kDecodeKVPairFunc) |
116117
silkworm::views::caching;
117118
}

‎silkworm/db/datastore/snapshots/history_range_by_keys_query.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ struct HistoryRangeByKeysSegmentQuery {
7171
};
7272

7373
return std::ranges::subrange{std::move(begin_it), ii_reader.end()} |
74-
std::views::take_while([key_end = std::move(key_end)](auto&& ii_entry) { return ii_entry.first < key_end; }) |
74+
std::views::take_while([key_end = std::move(key_end)](auto&& ii_entry) { return key_end.empty() || ii_entry.first < key_end; }) |
7575
std::views::transform(std::move(lookup_kv_pair_func)) |
7676
silkworm::views::caching |
7777
std::views::filter([](const std::optional<ResultItem>& result_opt) { return result_opt.has_value(); }) |

0 commit comments

Comments
 (0)