Skip to content

Commit fa78c70

Browse files
authored
Merge branch 'master' into add_zlib_for_streaming
2 parents 9368b59 + 7fcccf2 commit fa78c70

File tree

79 files changed

+981
-430
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+981
-430
lines changed

.github/workflows/execution-test.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ on:
1010
description: 'Stage to stop at; none for all; sample values: "Execution", "IntermediateHashes", "HashState", "HistoryIndex"'
1111
required: false
1212
type: string
13+
clean_datadir:
14+
description: 'Remove datadir before running the test'
15+
type: boolean
16+
default: true
1317

1418
jobs:
1519
execution-test-suite:
@@ -33,6 +37,10 @@ jobs:
3337
- name: Clean Build Directory
3438
run: rm -rf ${{runner.workspace}}/silkworm/build
3539

40+
- name: Clean Database Directory
41+
if: ${{github.event.inputs.clean_datadir}}
42+
run: rm -rf ${{runner.workspace}}/silkworm_datadir
43+
3644
- name: Create Build Environment
3745
run: cmake -E make_directory ${{runner.workspace}}/silkworm/build
3846

@@ -55,7 +63,7 @@ jobs:
5563
run: |
5664
set +e # Disable exit on error
5765
58-
# Run Erigon, wait sync and check ability to maintain sync
66+
# Run Silkworm, wait until the end of historical execution and check the result
5967
python3 $ERIGON_QA_PATH/test_system/qa-tests/silkworm-execution/run_and_check_execution.py \
6068
${{runner.workspace}}/silkworm/build/cmd ${{runner.workspace}}/silkworm_datadir $TRACKING_TIME_SECONDS $TOTAL_TIME_SECONDS $CHAIN
6169

.github/workflows/rpc-integration-tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ jobs:
3333
- name: Checkout RPC Tests Repository & Install Requirements
3434
run: |
3535
rm -rf ${{runner.workspace}}/rpc-tests
36-
git -c advice.detachedHead=false clone --depth 1 --branch v1.56.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
36+
git -c advice.detachedHead=false clone --depth 1 --branch v1.57.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
3737
cd ${{runner.workspace}}/rpc-tests
3838
pip3 install -r requirements.txt --break-system-packages
3939

silkworm/capi/instance.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ struct SilkwormInstance {
5151

5252
// TODO: This has to be changed and encapsulated by a proper block caching state
5353
struct ExecutionResult {
54-
silkworm::TxnId tx_id = 0;
54+
silkworm::TxnId txn_id = 0;
5555
uint64_t blob_gas_used = 0;
5656
silkworm::Receipt receipt;
5757
uint64_t log_index = 0;

silkworm/capi/silkworm.cpp

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -612,14 +612,14 @@ SILKWORM_EXPORT int silkworm_execute_txn(SilkwormHandle handle, MDBX_txn* mdbx_t
612612
silkworm::Hash block_header_hash{};
613613
memcpy(block_header_hash.bytes, block_hash.bytes, sizeof(block_hash.bytes));
614614
BlockNum block_number{block_num};
615-
TxnId txn_id_{txn_num};
615+
TxnId txn_id{txn_num};
616616

617617
auto unmanaged_tx = datastore::kvdb::RWTxnUnmanaged{mdbx_tx};
618618
auto unmanaged_env = silkworm::datastore::kvdb::EnvUnmanaged{::mdbx_txn_env(mdbx_tx)};
619619
auto chain_db = db::DataStore::make_chaindata_database(std::move(unmanaged_env));
620620
auto db_ref = chain_db.ref();
621621
silkworm::execution::DomainState state{
622-
txn_id_,
622+
txn_id,
623623
unmanaged_tx,
624624
db_ref,
625625
*handle->blocks_repository,
@@ -662,13 +662,13 @@ SILKWORM_EXPORT int silkworm_execute_txn(SilkwormHandle handle, MDBX_txn* mdbx_t
662662
<< " TxnHash " << silkworm::to_hex(transaction.hash().bytes, true)
663663
<< " Sender " << silkworm::to_hex(transaction.sender().value_or(evmc::address{}).bytes, true);
664664

665-
auto protocol_rule_set_{protocol::rule_set_factory(*handle->chain_config)};
666-
if (!protocol_rule_set_) {
665+
auto protocol_rule_set{protocol::rule_set_factory(*handle->chain_config)};
666+
if (!protocol_rule_set) {
667667
SILK_ERROR << "Protocol rule set not created";
668668
return SILKWORM_INTERNAL_ERROR;
669669
}
670670

671-
ExecutionProcessor processor{block, *protocol_rule_set_, state, *handle->chain_config, false};
671+
ExecutionProcessor processor{block, *protocol_rule_set, state, *handle->chain_config, false};
672672
// TODO: add analysis cache, check block exec for more
673673

674674
silkworm::Receipt receipt{};
@@ -685,7 +685,7 @@ SILKWORM_EXPORT int silkworm_execute_txn(SilkwormHandle handle, MDBX_txn* mdbx_t
685685
try {
686686
processor.flush_state();
687687
SilkwormInstance::ExecutionResult exec_result{
688-
.tx_id = txn_id_,
688+
.txn_id = txn_id,
689689
.blob_gas_used = transaction.total_blob_gas(),
690690
.receipt = receipt,
691691
.log_index = 0};
@@ -696,7 +696,7 @@ SILKWORM_EXPORT int silkworm_execute_txn(SilkwormHandle handle, MDBX_txn* mdbx_t
696696
exec_result.receipt.cumulative_gas_used += prev.receipt.cumulative_gas_used;
697697
exec_result.log_index += std::size(prev.receipt.logs);
698698
}
699-
handle->executions_in_block.push_back(std::move(exec_result));
699+
handle->executions_in_block.emplace_back(std::move(exec_result));
700700
} catch (const std::exception& ex) {
701701
SILK_ERROR << "transaction post-processing failed: " << ex.what();
702702
return SILKWORM_INTERNAL_ERROR;
@@ -762,10 +762,10 @@ SILKWORM_EXPORT int silkworm_block_exec_end(SilkwormHandle handle, MDBX_txn* mdb
762762

763763
rw_in_mem_cursor->insert(datastore::kvdb::Slice(key), datastore::kvdb::Slice(rlp_encoded));
764764

765-
const auto& tx_id = handle->executions_in_block[index].tx_id;
765+
const auto& txn_id = handle->executions_in_block[index].txn_id;
766766

767767
execution::DomainState state{
768-
tx_id,
768+
txn_id,
769769
unmanaged_tx,
770770
db_ref,
771771
*handle->blocks_repository,

silkworm/db/chain/remote_chain_storage_test.cpp

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
#include <catch2/catch_test_macros.hpp>
99
#include <catch2/matchers/catch_matchers_exception.hpp>
10-
#include <gmock/gmock.h>
1110
#include <nlohmann/json.hpp>
1211

1312
#include <silkworm/core/common/util.hpp>
@@ -55,16 +54,15 @@ struct RemoteChainStorageTest : public silkworm::test_util::ContextTestBase {
5554
chain::Providers& providers{storage.providers()};
5655
};
5756

57+
// Exclude on MSVC due to error LNK2001: unresolved external symbol testing::Matcher<class std::basic_string_view...
58+
// See also https://github.com/google/googletest/issues/4357
59+
#ifndef _WIN32
5860
TEST_CASE_METHOD(RemoteChainStorageTest, "read_chain_config") {
5961
SECTION("empty chain data") {
6062
EXPECT_CALL(transaction, get_one(table::kConfigName, _)).WillOnce(InvokeWithoutArgs([]() -> Task<Bytes> {
6163
co_return Bytes{};
6264
}));
63-
#ifdef SILKWORM_SANITIZE // Avoid comparison against exception message: it triggers a TSAN data race seemingly related to libstdc++ string implementation
6465
CHECK_THROWS_AS(spawn_and_wait(storage.read_chain_config()), std::invalid_argument);
65-
#else
66-
CHECK_THROWS_MATCHES(spawn_and_wait(storage.read_chain_config()), std::invalid_argument, Message("empty chain config data in read_chain_config"));
67-
#endif // SILKWORM_SANITIZE
6866
}
6967

7068
SECTION("invalid JSON chain data") {
@@ -98,6 +96,7 @@ TEST_CASE_METHOD(RemoteChainStorageTest, "read_chain_config") {
9896
})"_json);
9997
}
10098
}
99+
#endif // _WIN32
101100

102101
TEST_CASE_METHOD(RemoteChainStorageTest, "read_transaction_by_idx_in_block") {
103102
SECTION("not found") {
@@ -107,9 +106,9 @@ TEST_CASE_METHOD(RemoteChainStorageTest, "read_transaction_by_idx_in_block") {
107106
SECTION("found") {
108107
Block block = silkworm::test_util::sample_block();
109108
providers.block = [&](BlockNum, HashAsSpan, bool, Block& b) -> Task<bool> { b = block; co_return true; };
110-
const auto txn0 = spawn_and_wait(storage.read_transaction_by_idx_in_block(block.header.number, /*txn_id=*/0));
109+
const auto txn0 = spawn_and_wait(storage.read_transaction_by_idx_in_block(block.header.number, /*txn_idx=*/0));
111110
REQUIRE(txn0 == silkworm::test_util::kSampleTx0);
112-
const auto txn1 = spawn_and_wait(storage.read_transaction_by_idx_in_block(block.header.number, /*txn_id=*/1));
111+
const auto txn1 = spawn_and_wait(storage.read_transaction_by_idx_in_block(block.header.number, /*txn_idx=*/1));
113112
REQUIRE(txn1 == silkworm::test_util::kSampleTx1);
114113
}
115114
}

silkworm/db/cli/db_toolbox.cpp

Lines changed: 19 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@
22
// SPDX-License-Identifier: Apache-2.0
33

44
#include <algorithm>
5-
#include <bit>
6-
#include <bitset>
75
#include <filesystem>
86
#include <fstream>
97
#include <iostream>
8+
#include <ranges>
109
#include <regex>
1110
#include <stdexcept>
1211
#include <string>
@@ -251,7 +250,7 @@ void table_get(EnvConfig& config, const std::string& table, const std::optional<
251250
ensure(k.has_value() || block_num.has_value(), "You must specify either --key or --block");
252251
auto env = open_env(config);
253252
auto txn = env.start_read();
254-
ensure(has_map(txn, table.c_str()), [&table]() { return "Table " + table + " not found"; });
253+
ensure(has_map(txn, table), [&table]() { return "Table " + table + " not found"; });
255254
::mdbx::map_handle table_map = txn.open_map(table);
256255
const std::string_view key_identifier = block_num ? "block_key" : "key";
257256
const Bytes key = k.value_or(block_key(*block_num));
@@ -277,8 +276,7 @@ void table_get(EnvConfig& config, const std::string& table, const std::optional<
277276
std::cout << "\n";
278277
}
279278

280-
void do_clear(EnvConfig& config, bool dry, bool always_yes, const std::vector<std::string>& table_names,
281-
bool drop) {
279+
void do_clear(EnvConfig& config, bool dry, bool always_yes, const std::vector<std::string>& table_names, bool drop) {
282280
config.readonly = false;
283281

284282
if (!config.exclusive) {
@@ -288,22 +286,22 @@ void do_clear(EnvConfig& config, bool dry, bool always_yes, const std::vector<st
288286
auto env{open_env(config)};
289287
auto txn{env.start_write()};
290288

291-
for (const auto& tablename : table_names) {
292-
if (!has_map(txn, tablename.c_str())) {
293-
std::cout << "Table " << tablename << " not found\n";
289+
for (const auto& table : table_names) {
290+
if (!has_map(txn, table)) {
291+
std::cout << "Table " << table << " not found\n";
294292
continue;
295293
}
296294

297-
mdbx::map_handle table_map{txn.open_map(tablename)};
295+
mdbx::map_handle table_map{txn.open_map(table)};
298296
size_t rcount{txn.get_map_stat(table_map).ms_entries};
299297

300298
if (!rcount && !drop) {
301-
std::cout << " Table " << tablename << " is already empty. Skipping\n";
299+
std::cout << " Table " << table << " is already empty. Skipping\n";
302300
continue;
303301
}
304302

305303
std::cout << "\n"
306-
<< (drop ? "Dropping" : "Emptying") << " table " << tablename << " (" << rcount << " records) "
304+
<< (drop ? "Dropping" : "Emptying") << " table " << table << " (" << rcount << " records) "
307305
<< std::flush;
308306

309307
if (!always_yes) {
@@ -1003,12 +1001,12 @@ static void print_table_diff(ROTxn& txn1, ROTxn& txn2, const DbTableInfo& table1
10031001
[&]() { return "value_mode mismatch: " + std::to_string(static_cast<int>(table1.info.value_mode())) + " vs " + std::to_string(static_cast<int>(table2.info.value_mode())); });
10041002

10051003
MapConfig table1_config{
1006-
.name = table1.name.c_str(),
1004+
.name = table1.name,
10071005
.key_mode = table1.info.key_mode(),
10081006
.value_mode = table1.info.value_mode(),
10091007
};
10101008
MapConfig table2_config{
1011-
.name = table2.name.c_str(),
1009+
.name = table2.name,
10121010
.key_mode = table2.info.key_mode(),
10131011
.value_mode = table2.info.value_mode(),
10141012
};
@@ -1024,7 +1022,7 @@ static void print_table_diff(ROTxn& txn1, ROTxn& txn2, const DbTableInfo& table1
10241022
"MAIN_DBI"sv,
10251023
"DbInfo"sv,
10261024
};
1027-
std::any_of(kIrrelevantTables.begin(), kIrrelevantTables.end(), [&table1](const std::string_view table_name) { return table_name == table1.name; })) {
1025+
std::ranges::any_of(kIrrelevantTables, [&table1](const std::string_view table_name) { return table_name == table1.name; })) {
10281026
std::cout << "Skipping irrelevant table: " << table1.name << "\n";
10291027
return;
10301028
}
@@ -1050,7 +1048,7 @@ static void print_table_diff(ROTxn& txn1, ROTxn& txn2, const DbTableInfo& table1
10501048

10511049
static std::optional<DbTableInfo> find_table(const DbInfo& db_info, std::string_view table) {
10521050
const auto& db_tables{db_info.tables};
1053-
const auto it{std::find_if(db_tables.begin(), db_tables.end(), [=](const auto& t) { return t.name == table; })};
1051+
const auto it{std::ranges::find_if(db_tables, [=](const auto& t) { return t.name == table; })};
10541052
return it != db_tables.end() ? std::make_optional<DbTableInfo>(*it) : std::nullopt;
10551053
}
10561054

@@ -1066,12 +1064,12 @@ static DbComparisonResult compare_db_schema(const DbInfo& db1_info, const DbInfo
10661064

10671065
// Check both databases have the same table names
10681066
for (auto& db1_table : db1_tables) {
1069-
if (std::find(db2_tables.begin(), db2_tables.end(), db1_table) == db2_tables.end()) {
1067+
if (std::ranges::find(db2_tables, db1_table) == db2_tables.end()) {
10701068
return tl::make_unexpected("db1 table " + db1_table.name + " not present in db2\n");
10711069
}
10721070
}
10731071
for (auto& db2_table : db2_tables) {
1074-
if (std::find(db1_tables.begin(), db1_tables.end(), db2_table) == db1_tables.end()) {
1072+
if (std::ranges::find(db1_tables, db2_table) == db1_tables.end()) {
10751073
return tl::make_unexpected("db2 table " + db2_table.name + " not present in db1\n");
10761074
}
10771075
}
@@ -1374,12 +1372,10 @@ void do_first_byte_analysis(EnvConfig& config) {
13741372

13751373
// Sort histogram by usage (from most used to less used)
13761374
std::vector<std::pair<uint8_t, size_t>> histogram_sorted;
1377-
std::copy(histogram.begin(), histogram.end(),
1378-
std::back_inserter<std::vector<std::pair<uint8_t, size_t>>>(histogram_sorted));
1379-
std::sort(histogram_sorted.begin(), histogram_sorted.end(),
1380-
[](std::pair<uint8_t, size_t>& a, std::pair<uint8_t, size_t>& b) -> bool {
1381-
return a.second == b.second ? a.first < b.first : a.second > b.second;
1382-
});
1375+
std::ranges::copy(histogram, std::back_inserter<std::vector<std::pair<uint8_t, size_t>>>(histogram_sorted));
1376+
std::ranges::sort(histogram_sorted, [](std::pair<uint8_t, size_t>& a, std::pair<uint8_t, size_t>& b) -> bool {
1377+
return a.second == b.second ? a.first < b.first : a.second > b.second;
1378+
});
13831379

13841380
if (!histogram_sorted.empty()) {
13851381
std::cout << (boost::format(" %-4s %8s") % "Byte" % "Count") << "\n"

silkworm/db/datastore/common/ranges/caching_view.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,6 @@ struct CachingViewFactory {
105105
}
106106
};
107107

108-
inline constexpr CachingViewFactory caching;
108+
inline constexpr CachingViewFactory caching; // NOLINT(*-identifier-naming)
109109

110110
} // namespace silkworm::views

silkworm/db/datastore/common/ranges/if_view.hpp

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,13 @@ class IfView : public std::ranges::view_interface<IfView<Range1, Range2>> {
9898
std::nullopt,
9999
std::nullopt,
100100
};
101-
} else {
102-
return Iterator{
103-
std::nullopt,
104-
std::nullopt,
105-
std::ranges::begin(range2_),
106-
std::ranges::end(range2_),
107-
};
108101
}
102+
return Iterator{
103+
std::nullopt,
104+
std::nullopt,
105+
std::ranges::begin(range2_),
106+
std::ranges::end(range2_),
107+
};
109108
}
110109

111110
std::default_sentinel_t end() const { return std::default_sentinel; }

silkworm/db/datastore/common/ranges/lazy_view.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class LazyView : public std::ranges::view_interface<LazyView<TRangeFactory, TRan
2121
LazyView& operator=(LazyView&& other) noexcept {
2222
range_factory_ = std::exchange(std::move(other.range_factory_), std::nullopt);
2323
range_ = std::exchange(std::move(other.range_), std::nullopt);
24-
return this;
24+
return *this;
2525
};
2626

2727
std::ranges::iterator_t<TRange> begin() { return std::ranges::begin(range()); }

silkworm/db/datastore/common/ranges/merge_unique_view.hpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,10 +135,10 @@ class MergeUniqueView : public std::ranges::view_interface<MergeUniqueView<Range
135135
}
136136
}
137137

138-
Range1Iterator it1_;
139-
Range1Sentinel sentinel1_;
140-
Range2Iterator it2_;
141-
Range2Sentinel sentinel2_;
138+
Range1Iterator it1_{};
139+
Range1Sentinel sentinel1_{};
140+
Range2Iterator it2_{};
141+
Range2Sentinel sentinel2_{};
142142
const Comp* comp_{nullptr};
143143
Proj1 proj1_;
144144
Proj2 proj2_;

0 commit comments

Comments
 (0)