diff --git a/eth/chains/base.py b/eth/chains/base.py index 6497e58d4b..1fb9583cef 100644 --- a/eth/chains/base.py +++ b/eth/chains/base.py @@ -47,6 +47,7 @@ from eth.db.header import ( HeaderDB, ) +from eth.db.schema import Schemas from eth.estimators import ( get_gas_estimator, @@ -402,7 +403,10 @@ def from_genesis(cls, genesis_vm_class = cls.get_vm_class_for_block_number(BlockNumber(0)) pre_genesis_header = BlockHeader(difficulty=0, block_number=-1, gas_limit=0) - state = genesis_vm_class.build_state(base_db, pre_genesis_header) + state = genesis_vm_class.build_state( + base_db, pre_genesis_header, + expected_schema=Schemas.DEFAULT, + ) if genesis_state is None: genesis_state = {} @@ -580,7 +584,11 @@ def build_block_with_transactions( :param parent_header: parent of the new block -- or canonical head if ``None`` :return: (new block, receipts, computations) """ - base_header = self.ensure_header(parent_header) + if parent_header is None: + base_header = self.ensure_header() + else: + base_header = self.create_header_from_parent(parent_header) + vm = self.get_vm(base_header) new_header, receipts, computations = vm.apply_all_transactions(transactions, base_header) @@ -680,8 +688,6 @@ def estimate_gas( Returns an estimation of the amount of gas the given transaction will use if executed on top of the block specified by the given header. """ - if at_header is None: - at_header = self.get_canonical_head() with self.get_vm(at_header).state_in_temp_block() as state: return self.gas_estimator(state, transaction) @@ -869,10 +875,12 @@ def _extract_uncle_hashes(blocks: Iterable[BaseBlock]) -> Iterable[Hash32]: class MiningChain(Chain): header: BlockHeader = None + vm: BaseVM = None def __init__(self, base_db: BaseAtomicDB, header: BlockHeader=None) -> None: super().__init__(base_db) self.header = self.ensure_header(header) + self.vm = super().get_vm(self.header) def apply_transaction(self, transaction: BaseTransaction @@ -883,23 +891,33 @@ def apply_transaction(self, WARNING: Receipt and Transaction trie generation is computationally heavy and incurs significant performance overhead. """ - vm = self.get_vm(self.header) - base_block = vm.get_block() + base_block = self.vm.get_block() - receipt, computation = vm.apply_transaction(base_block.header, transaction) - header_with_receipt = vm.add_receipt_to_header(base_block.header, receipt) + receipt, computation = self.vm.apply_transaction(base_block.header, transaction) + header_with_receipt = self.vm.add_receipt_to_header(base_block.header, receipt) # since we are building the block locally, we have to persist all the incremental state - vm.state.persist() - new_header = header_with_receipt.copy(state_root=vm.state.state_root) + self.vm.state.persist() + new_header = header_with_receipt.copy(state_root=self.vm.state.state_root) transactions = base_block.transactions + (transaction, ) receipts = base_block.get_receipts(self.chaindb) + (receipt, ) - new_block = vm.set_block_transactions(base_block, new_header, transactions, receipts) + new_block = self.vm.set_block_transactions( + base_block, new_header, transactions, receipts + ) self.header = new_block.header + # We need to create a new VM so that the next transaction to be processed builds + # on top of the result of processing this transaction. However, in order to save + # block diffs, we need to have a single instance of _state which is reused between + # all calls to `apply_transaction`. This is a bit of a kludge but it works. + new_vm_class = self.get_vm_class_for_block_number(self.header.block_number) + new_vm = new_vm_class(header=self.header, chaindb=self.vm.chaindb) + new_vm._state = self.vm._state + self.vm = new_vm + return new_block, receipt, computation def import_block(self, @@ -910,6 +928,8 @@ def import_block(self, block, perform_validation) self.header = self.ensure_header() + self.vm = super().get_vm(self.header) + return imported_block, new_canonical_blocks, old_canonical_blocks def mine_block(self, *args: Any, **kwargs: Any) -> BaseBlock: @@ -917,16 +937,17 @@ def mine_block(self, *args: Any, **kwargs: Any) -> BaseBlock: Mines the current block. Proxies to the current Virtual Machine. See VM. :meth:`~eth.vm.base.VM.mine_block` """ - mined_block = self.get_vm(self.header).mine_block(*args, **kwargs) + mined_block = self.vm.mine_block(*args, **kwargs) self.validate_block(mined_block) self.chaindb.persist_block(mined_block) self.header = self.create_header_from_parent(mined_block.header) + self.vm = super().get_vm(self.header) return mined_block def get_vm(self, at_header: BlockHeader=None) -> 'BaseVM': if at_header is None: - at_header = self.header + return self.vm return super().get_vm(at_header) diff --git a/eth/db/account.py b/eth/db/account.py index 4acc7a3df3..2b7b245503 100644 --- a/eth/db/account.py +++ b/eth/db/account.py @@ -19,7 +19,9 @@ ) from eth_utils import ( + big_endian_to_int, encode_hex, + int_to_big_endian, to_checksum_address, to_tuple, ValidationError, @@ -41,18 +43,32 @@ from eth.db.batch import ( BatchDB, ) +from eth.db.block_diff import ( + BlockDiff, +) from eth.db.cache import ( CacheDB, ) from eth.db.diff import ( DBDiff, ) +from eth.db.header import ( + HeaderDB, +) from eth.db.journal import ( JournalDB, ) +from eth.db.schema import ( + Schemas, + SchemaTurbo, + ensure_schema, + get_schema, +) from eth.db.storage import ( AccountStorageDB, + StorageLookup, ) +from eth.db.turbo import TurboDatabase, TurboBaseDB from eth.db.typing import ( JournalDBCheckpoint, ) @@ -63,6 +79,9 @@ from eth.rlp.accounts import ( Account, ) +from eth.rlp.headers import ( + BlockHeader, +) from eth.validation import ( validate_is_bytes, validate_uint256, @@ -210,7 +229,7 @@ class AccountDB(BaseAccountDB): logger = cast(ExtendedDebugLogger, logging.getLogger('eth.db.account.AccountDB')) - def __init__(self, db: BaseAtomicDB, state_root: Hash32=BLANK_ROOT_HASH) -> None: + def __init__(self, db: BaseAtomicDB, header: BlockHeader = None) -> None: r""" Internal implementation details (subject to rapid change): Database entries go through several pipes, like so... @@ -248,6 +267,11 @@ def __init__(self, db: BaseAtomicDB, state_root: Hash32=BLANK_ROOT_HASH) -> None AccountDB synchronizes the snapshot/revert/persist of both of the journals. """ + if header: + state_root = header.state_root + else: + state_root = BLANK_ROOT_HASH + self._raw_store_db = db self._batchdb = BatchDB(db) self._batchtrie = BatchDB(db, read_through_deletes=True) @@ -260,6 +284,16 @@ def __init__(self, db: BaseAtomicDB, state_root: Hash32=BLANK_ROOT_HASH) -> None self._dirty_accounts: Set[Address] = set() self._root_hash_at_last_persist = state_root + self._dirty_account_rlps: Set[Address] = set() + self._deleted_accounts: Set[Address] = set() + + self._turbodb = None + if get_schema(db) == Schemas.TURBO: + headerdb = HeaderDB(db) + self._turbodb = TurboDatabase(headerdb, header) + self._turbobase = TurboBaseDB(self._turbodb) + self._turbojournal = JournalDB(self._turbobase) + @property def state_root(self) -> Hash32: return self._trie.root_hash @@ -434,8 +468,11 @@ def delete_account(self, address: Address) -> None: if address in self._account_cache: del self._account_cache[address] del self._journaltrie[address] + if self._turbodb: + del self._turbojournal[address] self._wipe_storage(address) + self._deleted_accounts.add(address) def account_exists(self, address: Address) -> bool: validate_canonical_address(address, title="Storage Address") @@ -455,6 +492,26 @@ def account_is_empty(self, address: Address) -> bool: # Internal # def _get_encoded_account(self, address: Address, from_journal: bool=True) -> bytes: + trie_result = self._get_encoded_account_from_trie(address, from_journal) + if self._turbodb is None: + return trie_result + + turbo_result = self._get_encoded_account_from_turbodb(address, from_journal) + + # TODO: remove this once enough tests have been run to ensure this class works + # TODO: raise a better error message here + if trie_result != turbo_result: + # Set some variables for easier debugging + if trie_result != b'': + trie_account = rlp.decode(trie_result, sedes=Account) + if turbo_result != b'': + turbo_account = rlp.decode(turbo_result, sedes=Account) + assert False + + return turbo_result + + def _get_encoded_account_from_trie(self, address: Address, + from_journal: bool=True) -> bytes: lookup_trie = self._journaltrie if from_journal else self._trie_cache try: @@ -465,6 +522,17 @@ def _get_encoded_account(self, address: Address, from_journal: bool=True) -> byt # In case the account is deleted in the JournalDB return b'' + def _get_encoded_account_from_turbodb(self, address: Address, + from_journal: bool=False) -> bytes: + source = self._turbojournal if from_journal else self._turbobase + + try: + return source[address] + except KeyError: + return b'' + + assert False + def _get_account(self, address: Address, from_journal: bool=True) -> Account: if from_journal and address in self._account_cache: return self._account_cache[address] @@ -483,6 +551,10 @@ def _set_account(self, address: Address, account: Account) -> None: self._account_cache[address] = account rlp_account = rlp.encode(account, sedes=Account) self._journaltrie[address] = rlp_account + if self._turbodb: + self._turbojournal[address] = rlp_account + + self._dirty_account_rlps.add(address) # # Record and discard API @@ -490,6 +562,8 @@ def _set_account(self, address: Address, account: Account) -> None: def record(self) -> JournalDBCheckpoint: checkpoint = self._journaldb.record() self._journaltrie.record(checkpoint) + if self._turbodb: + self._turbojournal.record(checkpoint) for _, store in self._dirty_account_stores(): store.record(checkpoint) @@ -498,6 +572,8 @@ def record(self) -> JournalDBCheckpoint: def discard(self, checkpoint: JournalDBCheckpoint) -> None: self._journaldb.discard(checkpoint) self._journaltrie.discard(checkpoint) + if self._turbodb: + self._turbojournal.discard(checkpoint) self._account_cache.clear() for _, store in self._dirty_account_stores(): store.discard(checkpoint) @@ -505,6 +581,8 @@ def discard(self, checkpoint: JournalDBCheckpoint) -> None: def commit(self, checkpoint: JournalDBCheckpoint) -> None: self._journaldb.commit(checkpoint) self._journaltrie.commit(checkpoint) + if self._turbodb: + self._turbojournal.commit(checkpoint) for _, store in self._dirty_account_stores(): store.commit(checkpoint) @@ -529,6 +607,10 @@ def make_state_root(self) -> Hash32: self._apply_account_diff_without_proof(diff, memory_trie) self._journaltrie.reset() + if self._turbodb: + # Don't reset, that blows away all changes (they weren't persisted like the + # changes in _journaltrie were). + self._turbojournal.flatten() self._trie_cache.reset_cache() return self.state_root @@ -555,6 +637,8 @@ def persist(self) -> None: # reset local storage trackers self._account_stores = {} self._dirty_accounts = set() + self._dirty_account_rlps = set() + self._deleted_accounts = set() # persist accounts self._validate_generated_root() @@ -565,6 +649,118 @@ def persist(self) -> None: self._batchdb.commit_to(write_batch, apply_deletes=False) self._root_hash_at_last_persist = new_root_hash + def persist_returning_block_diff(self, parent_state_root: Hash32) -> BlockDiff: + """ + Persists, including a diff which can be used to unwind/replay the changes this block makes. + """ + block_diff = BlockDiff() + + # 1. Grab all the changed accounts and their previous values + + # pre-Byzantium make_storage_root is called at the end of every transaction, and + # it blows away all the changes. Create an old_trie here so we can peer into the + # state as it was at the beginning of the block. + + # TODO: In a world where the trie isn't being stored this won't work. We should + # instead construct an old_turbodb + old_trie = CacheDB(HashTrie(HexaryTrie( + self._raw_store_db, parent_state_root, prune=False + ))) + + for deleted_address in self._deleted_accounts: + # TODO: this might raise a KeyError + old_value = old_trie[deleted_address] + block_diff.set_account_changed(deleted_address, old_value, b'') + + for address in self._dirty_account_rlps: + old_value = old_trie[address] + # TODO: This should be _get_encoded_account_from_turbodb, but + # make_storage_root() isn't correctly handled yet. + new_value = self._get_encoded_account_from_turbodb(address, from_journal=True) + block_diff.set_account_changed(address, old_value, new_value) + + # 2. Grab all the changed storage items and their previous values. + dirty_stores = tuple(self._dirty_account_stores()) + for address, store in dirty_stores: + diff = store.diff() + + for key in diff.deleted_keys(): + slot = big_endian_to_int(key) + current_slot_value = store.get(slot) + current_slot_value_bytes = int_to_big_endian(current_slot_value) + # TODO: Is b'' a valid value for a storage slot? 0 might be better + # TODO: this line is untested + block_diff.set_storage_changed(address, slot, current_slot_value_bytes, b'') + + encoded_account = old_trie[address] + if encoded_account: + old_account = rlp.decode(encoded_account, sedes=Account) + else: + old_account = Account() + fresh_store = StorageLookup( + self._raw_store_db, + old_account.storage_root, + address + ) + + for key, new_value in diff.pending_items(): + slot = big_endian_to_int(key) + + # make a new StorageLookup because, pre-Byzantium, make_state_root is + # called at the end of every transaction, and making the state root blows + # away all changes. If we were to ask the store for the old value it would + # tell us the state as of the beginning of the last txn, not the state as + # of the beginnig of the block. + + old_value_bytes = fresh_store.get(key) + + block_diff.set_storage_changed(address, slot, old_value_bytes, new_value) + + # 3. Persist! + self.persist() + + # 4. Grab the new storage roots + for address, _store in dirty_stores: + # TODO: This reads from old_trie, but it can't do that in a future where only + # the TurboDB is available. Check that the turbodb gives the same + # result, by calling _get_encoded_account_from_turbodb, or something + # like it, + # old_value_by_turbo = self._get_encoded_account_from_turbodb(address) + old_account_value = old_trie[address] + # assert old_value_by_turbo == old_account_value + + # TODO: Call self._get_encoded_account, so we read from both the trie and from + # the turbodb. This will cause problems, because the above call to + # self.persist() calls self.make_state_root() which blows away the + # journal. The _journaltrie path doesn't mind because the intermediate + # state is persisted, but the _turbojournal path will return the wrong + # result, because the intermediate state is lost when the journal is + # blown. + new_account_value = self._get_encoded_account_from_trie(address, from_journal=True) + block_diff.set_account_changed(address, old_account_value, new_account_value) + + # 5. return the block diff + return block_diff + + def _changed_accounts(self) -> DBDiff: + """ + Returns all the accounts which will be written to the db when persist() is called. + + Careful! If some storage items have changed then the storage roots for some accounts + should also change but those accounts will not show up here unless something else about + them also changed. + """ + return self._journaltrie.diff() + + def _changed_storage_items(self) -> Dict[Address, DBDiff]: + """ + Returns all the storage items which will be written to the db when persist() is called. + """ + return { + address: store.diff() + for address, store in self._dirty_account_stores() + } + def _validate_generated_root(self) -> None: db_diff = self._journaldb.diff() if len(db_diff): diff --git a/eth/db/block_diff.py b/eth/db/block_diff.py new file mode 100644 index 0000000000..3248037bde --- /dev/null +++ b/eth/db/block_diff.py @@ -0,0 +1,177 @@ +from collections import defaultdict +from typing import ( + Dict, + Iterable, + Optional, + Set, + Tuple, +) + +from eth_hash.auto import keccak +from eth_typing import ( + Address, + Hash32, +) +from eth_utils import ( + big_endian_to_int, + to_tuple +) +import rlp + +from eth.db.backends.base import BaseDB +from eth.db.atomic import AtomicDB +from eth.db.schema import SchemaTurbo, Schemas, get_schema +from eth.rlp.accounts import Account + + +""" +TODO: Decide on the best interface for returning changes: +- diff.get_slot_change() -> [old, new] +- diff.get_slot_change(new=FAlse) -> old +- diff.get_slot_change(kind=BlockDiff.OLD) -> old +- diff.get_old_slot_value() & diff.get_new_slot_value() +""" + + +class BlockDiff: + + def __init__(self) -> None: + self.old_account_values: Dict[Address, Optional[bytes]] = dict() + self.new_account_values: Dict[Address, Optional[bytes]] = dict() + + SLOT_TO_VALUE = Dict[int, bytes] + self.old_storage_items: Dict[Address, SLOT_TO_VALUE] = defaultdict(dict) + self.new_storage_items: Dict[Address, SLOT_TO_VALUE] = defaultdict(dict) + + def set_account_changed(self, address: Address, old_value: bytes, new_value: bytes) -> None: + self.old_account_values[address] = old_value + self.new_account_values[address] = new_value + + def set_storage_changed(self, address: Address, slot: int, + old_value: bytes, new_value: bytes) -> None: + self.old_storage_items[address][slot] = old_value + self.new_storage_items[address][slot] = new_value + + def get_changed_accounts(self) -> Set[Address]: + return set(self.old_account_values.keys()) | set(self.old_storage_items.keys()) + + @to_tuple + def get_changed_storage_items(self) -> Iterable[Tuple[Address, int, bytes, bytes]]: + for address in self.old_storage_items.keys(): + new_items = self.new_storage_items[address] + old_items = self.old_storage_items[address] + for slot in old_items.keys(): + yield address, slot, old_items[slot], new_items[slot] + + def get_changed_slots(self, address: Address) -> Set[int]: + """ + Returns which slots changed for the given account. + """ + if address not in self.old_storage_items.keys(): + return set() + + return set(self.old_storage_items[address].keys()) + + def get_slot_change(self, address: Address, slot: int) -> Tuple[int, int]: + if address not in self.old_storage_items: + raise Exception(f'account {address} did not change') + old_values = self.old_storage_items[address] + + if slot not in old_values: + raise Exception(f"{address}'s slot {slot} did not change") + + new_values = self.new_storage_items[address] + return big_endian_to_int(old_values[slot]), big_endian_to_int(new_values[slot]) + + def get_account(self, address: Address, new: bool = True) -> bytes: + dictionary = self.new_account_values if new else self.old_account_values + return dictionary[address] + + def get_decoded_account(self, address: Address, new: bool = True) -> Optional[Account]: + encoded = self.get_account(address, new) + if encoded == b'': + return None # this means the account used to or currently does not exist + return rlp.decode(encoded, sedes=Account) + + @classmethod + def from_db(cls, db: BaseDB, state_root: Hash32) -> 'BlockDiff': + """ + KeyError is thrown if a diff was not saved for the provided {state_root} + """ + + encoded_diff = db[SchemaTurbo.make_block_diff_lookup_key(state_root)] + diff = rlp.decode(encoded_diff) + + accounts, storage_items = diff + + block_diff = cls() + + for key, old, new in accounts: + block_diff.set_account_changed(key, old, new) + + for key, slot, old, new in storage_items: + decoded_slot = big_endian_to_int(slot) # rlp.encode turns our ints into bytes + block_diff.set_storage_changed(key, decoded_slot, old, new) + + return block_diff + + def write_to(self, db: BaseDB, state_root: Hash32) -> None: + # TODO: this should probably verify that the state roots have all been added + + accounts = [ + [address, self.old_account_values[address], self.new_account_values[address]] + for address in self.old_account_values.keys() + ] + + storage_items = self.get_changed_storage_items() + + diff = [ + accounts, + storage_items + ] + + encoded_diff = rlp.encode(diff) + db[SchemaTurbo.make_block_diff_lookup_key(state_root)] = encoded_diff + + @classmethod + def apply_to(cls, db: BaseDB, parent_state_root: Hash32, + state_root: Hash32, forward: bool = True) -> None: + """ + Looks up the BlockDif for the given hash and applies it to the databae + """ + + if get_schema(db) != Schemas.TURBO: + return + + # 1. Verify the database is in the correct state + if forward: + assert db[SchemaTurbo.current_state_root_key] == parent_state_root + else: + assert db[SchemaTurbo.current_state_root_key] == state_root + + # 2. Lookup the diff (throws KeyError if it does not exist) + diff = cls.from_db(db, state_root) + + # Sadly, AtomicDB.atomic_batch() databases are not themselves atomic, so the rest + # of this method cannot be wrapped in an atomic_batch context manager. + + # TODO: also keep track of storage items! + for address in diff.get_changed_accounts(): + old_value = diff.get_account(address, new=False) + new_value = diff.get_account(address, new=True) + + key = SchemaTurbo.make_account_state_lookup_key(keccak(address)) + + if forward: + if old_value != b'': + assert db[key] == old_value + db[key] = new_value + else: + if new_value != b'': + assert db[key] == new_value + db[key] = old_value + + if forward: + db[SchemaTurbo.current_state_root_key] = state_root + else: + db[SchemaTurbo.current_state_root_key] = parent_state_root diff --git a/eth/db/chain.py b/eth/db/chain.py index fa480d18dd..39540add70 100644 --- a/eth/db/chain.py +++ b/eth/db/chain.py @@ -24,19 +24,23 @@ from eth_hash.auto import keccak from eth.constants import ( + BLANK_ROOT_HASH, EMPTY_UNCLE_HASH, ) from eth.exceptions import ( + CanonicalHeadNotFound, HeaderNotFound, ReceiptNotFound, TransactionNotFound, ) +from eth.db.block_diff import BlockDiff from eth.db.header import BaseHeaderDB, HeaderDB from eth.db.backends.base import ( BaseAtomicDB, BaseDB, ) -from eth.db.schema import SchemaV1 +from eth.db import schema +from eth.db.trie import iterate_leaves from eth.rlp.headers import ( BlockHeader, ) @@ -75,10 +79,6 @@ class TransactionKey(rlp.Serializable): class BaseChainDB(BaseHeaderDB): db: BaseAtomicDB = None - @abstractmethod - def __init__(self, db: BaseAtomicDB) -> None: - raise NotImplementedError("ChainDB classes must implement this method") - # # Header API # @@ -166,9 +166,6 @@ def persist_trie_data_dict(self, trie_data_dict: Dict[Hash32, bytes]) -> None: class ChainDB(HeaderDB, BaseChainDB): - def __init__(self, db: BaseAtomicDB) -> None: - self.db = db - # # Header API # @@ -218,7 +215,7 @@ def _set_as_canonical_chain_head(cls, for h in new_canonical_headers: cls._add_block_number_to_hash_lookup(db, h) - db.set(SchemaV1.make_canonical_head_hash_lookup_key(), header.hash) + db.set(schema.SchemaV1.make_canonical_head_hash_lookup_key(), header.hash) return new_canonical_headers, tuple(old_canonical_headers) @@ -268,6 +265,9 @@ def _persist_block( old_canonical_hashes = tuple( header.hash for header in old_canonical_headers) + # TODO: Only do this if we're in Turbo mode + cls._update_turbo_db(db, new_canonical_headers, old_canonical_headers) + return new_canonical_hashes, old_canonical_hashes def persist_uncles(self, uncles: Tuple[BlockHeader]) -> Hash32: @@ -286,6 +286,22 @@ def _persist_uncles(db: BaseDB, uncles: Tuple[BlockHeader]) -> Hash32: rlp.encode(uncles, sedes=rlp.sedes.CountableList(BlockHeader))) return uncles_hash + @classmethod + def _update_turbo_db(cls, + db: 'BaseDB', + new_canonical_headers: Tuple[Tuple[BlockHeader]], + old_canonical_headers: Tuple[Tuple[BlockHeader]]) -> None: + # A - B - C - D (B, C, D is new_canonical_headers) + # \- E - F - G (E, F, G is old_canonical_headers) + + for old_header in reversed(old_canonical_headers): + parent = cls._get_block_header_by_hash(db, old_header.parent_hash) + BlockDiff.apply_to(db, parent.state_root, old_header.state_root, forward=False) + + for new_header in new_canonical_headers: + parent = cls._get_block_header_by_hash(db, new_header.parent_hash) + BlockDiff.apply_to(db, parent.state_root, new_header.state_root) + # # Transaction API # @@ -392,7 +408,7 @@ def get_transaction_index(self, transaction_hash: Hash32) -> Tuple[BlockNumber, Raises TransactionNotFound if the transaction_hash is not found in the canonical chain. """ - key = SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash) + key = schema.SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash) try: encoded_key = self.db[key] except KeyError: @@ -454,7 +470,7 @@ def _remove_transaction_from_canonical_chain(db: BaseDB, transaction_hash: Hash3 Removes the transaction specified by the given hash from the canonical chain. """ - db.delete(SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash)) + db.delete(schema.SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash)) @staticmethod def _add_transaction_to_canonical_chain(db: BaseDB, @@ -470,7 +486,7 @@ def _add_transaction_to_canonical_chain(db: BaseDB, """ transaction_key = TransactionKey(block_header.block_number, index) db.set( - SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash), + schema.SchemaV1.make_transaction_hash_to_block_lookup_key(transaction_hash), rlp.encode(transaction_key), ) @@ -496,3 +512,35 @@ def persist_trie_data_dict(self, trie_data_dict: Dict[Hash32, bytes]) -> None: with self.db.atomic_batch() as db: for key, value in trie_data_dict.items(): db[key] = value + + # + # Migration API + # + @classmethod + def upgrade_to_turbo_schema(cls, db: BaseDB) -> None: + # Takes a database and upgrades it to the new schema + + # TODO: For a large database this might take a while. Make it interruptable? + # If you keep track of the last account to be imported you can quickly + # resume. + + schema.ensure_schema(db, schema.Schemas.DEFAULT) + + # 1. Find the canonical head + try: + canonical_header: BlockHeader = cls._get_canonical_head(db) + except CanonicalHeadNotFound: + db[schema.SchemaTurbo.current_state_root_key] = BLANK_ROOT_HASH + schema.set_schema(db, schema.Schemas.TURBO) + return + + # 2. Iterate over the account trie as of that head; insert all accounts into the db + canonical_state_root = canonical_header.state_root + for full_path, account_rlp in iterate_leaves(db, canonical_header.state_root): + db[schema.SchemaTurbo.make_account_state_lookup_key(full_path)] = account_rlp + + # 3. Set the correct current_state_root_key + db[schema.SchemaTurbo.current_state_root_key] = canonical_header.state_root + + # 4. We're done! + schema.set_schema(db, schema.Schemas.TURBO) diff --git a/eth/db/header.py b/eth/db/header.py index 8e35f5f648..ecfe5206ab 100644 --- a/eth/db/header.py +++ b/eth/db/header.py @@ -33,7 +33,7 @@ BaseAtomicDB, BaseDB, ) -from eth.db.schema import SchemaV1 +from eth.db.schema import SchemaV1, Schemas, ensure_schema from eth.rlp.headers import BlockHeader from eth.validation import ( validate_block_number, @@ -44,8 +44,13 @@ class BaseHeaderDB(ABC): db: BaseAtomicDB = None - def __init__(self, db: BaseAtomicDB) -> None: + def __init__(self, db: BaseAtomicDB, + # expected_schema: Schemas = Schemas.DEFAULT) -> None: + expected_schema: Schemas = Schemas.TURBO) -> None: self.db = db + self.schema = expected_schema + + # ensure_schema(db, expected_schema) # # Canonical Chain API diff --git a/eth/db/schema.py b/eth/db/schema.py index 05e868b97c..ecf3783d72 100644 --- a/eth/db/schema.py +++ b/eth/db/schema.py @@ -1,4 +1,12 @@ from abc import ABC, abstractmethod +from enum import Enum + +from eth.exceptions import ( + SchemaDoesNotMatchError, + SchemaNotRecognizedError, +) + +from eth.db.backends.base import BaseDB from eth_typing import ( BlockNumber, @@ -6,6 +14,11 @@ ) +class Schemas(Enum): + DEFAULT = b'default' + TURBO = b'turbo' + + class BaseSchema(ABC): @staticmethod @abstractmethod @@ -45,3 +58,41 @@ def make_block_hash_to_score_lookup_key(block_hash: Hash32) -> bytes: @staticmethod def make_transaction_hash_to_block_lookup_key(transaction_hash: Hash32) -> bytes: return b'transaction-hash-to-block:%s' % transaction_hash + + +class SchemaTurbo(SchemaV1): + current_schema_lookup_key: bytes = b'current-schema' + _block_diff_prefix = b'block-diff' + + current_state_root_key: bytes = b'current-turbo-state' + + @classmethod + def make_block_diff_lookup_key(cls, state_root: Hash32) -> bytes: + # TODO: look at all callers, they should be using a state root! + return cls._block_diff_prefix + b':' + state_root + + @classmethod + def make_account_state_lookup_key(cls, address_hash: Hash32) -> bytes: + return cls.current_state_root_key + b':' + address_hash + + +def get_schema(db: BaseDB) -> Schemas: + try: + current_schema = db[SchemaTurbo.current_schema_lookup_key] + except KeyError: + return Schemas.DEFAULT + + try: + return Schemas(current_schema) + except ValueError: + raise SchemaNotRecognizedError(current_schema) + + +def set_schema(db: BaseDB, schema: Schemas) -> None: + db.set(SchemaTurbo.current_schema_lookup_key, schema.value) + + +def ensure_schema(db: BaseDB, expected_schema: Schemas) -> None: + reported_schema = get_schema(db) + if reported_schema != expected_schema: + raise SchemaDoesNotMatchError(reported_schema) diff --git a/eth/db/storage.py b/eth/db/storage.py index 3b5d79b065..d881212f25 100644 --- a/eth/db/storage.py +++ b/eth/db/storage.py @@ -31,6 +31,9 @@ from eth.db.cache import ( CacheDB, ) +from eth.db.diff import ( + DBDiff +) from eth.db.journal import ( JournalDB, ) @@ -170,7 +173,7 @@ def __init__(self, db: BaseAtomicDB, storage_root: Hash32, address: Address) -> Keys are stored as node hashes and rlp-encoded node values. _storage_lookup is itself a pair of databases: (BatchDB -> HexaryTrie), - writes to storage lookup *are* immeditaely applied to a trie, generating + writes to storage lookup *are* immediately applied to a trie, generating the appropriate trie nodes and and root hash (via the HexaryTrie). The writes are *not* persisted to db, until _storage_lookup is explicitly instructed to, via :meth:`StorageLookup.commit_to` @@ -191,6 +194,7 @@ def __init__(self, db: BaseAtomicDB, storage_root: Hash32, address: Address) -> self._storage_lookup = StorageLookup(db, storage_root, address) self._storage_cache = CacheDB(self._storage_lookup) self._journal_storage = JournalDB(self._storage_cache) + self._changes_since_last_persist = JournalDB(dict()) def get(self, slot: int, from_journal: bool=True) -> int: key = int_to_big_endian(slot) @@ -211,8 +215,15 @@ def set(self, slot: int, value: int) -> None: key = int_to_big_endian(slot) if value: self._journal_storage[key] = rlp.encode(value) + self._changes_since_last_persist[key] = rlp.encode(value) else: del self._journal_storage[key] + try: + del self._changes_since_last_persist[key] + except KeyError: + self._changes_since_last_persist[key] = b'' + del self._changes_since_last_persist[key] + def delete(self) -> None: self.logger.debug2( @@ -221,28 +232,34 @@ def delete(self) -> None: keccak(self._address).hex(), ) self._journal_storage.clear() + self._changes_since_last_persist.clear() self._storage_cache.reset_cache() def record(self, checkpoint: JournalDBCheckpoint) -> None: self._journal_storage.record(checkpoint) + self._changes_since_last_persist.record(checkpoint) def discard(self, checkpoint: JournalDBCheckpoint) -> None: self.logger.debug2('discard checkpoint %r', checkpoint) if self._journal_storage.has_checkpoint(checkpoint): self._journal_storage.discard(checkpoint) + self._changes_since_last_persist.discard(checkpoint) else: # if the checkpoint comes before this account started tracking, # then simply reset to the beginning self._journal_storage.reset() + self._changes_since_last_persist.reset() self._storage_cache.reset_cache() def commit(self, checkpoint: JournalDBCheckpoint) -> None: if self._journal_storage.has_checkpoint(checkpoint): self._journal_storage.commit(checkpoint) + self._changes_since_last_persist.commit(checkpoint) else: # if the checkpoint comes before this account started tracking, # then flatten all changes, without persisting self._journal_storage.flatten() + self._changes_since_last_persist.flatten() def make_storage_root(self) -> None: """ @@ -271,3 +288,14 @@ def persist(self, db: BaseDB) -> None: self._validate_flushed() if self._storage_lookup.has_changed_root: self._storage_lookup.commit_to(db) + + self._changes_since_last_persist.clear() + + def diff(self) -> DBDiff: + """ + Returns all the changes that would be saved if persist() were called. + + Note: Calling make_storage_root() wipes away changes, after it is called this method will + return an empty diff. + """ + return self._changes_since_last_persist.diff() diff --git a/eth/db/trie.py b/eth/db/trie.py index f42468a0a2..0a91730d51 100644 --- a/eth/db/trie.py +++ b/eth/db/trie.py @@ -1,16 +1,34 @@ +import enum import functools -from typing import Dict, Tuple, Union +from typing import cast, Dict, Tuple, Union, NamedTuple, List, Iterable import rlp from trie import ( HexaryTrie, ) +from trie.constants import ( + BLANK_NODE_HASH, + NODE_TYPE_BLANK, + NODE_TYPE_BRANCH, + NODE_TYPE_EXTENSION, + NODE_TYPE_LEAF, +) +from trie.utils.nodes import ( + decode_node, + extract_key, + get_common_prefix_length, + get_node_type, +) +from trie.utils.nibbles import nibbles_to_bytes + from eth_typing import Hash32 +from eth_utils import to_tuple from eth.constants import ( BLANK_ROOT_HASH, ) +from eth.db.backends.base import BaseDB from eth.rlp.receipts import Receipt from eth.rlp.transactions import BaseTransaction @@ -35,3 +53,149 @@ def _make_trie_root_and_nodes(items: Tuple[bytes, ...]) -> TrieRootAndData: index_key = rlp.encode(index, sedes=rlp.sedes.big_endian_int) memory_trie[index_key] = item return trie.root_hash, kv_store + + +### Copied over from lithp#trinity:lithp/firehose-protocol +### The best long-term home for this might be ethereum#py-trie? + + +Nibbles = Tuple[int, ...] + + +class NodeKind(enum.Enum): + BLANK = NODE_TYPE_BLANK + LEAF = NODE_TYPE_LEAF + EXTENSION = NODE_TYPE_EXTENSION + BRANCH = NODE_TYPE_BRANCH + + +class TrieNode(NamedTuple): + kind: NodeKind + rlp: bytes + obj: List[bytes] # this type is wrong but mypy doesn't support recursive types + keccak: Hash32 + + def __str__(self) -> str: + if self.kind == NodeKind.EXTENSION: + return ( + "TrieNode(Extension," + f" hash={self.keccak.hex()}" + f" path={self.path_rest}" + f" child={self.obj[1].hex()}" + ")" + ) + if self.kind == NodeKind.LEAF: + return ( + "TrieNode(Leaf," + f" hash={self.keccak.hex()}" + f" path={self.path_rest[:10]}..." + ")" + ) + return f"TrieNode(kind={self.kind.name} hash={self.keccak.hex()})" + + @property + def path_rest(self) -> Nibbles: + # careful: this doesn't make any sense for branches + return cast(Nibbles, extract_key(self.obj)) + + +def is_subtree(prefix: Nibbles, nibbles: Nibbles) -> bool: + """ + Returns True if {nibbles} represents a subtree of {prefix}. + """ + if len(nibbles) < len(prefix): + # nibbles represents a bigger tree than prefix does + return False + return get_common_prefix_length(prefix, nibbles) == len(prefix) + + +@to_tuple +def _get_children_with_nibbles(node: TrieNode, prefix: Nibbles) -> Iterable[Tuple[Nibbles, Hash32]]: + """ + Return the children of the given node at the given path, including their full paths + """ + if node.kind == NodeKind.BLANK: + return + elif node.kind == NodeKind.LEAF: + full_path = prefix + node.path_rest + yield (full_path, cast(Hash32, node.obj[1])) + elif node.kind == NodeKind.EXTENSION: + full_path = prefix + node.path_rest + # TODO: this cast to a Hash32 is not right, nodes smaller than 32 are inlined + yield (full_path, cast(Hash32, node.obj[1])) + elif node.kind == NodeKind.BRANCH: + for i in range(17): + full_path = prefix + (i,) + yield (full_path, cast(Hash32, node.obj[i])) + + +def _get_node(db: BaseDB, node_hash: Hash32) -> TrieNode: + if len(node_hash) < 32: + node_rlp = node_hash + else: + node_rlp = db.get(node_hash) + + node = decode_node(node_rlp) + node_type = get_node_type(node) + + return TrieNode(kind=NodeKind(node_type), rlp=node_rlp, obj=node, keccak=node_hash) + + +def _iterate_trie(db: BaseDB, + node: TrieNode, # the node we should look at + sub_trie: Nibbles, # which sub_trie to return nodes from + prefix: Nibbles, # our current path in the trie + ) -> Iterable[Tuple[Nibbles, TrieNode]]: + + if node.kind == NodeKind.BLANK: + return + + if node.kind == NodeKind.LEAF: + full_path = prefix + node.path_rest + + if is_subtree(sub_trie, prefix) or is_subtree(sub_trie, full_path): + # also check full_path because either the node or the item the node points to + # might be part of the desired subtree + yield (prefix, node) + + # there's no need to recur, this is a leaf + return + + child_of_sub_trie = is_subtree(sub_trie, prefix) + + if child_of_sub_trie: + # this node is part of the subtrie which should be returned + yield (prefix, node) + + parent_of_sub_trie = is_subtree(prefix, sub_trie) + + if child_of_sub_trie or parent_of_sub_trie: + for path, child_hash in _get_children_with_nibbles(node, prefix): + child_node = _get_node(db, child_hash) + yield from _iterate_trie(db, child_node, sub_trie, path) + + +def iterate_trie(db: BaseDB, root_hash: Hash32, + sub_trie: Nibbles = ()) -> Iterable[Tuple[Nibbles, TrieNode]]: + + if root_hash == BLANK_NODE_HASH: + return + + root_node = _get_node(db, root_hash) + + yield from _iterate_trie( + db, root_node, sub_trie, + prefix=(), + ) + + +def iterate_leaves(db: BaseDB, root_hash: Hash32, + sub_trie: Nibbles = ()) -> Iterable[Tuple[Nibbles, bytes]]: + "This returns all of the leaves in the trie, along with their full paths" + + node_iterator = iterate_trie(db, root_hash, sub_trie) + + for path, node in node_iterator: + if node.kind == NodeKind.LEAF: + full_path = nibbles_to_bytes(path + node.path_rest) + yield (full_path, node.obj[1]) diff --git a/eth/db/turbo.py b/eth/db/turbo.py new file mode 100644 index 0000000000..1ca136fc2e --- /dev/null +++ b/eth/db/turbo.py @@ -0,0 +1,203 @@ +from typing import Iterable, Tuple + +import rlp + +from eth_hash.auto import keccak +from eth_typing import Address, Hash32 +from eth_utils import to_tuple + +from eth.db.backends.base import BaseDB +from eth.db.block_diff import BlockDiff +from eth.db.header import HeaderDB +from eth.db.schema import Schemas, SchemaTurbo, ensure_schema + +from eth.rlp.accounts import Account +from eth.rlp.headers import BlockHeader + + +def find_header_path(db: HeaderDB, source: BlockHeader, dest: BlockHeader) \ + -> Tuple[Tuple[BlockHeader], Tuple[BlockHeader]]: + """ + Returns the headers which must be unapplied in order to reach dest, followed by + the headers which must be applied. + """ + + if source == dest: + return ((), ()) + + ancestor = find_greatest_common_ancestor(db, source, dest) + + if ancestor == source: + forward_headers = tuple(reversed(build_header_chain(db, dest, source))) + reverse_headers = () + return reverse_headers, forward_headers + + if ancestor == dest: + reverse_headers = build_header_chain(db, source, dest) + forward_headers = () + return reverse_headers, forward_headers + + reverse_headers = build_header_chain(db, source, ancestor) + forward_headers = tuple(reversed(build_header_chain(db, dest, ancestor))) + return reverse_headers, forward_headers + + +def find_greatest_common_ancestor(db: HeaderDB, source: BlockHeader, + dest: BlockHeader) -> BlockHeader: + "If you view the header chain as a meet-semilattice: this returns the meet" + if source.block_number > dest.block_number: + source, dest = dest, source + + assert source.block_number <= dest.block_number + + while dest.block_number > source.block_number: + parent = db.get_block_header_by_hash(dest.parent_hash) + dest = parent + + assert source.block_number == dest.block_number + + while dest.block_number >= 0: + if dest.hash == source.hash: + return dest + + dest_parent = db.get_block_header_by_hash(dest.parent_hash) + dest = dest_parent + + source_parent = db.get_block_header_by_hash(source.parent_hash) + source = source_parent + + assert False, "These headers do not share a genesis?" + + +@to_tuple +def build_header_chain(db: HeaderDB, tail: BlockHeader, + head: BlockHeader) -> Iterable[BlockHeader]: + """ + Returns a chain of headers beginning with {tail} and ending with {head}. + + Will loop forever if {head} is not an ancestor of {tail}. + """ + current_header = tail + + while True: + yield current_header + + parent = db.get_block_header_by_hash(current_header.parent_hash) + if parent == head: + return + current_header = parent + + +class TurboDatabase: + """ + A helper for accessing data from the TurboDB. + """ + + def __init__(self, db: HeaderDB, header: BlockHeader = None) -> None: + """ + {header} specifies which state to read from. If {header} is not provided then the + most recent state is used. + """ + self.db = db + base_db = db.db + + ensure_schema(base_db, Schemas.TURBO) + + self.reverse_diffs = () + self.forward_diffs = () + + + if header is None: + return + + if base_db[SchemaTurbo.current_state_root_key] == header.state_root: + return + + # we've been asked to return some state which is not the current state + + # first, double-check that the turbodb hasn't gotten out of sync: + current_header = db.get_canonical_head() + assert base_db[SchemaTurbo.current_state_root_key] == current_header.state_root + + # next, we need to lookup the series of block diffs to get to {header} + reverse_headers, forward_headers = find_header_path(db, current_header, header) + + # TODO: throw a better exception when the block diff is not found + self.reverse_diffs = tuple( + BlockDiff.from_db(base_db, header.state_root) + for header in reverse_headers + ) + self.forward_diffs = tuple( + BlockDiff.from_db(base_db, header.state_root) + for header in forward_headers + ) + + def get_encoded_account(self, address: Address) -> bytes: + for diff in reversed(self.forward_diffs): + if address in diff.get_changed_accounts(): + return diff.get_account(address, new=True) + + for diff in reversed(self.reverse_diffs): + if address in diff.get_changed_accounts(): + return diff.get_account(address, new=False) + + # The account was apparently never changed, return the current value + return self._get_encoded_account(self.db.db, address) + + def get_account(self, address: Address) -> Account: + # TODO: merge this with _get_account + try: + account_rlp = self.get_encoded_account(address) + + if account_rlp == b'': + return Account() + + return rlp.decode(account_rlp, sedes=Account) + except KeyError: + return Account() + + @staticmethod + def _get_encoded_account(db: BaseDB, address: Address) -> bytes: + ensure_schema(db, Schemas.TURBO) + + key = SchemaTurbo.make_account_state_lookup_key(keccak(address)) + return db[key] + + @classmethod + def _get_account(cls, db: BaseDB, address: Address) -> Account: + try: + account_rlp = cls._get_encoded_account(db, address) + + if account_rlp == b'': + return Account() + + return rlp.decode(account_rlp, sedes=Account) + except KeyError: + return Account() + + +class TurboBaseDB(BaseDB): + """ + A helper so TurboDatabase can be used from inside JournalDB + + Needs: __contains__, __delitem__, __setitem__, __getitem__ + """ + + def __init__(self, turbodb: TurboDatabase) -> None: + self._turbodb = turbodb + + def _exists(self, key: bytes) -> bool: + try: + self.__getitem__(key) + return True + except KeyError: + return False + + def __getitem__(self, key: bytes) -> bytes: + return self._turbodb.get_encoded_account(key) + + def __setitem__(self, key: bytes, value: bytes) -> None: + raise NotImplemented("TurboBaseDB objects cannot be mutated") + + def __delitem__(self, key: bytes) -> None: + raise NotImplemented("TurboBaseDB objects cannot be mutated") diff --git a/eth/exceptions.py b/eth/exceptions.py index 87c0775618..6287ec687c 100644 --- a/eth/exceptions.py +++ b/eth/exceptions.py @@ -8,6 +8,18 @@ class PyEVMError(Exception): pass +class SchemaNotRecognizedError(PyEVMError): + """ + The database uses a schema which this version of py-evm does not support. + """ + + +class SchemaDoesNotMatchError(PyEVMError): + """ + The database schema does not match the expected schema. It might need to be migrated. + """ + + class VMNotFound(PyEVMError): """ Raised when no VM is available for the provided block number. diff --git a/eth/tools/builder/chain/__init__.py b/eth/tools/builder/chain/__init__.py index ab42c8486c..b37fc44d9a 100644 --- a/eth/tools/builder/chain/__init__.py +++ b/eth/tools/builder/chain/__init__.py @@ -15,6 +15,7 @@ mine_block, mine_blocks, name, + upgrade_to_turbo, ) from .builders import ( # noqa: F401 byzantium_at, @@ -96,5 +97,7 @@ class API: chain_split = staticmethod(chain_split) at_block_number = staticmethod(at_block_number) + upgrade_to_turbo = staticmethod(upgrade_to_turbo) + api = API() diff --git a/eth/tools/builder/chain/builders.py b/eth/tools/builder/chain/builders.py index 7accd7958d..2706cc23c8 100644 --- a/eth/tools/builder/chain/builders.py +++ b/eth/tools/builder/chain/builders.py @@ -41,6 +41,7 @@ from eth.db.backends.memory import ( MemoryDB, ) +from eth.db.chain import ChainDB from eth.rlp.blocks import ( BaseBlock, ) @@ -513,3 +514,9 @@ def at_block_number(block_number: BlockNumber, chain: MiningChain) -> MiningChai db = chain.chaindb.db chain_at_block = type(chain)(db, chain.create_header_from_parent(at_block.header)) return chain_at_block + + +def upgrade_to_turbo(chain: BaseChain) -> BaseChain: + db = chain.chaindb.db + ChainDB.upgrade_to_turbo_schema(db) + return chain diff --git a/eth/vm/base.py b/eth/vm/base.py index a932e0d0ee..0980d1e515 100644 --- a/eth/vm/base.py +++ b/eth/vm/base.py @@ -23,6 +23,7 @@ Hash32, ) from eth_utils import ( + decode_hex, ValidationError, ) import rlp @@ -40,6 +41,7 @@ ) from eth.db.trie import make_trie_root_and_nodes from eth.db.chain import BaseChainDB +from eth.db.schema import Schemas from eth.exceptions import ( HeaderNotFound, ) @@ -429,7 +431,8 @@ def state(self) -> BaseState: def build_state(cls, db: BaseAtomicDB, header: BlockHeader, - previous_hashes: Iterable[Hash32] = () + previous_hashes: Iterable[Hash32] = (), + expected_schema: Schemas = Schemas.TURBO, ) -> BaseState: """ You probably want `VM().state` instead of this. @@ -439,7 +442,9 @@ def build_state(cls, """ execution_context = header.create_execution_context(previous_hashes) - return cls.get_state_class()(db, execution_context, header.state_root) + return cls.get_state_class()( + db, execution_context, header, expected_schema + ) # # Logging @@ -587,10 +592,39 @@ def import_block(self, block: BaseBlock) -> BaseBlock: ) # we need to re-initialize the `state` to update the execution context. - self._state = self.build_state(self.chaindb.db, self.get_header(), self.previous_hashes) + header = self.get_header() + parent_header_hash = header.parent_hash + parent_header = self.chaindb.get_block_header_by_hash(parent_header_hash) + execution_context = header.create_execution_context(self.previous_hashes) + self._state = self.get_state_class()( + self.chaindb.db, execution_context, parent_header, Schemas.TURBO, + ) + + # In geth the state is modified for the DAO fork block before any transactions are + # applied. Doing it here is the closest we can get to that. + block_number = header.block_number + supports_dao_fork = hasattr(self, 'support_dao_fork') and self.support_dao_fork + if supports_dao_fork and block_number == self.get_dao_fork_block_number(): + + for hex_account in dao_drain_list: + address = Address(decode_hex(hex_account)) + balance = self._state.get_balance(address) + self._state.delta_balance(dao_refund_contract, balance) + self._state.set_balance(address, 0) + + # Persist the changes to the database + # self._state.persist() + + base_header = header.copy( + state_root=self._state.state_root + ) + else: + base_header = header # run all of the transactions. - new_header, receipts, _ = self.apply_all_transactions(block.transactions, self.get_header()) + new_header, receipts, _ = self.apply_all_transactions( + block.transactions, base_header + ) self._block = self.set_block_transactions( self.get_block(), @@ -675,9 +709,19 @@ def finalize_block(self, block: BaseBlock) -> BaseBlock: # We need to call `persist` here since the state db batches # all writes until we tell it to write to the underlying db - self.state.persist() + # self.state.persist() + + # TODO: only do this if we're in turbo mode + # TODO: will we always know the hash here? + parent_hash = block.header.parent_hash + parent_header = self.chaindb.get_block_header_by_hash(parent_hash) + block_diff = self.state.persist_returning_block_diff(parent_header.state_root) - return block.copy(header=block.header.copy(state_root=self.state.state_root)) + result = block.copy(header=block.header.copy(state_root=self.state.state_root)) + + basedb = self.chaindb.db + block_diff.write_to(basedb, result.header.state_root) + return result def pack_block(self, block: BaseBlock, *args: Any, **kwargs: Any) -> BaseBlock: """ @@ -983,12 +1027,137 @@ def get_state_class(cls) -> Type[BaseState]: @contextlib.contextmanager def state_in_temp_block(self) -> Iterator[BaseState]: - header = self.get_header() - temp_block = self.generate_block_from_parent_header_and_coinbase(header, header.coinbase) - prev_hashes = itertools.chain((header.hash,), self.previous_hashes) - - state = self.build_state(self.chaindb.db, temp_block.header, prev_hashes) - - snapshot = state.snapshot() - yield state - state.revert(snapshot) + snapshot = self.state.snapshot() + yield self.state + self.state.revert(snapshot) + + # header = self.get_header() + # temp_block = self.generate_block_from_parent_header_and_coinbase(header, header.coinbase) + # prev_hashes = itertools.chain((header.hash,), self.previous_hashes) + + # state = self.build_state(self.chaindb.db, temp_block.header, prev_hashes) + + # snapshot = state.snapshot() + # yield state + # state.revert(snapshot) + + +dao_refund_contract = Address(decode_hex('0xbf4ed7b27f1d666546e30d74d50d173d20bca754')) +dao_drain_list = [ + "0xd4fe7bc31cedb7bfb8a345f31e668033056b2728", + "0xb3fb0e5aba0e20e5c49d252dfd30e102b171a425", + "0x2c19c7f9ae8b751e37aeb2d93a699722395ae18f", + "0xecd135fa4f61a655311e86238c92adcd779555d2", + "0x1975bd06d486162d5dc297798dfc41edd5d160a7", + "0xa3acf3a1e16b1d7c315e23510fdd7847b48234f6", + "0x319f70bab6845585f412ec7724b744fec6095c85", + "0x06706dd3f2c9abf0a21ddcc6941d9b86f0596936", + "0x5c8536898fbb74fc7445814902fd08422eac56d0", + "0x6966ab0d485353095148a2155858910e0965b6f9", + "0x779543a0491a837ca36ce8c635d6154e3c4911a6", + "0x2a5ed960395e2a49b1c758cef4aa15213cfd874c", + "0x5c6e67ccd5849c0d29219c4f95f1a7a93b3f5dc5", + "0x9c50426be05db97f5d64fc54bf89eff947f0a321", + "0x200450f06520bdd6c527622a273333384d870efb", + "0xbe8539bfe837b67d1282b2b1d61c3f723966f049", + "0x6b0c4d41ba9ab8d8cfb5d379c69a612f2ced8ecb", + "0xf1385fb24aad0cd7432824085e42aff90886fef5", + "0xd1ac8b1ef1b69ff51d1d401a476e7e612414f091", + "0x8163e7fb499e90f8544ea62bbf80d21cd26d9efd", + "0x51e0ddd9998364a2eb38588679f0d2c42653e4a6", + "0x627a0a960c079c21c34f7612d5d230e01b4ad4c7", + "0xf0b1aa0eb660754448a7937c022e30aa692fe0c5", + "0x24c4d950dfd4dd1902bbed3508144a54542bba94", + "0x9f27daea7aca0aa0446220b98d028715e3bc803d", + "0xa5dc5acd6a7968a4554d89d65e59b7fd3bff0f90", + "0xd9aef3a1e38a39c16b31d1ace71bca8ef58d315b", + "0x63ed5a272de2f6d968408b4acb9024f4cc208ebf", + "0x6f6704e5a10332af6672e50b3d9754dc460dfa4d", + "0x77ca7b50b6cd7e2f3fa008e24ab793fd56cb15f6", + "0x492ea3bb0f3315521c31f273e565b868fc090f17", + "0x0ff30d6de14a8224aa97b78aea5388d1c51c1f00", + "0x9ea779f907f0b315b364b0cfc39a0fde5b02a416", + "0xceaeb481747ca6c540a000c1f3641f8cef161fa7", + "0xcc34673c6c40e791051898567a1222daf90be287", + "0x579a80d909f346fbfb1189493f521d7f48d52238", + "0xe308bd1ac5fda103967359b2712dd89deffb7973", + "0x4cb31628079fb14e4bc3cd5e30c2f7489b00960c", + "0xac1ecab32727358dba8962a0f3b261731aad9723", + "0x4fd6ace747f06ece9c49699c7cabc62d02211f75", + "0x440c59b325d2997a134c2c7c60a8c61611212bad", + "0x4486a3d68fac6967006d7a517b889fd3f98c102b", + "0x9c15b54878ba618f494b38f0ae7443db6af648ba", + "0x27b137a85656544b1ccb5a0f2e561a5703c6a68f", + "0x21c7fdb9ed8d291d79ffd82eb2c4356ec0d81241", + "0x23b75c2f6791eef49c69684db4c6c1f93bf49a50", + "0x1ca6abd14d30affe533b24d7a21bff4c2d5e1f3b", + "0xb9637156d330c0d605a791f1c31ba5890582fe1c", + "0x6131c42fa982e56929107413a9d526fd99405560", + "0x1591fc0f688c81fbeb17f5426a162a7024d430c2", + "0x542a9515200d14b68e934e9830d91645a980dd7a", + "0xc4bbd073882dd2add2424cf47d35213405b01324", + "0x782495b7b3355efb2833d56ecb34dc22ad7dfcc4", + "0x58b95c9a9d5d26825e70a82b6adb139d3fd829eb", + "0x3ba4d81db016dc2890c81f3acec2454bff5aada5", + "0xb52042c8ca3f8aa246fa79c3feaa3d959347c0ab", + "0xe4ae1efdfc53b73893af49113d8694a057b9c0d1", + "0x3c02a7bc0391e86d91b7d144e61c2c01a25a79c5", + "0x0737a6b837f97f46ebade41b9bc3e1c509c85c53", + "0x97f43a37f595ab5dd318fb46e7a155eae057317a", + "0x52c5317c848ba20c7504cb2c8052abd1fde29d03", + "0x4863226780fe7c0356454236d3b1c8792785748d", + "0x5d2b2e6fcbe3b11d26b525e085ff818dae332479", + "0x5f9f3392e9f62f63b8eac0beb55541fc8627f42c", + "0x057b56736d32b86616a10f619859c6cd6f59092a", + "0x9aa008f65de0b923a2a4f02012ad034a5e2e2192", + "0x304a554a310c7e546dfe434669c62820b7d83490", + "0x914d1b8b43e92723e64fd0a06f5bdb8dd9b10c79", + "0x4deb0033bb26bc534b197e61d19e0733e5679784", + "0x07f5c1e1bc2c93e0402f23341973a0e043f7bf8a", + "0x35a051a0010aba705c9008d7a7eff6fb88f6ea7b", + "0x4fa802324e929786dbda3b8820dc7834e9134a2a", + "0x9da397b9e80755301a3b32173283a91c0ef6c87e", + "0x8d9edb3054ce5c5774a420ac37ebae0ac02343c6", + "0x0101f3be8ebb4bbd39a2e3b9a3639d4259832fd9", + "0x5dc28b15dffed94048d73806ce4b7a4612a1d48f", + "0xbcf899e6c7d9d5a215ab1e3444c86806fa854c76", + "0x12e626b0eebfe86a56d633b9864e389b45dcb260", + "0xa2f1ccba9395d7fcb155bba8bc92db9bafaeade7", + "0xec8e57756626fdc07c63ad2eafbd28d08e7b0ca5", + "0xd164b088bd9108b60d0ca3751da4bceb207b0782", + "0x6231b6d0d5e77fe001c2a460bd9584fee60d409b", + "0x1cba23d343a983e9b5cfd19496b9a9701ada385f", + "0xa82f360a8d3455c5c41366975bde739c37bfeb8a", + "0x9fcd2deaff372a39cc679d5c5e4de7bafb0b1339", + "0x005f5cee7a43331d5a3d3eec71305925a62f34b6", + "0x0e0da70933f4c7849fc0d203f5d1d43b9ae4532d", + "0xd131637d5275fd1a68a3200f4ad25c71a2a9522e", + "0xbc07118b9ac290e4622f5e77a0853539789effbe", + "0x47e7aa56d6bdf3f36be34619660de61275420af8", + "0xacd87e28b0c9d1254e868b81cba4cc20d9a32225", + "0xadf80daec7ba8dcf15392f1ac611fff65d94f880", + "0x5524c55fb03cf21f549444ccbecb664d0acad706", + "0x40b803a9abce16f50f36a77ba41180eb90023925", + "0xfe24cdd8648121a43a7c86d289be4dd2951ed49f", + "0x17802f43a0137c506ba92291391a8a8f207f487d", + "0x253488078a4edf4d6f42f113d1e62836a942cf1a", + "0x86af3e9626fce1957c82e88cbf04ddf3a2ed7915", + "0xb136707642a4ea12fb4bae820f03d2562ebff487", + "0xdbe9b615a3ae8709af8b93336ce9b477e4ac0940", + "0xf14c14075d6c4ed84b86798af0956deef67365b5", + "0xca544e5c4687d109611d0f8f928b53a25af72448", + "0xaeeb8ff27288bdabc0fa5ebb731b6f409507516c", + "0xcbb9d3703e651b0d496cdefb8b92c25aeb2171f7", + "0x6d87578288b6cb5549d5076a207456a1f6a63dc0", + "0xb2c6f0dfbb716ac562e2d85d6cb2f8d5ee87603e", + "0xaccc230e8a6e5be9160b8cdf2864dd2a001c28b6", + "0x2b3455ec7fedf16e646268bf88846bd7a2319bb2", + "0x4613f3bca5c44ea06337a9e439fbc6d42e501d0a", + "0xd343b217de44030afaa275f54d31a9317c7f441e", + "0x84ef4b2357079cd7a7c69fd7a37cd0609a679106", + "0xda2fef9e4a3230988ff17df2165440f37e8b1708", + "0xf4c64518ea10f995918a454158c6b61407ea345c", + "0x7602b46df5390e432ef1c307d4f2c9ff6d65cc97", + "0xbb9bc244d798123fde783fcc1c72d3bb8c189413", + "0x807640a13483f8ac783c557fcdf27be11ea4ac7a", +] diff --git a/eth/vm/forks/frontier/validation.py b/eth/vm/forks/frontier/validation.py index ddfaa0fca0..1c83f2887f 100644 --- a/eth/vm/forks/frontier/validation.py +++ b/eth/vm/forks/frontier/validation.py @@ -31,7 +31,8 @@ def validate_frontier_transaction(state: BaseState, raise ValidationError("Sender account balance cannot afford txn") if state.get_nonce(transaction.sender) != transaction.nonce: - raise ValidationError("Invalid transaction nonce") + expected = state.get_nonce(transaction.sender) + raise ValidationError(f"Invalid transaction nonce. Expected {expected} but transaction had {transaction.nonce}") def validate_frontier_transaction_against_header(_vm: BaseVM, diff --git a/eth/vm/forks/homestead/headers.py b/eth/vm/forks/homestead/headers.py index 167949418c..1c58ecec7c 100644 --- a/eth/vm/forks/homestead/headers.py +++ b/eth/vm/forks/homestead/headers.py @@ -82,146 +82,5 @@ def configure_homestead_header(vm: "HomesteadVM", **header_params: Any) -> Block header_params['timestamp'], ) - # In geth the modification of the state in the DAO fork block is performed - # before any transactions are applied, so doing it here is the closest we - # get to that. Another alternative would be to do it in Block.mine(), but - # there we'd need to manually instantiate the State and update - # header.state_root after we're done. - if vm.support_dao_fork and changeset.block_number == vm.get_dao_fork_block_number(): - state = vm.state - - for hex_account in dao_drain_list: - address = Address(decode_hex(hex_account)) - balance = state.get_balance(address) - state.delta_balance(dao_refund_contract, balance) - state.set_balance(address, 0) - - # Persist the changes to the database - state.persist() - - # Update state_root manually - changeset.state_root = state.state_root - header = changeset.commit() return header - - -dao_refund_contract = Address(decode_hex('0xbf4ed7b27f1d666546e30d74d50d173d20bca754')) -dao_drain_list = [ - "0xd4fe7bc31cedb7bfb8a345f31e668033056b2728", - "0xb3fb0e5aba0e20e5c49d252dfd30e102b171a425", - "0x2c19c7f9ae8b751e37aeb2d93a699722395ae18f", - "0xecd135fa4f61a655311e86238c92adcd779555d2", - "0x1975bd06d486162d5dc297798dfc41edd5d160a7", - "0xa3acf3a1e16b1d7c315e23510fdd7847b48234f6", - "0x319f70bab6845585f412ec7724b744fec6095c85", - "0x06706dd3f2c9abf0a21ddcc6941d9b86f0596936", - "0x5c8536898fbb74fc7445814902fd08422eac56d0", - "0x6966ab0d485353095148a2155858910e0965b6f9", - "0x779543a0491a837ca36ce8c635d6154e3c4911a6", - "0x2a5ed960395e2a49b1c758cef4aa15213cfd874c", - "0x5c6e67ccd5849c0d29219c4f95f1a7a93b3f5dc5", - "0x9c50426be05db97f5d64fc54bf89eff947f0a321", - "0x200450f06520bdd6c527622a273333384d870efb", - "0xbe8539bfe837b67d1282b2b1d61c3f723966f049", - "0x6b0c4d41ba9ab8d8cfb5d379c69a612f2ced8ecb", - "0xf1385fb24aad0cd7432824085e42aff90886fef5", - "0xd1ac8b1ef1b69ff51d1d401a476e7e612414f091", - "0x8163e7fb499e90f8544ea62bbf80d21cd26d9efd", - "0x51e0ddd9998364a2eb38588679f0d2c42653e4a6", - "0x627a0a960c079c21c34f7612d5d230e01b4ad4c7", - "0xf0b1aa0eb660754448a7937c022e30aa692fe0c5", - "0x24c4d950dfd4dd1902bbed3508144a54542bba94", - "0x9f27daea7aca0aa0446220b98d028715e3bc803d", - "0xa5dc5acd6a7968a4554d89d65e59b7fd3bff0f90", - "0xd9aef3a1e38a39c16b31d1ace71bca8ef58d315b", - "0x63ed5a272de2f6d968408b4acb9024f4cc208ebf", - "0x6f6704e5a10332af6672e50b3d9754dc460dfa4d", - "0x77ca7b50b6cd7e2f3fa008e24ab793fd56cb15f6", - "0x492ea3bb0f3315521c31f273e565b868fc090f17", - "0x0ff30d6de14a8224aa97b78aea5388d1c51c1f00", - "0x9ea779f907f0b315b364b0cfc39a0fde5b02a416", - "0xceaeb481747ca6c540a000c1f3641f8cef161fa7", - "0xcc34673c6c40e791051898567a1222daf90be287", - "0x579a80d909f346fbfb1189493f521d7f48d52238", - "0xe308bd1ac5fda103967359b2712dd89deffb7973", - "0x4cb31628079fb14e4bc3cd5e30c2f7489b00960c", - "0xac1ecab32727358dba8962a0f3b261731aad9723", - "0x4fd6ace747f06ece9c49699c7cabc62d02211f75", - "0x440c59b325d2997a134c2c7c60a8c61611212bad", - "0x4486a3d68fac6967006d7a517b889fd3f98c102b", - "0x9c15b54878ba618f494b38f0ae7443db6af648ba", - "0x27b137a85656544b1ccb5a0f2e561a5703c6a68f", - "0x21c7fdb9ed8d291d79ffd82eb2c4356ec0d81241", - "0x23b75c2f6791eef49c69684db4c6c1f93bf49a50", - "0x1ca6abd14d30affe533b24d7a21bff4c2d5e1f3b", - "0xb9637156d330c0d605a791f1c31ba5890582fe1c", - "0x6131c42fa982e56929107413a9d526fd99405560", - "0x1591fc0f688c81fbeb17f5426a162a7024d430c2", - "0x542a9515200d14b68e934e9830d91645a980dd7a", - "0xc4bbd073882dd2add2424cf47d35213405b01324", - "0x782495b7b3355efb2833d56ecb34dc22ad7dfcc4", - "0x58b95c9a9d5d26825e70a82b6adb139d3fd829eb", - "0x3ba4d81db016dc2890c81f3acec2454bff5aada5", - "0xb52042c8ca3f8aa246fa79c3feaa3d959347c0ab", - "0xe4ae1efdfc53b73893af49113d8694a057b9c0d1", - "0x3c02a7bc0391e86d91b7d144e61c2c01a25a79c5", - "0x0737a6b837f97f46ebade41b9bc3e1c509c85c53", - "0x97f43a37f595ab5dd318fb46e7a155eae057317a", - "0x52c5317c848ba20c7504cb2c8052abd1fde29d03", - "0x4863226780fe7c0356454236d3b1c8792785748d", - "0x5d2b2e6fcbe3b11d26b525e085ff818dae332479", - "0x5f9f3392e9f62f63b8eac0beb55541fc8627f42c", - "0x057b56736d32b86616a10f619859c6cd6f59092a", - "0x9aa008f65de0b923a2a4f02012ad034a5e2e2192", - "0x304a554a310c7e546dfe434669c62820b7d83490", - "0x914d1b8b43e92723e64fd0a06f5bdb8dd9b10c79", - "0x4deb0033bb26bc534b197e61d19e0733e5679784", - "0x07f5c1e1bc2c93e0402f23341973a0e043f7bf8a", - "0x35a051a0010aba705c9008d7a7eff6fb88f6ea7b", - "0x4fa802324e929786dbda3b8820dc7834e9134a2a", - "0x9da397b9e80755301a3b32173283a91c0ef6c87e", - "0x8d9edb3054ce5c5774a420ac37ebae0ac02343c6", - "0x0101f3be8ebb4bbd39a2e3b9a3639d4259832fd9", - "0x5dc28b15dffed94048d73806ce4b7a4612a1d48f", - "0xbcf899e6c7d9d5a215ab1e3444c86806fa854c76", - "0x12e626b0eebfe86a56d633b9864e389b45dcb260", - "0xa2f1ccba9395d7fcb155bba8bc92db9bafaeade7", - "0xec8e57756626fdc07c63ad2eafbd28d08e7b0ca5", - "0xd164b088bd9108b60d0ca3751da4bceb207b0782", - "0x6231b6d0d5e77fe001c2a460bd9584fee60d409b", - "0x1cba23d343a983e9b5cfd19496b9a9701ada385f", - "0xa82f360a8d3455c5c41366975bde739c37bfeb8a", - "0x9fcd2deaff372a39cc679d5c5e4de7bafb0b1339", - "0x005f5cee7a43331d5a3d3eec71305925a62f34b6", - "0x0e0da70933f4c7849fc0d203f5d1d43b9ae4532d", - "0xd131637d5275fd1a68a3200f4ad25c71a2a9522e", - "0xbc07118b9ac290e4622f5e77a0853539789effbe", - "0x47e7aa56d6bdf3f36be34619660de61275420af8", - "0xacd87e28b0c9d1254e868b81cba4cc20d9a32225", - "0xadf80daec7ba8dcf15392f1ac611fff65d94f880", - "0x5524c55fb03cf21f549444ccbecb664d0acad706", - "0x40b803a9abce16f50f36a77ba41180eb90023925", - "0xfe24cdd8648121a43a7c86d289be4dd2951ed49f", - "0x17802f43a0137c506ba92291391a8a8f207f487d", - "0x253488078a4edf4d6f42f113d1e62836a942cf1a", - "0x86af3e9626fce1957c82e88cbf04ddf3a2ed7915", - "0xb136707642a4ea12fb4bae820f03d2562ebff487", - "0xdbe9b615a3ae8709af8b93336ce9b477e4ac0940", - "0xf14c14075d6c4ed84b86798af0956deef67365b5", - "0xca544e5c4687d109611d0f8f928b53a25af72448", - "0xaeeb8ff27288bdabc0fa5ebb731b6f409507516c", - "0xcbb9d3703e651b0d496cdefb8b92c25aeb2171f7", - "0x6d87578288b6cb5549d5076a207456a1f6a63dc0", - "0xb2c6f0dfbb716ac562e2d85d6cb2f8d5ee87603e", - "0xaccc230e8a6e5be9160b8cdf2864dd2a001c28b6", - "0x2b3455ec7fedf16e646268bf88846bd7a2319bb2", - "0x4613f3bca5c44ea06337a9e439fbc6d42e501d0a", - "0xd343b217de44030afaa275f54d31a9317c7f441e", - "0x84ef4b2357079cd7a7c69fd7a37cd0609a679106", - "0xda2fef9e4a3230988ff17df2165440f37e8b1708", - "0xf4c64518ea10f995918a454158c6b61407ea345c", - "0x7602b46df5390e432ef1c307d4f2c9ff6d65cc97", - "0xbb9bc244d798123fde783fcc1c72d3bb8c189413", - "0x807640a13483f8ac783c557fcdf27be11ea4ac7a", -] diff --git a/eth/vm/state.py b/eth/vm/state.py index d3600cd403..392949e8c7 100644 --- a/eth/vm/state.py +++ b/eth/vm/state.py @@ -25,11 +25,15 @@ ) from eth.db.account import ( BaseAccountDB, + AccountDB, ) +from eth.db.block_diff import BlockDiff from eth.db.backends.base import ( BaseAtomicDB, ) +from eth.db.schema import ensure_schema, Schemas from eth.exceptions import StateRootNotFound +from eth.rlp.headers import BlockHeader from eth.tools.logging import ( ExtendedDebugLogger, ) @@ -86,10 +90,19 @@ def __init__( self, db: BaseAtomicDB, execution_context: ExecutionContext, - state_root: bytes) -> None: + header: BlockHeader, + # expected_schema: Schemas = Schemas.DEFAULT) -> None: + expected_schema: Schemas = Schemas.TURBO) -> None: self._db = db self.execution_context = execution_context - self._account_db = self.get_account_db_class()(db, state_root) + + # TODO: Instead of calling ensure_schema, pick our account_db_class based on + # expected_schema? + + # TODO: somehow integrate with self.get_account_db_class() + self._account_db = AccountDB(db, header) + + # ensure_schema(db, expected_schema) # # Logging @@ -249,6 +262,12 @@ def commit(self, snapshot: Tuple[Hash32, UUID]) -> None: def persist(self) -> None: self._account_db.persist() + def persist_returning_block_diff(self, parent_state_root: Hash32) -> BlockDiff: + """ + Persists all changes and also saves a record of them to the database. + """ + return self._account_db.persist_returning_block_diff(parent_state_root) + # # Access self.prev_hashes (Read-only) # diff --git a/setup.py b/setup.py index 17dcbf8c8e..468df65226 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ "pytest-cov==2.5.1", "pytest-watch>=4.1.0,<5", "pytest-xdist==1.18.1", + "vyper==0.1.0b11", ], 'lint': [ "flake8==3.5.0", diff --git a/tests/conftest.py b/tests/conftest.py index f2dc1c28f8..02e45e4e6e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ MiningChain, ) from eth.db.atomic import AtomicDB +from eth.db.chain import ChainDB from eth.rlp.headers import BlockHeader from eth.vm.forks import ( FrontierVM, @@ -144,6 +145,10 @@ def _chain_with_block_validation(VM, base_db, genesis_state, chain_cls=Chain): chain_id=1337, ) chain = klass.from_genesis(base_db, genesis_params, genesis_state) + + db = chain.chaindb.db + ChainDB.upgrade_to_turbo_schema(db) + return chain @@ -227,6 +232,10 @@ def chain_without_block_validation( 'timestamp': 1501851927, } chain = klass.from_genesis(base_db, genesis_params, genesis_state) + + db = chain.chaindb.db + ChainDB.upgrade_to_turbo_schema(db) + return chain diff --git a/tests/core/builder-tools/test_chain_builder.py b/tests/core/builder-tools/test_chain_builder.py index 6f1e6e8b4c..2f410b4f33 100644 --- a/tests/core/builder-tools/test_chain_builder.py +++ b/tests/core/builder-tools/test_chain_builder.py @@ -23,6 +23,7 @@ import_blocks, mine_block, mine_blocks, + upgrade_to_turbo, ) from eth.tools.builder.chain.builders import ( NoChainSealValidationMixin, @@ -34,6 +35,7 @@ frontier_at(0), disable_pow_check, genesis(), + upgrade_to_turbo, ) diff --git a/tests/core/builder-tools/test_chain_construction.py b/tests/core/builder-tools/test_chain_construction.py index 29d0db466f..7b6260ae38 100644 --- a/tests/core/builder-tools/test_chain_construction.py +++ b/tests/core/builder-tools/test_chain_construction.py @@ -21,6 +21,7 @@ petersburg_at, spurious_dragon_at, tangerine_whistle_at, + upgrade_to_turbo, ) from eth.vm.forks import ( FrontierVM, @@ -113,6 +114,7 @@ def test_chain_builder_enable_pow_mining(): frontier_at(0), enable_pow_mining(), genesis(), + upgrade_to_turbo, ) block = chain.mine_block() check_pow( @@ -129,6 +131,7 @@ def test_chain_builder_without_any_mining_config(): MiningChain, frontier_at(0), genesis(), + upgrade_to_turbo, ) with pytest.raises(ValidationError, match='mix hash mismatch'): chain.mine_block() @@ -140,6 +143,7 @@ def test_chain_builder_disable_pow_check(): frontier_at(0), disable_pow_check(), genesis(), + upgrade_to_turbo, ) block = chain.mine_block() with pytest.raises(ValidationError, match='mix hash mismatch'): diff --git a/tests/core/builder-tools/test_chain_initializer.py b/tests/core/builder-tools/test_chain_initializer.py index 9cb653703a..e7991f5491 100644 --- a/tests/core/builder-tools/test_chain_initializer.py +++ b/tests/core/builder-tools/test_chain_initializer.py @@ -9,6 +9,7 @@ from eth.tools.builder.chain import ( frontier_at, genesis, + upgrade_to_turbo, ) @@ -54,7 +55,8 @@ def test_chain_builder_initialize_chain_with_state_simple(chain_class): chain_class, genesis( state=((ADDRESS_A, 'balance', 1),), - ) + ), + upgrade_to_turbo, ) header = chain.get_canonical_head() @@ -71,7 +73,8 @@ def test_chain_builder_initialize_chain_with_state_multiple(chain_class): chain_class, genesis( state=((ADDRESS_A, 'balance', 1), (ADDRESS_B, 'balance', 2)), - ) + ), + upgrade_to_turbo, ) header = chain.get_canonical_head() @@ -89,7 +92,8 @@ def test_chain_builder_initialize_chain_with_params(chain_class): chain_class, genesis( params={'difficulty': 12345}, - ) + ), + upgrade_to_turbo, ) header = chain.get_canonical_head() @@ -104,7 +108,8 @@ def test_chain_builder_initialize_chain_with_params_and_state(chain_class): genesis( params={'difficulty': 12345}, state=((ADDRESS_A, 'balance', 1),), - ) + ), + upgrade_to_turbo, ) header = chain.get_canonical_head() diff --git a/tests/core/chain-object/test_chain_reorganization.py b/tests/core/chain-object/test_chain_reorganization.py index 57789ea79b..200fb0ea4d 100644 --- a/tests/core/chain-object/test_chain_reorganization.py +++ b/tests/core/chain-object/test_chain_reorganization.py @@ -26,6 +26,7 @@ def base_chain(request): def chain(base_chain): chain = api.build( base_chain, + api.upgrade_to_turbo, api.mine_blocks(3), ) assert chain.get_canonical_head().block_number == 3 diff --git a/tests/core/chain-object/test_gas_estimation.py b/tests/core/chain-object/test_gas_estimation.py index 8f7f822fd7..33e7c67d39 100644 --- a/tests/core/chain-object/test_gas_estimation.py +++ b/tests/core/chain-object/test_gas_estimation.py @@ -23,28 +23,22 @@ def chain(chain_without_block_validation): 'should_sign_tx', (True, False), ) @pytest.mark.parametrize( - 'data, gas_estimator, to, on_pending, expected', + 'data, gas_estimator, to, expected', ( - (b'', None, ADDR_1010, True, 21000), - (b'', None, ADDR_1010, False, 21000), - (b'\xff' * 10, None, ADDR_1010, True, 21680), - (b'\xff' * 10, None, ADDR_1010, False, 21680), + (b'', None, ADDR_1010, 21000), + (b'\xff' * 10, None, ADDR_1010, 21680), # sha3 precompile - (b'\xff' * 32, None, ADDRESS_2, True, 35333), - (b'\xff' * 32, None, ADDRESS_2, False, 35345), - (b'\xff' * 320, None, ADDRESS_2, True, 54840), + (b'\xff' * 32, None, ADDRESS_2, 35345), + (b'\xff' * 320, None, ADDRESS_2, 54852), # 1000_tolerance binary search - (b'\xff' * 32, binary_gas_search_1000_tolerance, ADDRESS_2, True, 23935), + (b'\xff' * 32, binary_gas_search_1000_tolerance, ADDRESS_2, 23936), ), ids=[ - 'simple default pending', 'simple default', - '10 bytes default pending', '10 bytes default', - 'sha3 precompile 32 bytes default pending', 'sha3 precompile 32 bytes default', - 'sha3 precompile 320 bytes default pending', - 'sha3 precompile 32 bytes 1000_tolerance binary pending', + 'sha3 precompile 320 bytes default', + 'sha3 precompile 32 bytes 1000_tolerance binary', ], ) def test_estimate_gas( @@ -52,7 +46,6 @@ def test_estimate_gas( data, gas_estimator, to, - on_pending, expected, funded_address, funded_address_private_key, @@ -77,15 +70,10 @@ def test_estimate_gas( else: tx = new_transaction(**tx_params) - if on_pending: - # estimate on *pending* block - pending_header = chain.create_header_from_parent(chain.get_canonical_head()) - assert chain.estimate_gas(tx, pending_header) == expected - else: - # estimates on top of *latest* block - assert chain.estimate_gas(tx) == expected - # these are long, so now that we know the exact numbers let's skip the repeat test - # assert chain.estimate_gas(tx, chain.get_canonical_head()) == expected + # estimates on top of *latest* block + assert chain.estimate_gas(tx) == expected + # these are long, so now that we know the exact numbers let's skip the repeat test + # assert chain.estimate_gas(tx, chain.get_canonical_head()) == expected def test_estimate_gas_on_full_block(chain, funded_address_private_key, funded_address): @@ -125,4 +113,4 @@ def mk_estimation_txn(chain, from_, from_key, data): # build a transaction to estimate gas for next_pending_tx = mk_estimation_txn(chain, from_, from_key, data=garbage_data * 2) - assert chain.estimate_gas(next_pending_tx, chain.header) == 722760 + assert chain.estimate_gas(next_pending_tx) == 722760 diff --git a/tests/core/chain-object/test_turbo_chain.py b/tests/core/chain-object/test_turbo_chain.py new file mode 100644 index 0000000000..05008b4588 --- /dev/null +++ b/tests/core/chain-object/test_turbo_chain.py @@ -0,0 +1,323 @@ +""" +some tests that chain correctly manipulates the turbo database +""" +import pytest + +from eth_utils.toolz import ( + assoc, +) +from eth_hash.auto import keccak + +import rlp + +from eth.rlp.accounts import Account + +from eth.chains.base import MiningChain +from eth.db.block_diff import BlockDiff +from eth.db.schema import SchemaTurbo +from eth.tools._utils.vyper import ( + compile_vyper_lll, +) + +from tests.core.helpers import ( + new_transaction, +) + + +CONTRACT_ADDRESS = b'\x10' * 20 + + +@pytest.fixture +def genesis_state(base_genesis_state): + """ + A little bit of magic, this overrides the genesis_state fixture which was defined elsewhere so + chain_without_block_validation uses the genesis state specified here. + """ + + # 1. when called this contract makes a simple change to the state + code = ['SSTORE', 0, 42] + bytecode = compile_vyper_lll(code)[0] + + # 2. put that code somewhere useful + return assoc( + base_genesis_state, + CONTRACT_ADDRESS, + { + 'balance': 0, + 'nonce': 0, + 'code': bytecode, + 'storage': {}, + } + ) + + +@pytest.fixture +def chain(chain_without_block_validation): + # make things a little less verbose + return chain_without_block_validation + + +def test_import_block_saves_block_diff(chain, funded_address, funded_address_private_key): + tx = new_transaction( + chain.get_vm(), + funded_address, + CONTRACT_ADDRESS, + data=b'', + private_key=funded_address_private_key, + ) + + new_block, _, _ = chain.build_block_with_transactions([tx]) + imported_block, _, _ = chain.import_block(new_block) + + imported_header = imported_block.header + imported_block_state_root = imported_header.state_root + + # sanity check, did the transaction go through? + assert len(imported_block.transactions) == 1 + state = chain.get_vm(imported_header).state + assert state.get_storage(CONTRACT_ADDRESS, 0) == 42 + + # the actual test, did we write out all the changes which happened? + base_db = chain.chaindb.db + diff = BlockDiff.from_db(base_db, imported_block_state_root) + assert len(diff.get_changed_accounts()) == 3 + assert CONTRACT_ADDRESS in diff.get_changed_accounts() + assert imported_header.coinbase in diff.get_changed_accounts() + assert funded_address in diff.get_changed_accounts() + + assert diff.get_changed_slots(CONTRACT_ADDRESS) == {0} + assert diff.get_slot_change(CONTRACT_ADDRESS, 0) == (0, 42) + + assert diff.get_changed_slots(funded_address) == set() + assert diff.get_changed_slots(imported_header.coinbase) == set() + + # do some spot checks to make sure different fields were saved + + assert diff.get_decoded_account(imported_header.coinbase, new=False) is None + new_coinbase_balance = diff.get_decoded_account(imported_header.coinbase, new=True).balance + assert new_coinbase_balance > 0 + + old_sender_balance = diff.get_decoded_account(funded_address, new=False).balance + new_sender_balance = diff.get_decoded_account(funded_address, new=True).balance + assert old_sender_balance > new_sender_balance + + old_contract_nonce = diff.get_decoded_account(CONTRACT_ADDRESS, new=False).nonce + new_contract_nonce = diff.get_decoded_account(CONTRACT_ADDRESS, new=True).nonce + assert old_contract_nonce == 0 + assert new_contract_nonce == 0 + + +# TODO: This belongs somewhere in the actual code, not in the tests. What kind of +# interface would make interacting with the turbodb easier? +def read_account_from_db(base_db, address): + key = SchemaTurbo.make_account_state_lookup_key(keccak(address)) + + try: + account_rlp = base_db[key] + except KeyError: + return Account() + + if account_rlp == b'': + return Account() + + return rlp.decode(account_rlp, sedes=Account) + + +def test_import_multiple_txns_saves_complete_block_diff(chain, funded_address, funded_address_private_key): + """ + MiningChain builds a new VM each time a method (such as apply_transaction) is called. + + block diffs are created by AccountDB, and there isn't a good way of tracking changes + over multiple instance of AccountDB. This means that block diffs created by the later + VM instances will be incomplete, they'll miss any accounts which changed in previous + VM instances. + + It turns out that this isn't actually a problem, because the VM which builds the block + diff is created during `chain.import_block`, it re-runs every transaction and then + saves the block diff. + + Keeping this test because that's kind of an inefficient implementation detail and + might change. Once it changes this test will start failing. I think the right fix is + a refactor to MiningChain. It ought to keep around a single pending VM and apply all + transactions to that VM. This way the AccountDB will have seen all changes, so it can + emit an accurate block diff. This refactor can also be expected to improve test + performance. + """ + if not isinstance(chain, MiningChain): + pytest.skip('this test checks that MiningChain works properly') + + FIRST_ADDRESS = b'\x10' * 20 + SECOND_ADDRESS = b'\x20' * 20 + + # 1. Make a txn which changes one account + first_txn = new_transaction( + chain.get_vm(), + funded_address, + FIRST_ADDRESS, + data=b'', + private_key=funded_address_private_key, + amount=1000, + ) + chain.apply_transaction(first_txn) + + # 2. Make a txn which changes a second account + second_txn = new_transaction( + chain.get_vm(), + funded_address, + SECOND_ADDRESS, + data=b'', + private_key=funded_address_private_key, + amount=1000, + ) + new_block, _receipt, _computation = chain.apply_transaction(second_txn) + mined_block, _, _ = chain.import_block(new_block) + + # did the transactions go through? + assert mined_block.transactions == (first_txn, second_txn) + + # what does the block diff say? + base_db = chain.chaindb.db + diff = BlockDiff.from_db(base_db, mined_block.header.state_root) + assert diff.get_changed_accounts() == { + funded_address, + FIRST_ADDRESS, + SECOND_ADDRESS, + mined_block.header.coinbase, + } + + first_account = read_account_from_db(base_db, FIRST_ADDRESS) + assert first_account.balance == 1000 + + +def test_block_reorgs_disjoin_state(chain, funded_address, funded_address_private_key): + """ + A1 - B1 + \ B2 - C2 + + Each block manipulates a different part of the state. + """ + base_db = chain.chaindb.db + + A1_ADDRESS = b'\x10' * 20 + B1_ADDRESS = b'\x20' * 20 + B2_ADDRESS = b'\x30' * 20 + C2_ADDRESS = b'\x40' * 20 + + def make_transaction_to(destination_address, amount, parent_header=None): + return new_transaction( + chain.get_vm(parent_header), + funded_address, + destination_address, + data=b'', + private_key=funded_address_private_key, + amount=amount, + ) + + # The account starts out empty + assert read_account_from_db(base_db, A1_ADDRESS).balance == 0 + + # After we import A1 the address has some wei + A1_transaction = make_transaction_to(A1_ADDRESS, 1000) + new_A1, _, _ = chain.build_block_with_transactions([A1_transaction]) + imported_A1, _, _ = chain.import_block(new_A1) + assert read_account_from_db(base_db, A1_ADDRESS).balance == 1000 + assert read_account_from_db(base_db, funded_address).nonce == 1 + + # After we import B1 the second address also has some wei + B1_transaction = make_transaction_to(B1_ADDRESS, 1000) + new_B1, _, _ = chain.build_block_with_transactions([B1_transaction]) + imported_B1, _, _ = chain.import_block(new_B1) + assert read_account_from_db(base_db, A1_ADDRESS).balance == 1000 + assert read_account_from_db(base_db, B1_ADDRESS).balance == 1000 + assert read_account_from_db(base_db, funded_address).nonce == 2 + + # Import a competing block. No asserts because it's not clear which one should win + B2_transaction = make_transaction_to(B2_ADDRESS, 1000, imported_A1.header) + new_B2, _, _ = chain.build_block_with_transactions( + [B2_transaction], parent_header=imported_A1.header + ) + imported_B2, _, _ = chain.import_block(new_B2) + + # Importing C2 causes a re-org which should reshuffle some state around + C2_transaction = make_transaction_to(C2_ADDRESS, 1000, imported_B2.header) + new_C2, _, _ = chain.build_block_with_transactions( + [C2_transaction], parent_header=imported_B2.header + ) + imported_C2, _, _ = chain.import_block(new_C2) + assert read_account_from_db(base_db, A1_ADDRESS).balance == 1000 + assert read_account_from_db(base_db, B1_ADDRESS).balance == 0 + assert read_account_from_db(base_db, B2_ADDRESS).balance == 1000 + assert read_account_from_db(base_db, C2_ADDRESS).balance == 1000 + assert read_account_from_db(base_db, funded_address).nonce == 3 + + +def test_block_reorgs_same_account(chain, funded_address, funded_address_private_key): + """ + A1 - B1 + \ B2 - C2 + + Each block touches the same part of the state. In order for this test to pass the + TurboDB must be able to read states which are not the canonical chain tip. + """ + base_db = chain.chaindb.db + + ADDRESS = b'\x10' * 20 + + def make_transaction_to(destination_address, amount, parent_header=None): + return new_transaction( + chain.get_vm(parent_header), + funded_address, + destination_address, + data=b'', + private_key=funded_address_private_key, + amount=amount, + ) + + assert read_account_from_db(base_db, ADDRESS).balance == 0 + + A1_transaction = make_transaction_to(ADDRESS, 1000) + new_A1, _, _ = chain.build_block_with_transactions([A1_transaction]) + imported_A1, _, _ = chain.import_block(new_A1) + assert read_account_from_db(base_db, ADDRESS).balance == 1000 + assert read_account_from_db(base_db, funded_address).nonce == 1 + + diff = BlockDiff.from_db(base_db, imported_A1.header.state_root) + assert diff.get_decoded_account(ADDRESS, new=False).balance == 0 + assert diff.get_decoded_account(ADDRESS, new=True).balance == 1000 + + B1_transaction = make_transaction_to(ADDRESS, 2000) + new_B1, _, _ = chain.build_block_with_transactions([B1_transaction]) + imported_B1, _, _ = chain.import_block(new_B1) + assert read_account_from_db(base_db, ADDRESS).balance == 3000 + assert read_account_from_db(base_db, funded_address).nonce == 2 + + diff = BlockDiff.from_db(base_db, imported_B1.header.state_root) + assert diff.get_decoded_account(ADDRESS, new=False).balance == 1000 + assert diff.get_decoded_account(ADDRESS, new=True).balance == 3000 + + B2_transaction = make_transaction_to(ADDRESS, 3000, imported_A1.header) + new_B2, _, _ = chain.build_block_with_transactions( + [B2_transaction], parent_header=imported_A1.header + ) + imported_B2, _, _ = chain.import_block(new_B2) + assert read_account_from_db(base_db, funded_address).nonce == 2 + + diff = BlockDiff.from_db(base_db, imported_B2.header.state_root) + assert diff.get_decoded_account(ADDRESS, new=False).balance == 1000 + assert diff.get_decoded_account(ADDRESS, new=True).balance == 4000 + + # the update has not happened yet, B1 is still canonical + assert base_db[SchemaTurbo.current_state_root_key] == imported_B1.header.state_root + assert read_account_from_db(base_db, ADDRESS).balance == 3000 + + C2_transaction = make_transaction_to(ADDRESS, 4000, imported_B2.header) + new_C2, _, _ = chain.build_block_with_transactions( + [C2_transaction], parent_header=imported_B2.header + ) + imported_C2, _, _ = chain.import_block(new_C2) + assert read_account_from_db(base_db, ADDRESS).balance == 8000 + assert read_account_from_db(base_db, funded_address).nonce == 3 + + diff = BlockDiff.from_db(base_db, imported_C2.header.state_root) + assert diff.get_decoded_account(ADDRESS, new=False).balance == 4000 + assert diff.get_decoded_account(ADDRESS, new=True).balance == 8000 diff --git a/tests/core/consensus/test_pow_mining.py b/tests/core/consensus/test_pow_mining.py index 3a8e10c82a..a3c1ae4ac4 100644 --- a/tests/core/consensus/test_pow_mining.py +++ b/tests/core/consensus/test_pow_mining.py @@ -14,6 +14,7 @@ from eth.tools.mining import POWMiningMixin from eth.tools.builder.chain import ( genesis, + upgrade_to_turbo, ) @@ -75,7 +76,7 @@ class ChainClass(MiningChain): (0, vm_class), ) - chain = genesis(ChainClass) + chain = upgrade_to_turbo(genesis(ChainClass)) block = chain.mine_block() check_pow( diff --git a/tests/core/vm/test_block_diffs.py b/tests/core/vm/test_block_diffs.py new file mode 100644 index 0000000000..fcbf8cdaf4 --- /dev/null +++ b/tests/core/vm/test_block_diffs.py @@ -0,0 +1,189 @@ +import pytest + +from eth_utils import int_to_big_endian + +from eth_hash.auto import keccak + +from eth.constants import BLANK_ROOT_HASH +from eth.db.atomic import AtomicDB +from eth.db.chain import ChainDB +from eth.db.block_diff import BlockDiff +from eth.db.account import AccountDB +from eth.db.storage import StorageLookup + +ACCOUNT = b'\xaa' * 20 +BLOCK_HASH = keccak(b'one') + +""" +TODO: Some tests remain to be written: +- Test that this behavior is trigger during block import (if Turbo-mode is enabled) +- Test that this works even under calls to things like commit() and snapshot() +- Test that these diffs can be applied to something and the correct resulting state obtained +""" + + +@pytest.fixture +def base_db(): + db = AtomicDB() + ChainDB.upgrade_to_turbo_schema(db) + return db + + +@pytest.fixture +def account_db(base_db): + return AccountDB(base_db) + + +# Some basic tests that BlockDiff works as expected and can round-trip data to the database + + +def test_no_such_diff_raises_key_error(base_db): + with pytest.raises(KeyError): + BlockDiff.from_db(base_db, BLOCK_HASH) + + +def test_can_persist_empty_block_diff(base_db): + orig = BlockDiff() + orig.write_to(base_db, BLOCK_HASH) + + block_diff = BlockDiff.from_db(base_db, BLOCK_HASH) + assert len(block_diff.get_changed_accounts()) == 0 + + +def test_can_persist_changed_account(base_db): + orig = BlockDiff() + orig.set_account_changed(ACCOUNT, b'old', b'new') # TODO: more realistic data + orig.write_to(base_db, BLOCK_HASH) + + block_diff = BlockDiff.from_db(base_db, BLOCK_HASH) + assert block_diff.get_changed_accounts() == {ACCOUNT} + assert block_diff.get_account(ACCOUNT, new=True) == b'new' + assert block_diff.get_account(ACCOUNT, new=False) == b'old' + + +# Some tests that AccountDB saves a block diff when persist()ing + + +def save_block_diff(account_db, block_hash): + parent_state_root = account_db._root_hash_at_last_persist + diff = account_db.persist_returning_block_diff(parent_state_root) + diff.write_to(account_db._raw_store_db, block_hash) + + +def test_account_diffs(account_db): + account_db.set_nonce(ACCOUNT, 10) + save_block_diff(account_db, BLOCK_HASH) + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_accounts() == {ACCOUNT} + new_account = diff.get_decoded_account(ACCOUNT, new=True) + assert new_account.nonce == 10 + + assert diff.get_decoded_account(ACCOUNT, new=False) is None + + +def test_persists_storage_changes(account_db): + account_db.set_storage(ACCOUNT, 1, 10) + save_block_diff(account_db, BLOCK_HASH) + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_accounts() == {ACCOUNT} + + assert diff.get_changed_slots(ACCOUNT) == {1} + assert diff.get_slot_change(ACCOUNT, 1) == (0, 10) + + +def test_persists_state_root(account_db): + """ + When the storage items change the account's storage root also changes and that change also + needs to be persisted. + """ + + # First, compute the expected new storage root + db = AtomicDB() + example_lookup = StorageLookup(db, BLANK_ROOT_HASH, ACCOUNT) + key = int_to_big_endian(1) + example_lookup[key] = int_to_big_endian(10) + expected_root = example_lookup.get_changed_root() + + # Next, make the same change to out storage + account_db.set_storage(ACCOUNT, 1, 10) + save_block_diff(account_db, BLOCK_HASH) + + # The new state root should have been included as part of the diff. + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_accounts() == {ACCOUNT} + new_account = diff.get_decoded_account(ACCOUNT, new=True) + assert new_account.storage_root == expected_root + + +@pytest.mark.skip('persist() should not be allowed if you will save a block diff') +def test_two_storage_changes(account_db): + account_db.set_storage(ACCOUNT, 1, 10) + account_db.persist() + + account_db.set_storage(ACCOUNT, 1, 20) + save_block_diff(account_db, BLOCK_HASH) + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_accounts() == {ACCOUNT} + + assert diff.get_changed_slots(ACCOUNT) == {1} + assert diff.get_slot_change(ACCOUNT, 1) == (10, 20) + + +def test_account_and_storage_change(account_db): + account_db.set_balance(ACCOUNT, 100) + account_db.set_storage(ACCOUNT, 1, 10) + + save_block_diff(account_db, BLOCK_HASH) + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_accounts() == {ACCOUNT} + + old_account = diff.get_decoded_account(ACCOUNT, new=False) + assert old_account is None + + new_account = diff.get_decoded_account(ACCOUNT, new=True) + assert new_account.storage_root != BLANK_ROOT_HASH + assert new_account.balance == 100 + + assert diff.get_changed_slots(ACCOUNT) == {1} + assert diff.get_slot_change(ACCOUNT, 1) == (0, 10) + + +@pytest.mark.skip('persist() should not be allowed if you will save a block diff') +def test_delete_account(account_db): + account_db.set_balance(ACCOUNT, 100) + account_db.persist() + + account_db.delete_account(ACCOUNT) + save_block_diff(account_db, BLOCK_HASH) + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_accounts() == {ACCOUNT} + old_account = diff.get_decoded_account(ACCOUNT, new=False) + new_account = diff.get_decoded_account(ACCOUNT, new=True) + + assert old_account.balance == 100 + assert new_account is None + + +def test_delete_storage(account_db): + """ + This is only called before a CREATE message is processed, and CREATE messages are not + allowed to overwrite non-empty storage tries (search for "collisions" in EIP 1014), so + this operation *should* always be a no-op. Here's a quick check to ensure block diff + handles it gracefully. + """ + + account_db.set_balance(ACCOUNT, 10) + account_db.delete_storage(ACCOUNT) + save_block_diff(account_db, BLOCK_HASH) + + diff = BlockDiff.from_db(account_db._raw_store_db, BLOCK_HASH) + assert diff.get_changed_slots(ACCOUNT) == set() + + new_account = diff.get_decoded_account(ACCOUNT, new=True) + assert new_account.balance == 10 diff --git a/tests/core/vm/test_interrupt.py b/tests/core/vm/test_interrupt.py index 4a777b994a..c737f4a299 100644 --- a/tests/core/vm/test_interrupt.py +++ b/tests/core/vm/test_interrupt.py @@ -104,8 +104,9 @@ def test_bytecode_missing_interrupt(chain, bytecode, bytecode_hash, address_with def test_account_missing_interrupt(chain, balance, address_with_balance, address_with_balance_hash): # confirm test setup - retrieved_balance = chain.get_vm().state.get_balance(address_with_balance) - assert retrieved_balance == balance + # commented out because checking the balance causes it to be cached + # retrieved_balance = chain.get_vm().state.get_balance(address_with_balance) + # assert retrieved_balance == balance expected_state_root = chain.get_vm().state.state_root # manually remove trie node with account from database @@ -125,8 +126,9 @@ def test_account_missing_interrupt(chain, balance, address_with_balance, address def test_storage_missing_interrupt(chain, address_with_storage, address_with_storage_hash): # confirm test setup test_slot = 42 - retrieved_storage_value = chain.get_vm().state.get_storage(address_with_storage, test_slot) - assert retrieved_storage_value == test_slot + # commented out because checking the balance causes it to be cached + # retrieved_storage_value = chain.get_vm().state.get_storage(address_with_storage, test_slot) + # assert retrieved_storage_value == test_slot expected_storage_root = chain.get_vm().state._account_db._get_storage_root(address_with_storage) expected_slot_hash = keccak(int_to_big_endian(test_slot).rjust(32, b'\0')) diff --git a/tests/core/vm/test_rewards.py b/tests/core/vm/test_rewards.py index 16234f723a..868d45cb98 100644 --- a/tests/core/vm/test_rewards.py +++ b/tests/core/vm/test_rewards.py @@ -22,6 +22,7 @@ constantinople_at, petersburg_at, genesis, + upgrade_to_turbo, ) @@ -47,6 +48,7 @@ def test_rewards(vm_fn, miner_1_balance, miner_2_balance): vm_fn(0), disable_pow_check(), genesis(), + upgrade_to_turbo, mine_block(), # 1 mine_block(), # 2 ) @@ -154,6 +156,7 @@ def test_rewards_uncle_created_at_different_generations( vm_fn(0), disable_pow_check(), genesis(), + upgrade_to_turbo, mine_blocks(TOTAL_BLOCKS_CANONICAL_CHAIN - 1), ) @@ -215,6 +218,7 @@ def test_uncle_block_inclusion_validity(vm_fn): vm_fn(0), disable_pow_check(), genesis(), + upgrade_to_turbo, mine_block(), # 1 mine_block(), # 2 ) @@ -265,6 +269,7 @@ def test_rewards_nephew_uncle_different_vm( vm_fn_nephew(VM_CHANGE_BLOCK_NUMBER), disable_pow_check(), genesis(), + upgrade_to_turbo, mine_blocks(TOTAL_BLOCKS_CANONICAL_CHAIN - 1), ) diff --git a/tests/core/vm/test_vm_state.py b/tests/core/vm/test_vm_state.py index 944a2d608d..18c071f6a7 100644 --- a/tests/core/vm/test_vm_state.py +++ b/tests/core/vm/test_vm_state.py @@ -28,6 +28,7 @@ def test_block_properties(chain_without_block_validation): assert vm.state.gas_limit == block.header.gas_limit +@pytest.mark.skip('State accepts a header, this test should build an invalid header') def test_missing_state_root(): context = None state = FrontierState(MemoryDB(), context, b'\x0f' * 32) diff --git a/tests/database/test_account_db.py b/tests/database/test_account_db.py index 975bddf164..f6d053c947 100644 --- a/tests/database/test_account_db.py +++ b/tests/database/test_account_db.py @@ -11,9 +11,16 @@ from eth.db.account import ( AccountDB, ) +from eth.db.schema import ( + set_schema, + Schemas, + SchemaTurbo, +) + from eth.constants import ( EMPTY_SHA3, + BLANK_ROOT_HASH, ) @@ -32,8 +39,18 @@ def account_db(base_db): return AccountDB(base_db) +def turbo_account_db(): + base_db = MemoryDB() + + set_schema(base_db, Schemas.TURBO) + base_db[SchemaTurbo.current_state_root_key] = BLANK_ROOT_HASH + + return AccountDB(base_db) + + @pytest.mark.parametrize("state", [ AccountDB(MemoryDB()), + turbo_account_db(), ]) def test_balance(state): assert state.get_balance(ADDRESS) == 0 @@ -52,6 +69,7 @@ def test_balance(state): @pytest.mark.parametrize("state", [ AccountDB(MemoryDB()), + turbo_account_db(), ]) def test_nonce(state): assert state.get_nonce(ADDRESS) == 0 @@ -76,6 +94,7 @@ def test_nonce(state): @pytest.mark.parametrize("state", [ AccountDB(MemoryDB()), + turbo_account_db(), ]) def test_code(state): assert state.get_code(ADDRESS) == b'' @@ -96,6 +115,7 @@ def test_code(state): @pytest.mark.parametrize("state", [ AccountDB(MemoryDB()), + turbo_account_db(), ]) def test_accounts(state): assert not state.account_exists(ADDRESS) diff --git a/tests/database/test_turbo.py b/tests/database/test_turbo.py new file mode 100644 index 0000000000..cde676e23d --- /dev/null +++ b/tests/database/test_turbo.py @@ -0,0 +1,50 @@ +"Test that we can read from non-canonical states by looking at block diffs" +import pytest + +from eth.db.turbo import TurboDatabase + +from tests.core.helpers import ( + new_transaction, +) + + +@pytest.fixture +def chain(chain_without_block_validation): + # make things a little less verbose + return chain_without_block_validation + + +def test_read_older_state(chain, funded_address, funded_address_private_key): + # Add two blocks to the chain. + # Check that the turbodb has the newest state + # Open a TurboDatabase from the older state and check that you get the correct result + base_db = chain.chaindb.db + ADDRESS = b'\x10' * 20 + + def make_transaction_to(destination_address, amount, parent_header=None): + return new_transaction( + chain.get_vm(parent_header), + funded_address, + destination_address, + data=b'', + private_key=funded_address_private_key, + amount=amount, + ) + + assert TurboDatabase._get_account(base_db, ADDRESS).balance == 0 + + first_transaction = make_transaction_to(ADDRESS, 1000) + first_new_block, _, _ = chain.build_block_with_transactions([first_transaction]) + first_imported_block, _, _ = chain.import_block(first_new_block) + assert TurboDatabase._get_account(base_db, ADDRESS).balance == 1000 + assert TurboDatabase._get_account(base_db, funded_address).nonce == 1 + + second_transaction = make_transaction_to(ADDRESS, 2000) + second_new_block, _, _ = chain.build_block_with_transactions([second_transaction]) + second_imported_block, _, _ = chain.import_block(second_new_block) + assert TurboDatabase._get_account(base_db, ADDRESS).balance == 3000 + assert TurboDatabase._get_account(base_db, funded_address).nonce == 2 + + turbo = TurboDatabase(chain.chaindb, first_imported_block.header) + assert turbo.get_account(ADDRESS).balance == 1000 + assert turbo.get_account(funded_address).nonce == 1 diff --git a/tests/database/test_turbo_traversal.py b/tests/database/test_turbo_traversal.py new file mode 100644 index 0000000000..57de6c455e --- /dev/null +++ b/tests/database/test_turbo_traversal.py @@ -0,0 +1,119 @@ +import random + +import pytest + +from eth_utils import to_tuple, keccak + +from eth.constants import ( + GENESIS_BLOCK_NUMBER, + GENESIS_DIFFICULTY, + GENESIS_GAS_LIMIT, +) +from eth.db.header import HeaderDB +from eth.db.turbo import TurboDatabase, find_header_path +from eth.rlp.headers import ( + BlockHeader, +) + + +@pytest.fixture +def headerdb(base_db): + return HeaderDB(base_db) + + +@pytest.fixture +def genesis_header(): + return BlockHeader( + difficulty=GENESIS_DIFFICULTY, + block_number=GENESIS_BLOCK_NUMBER, + gas_limit=GENESIS_GAS_LIMIT, + ) + + +# copied from tests/database/test_header_db, maybe this belongs in some util package +def make_header(previous_header): + return BlockHeader.from_parent( + parent=previous_header, + timestamp=previous_header.timestamp + 1, + gas_limit=previous_header.gas_limit, + difficulty=previous_header.difficulty, + extra_data=keccak(random.randint(0, 1e18)), + ) + + +@to_tuple +def mk_header_chain(base_header, length): + previous_header = base_header + for _ in range(length): + next_header = make_header(previous_header) + yield next_header + previous_header = next_header + + +def test_genesis_header_path(headerdb, genesis_header): + headerdb.persist_header(genesis_header) + + reverse, forward = find_header_path(headerdb, genesis_header, genesis_header) + assert reverse == () + assert forward == () + + +def test_same_chain_single_header_path(headerdb, genesis_header): + next_header = make_header(genesis_header) + headerdb.persist_header_chain((genesis_header, next_header)) + + reverse, forward = find_header_path(headerdb, genesis_header, next_header) + assert reverse == () + assert forward == (next_header,) + + reverse, forward = find_header_path(headerdb, next_header, genesis_header) + assert reverse == (next_header,) + assert forward == () + + +def test_small_fork(headerdb, genesis_header): + left_header = make_header(genesis_header) + right_header = make_header(genesis_header) + + for header in (genesis_header, left_header, right_header): + headerdb.persist_header(header) + + reverse, forward = find_header_path(headerdb, left_header, right_header) + assert reverse == (left_header,) + assert forward == (right_header,) + + reverse, forward = find_header_path(headerdb, right_header, left_header) + assert reverse == (right_header,) + assert forward == (left_header,) + + +def test_fork_one_side_longer(headerdb, genesis_header): + headerdb.persist_header(genesis_header) + + left_tip = genesis_header + for _ in range(5): + next_left = make_header(left_tip) + headerdb.persist_header(next_left) + left_tip = next_left + + right_tip = genesis_header + for _ in range(10): + next_right = make_header(right_tip) + headerdb.persist_header(next_right) + right_tip = next_right + + # TODO: Make this test more thorough by checking that every header matches. Some kind + # of property-based testing might be ideal! + reverse, forward = find_header_path(headerdb, left_tip, right_tip) + assert len(reverse) == 5 + assert len(forward) == 10 + + assert reverse[0] == left_tip + assert forward[-1] == right_tip + + reverse, forward = find_header_path(headerdb, right_tip, left_tip) + assert len(reverse) == 10 + assert len(forward) == 5 + + assert reverse[0] == right_tip + assert forward[-1] == left_tip diff --git a/tests/json-fixtures/test_blockchain.py b/tests/json-fixtures/test_blockchain.py index a6c02844d2..176ba6725e 100644 --- a/tests/json-fixtures/test_blockchain.py +++ b/tests/json-fixtures/test_blockchain.py @@ -7,6 +7,7 @@ ValidationError, ) +from eth.db.chain import ChainDB from eth.rlp.headers import ( BlockHeader, ) @@ -272,6 +273,9 @@ def fixture(fixture_data): def test_blockchain_fixtures(fixture_data, fixture): try: chain = new_chain_from_fixture(fixture) + + db = chain.chaindb.db + ChainDB.upgrade_to_turbo_schema(db) except ValueError as e: raise AssertionError("could not load chain for {}".format((fixture_data,))) from e