diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index 9fcaf5d0196..c736de64ff7 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -229,7 +229,7 @@ namespace epee case SERIALIZE_TYPE_OBJECT: return read_ae
(); case SERIALIZE_TYPE_ARRAY: return read_ae(); default: - CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << type); + CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << static_cast(type)); } return read_ae(); // unreachable, dummy return to avoid compiler warning } @@ -321,7 +321,7 @@ namespace epee case SERIALIZE_TYPE_OBJECT: return read_se
(); case SERIALIZE_TYPE_ARRAY: return read_se(); default: - CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << ent_type); + CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << static_cast(ent_type)); } return read_se(); // unreachable, dummy return to avoid compiler warning } diff --git a/tests/functional_tests/blockchain.py b/tests/functional_tests/blockchain.py index c1b610596d2..be5d8d0ffda 100755 --- a/tests/functional_tests/blockchain.py +++ b/tests/functional_tests/blockchain.py @@ -43,12 +43,14 @@ """ from framework.daemon import Daemon +from framework.wallet import Wallet class BlockchainTest(): def run_test(self): self.reset() self._test_generateblocks(5) self._test_alt_chains() + self.test_get_blocks_fast() def reset(self): print('Resetting blockchain') @@ -340,6 +342,81 @@ def _test_alt_chains(self): print('Saving blockchain explicitely') daemon.save_bc() + def test_get_blocks_fast(self): + print('Testing the /get_blocks.bin RPC endpoint') + + daemon = Daemon() + + n_blocks = daemon.get_height()['height'] + assert(n_blocks >= 3) + + genesis_block_id = daemon.getblockheaderbyheight(0)['block_header']['hash'] + + target_block_index = n_blocks-1 + target_block_id = daemon.getblockheaderbyheight(target_block_index)['block_header']['hash'] + + print('First, mining to wallet and creating 2 transactions in the top block') + + wallet = Wallet() + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + main_address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + + try: wallet.close_wallet() + except: pass + wallet.auto_refresh(enable = False) + wallet.restore_deterministic_wallet(seed) + assert wallet.get_transfers() == {} + assert wallet.get_address().address == main_address + + wallet.refresh() + res = wallet.get_transfers() + assert len(res['in']) > 0 + first_recv_height = res['in'][0]['height'] + + N_TO_MINE = 8 + max(0, first_recv_height + 60 - n_blocks) + daemon.generateblocks(main_address, N_TO_MINE) + wallet.refresh() + + dst = {'address': '8BQKgTSSqJjP14AKnZUBwnXWj46MuNmLvHfPTpmry52DbfNjjHVvHUk4mczU8nj8yZ57zBhksTJ8kM5xKeJXw55kCMVqyG7', 'amount': 1000000000000} + + res = wallet.get_transfers() + assert len(res['in']) >= N_TO_MINE + last_recv_height = max(x['height'] for x in res['in']) + assert last_recv_height == n_blocks + N_TO_MINE - 1, last_recv_height + assert wallet.get_balance().unlocked_balance > dst['amount'] * 2 + + res = wallet.transfer([dst]) + balance_1 = wallet.get_balance().unlocked_balance + res = wallet.transfer([dst]) + + assert len(daemon.get_transaction_pool_hashes()['tx_hashes']) == 2 + + daemon.generateblocks(main_address, 1) + + print('Calling /get_blocks.bin and testing response...') + + res = daemon.get_blocks_fast(0, [target_block_id, genesis_block_id]) + assert len(res.blocks) == N_TO_MINE + 1 + 1 + assert len(res.blocks) == len(res.output_indices) + + for i in range(len(res.blocks)): + is_last = i == len(res.blocks) - 1 + block = res.blocks[i] + assert block.pruned + block_indices = res.output_indices[i]['indices'] + n_txs = len(block['txs']) if 'txs' in block else 0 + if is_last: + assert n_txs == 2 + assert len(block_indices) == 3 + else: + assert n_txs == 0 + assert len(block_indices) == 1 + for tx_idx in range(n_txs): + tx = block.txs[tx_idx] + tx_indices = block_indices[1 + tx_idx]['indices'] + assert tx.prunable_hash != (b'\0' * 32) # non-null + assert len(tx_indices) == 2 + if __name__ == '__main__': BlockchainTest().run_test() diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 917e68c8e53..780111d0fb1 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -29,8 +29,17 @@ """Daemon class to make rpc calls and store state.""" +from . import epee_binary from .rpc import JSONRPC +class DaemonBinary: + def hash_list_to_blob(hash_list): + blob = bytes() + for h in hash_list: + assert len(h) == 64 + blob += bytes.fromhex(h) + return blob + class Daemon(object): def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0, restricted_rpc = False, username=None, password=None): @@ -179,6 +188,22 @@ def getblockheadersrange(self, start_height, end_height, fill_pow_hash = False, return self.rpc.send_json_rpc_request(getblockheadersrange) get_block_headers_range = getblockheadersrange + def get_blocks_fast(self, start_height, block_ids, requested_info = 0, + pool_info_since = 0, prune = True, no_miner_tx = False, max_block_count = 0): + get_blocks_fast = { + 'requested_info': requested_info, + 'block_ids': DaemonBinary.hash_list_to_blob(block_ids), + 'start_height': start_height, + 'prune': prune, + 'no_miner_tx': no_miner_tx, + 'pool_info_since': pool_info_since, + 'max_block_count': max_block_count + } + res = self.rpc.send_binary_request("/getblocks.bin", get_blocks_fast) + if 'top_block_hash' in res: + res['top_block_hash'] = res['top_block_hash'].hex() + return res + def get_connections(self, client = ""): get_connections = { 'client': client, diff --git a/utils/python-rpc/framework/epee_binary.py b/utils/python-rpc/framework/epee_binary.py new file mode 100644 index 00000000000..d4eeac4064b --- /dev/null +++ b/utils/python-rpc/framework/epee_binary.py @@ -0,0 +1,319 @@ +import io + +PORTABLE_STORAGE_SIGNATURE = bytes.fromhex('0111010101010201') # bender's nightmare +PORTABLE_STORAGE_FORMAT_VER = 1 + +PORTABLE_RAW_SIZE_MARK_MASK = 0x03 +PORTABLE_RAW_SIZE_MARK_BYTE = 0 +PORTABLE_RAW_SIZE_MARK_WORD = 1 +PORTABLE_RAW_SIZE_MARK_DWORD = 2 +PORTABLE_RAW_SIZE_MARK_INT64 = 3 + +SERIALIZE_TYPE_INT64 = 1 +SERIALIZE_TYPE_INT32 = 2 +SERIALIZE_TYPE_INT16 = 3 +SERIALIZE_TYPE_INT8 = 4 +SERIALIZE_TYPE_UINT64 = 5 +SERIALIZE_TYPE_UINT32 = 6 +SERIALIZE_TYPE_UINT16 = 7 +SERIALIZE_TYPE_UINT8 = 8 +SERIALIZE_TYPE_DOUBLE = 9 +SERIALIZE_TYPE_STRING = 10 +SERIALIZE_TYPE_BOOL = 11 +SERIALIZE_TYPE_OBJECT = 12 +#SERIALIZE_TYPE_ARRAY = 13 + +SERIALIZE_FLAG_ARRAY = 0x80 + +class Serializer: + def __init__(self, outf = None): + self.outf = outf if outf is not None else io.BytesIO() + + def serialize_stream(self, obj): + self.outf.write(PORTABLE_STORAGE_SIGNATURE) + self.outf.write(bytes([PORTABLE_STORAGE_FORMAT_VER])) + self.__serialize_section(obj, include_type=False) + + def serialize(self, obj): + assert(isinstance(self.outf, io.BytesIO)) + self.serialize_stream(obj) + return self.outf.getvalue() + + @classmethod + def __is_maybe_dict_like(cls, x): + return hasattr(x, 'keys') + + @classmethod + def __is_maybe_array_like(cls, x): + return hasattr(x, '__iter__') \ + and not hasattr(x, 'encode') \ + and not hasattr(x, 'decode') \ + and not cls.__is_maybe_dict_like(x) + + @classmethod + def __get_int_serialize_type(cls, signed, byte_size): + if byte_size == 8: + int_type = 1 + elif byte_size == 4: + int_type = 2 + elif byte_size == 2: + int_type = 3 + elif byte_size == 1: + int_type = 4 + else: + raise ValueError("Unrecognized serialized int byte size: " + str(byte_size)) + if not signed: + int_type += 4 + return int_type + + def __serialize_fixed_int(self, x, signed, byte_size, include_type): + int_type = self.__get_int_serialize_type(signed, byte_size) + + ones_mask = 2**(byte_size*8) - 1 + if signed: + max_val = ones_mask // 2 + min_val = -(max_val+1) + else: + max_val = ones_mask + min_val = 0 + + if not (min_val <= x <= max_val): + raise ValueError("Cannot serialize {}integer {} in {} bytes".format('un' if not signed else '', x, byte_size)) + + if signed and x < 0: + twos_comp = ones_mask+x+1 + else: + twos_comp = x + + if include_type: + self.outf.write(bytes([int_type])) + + int_bytes = twos_comp.to_bytes(byte_size, byteorder='little') + self.outf.write(int_bytes) + + @classmethod + def __get_auto_int_properties(cls, x): + signed = x < 0 + if signed: + x_lim = -2 * x + else: + x_lim = x + + byte_size = 1 + while byte_size <= 4: + if x_lim <= 2**(byte_size*8)-1: + break + byte_size *= 2 + + return signed, byte_size + + def __serialize_fixed_int_auto(self, x): + signed, byte_size = self.__get_auto_int_properties(x) + self.__serialize_fixed_int(x, signed, byte_size, include_type=True) + + def __serialize_varint(self, x): + if x < 0: + raise ValueError("Negative values cannot be serialized as a varint") + for size_marker in range(4): + byte_size = 1 << size_marker + max_val = 2**(byte_size*8-2)-1 + if x > max_val: + continue + x_serial = (x << 2) + size_marker + self.__serialize_fixed_int(x_serial, False, byte_size, False) + return + raise ValueError("Variant too large to be serialized") + + def __serialize_bool(self, x, include_type): + if include_type: + self.outf.write(bytes([SERIALIZE_TYPE_BOOL])) + self.outf.write(bytes([1 if x else 0])) + + def __serialize_float(self, x, include_type): + raise ValueError("Floating point numbers not supported yet") + + def __serialize_string(self, x, include_type): + if include_type: + self.outf.write(bytes([SERIALIZE_TYPE_STRING])) + self.__serialize_varint(len(x)) + if isinstance(x, str): + x = x.encode() + self.outf.write(x) + + def __dispatch_serialize_scalar(self, x, include_type, signed_override = None, byte_size_override = None): + if isinstance(x, bool): + self.__serialize_bool(x, include_type) + elif isinstance(x, int): + if signed_override is None: + self.__serialize_fixed_int_auto(x) + else: + self.__serialize_fixed_int(x, signed_override, byte_size_override, include_type) + elif isinstance(x, float): + self.__serialize_float(x, include_type) + elif isinstance(x, bytes) or isinstance(x, str): + self.__serialize_string(x, include_type) + elif self.__is_maybe_dict_like(x): + self.__serialize_section(x, include_type=True) + else: + raise ValueError("Cannot decide how to dispatch serialization for type {}".format(type(x))) + + def __serialize_section_key(self, x): + if isinstance(x, str): + x = x.encode() + if len(x) > 255: + raise ValueError("Object/section key name cannot be longer than 255 ASCII characters") + self.__serialize_fixed_int(len(x), False, 1, False) + self.outf.write(x) + + def __serialize_section_value(self, x): + if self.__is_maybe_array_like(x): + self.__serialize_array(x) + else: + self.__dispatch_serialize_scalar(x, include_type = True) + + def __serialize_section(self, x, include_type): + if include_type: + self.outf.write(bytes([SERIALIZE_TYPE_OBJECT])) + self.__serialize_varint(len(x)) + for key, value in x.items(): + self.__serialize_section_key(key) + self.__serialize_section_value(value) + + def __serialize_array(self, x): + # Cannot determine the serialization type of a dynamically typed empty array + # Thankfully, monero deserialization code allows coercing empty arrays to any type + if len(x) == 0: + EMPTY_U8_ARRAY_FLAG = SERIALIZE_FLAG_ARRAY | SERIALIZE_TYPE_UINT8 + self.__serialize_fixed_int(EMPTY_U8_ARRAY_FLAG, signed = False, byte_size = 1, include_type = False) #type + self.__serialize_varint(0) # length + + signed = False + byte_size = 1 + base_type = None + + # TODO: This doesn't assert that arrays are homogenous + + if isinstance[x[0], bool]: + base_type = SERIALIZE_TYPE_BOOL + elif isinstance(x[0], int): + # Get automatic int type for entire array + for elem in x: + s, b = self.__get_auto_int_properties(elem) + if s: + signed = True + byte_size = max(byte_size, b) + base_type = self.__get_int_serialize_type(signed, byte_size) + elif isinstance(x[0], float): + base_type = SERIALIZE_TYPE_DOUBLE + elif isinstance(x[0], bytes) or isinstance(x[0], str): + base_type = SERIALIZE_TYPE_STRING + elif self.__is_maybe_dict_like(x[0]): + base_type = SERIALIZE_TYPE_OBJECT + else: + raise ValueError("Cannot determine array element type for Python type {}".format(type(x[0]))) + + array_type = base_type | SERIALIZE_FLAG_ARRAY + self.__serialize_fixed_int(array_type, signed = False, byte_size = 1, include_type = False) #type + self.__serialize_varint(len(x)) # length + + for elem in x: + self.__dispatch_serialize_scalar(elem, include_type = False, + signed_override = signed, byte_size_override = byte_size) + +class Deserializer: + def __init__(self, inf): + if isinstance(inf, bytes): + self.inf = io.BytesIO(inf) + else: + self.inf = inf + + def deserialize(self): + assert(self.inf.read(len(PORTABLE_STORAGE_SIGNATURE)) == PORTABLE_STORAGE_SIGNATURE) + assert(self.inf.read(1)[0] == PORTABLE_STORAGE_FORMAT_VER) + return self.__deserialize_section() + + def __deserialize_int(self, int_type): + assert(int_type != 0) + assert(int_type <= SERIALIZE_TYPE_UINT8) + signed = int_type <= SERIALIZE_TYPE_INT8 + if int_type > SERIALIZE_TYPE_INT8: + int_type -= 4 + assert(int_type <= SERIALIZE_TYPE_INT8) + byte_size = 1 << (SERIALIZE_TYPE_INT8-int_type) + int_bytes = self.inf.read(byte_size) + assert(len(int_bytes) == byte_size) + val = int.from_bytes(int_bytes, 'little') + if signed: + ones_mask = (1 << (byte_size*8)) - 1 + return val - ones_mask - 1 + else: + return val + + def __deserialize_varint(self): + int_bytes = self.inf.read(1) + byte_size = 1 << (int_bytes[0] & PORTABLE_RAW_SIZE_MARK_MASK) + if byte_size > 1: + int_bytes += self.inf.read(byte_size - 1) + assert(len(int_bytes) == byte_size) + return int.from_bytes(int_bytes, 'little') >> 2 + + def __deserialize_float(self): + raise ValueError("Floating point numbers not supported yet") + + def __deserialize_string(self): + length = self.__deserialize_varint() + string = self.inf.read(length) + assert(len(string) == length) + return string + + def __deserialize_bool(self): + b = self.inf.read(1)[0] + return b != 0 + + def __deserialize_section_key(self): + key_length = self.__deserialize_int(SERIALIZE_TYPE_UINT8) + key_bytes = self.inf.read(key_length) + assert(len(key_bytes) == key_length) + key_string = key_bytes.decode() + assert(len(key_string) == key_length) + return key_string + + def __deserialize_section_value(self): + type_tag = self.__deserialize_int(SERIALIZE_TYPE_UINT8) + is_array = (type_tag & SERIALIZE_FLAG_ARRAY) != 0 + base_type = type_tag & ~(SERIALIZE_FLAG_ARRAY) + assert(base_type != 0) + assert(base_type <= SERIALIZE_TYPE_OBJECT) + if is_array: + array_length = self.__deserialize_varint() + val = [] + for _ in range(array_length): + val.append(self.__deserialize_of_type(base_type)) + return val + else: + return self.__deserialize_of_type(base_type) + + def __deserialize_section(self): + obj_length = self.__deserialize_varint() + obj = {} + for _ in range(obj_length): + key = self.__deserialize_section_key() + value = self.__deserialize_section_value() + obj[key] = value + return obj + + def __deserialize_of_type(self, serialize_type): + assert(serialize_type > 0) + assert(serialize_type <= SERIALIZE_TYPE_OBJECT) + if serialize_type <= SERIALIZE_TYPE_UINT8: + return self.__deserialize_int(serialize_type) + elif serialize_type == SERIALIZE_TYPE_DOUBLE: + return self.__deserialize_float() + elif serialize_type == SERIALIZE_TYPE_STRING: + return self.__deserialize_string() + elif serialize_type == SERIALIZE_TYPE_BOOL: + return self.__deserialize_bool() + elif serialize_type == SERIALIZE_TYPE_OBJECT: + return self.__deserialize_section() + else: + raise ValueError("Unrecognized serialize type: " + str(serialize_type)) diff --git a/utils/python-rpc/framework/rpc.py b/utils/python-rpc/framework/rpc.py index f476b93bf3d..b9f35bb68a2 100644 --- a/utils/python-rpc/framework/rpc.py +++ b/utils/python-rpc/framework/rpc.py @@ -31,6 +31,8 @@ from requests.auth import HTTPDigestAuth import json +from . import epee_binary + class Response(dict): def __init__(self, d): for k, v in d.items(): @@ -74,5 +76,20 @@ def send_request(self, path, inputs, result_field = None): def send_json_rpc_request(self, inputs): return self.send_request("/json_rpc", inputs, 'result') + def send_binary_request(self, path, inputs): + binary_req = epee_binary.Serializer().serialize(inputs) + + res = requests.post( + self.url + path, + data=binary_req, + headers={'content-type': 'application/octet-stream'}, + auth=HTTPDigestAuth(self.username, self.password) if self.username is not None else None, + stream=True) + assert res.status_code == 200, res.status_code + res = epee_binary.Deserializer(res.raw).deserialize() + + assert res['status'] == b'OK', res['status'] + + return Response(res)