Merge pull request nbd-wtf#12 from nbd-wtf/master

update miner so we can get to descriptor wallets.

Author: rsafier

miner

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2016-2021 The Bitcoin Core developers
+# Copyright (c) 2016-2022 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Encode and decode Bitcoin addresses.
@@ -20,8 +20,17 @@
     sha256,
     taproot_construct,
 )
-from .segwit_addr import encode_segwit_address
 from .util import assert_equal
+from test_framework.script_util import (
+    keyhash_to_p2pkh_script,
+    program_to_witness_script,
+    scripthash_to_p2sh_script,
+)
+from test_framework.segwit_addr import (
+    decode_segwit_address,
+    encode_segwit_address,
+)
+
 
 ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj'
 ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97'
@@ -35,22 +44,23 @@ class AddressType(enum.Enum):
     legacy = 'legacy'  # P2PKH
 
 
-chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
 
-def create_deterministic_address_bcrt1_p2tr_op_true():
+def create_deterministic_address_bcrt1_p2tr_op_true(explicit_internal_key=None):
     """
     Generates a deterministic bech32m address (segwit v1 output) that
     can be spent with a witness stack of OP_TRUE and the control block
     with internal public key (script-path spending).
 
-    Returns a tuple with the generated address and the internal key.
+    Returns a tuple with the generated address and the TaprootInfo object.
     """
-    internal_key = (1).to_bytes(32, 'big')
-    scriptPubKey = taproot_construct(internal_key, [(None, CScript([OP_TRUE]))]).scriptPubKey
-    address = encode_segwit_address("bcrt", 1, scriptPubKey[2:])
-    assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka')
-    return (address, internal_key)
+    internal_key = explicit_internal_key or (1).to_bytes(32, 'big')
+    taproot_info = taproot_construct(internal_key, [("only-path", CScript([OP_TRUE]))])
+    address = output_key_to_p2tr(taproot_info.output_pubkey)
+    if explicit_internal_key is None:
+        assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka')
+    return (address, taproot_info)
 
 
 def byte_to_base58(b, version):
@@ -59,10 +69,10 @@ def byte_to_base58(b, version):
     b += hash256(b)[:4]       # append checksum
     value = int.from_bytes(b, 'big')
     while value > 0:
-        result = chars[value % 58] + result
+        result = b58chars[value % 58] + result
         value //= 58
     while b[0] == 0:
-        result = chars[0] + result
+        result = b58chars[0] + result
         b = b[1:]
     return result
 
@@ -76,23 +86,23 @@ def base58_to_byte(s):
     n = 0
     for c in s:
         n *= 58
-        assert c in chars
-        digit = chars.index(c)
+        assert c in b58chars
+        digit = b58chars.index(c)
         n += digit
     h = '%x' % n
     if len(h) % 2:
         h = '0' + h
     res = n.to_bytes((n.bit_length() + 7) // 8, 'big')
     pad = 0
     for c in s:
-        if c == chars[0]:
+        if c == b58chars[0]:
             pad += 1
         else:
             break
     res = b'\x00' * pad + res
 
-    # Assert if the checksum is invalid
-    assert_equal(hash256(res[:-4])[:4], res[-4:])
+    if hash256(res[:-4])[:4] != res[-4:]:
+        raise ValueError('Invalid Base58Check checksum')
 
     return res[1:-4], int(res[0])
 
@@ -141,6 +151,13 @@ def script_to_p2sh_p2wsh(script, main=False):
     p2shscript = CScript([OP_0, sha256(script)])
     return script_to_p2sh(p2shscript, main)
 
+def output_key_to_p2tr(key, main=False):
+    assert len(key) == 32
+    return program_to_witness(1, key, main)
+
+def p2a(main=False):
+    return program_to_witness(1, "4e73", main)
+
 def check_key(key):
     if (type(key) is str):
         key = bytes.fromhex(key)  # Assuming this is hex string
@@ -156,6 +173,31 @@ def check_script(script):
     assert False
 
 
+def bech32_to_bytes(address):
+    hrp = address.split('1')[0]
+    if hrp not in ['bc', 'tb', 'bcrt']:
+        return (None, None)
+    version, payload = decode_segwit_address(hrp, address)
+    if version is None:
+        return (None, None)
+    return version, bytearray(payload)
+
+
+def address_to_scriptpubkey(address):
+    """Converts a given address to the corresponding output script (scriptPubKey)."""
+    version, payload = bech32_to_bytes(address)
+    if version is not None:
+        return program_to_witness_script(version, payload) # testnet segwit scriptpubkey
+    payload, version = base58_to_byte(address)
+    if version == 111:  # testnet pubkey hash
+        return keyhash_to_p2pkh_script(payload)
+    elif version == 196:  # testnet script hash
+        return scripthash_to_p2sh_script(payload)
+    # TODO: also support other address formats
+    else:
+        assert False
+
+
 class TestFrameworkScript(unittest.TestCase):
     def test_base58encodedecode(self):
         def check_base58(data, version):
@@ -173,3 +215,18 @@ def check_base58(data, version):
         check_base58(bytes.fromhex('0041c1eaf111802559bad61b60d62b1f897c63928a'), 0)
         check_base58(bytes.fromhex('000041c1eaf111802559bad61b60d62b1f897c63928a'), 0)
         check_base58(bytes.fromhex('00000041c1eaf111802559bad61b60d62b1f897c63928a'), 0)
+
+
+    def test_bech32_decode(self):
+        def check_bech32_decode(payload, version):
+            hrp = "tb"
+            self.assertEqual(bech32_to_bytes(encode_segwit_address(hrp, version, payload)), (version, payload))
+
+        check_bech32_decode(bytes.fromhex('36e3e2a33f328de12e4b43c515a75fba2632ecc3'), 0)
+        check_bech32_decode(bytes.fromhex('823e9790fc1d1782321140d4f4aa61aabd5e045b'), 0)
+        check_bech32_decode(bytes.fromhex('79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798'), 1)
+        check_bech32_decode(bytes.fromhex('39cf8ebd95134f431c39db0220770bd127f5dd3cc103c988b7dcd577ae34e354'), 1)
+        check_bech32_decode(bytes.fromhex('708244006d27c757f6f1fc6f853b6ec26268b727866f7ce632886e34eb5839a3'), 1)
+        check_bech32_decode(bytes.fromhex('616211ab00dffe0adcb6ce258d6d3fd8cbd901e2'), 0)
+        check_bech32_decode(bytes.fromhex('b6a7c98b482d7fb21c9fa8e65692a0890410ff22'), 0)
+        check_bech32_decode(bytes.fromhex('f0c2109cb1008cfa7b5a09cc56f7267cd8e50929'), 0)

miner_imports/test_framework/address.py

@@ -26,7 +26,7 @@
 
 - HTTP connections persist for the life of the AuthServiceProxy object
   (if server supports HTTP/1.1)
-- sends protocol 'version', per JSON-RPC 1.1
+- sends "jsonrpc":"2.0", per JSON-RPC 2.0
 - sends proper, incrementing 'id'
 - sends Basic HTTP authentication headers
 - parses all JSON numbers that look like floats as Decimal
@@ -39,7 +39,7 @@
 import http.client
 import json
 import logging
-import os
+import pathlib
 import socket
 import time
 import urllib.parse
@@ -60,9 +60,11 @@ def __init__(self, rpc_error, http_status=None):
         self.http_status = http_status
 
 
-def EncodeDecimal(o):
+def serialization_fallback(o):
     if isinstance(o, decimal.Decimal):
         return str(o)
+    if isinstance(o, pathlib.Path):
+        return str(o)
     raise TypeError(repr(o) + " is not JSON serializable")
 
 class AuthServiceProxy():
@@ -78,7 +80,10 @@ def __init__(self, service_url, service_name=None, timeout=HTTP_TIMEOUT, connect
         passwd = None if self.__url.password is None else self.__url.password.encode('utf8')
         authpair = user + b':' + passwd
         self.__auth_header = b'Basic ' + base64.b64encode(authpair)
-        self.timeout = timeout
+        # clamp the socket timeout, since larger values can cause an
+        # "Invalid argument" exception in Python's HTTP(S) client
+        # library on some operating systems (e.g. OpenBSD, FreeBSD)
+        self.timeout = min(timeout, 2147483)
         self._set_conn(connection)
 
     def __getattr__(self, name):
@@ -91,73 +96,71 @@ def __getattr__(self, name):
 
     def _request(self, method, path, postdata):
         '''
-        Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout).
-        This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5.
+        Do a HTTP request.
         '''
         headers = {'Host': self.__url.hostname,
                    'User-Agent': USER_AGENT,
                    'Authorization': self.__auth_header,
                    'Content-type': 'application/json'}
-        if os.name == 'nt':
-            # Windows somehow does not like to re-use connections
-            # TODO: Find out why the connection would disconnect occasionally and make it reusable on Windows
-            # Avoid "ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine"
-            self._set_conn()
-        try:
-            self.__conn.request(method, path, postdata, headers)
-            return self._get_response()
-        except (BrokenPipeError, ConnectionResetError):
-            # Python 3.5+ raises BrokenPipeError when the connection was reset
-            # ConnectionResetError happens on FreeBSD
-            self.__conn.close()
-            self.__conn.request(method, path, postdata, headers)
-            return self._get_response()
-        except OSError as e:
-            # Workaround for a bug on macOS. See https://bugs.python.org/issue33450
-            retry = '[Errno 41] Protocol wrong type for socket' in str(e)
-            if retry:
-                self.__conn.close()
-                self.__conn.request(method, path, postdata, headers)
-                return self._get_response()
-            else:
-                raise
+        self.__conn.request(method, path, postdata, headers)
+        return self._get_response()
+
+    def _json_dumps(self, obj):
+        return json.dumps(obj, default=serialization_fallback, ensure_ascii=self.ensure_ascii)
 
     def get_request(self, *args, **argsn):
         AuthServiceProxy.__id_count += 1
 
-        log.debug("-{}-> {} {}".format(
+        log.debug("-{}-> {} {} {}".format(
             AuthServiceProxy.__id_count,
             self._service_name,
-            json.dumps(args or argsn, default=EncodeDecimal, ensure_ascii=self.ensure_ascii),
+            self._json_dumps(args),
+            self._json_dumps(argsn),
         ))
+
         if args and argsn:
-            raise ValueError('Cannot handle both named and positional arguments')
-        return {'version': '1.1',
+            params = dict(args=args, **argsn)
+        else:
+            params = args or argsn
+        return {'jsonrpc': '2.0',
                 'method': self._service_name,
-                'params': args or argsn,
+                'params': params,
                 'id': AuthServiceProxy.__id_count}
 
     def __call__(self, *args, **argsn):
-        postdata = json.dumps(self.get_request(*args, **argsn), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
+        postdata = self._json_dumps(self.get_request(*args, **argsn))
         response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
-        if response['error'] is not None:
-            raise JSONRPCException(response['error'], status)
-        elif 'result' not in response:
-            raise JSONRPCException({
-                'code': -343, 'message': 'missing JSON-RPC result'}, status)
-        elif status != HTTPStatus.OK:
-            raise JSONRPCException({
-                'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
+        # For backwards compatibility tests, accept JSON RPC 1.1 responses
+        if 'jsonrpc' not in response:
+            if response['error'] is not None:
+                raise JSONRPCException(response['error'], status)
+            elif 'result' not in response:
+                raise JSONRPCException({
+                    'code': -343, 'message': 'missing JSON-RPC result'}, status)
+            elif status != HTTPStatus.OK:
+                raise JSONRPCException({
+                    'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
+            else:
+                return response['result']
         else:
+            assert response['jsonrpc'] == '2.0'
+            if status != HTTPStatus.OK:
+                raise JSONRPCException({
+                    'code': -342, 'message': 'non-200 HTTP status code'}, status)
+            if 'error' in response:
+                raise JSONRPCException(response['error'], status)
+            elif 'result' not in response:
+                raise JSONRPCException({
+                    'code': -343, 'message': 'missing JSON-RPC 2.0 result and error'}, status)
             return response['result']
 
     def batch(self, rpc_call_list):
-        postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal, ensure_ascii=self.ensure_ascii)
+        postdata = self._json_dumps(list(rpc_call_list))
         log.debug("--> " + postdata)
         response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
         if status != HTTPStatus.OK:
             raise JSONRPCException({
-                'code': -342, 'message': 'non-200 HTTP status code but no JSON-RPC error'}, status)
+                'code': -342, 'message': 'non-200 HTTP status code'}, status)
         return response
 
     def _get_response(self):
@@ -175,17 +178,31 @@ def _get_response(self):
             raise JSONRPCException({
                 'code': -342, 'message': 'missing HTTP response from server'})
 
+        # Check for no-content HTTP status code, which can be returned when an
+        # RPC client requests a JSON-RPC 2.0 "notification" with no response.
+        # Currently this is only possible if clients call the _request() method
+        # directly to send a raw request.
+        if http_response.status == HTTPStatus.NO_CONTENT:
+            if len(http_response.read()) != 0:
+                raise JSONRPCException({'code': -342, 'message': 'Content received with NO CONTENT status code'})
+            return None, http_response.status
+
         content_type = http_response.getheader('Content-Type')
         if content_type != 'application/json':
             raise JSONRPCException(
                 {'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)},
                 http_response.status)
 
-        responsedata = http_response.read().decode('utf8')
+        data = http_response.read()
+        try:
+            responsedata = data.decode('utf8')
+        except UnicodeDecodeError as e:
+            raise JSONRPCException({
+                'code': -342, 'message': f'Cannot decode response in utf8 format, content: {data}, exception: {e}'})
         response = json.loads(responsedata, parse_float=decimal.Decimal)
         elapsed = time.time() - req_start_time
         if "error" in response and response["error"] is None:
-            log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, json.dumps(response["result"], default=EncodeDecimal, ensure_ascii=self.ensure_ascii)))
+            log.debug("<-%s- [%.6f] %s" % (response["id"], elapsed, self._json_dumps(response["result"])))
         else:
             log.debug("<-- [%.6f] %s" % (elapsed, responsedata))
         return response, http_response.status