Merge branch 'main' into wallentx/chia-peer-command

Author: jack60612

.github/workflows/test-install-scripts.yml

@@ -108,14 +108,6 @@ jobs:
           type: ubuntu
           # https://packages.ubuntu.com/focal/python3 (20.04, 3.8)
           url: "docker://ubuntu:focal"
-        - name: ubuntu:hirsute (21.04)
-          type: ubuntu
-          # https://packages.ubuntu.com/hirsute/python3 (21.04, 3.9)
-          url: "docker://ubuntu:hirsute"
-        - name: ubuntu:impish (21.10)
-          type: ubuntu
-          # https://packages.ubuntu.com/impish/python3 (21.10, 3.9)
-          url: "docker://ubuntu:impish"
         - name: ubuntu:jammy (22.04)
           type: ubuntu
           # https://packages.ubuntu.com/jammy/python3 (22.04, 3.10)

chia/cmds/configure.py

@@ -101,6 +101,12 @@ def configure(
                 bootstrap_peers = ["testnet10-node.chia.net"]
                 testnet = "testnet10"
                 config["full_node"]["port"] = int(testnet_port)
+                if config["full_node"]["introducer_peer"] is None:
+                    config["full_node"]["introducer_peer"] = {}
+                assert config["full_node"]["introducer_peer"] is not None  # mypy
+                if config["wallet"]["introducer_peer"] is None:
+                    config["wallet"]["introducer_peer"] = {}
+                assert config["wallet"]["introducer_peer"] is not None  # mypy
                 config["full_node"]["introducer_peer"]["port"] = int(testnet_port)
                 config["farmer"]["full_node_peer"]["port"] = int(testnet_port)
                 config["timelord"]["full_node_peer"]["port"] = int(testnet_port)

chia/rpc/full_node_rpc_api.py

@@ -1,10 +1,14 @@
 from typing import Any, Dict, List, Optional
 
+from clvm.casts import int_from_bytes
+
 from chia.consensus.block_record import BlockRecord
 from chia.consensus.pos_quality import UI_ACTUAL_SPACE_CONSTANT_FACTOR
 from chia.full_node.full_node import FullNode
+from chia.full_node.generator import setup_generator_args
 from chia.full_node.mempool_check_conditions import get_puzzle_and_solution_for_coin
 from chia.rpc.rpc_server import Endpoint, EndpointResult
+from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.program import Program, SerializedProgram
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.coin_record import CoinRecord
@@ -18,6 +22,7 @@
 from chia.util.ints import uint32, uint64, uint128
 from chia.util.log_exceptions import log_exceptions
 from chia.util.ws_message import WsRpcMessage, create_payload_dict
+from chia.wallet.puzzles.decompress_block_spends import DECOMPRESS_BLOCK_SPENDS
 
 
 def coin_record_dict_backwards_compat(coin_record: Dict[str, Any]):
@@ -41,6 +46,7 @@ def get_routes(self) -> Dict[str, Endpoint]:
             "/get_block_record_by_height": self.get_block_record_by_height,
             "/get_block_record": self.get_block_record,
             "/get_block_records": self.get_block_records,
+            "/get_block_spends": self.get_block_spends,
             "/get_unfinished_block_headers": self.get_unfinished_block_headers,
             "/get_network_space": self.get_network_space,
             "/get_additions_and_removals": self.get_additions_and_removals,
@@ -399,6 +405,32 @@ async def get_block_records(self, request: Dict) -> EndpointResult:
             records.append(record)
         return {"block_records": records}
 
+    async def get_block_spends(self, request: Dict) -> EndpointResult:
+        if "header_hash" not in request:
+            raise ValueError("No header_hash in request")
+        header_hash = bytes32.from_hexstr(request["header_hash"])
+        full_block: Optional[FullBlock] = await self.service.block_store.get_full_block(header_hash)
+        if full_block is None or full_block.transactions_generator is None:
+            raise ValueError(f"Block {header_hash.hex()} not found or invalid block generator")
+
+        spends: List[CoinSpend] = []
+        block_generator = await self.service.blockchain.get_block_generator(full_block)
+        if block_generator is None:
+            return {"block_spends": spends}
+
+        block_program, block_program_args = setup_generator_args(block_generator)
+        _, coin_spends = DECOMPRESS_BLOCK_SPENDS.run_with_cost(
+            self.service.constants.MAX_BLOCK_COST_CLVM, block_program, block_program_args
+        )
+
+        for spend in coin_spends.as_iter():
+            parent, puzzle, amount, solution = spend.as_iter()
+            puzzle_hash = puzzle.get_tree_hash()
+            coin = Coin(parent.atom, puzzle_hash, int_from_bytes(amount.atom))
+            spends.append(CoinSpend(coin, puzzle, solution))
+
+        return {"block_spends": spends}
+
     async def get_block_record_by_height(self, request: Dict) -> EndpointResult:
         if "height" not in request:
             raise ValueError("No height in request")

chia/rpc/full_node_rpc_client.py

@@ -200,6 +200,16 @@ async def get_block_records(self, start: int, end: int) -> List:
         # TODO: return block records
         return response["block_records"]
 
+    async def get_block_spends(self, header_hash: bytes32) -> Optional[List[CoinSpend]]:
+        try:
+            response = await self.fetch("get_block_spends", {"header_hash": header_hash.hex()})
+            block_spends = []
+            for block_spend in response["block_spends"]:
+                block_spends.append(CoinSpend.from_json_dict(block_spend))
+            return block_spends
+        except Exception:
+            return None
+
     async def push_tx(self, spend_bundle: SpendBundle):
         return await self.fetch("push_tx", {"spend_bundle": spend_bundle.to_json_dict()})
 

chia/server/start_wallet.py

@@ -90,7 +90,7 @@ async def async_main() -> int:
     # This is simulator
     local_test = service_config["testing"]
     if local_test is True:
-        from tests.block_tools import test_constants
+        from chia.simulator.block_tools import test_constants
 
         constants = test_constants
         current = service_config["database_path"]

tests/block_tools.py renamed to chia/simulator/block_tools.py

@@ -1,7 +1,7 @@
 import asyncio
-import math
 import copy
 import logging
+import math
 import os
 import random
 import shutil
@@ -12,28 +12,18 @@
 from argparse import Namespace
 from dataclasses import replace
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Tuple, Any, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 from blspy import AugSchemeMPL, G1Element, G2Element, PrivateKey
+from chia_rs import compute_merkle_set_root
 from chiabip158 import PyBIP158
 
 from chia.cmds.init_funcs import create_all_ssl, create_default_chia_config
-from chia.daemon.keychain_proxy import connect_to_keychain_and_validate, wrap_local_keychain, KeychainProxy
-from chia.full_node.bundle_tools import (
-    best_solution_generator_from_template,
-    detect_potential_template_generator,
-    simple_solution_generator,
-)
-from chia.util.errors import Err
-from chia.full_node.generator import setup_generator_args
-from chia.full_node.mempool_check_conditions import GENERATOR_MOD
-from chia.plotting.create_plots import create_plots, PlotKeys
-from chia.plotting.util import add_plot_directory
 from chia.consensus.block_creation import unfinished_block_to_full_block
 from chia.consensus.block_record import BlockRecord
 from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward
 from chia.consensus.blockchain_interface import BlockchainInterface
-from chia.consensus.coinbase import create_puzzlehash_for_pk, create_farmer_coin, create_pool_coin
+from chia.consensus.coinbase import create_farmer_coin, create_pool_coin, create_puzzlehash_for_pk
 from chia.consensus.condition_costs import ConditionCost
 from chia.consensus.constants import ConsensusConstants
 from chia.consensus.default_constants import DEFAULT_CONSTANTS
@@ -48,10 +38,29 @@
     is_overflow_block,
 )
 from chia.consensus.vdf_info_computation import get_signage_point_vdf_info
+from chia.daemon.keychain_proxy import KeychainProxy, connect_to_keychain_and_validate, wrap_local_keychain
+from chia.full_node.bundle_tools import (
+    best_solution_generator_from_template,
+    detect_potential_template_generator,
+    simple_solution_generator,
+)
+from chia.full_node.generator import setup_generator_args
+from chia.full_node.mempool_check_conditions import GENERATOR_MOD
 from chia.full_node.signage_point import SignagePoint
-from chia.plotting.util import PlotsRefreshParameter, PlotRefreshResult, PlotRefreshEvents, parse_plot_info
+from chia.plotting.create_plots import PlotKeys, create_plots
 from chia.plotting.manager import PlotManager
+from chia.plotting.util import (
+    PlotRefreshEvents,
+    PlotRefreshResult,
+    PlotsRefreshParameter,
+    add_plot_directory,
+    parse_plot_info,
+)
 from chia.server.server import ssl_context_for_client
+from chia.simulator.socket import find_available_listen_port
+from chia.simulator.ssl_certs import get_next_nodes_certs_and_keys, get_next_private_ca_cert_and_key
+from chia.simulator.time_out_assert import time_out_assert_custom_interval
+from chia.simulator.wallet_tools import WalletTool
 from chia.types.blockchain_format.classgroup import ClassgroupElement
 from chia.types.blockchain_format.coin import Coin, hash_coin_ids
 from chia.types.blockchain_format.foliage import Foliage, FoliageBlockData, FoliageTransactionBlock, TransactionsInfo
@@ -76,24 +85,20 @@
 from chia.types.unfinished_block import UnfinishedBlock
 from chia.util.bech32m import encode_puzzle_hash
 from chia.util.block_cache import BlockCache
-from chia.util.config import load_config, lock_config, save_config, override_config
+from chia.util.config import load_config, lock_config, override_config, save_config
 from chia.util.default_root import DEFAULT_ROOT_PATH
+from chia.util.errors import Err
 from chia.util.hash import std_hash
 from chia.util.ints import uint8, uint16, uint32, uint64, uint128
 from chia.util.keychain import Keychain, bytes_to_mnemonic
 from chia.util.prev_transaction_block import get_prev_transaction_block
 from chia.util.vdf_prover import get_vdf_info_and_proof
-from tests.time_out_assert import time_out_assert_custom_interval
-from tests.wallet_tools import WalletTool
-from tests.util.socket import find_available_listen_port
-from tests.util.ssl_certs import get_next_nodes_certs_and_keys, get_next_private_ca_cert_and_key
 from chia.wallet.derive_keys import (
     master_sk_to_farmer_sk,
     master_sk_to_local_sk,
     master_sk_to_pool_sk,
     master_sk_to_wallet_sk,
 )
-from chia_rs import compute_merkle_set_root
 
 test_constants = DEFAULT_CONSTANTS.replace(
     **{
@@ -215,7 +220,7 @@ def test_callback(event: PlotRefreshEvents, update_result: PlotRefreshResult):
                 self.total_result.loaded += update_result.loaded
                 self.total_result.processed += update_result.processed
                 self.total_result.duration += update_result.duration
-                assert update_result.remaining == len(self.expected_plots) - self.total_result.processed
+                assert update_result.remaining >= len(self.expected_plots) - self.total_result.processed
                 assert len(update_result.loaded) <= self.plot_manager.refresh_parameter.batch_size
 
             if event == PlotRefreshEvents.done:
@@ -269,7 +274,10 @@ async def setup_keys(self, fingerprint: Optional[int] = None, reward_ph: Optiona
         else:
             self.farmer_ph = reward_ph
             self.pool_ph = reward_ph
-        self.all_sks: List[PrivateKey] = [sk for sk, _ in await self.keychain_proxy.get_all_private_keys()]
+        if self.automated_testing:
+            self.all_sks: List[PrivateKey] = [sk for sk, _ in await self.keychain_proxy.get_all_private_keys()]
+        else:
+            self.all_sks = [self.farmer_master_sk]  # we only want to include plots under the same fingerprint
         self.pool_pubkeys: List[G1Element] = [master_sk_to_pool_sk(sk).get_g1() for sk in self.all_sks]
 
         self.farmer_pubkeys: List[G1Element] = [master_sk_to_farmer_sk(sk).get_g1() for sk in self.all_sks]

chia/simulator/full_node_simulator.py

@@ -7,13 +7,13 @@
 from chia.full_node.full_node import FullNode
 from chia.full_node.full_node_api import FullNodeAPI
 from chia.protocols.full_node_protocol import RespondBlock
+from chia.simulator.block_tools import BlockTools
 from chia.simulator.simulator_protocol import FarmNewBlockProtocol, ReorgProtocol
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.full_block import FullBlock
 from chia.util.api_decorators import api_request
 from chia.util.config import lock_and_load_config, save_config
 from chia.util.ints import uint8
-from tests.block_tools import BlockTools
 
 
 class FullNodeSimulator(FullNodeAPI):

chia/simulator/simulator_constants.py

@@ -1,6 +1,6 @@
 if __name__ == "__main__":
     from chia.util.default_root import DEFAULT_ROOT_PATH
-    from tests.block_tools import create_block_tools, test_constants
+    from chia.simulator.block_tools import create_block_tools, test_constants
     from tests.util.keyring import TempKeyring
 
     with TempKeyring() as keychain:

tests/util/socket.py renamed to chia/simulator/socket.py

@@ -1,13 +1,13 @@
 import itertools
 from typing import Dict, List, Tuple
 
-from tests.util.ssl_certs_1 import SSL_TEST_NODE_CERTS_AND_KEYS_1, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_1
-from tests.util.ssl_certs_2 import SSL_TEST_NODE_CERTS_AND_KEYS_2, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_2
-from tests.util.ssl_certs_3 import SSL_TEST_NODE_CERTS_AND_KEYS_3, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_3
-from tests.util.ssl_certs_4 import SSL_TEST_NODE_CERTS_AND_KEYS_4, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_4
-from tests.util.ssl_certs_5 import SSL_TEST_NODE_CERTS_AND_KEYS_5, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_5
-from tests.util.ssl_certs_6 import SSL_TEST_NODE_CERTS_AND_KEYS_6, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_6
-from tests.util.ssl_certs_7 import SSL_TEST_NODE_CERTS_AND_KEYS_7, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_7
+from chia.simulator.ssl_certs_1 import SSL_TEST_NODE_CERTS_AND_KEYS_1, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_1
+from chia.simulator.ssl_certs_2 import SSL_TEST_NODE_CERTS_AND_KEYS_2, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_2
+from chia.simulator.ssl_certs_3 import SSL_TEST_NODE_CERTS_AND_KEYS_3, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_3
+from chia.simulator.ssl_certs_4 import SSL_TEST_NODE_CERTS_AND_KEYS_4, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_4
+from chia.simulator.ssl_certs_5 import SSL_TEST_NODE_CERTS_AND_KEYS_5, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_5
+from chia.simulator.ssl_certs_6 import SSL_TEST_NODE_CERTS_AND_KEYS_6, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_6
+from chia.simulator.ssl_certs_7 import SSL_TEST_NODE_CERTS_AND_KEYS_7, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_7
 
 # ---------------------------------------------------------------------------
 # Private CA certs/keys

tests/util/ssl_certs.py renamed to chia/simulator/ssl_certs.py

@@ -12,14 +12,16 @@
 from chia.util.config import load_config_cli, override_config
 from chia.util.default_root import DEFAULT_ROOT_PATH
 from chia.util.path import path_from_root
-from tests.block_tools import BlockTools, test_constants
+from chia.simulator.block_tools import BlockTools, test_constants
 from chia.util.ints import uint16
 from chia.simulator.full_node_simulator import FullNodeSimulator
 
 # See: https://bugs.python.org/issue29288
 "".encode("idna")
 
 SERVICE_NAME = "full_node"
+PLOTS = 3  # 3 plots should be enough
+PLOT_SIZE = 19  # anything under k19 is a bit buggy
 
 
 def create_full_node_simulator_service(
@@ -66,8 +68,6 @@ async def async_main(test_mode: bool = False, root_path: Path = DEFAULT_ROOT_PAT
     fingerprint: Optional[int] = None
     farming_puzzle_hash: Optional[bytes32] = None
     plot_dir: str = "simulator-plots"
-    plots = 3  # 3 plots should be enough
-    plot_size = 19  # anything under k19 is a bit buggy
     if "simulator" in config:
         overrides = {}
         plot_dir = config["simulator"].get("plot_directory", "simulator-plots")
@@ -93,7 +93,7 @@ async def async_main(test_mode: bool = False, root_path: Path = DEFAULT_ROOT_PAT
         plot_dir=plot_dir,
     )
     await bt.setup_keys(fingerprint=fingerprint, reward_ph=farming_puzzle_hash)
-    await bt.setup_plots(num_og_plots=plots, num_pool_plots=0, num_non_keychain_plots=0, plot_size=plot_size)
+    await bt.setup_plots(num_og_plots=PLOTS, num_pool_plots=0, num_non_keychain_plots=0, plot_size=PLOT_SIZE)
     service = create_full_node_simulator_service(root_path, override_config(config, overrides), bt)
     if test_mode:
         return service

tests/util/ssl_certs_1.py renamed to chia/simulator/ssl_certs_1.py

@@ -1,10 +1,9 @@
-from typing import Dict, List, Optional, Tuple, Any
+from typing import Any, Dict, List, Optional, Tuple
 
-from blspy import AugSchemeMPL, G2Element, PrivateKey, G1Element
+from blspy import AugSchemeMPL, G1Element, G2Element, PrivateKey
 from clvm.casts import int_from_bytes, int_to_bytes
 
 from chia.consensus.constants import ConsensusConstants
-from chia.util.hash import std_hash
 from chia.types.announcement import Announcement
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.program import Program, SerializedProgram
@@ -14,6 +13,7 @@
 from chia.types.condition_with_args import ConditionWithArgs
 from chia.types.spend_bundle import SpendBundle
 from chia.util.condition_tools import conditions_by_opcode, conditions_for_solution
+from chia.util.hash import std_hash
 from chia.util.ints import uint32, uint64
 from chia.wallet.derive_keys import master_sk_to_wallet_sk
 from chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle import (

tests/util/ssl_certs_2.py renamed to chia/simulator/ssl_certs_2.py

@@ -78,48 +78,6 @@ def dump(keyring_file, full_payload: bool, passphrase_file: Optional[TextIOWrapp
             break
 
 
-def dump_to_string(
-    keyring_file, full_payload: bool, passphrase_file: Optional[TextIOWrapper], pretty_print: bool
-) -> str:
-    saved_passphrase: Optional[str] = KeyringWrapper.get_shared_instance().get_master_passphrase_from_credential_store()
-    passphrase: str = saved_passphrase or DEFAULT_PASSPHRASE_IF_NO_MASTER_PASSPHRASE
-    prompt: str = get_passphrase_prompt(str(keyring_file))
-    data: Dict[str, Any] = {}
-
-    print(f"Attempting to dump contents of keyring file: {keyring_file}\n")
-
-    if passphrase_file is not None:
-        passphrase = read_passphrase_from_file(passphrase_file)
-
-    keyring_path = Path(keyring_file)
-    keyring = FileKeyring(keyring_path, FileKeyring.lockfile_path_for_file_path(keyring_path))
-
-    if full_payload:
-        keyring.load_outer_payload()
-        data = keyring.outer_payload_cache
-
-    s: str = ""
-    for i in range(5):
-        try:
-            keyring.load_keyring(passphrase)
-            if len(data) > 0:
-                data["data"] = keyring.payload_cache
-            else:
-                data = keyring.payload_cache
-
-            if pretty_print:
-                s = yaml.dump(data)
-            else:
-                s = str(data)
-            break
-        except (ValueError, InvalidTag):
-            passphrase = prompt_for_passphrase(prompt)
-        except Exception as e:
-            print(f"Unhandled exception: {e}")
-            break
-    return s
-
-
 def main():
     colorama.init()
     dump()  # pylint: disable=no-value-for-parameter