Move simulator test requirements over to new module (#12521)

* create new test.simulator module

to allow other packages to use the simulator without it crashing.
also tiny QOL fixes

* change user mode sim plot filter

Now only plots matching the user set fingerprint are loaded, this decreases the chance of user error.

* fix small oops in configure

* oops

* fix lint

* isort, who dis

* move folder over to chia/simulator

also change mypy settings again

Author: jack60612

chia/cmds/configure.py

@@ -101,6 +101,12 @@ def configure(
                 bootstrap_peers = ["testnet10-node.chia.net"]
                 testnet = "testnet10"
                 config["full_node"]["port"] = int(testnet_port)
+                if config["full_node"]["introducer_peer"] is None:
+                    config["full_node"]["introducer_peer"] = {}
+                assert config["full_node"]["introducer_peer"] is not None  # mypy
+                if config["wallet"]["introducer_peer"] is None:
+                    config["wallet"]["introducer_peer"] = {}
+                assert config["wallet"]["introducer_peer"] is not None  # mypy
                 config["full_node"]["introducer_peer"]["port"] = int(testnet_port)
                 config["farmer"]["full_node_peer"]["port"] = int(testnet_port)
                 config["timelord"]["full_node_peer"]["port"] = int(testnet_port)

chia/server/start_wallet.py

@@ -90,7 +90,7 @@ async def async_main() -> int:
     # This is simulator
     local_test = service_config["testing"]
     if local_test is True:
-        from tests.block_tools import test_constants
+        from chia.simulator.block_tools import test_constants
 
         constants = test_constants
         current = service_config["database_path"]

tests/block_tools.py renamed to chia/simulator/block_tools.py

@@ -1,7 +1,7 @@
 import asyncio
-import math
 import copy
 import logging
+import math
 import os
 import random
 import shutil
@@ -12,28 +12,18 @@
 from argparse import Namespace
 from dataclasses import replace
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Tuple, Any, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 from blspy import AugSchemeMPL, G1Element, G2Element, PrivateKey
+from chia_rs import compute_merkle_set_root
 from chiabip158 import PyBIP158
 
 from chia.cmds.init_funcs import create_all_ssl, create_default_chia_config
-from chia.daemon.keychain_proxy import connect_to_keychain_and_validate, wrap_local_keychain, KeychainProxy
-from chia.full_node.bundle_tools import (
-    best_solution_generator_from_template,
-    detect_potential_template_generator,
-    simple_solution_generator,
-)
-from chia.util.errors import Err
-from chia.full_node.generator import setup_generator_args
-from chia.full_node.mempool_check_conditions import GENERATOR_MOD
-from chia.plotting.create_plots import create_plots, PlotKeys
-from chia.plotting.util import add_plot_directory
 from chia.consensus.block_creation import unfinished_block_to_full_block
 from chia.consensus.block_record import BlockRecord
 from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward
 from chia.consensus.blockchain_interface import BlockchainInterface
-from chia.consensus.coinbase import create_puzzlehash_for_pk, create_farmer_coin, create_pool_coin
+from chia.consensus.coinbase import create_farmer_coin, create_pool_coin, create_puzzlehash_for_pk
 from chia.consensus.condition_costs import ConditionCost
 from chia.consensus.constants import ConsensusConstants
 from chia.consensus.default_constants import DEFAULT_CONSTANTS
@@ -48,10 +38,29 @@
     is_overflow_block,
 )
 from chia.consensus.vdf_info_computation import get_signage_point_vdf_info
+from chia.daemon.keychain_proxy import KeychainProxy, connect_to_keychain_and_validate, wrap_local_keychain
+from chia.full_node.bundle_tools import (
+    best_solution_generator_from_template,
+    detect_potential_template_generator,
+    simple_solution_generator,
+)
+from chia.full_node.generator import setup_generator_args
+from chia.full_node.mempool_check_conditions import GENERATOR_MOD
 from chia.full_node.signage_point import SignagePoint
-from chia.plotting.util import PlotsRefreshParameter, PlotRefreshResult, PlotRefreshEvents, parse_plot_info
+from chia.plotting.create_plots import PlotKeys, create_plots
 from chia.plotting.manager import PlotManager
+from chia.plotting.util import (
+    PlotRefreshEvents,
+    PlotRefreshResult,
+    PlotsRefreshParameter,
+    add_plot_directory,
+    parse_plot_info,
+)
 from chia.server.server import ssl_context_for_client
+from chia.simulator.socket import find_available_listen_port
+from chia.simulator.ssl_certs import get_next_nodes_certs_and_keys, get_next_private_ca_cert_and_key
+from chia.simulator.time_out_assert import time_out_assert_custom_interval
+from chia.simulator.wallet_tools import WalletTool
 from chia.types.blockchain_format.classgroup import ClassgroupElement
 from chia.types.blockchain_format.coin import Coin, hash_coin_ids
 from chia.types.blockchain_format.foliage import Foliage, FoliageBlockData, FoliageTransactionBlock, TransactionsInfo
@@ -76,24 +85,20 @@
 from chia.types.unfinished_block import UnfinishedBlock
 from chia.util.bech32m import encode_puzzle_hash
 from chia.util.block_cache import BlockCache
-from chia.util.config import load_config, lock_config, save_config, override_config
+from chia.util.config import load_config, lock_config, override_config, save_config
 from chia.util.default_root import DEFAULT_ROOT_PATH
+from chia.util.errors import Err
 from chia.util.hash import std_hash
 from chia.util.ints import uint8, uint16, uint32, uint64, uint128
 from chia.util.keychain import Keychain, bytes_to_mnemonic
 from chia.util.prev_transaction_block import get_prev_transaction_block
 from chia.util.vdf_prover import get_vdf_info_and_proof
-from tests.time_out_assert import time_out_assert_custom_interval
-from tests.wallet_tools import WalletTool
-from tests.util.socket import find_available_listen_port
-from tests.util.ssl_certs import get_next_nodes_certs_and_keys, get_next_private_ca_cert_and_key
 from chia.wallet.derive_keys import (
     master_sk_to_farmer_sk,
     master_sk_to_local_sk,
     master_sk_to_pool_sk,
     master_sk_to_wallet_sk,
 )
-from chia_rs import compute_merkle_set_root
 
 test_constants = DEFAULT_CONSTANTS.replace(
     **{
@@ -215,7 +220,7 @@ def test_callback(event: PlotRefreshEvents, update_result: PlotRefreshResult):
                 self.total_result.loaded += update_result.loaded
                 self.total_result.processed += update_result.processed
                 self.total_result.duration += update_result.duration
-                assert update_result.remaining == len(self.expected_plots) - self.total_result.processed
+                assert update_result.remaining >= len(self.expected_plots) - self.total_result.processed
                 assert len(update_result.loaded) <= self.plot_manager.refresh_parameter.batch_size
 
             if event == PlotRefreshEvents.done:
@@ -269,7 +274,10 @@ async def setup_keys(self, fingerprint: Optional[int] = None, reward_ph: Optiona
         else:
             self.farmer_ph = reward_ph
             self.pool_ph = reward_ph
-        self.all_sks: List[PrivateKey] = [sk for sk, _ in await self.keychain_proxy.get_all_private_keys()]
+        if self.automated_testing:
+            self.all_sks: List[PrivateKey] = [sk for sk, _ in await self.keychain_proxy.get_all_private_keys()]
+        else:
+            self.all_sks = [self.farmer_master_sk]  # we only want to include plots under the same fingerprint
         self.pool_pubkeys: List[G1Element] = [master_sk_to_pool_sk(sk).get_g1() for sk in self.all_sks]
 
         self.farmer_pubkeys: List[G1Element] = [master_sk_to_farmer_sk(sk).get_g1() for sk in self.all_sks]

chia/simulator/full_node_simulator.py

@@ -7,13 +7,13 @@
 from chia.full_node.full_node import FullNode
 from chia.full_node.full_node_api import FullNodeAPI
 from chia.protocols.full_node_protocol import RespondBlock
+from chia.simulator.block_tools import BlockTools
 from chia.simulator.simulator_protocol import FarmNewBlockProtocol, ReorgProtocol
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.types.full_block import FullBlock
 from chia.util.api_decorators import api_request
 from chia.util.config import lock_and_load_config, save_config
 from chia.util.ints import uint8
-from tests.block_tools import BlockTools
 
 
 class FullNodeSimulator(FullNodeAPI):

chia/simulator/simulator_constants.py

@@ -1,6 +1,6 @@
 if __name__ == "__main__":
     from chia.util.default_root import DEFAULT_ROOT_PATH
-    from tests.block_tools import create_block_tools, test_constants
+    from chia.simulator.block_tools import create_block_tools, test_constants
     from tests.util.keyring import TempKeyring
 
     with TempKeyring() as keychain:

tests/util/socket.py renamed to chia/simulator/socket.py

@@ -1,13 +1,13 @@
 import itertools
 from typing import Dict, List, Tuple
 
-from tests.util.ssl_certs_1 import SSL_TEST_NODE_CERTS_AND_KEYS_1, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_1
-from tests.util.ssl_certs_2 import SSL_TEST_NODE_CERTS_AND_KEYS_2, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_2
-from tests.util.ssl_certs_3 import SSL_TEST_NODE_CERTS_AND_KEYS_3, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_3
-from tests.util.ssl_certs_4 import SSL_TEST_NODE_CERTS_AND_KEYS_4, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_4
-from tests.util.ssl_certs_5 import SSL_TEST_NODE_CERTS_AND_KEYS_5, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_5
-from tests.util.ssl_certs_6 import SSL_TEST_NODE_CERTS_AND_KEYS_6, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_6
-from tests.util.ssl_certs_7 import SSL_TEST_NODE_CERTS_AND_KEYS_7, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_7
+from chia.simulator.ssl_certs_1 import SSL_TEST_NODE_CERTS_AND_KEYS_1, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_1
+from chia.simulator.ssl_certs_2 import SSL_TEST_NODE_CERTS_AND_KEYS_2, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_2
+from chia.simulator.ssl_certs_3 import SSL_TEST_NODE_CERTS_AND_KEYS_3, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_3
+from chia.simulator.ssl_certs_4 import SSL_TEST_NODE_CERTS_AND_KEYS_4, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_4
+from chia.simulator.ssl_certs_5 import SSL_TEST_NODE_CERTS_AND_KEYS_5, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_5
+from chia.simulator.ssl_certs_6 import SSL_TEST_NODE_CERTS_AND_KEYS_6, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_6
+from chia.simulator.ssl_certs_7 import SSL_TEST_NODE_CERTS_AND_KEYS_7, SSL_TEST_PRIVATE_CA_CERT_AND_KEY_7
 
 # ---------------------------------------------------------------------------
 # Private CA certs/keys

tests/util/ssl_certs.py renamed to chia/simulator/ssl_certs.py

@@ -12,14 +12,16 @@
 from chia.util.config import load_config_cli, override_config
 from chia.util.default_root import DEFAULT_ROOT_PATH
 from chia.util.path import path_from_root
-from tests.block_tools import BlockTools, test_constants
+from chia.simulator.block_tools import BlockTools, test_constants
 from chia.util.ints import uint16
 from chia.simulator.full_node_simulator import FullNodeSimulator
 
 # See: https://bugs.python.org/issue29288
 "".encode("idna")
 
 SERVICE_NAME = "full_node"
+PLOTS = 3  # 3 plots should be enough
+PLOT_SIZE = 19  # anything under k19 is a bit buggy
 
 
 def create_full_node_simulator_service(
@@ -66,8 +68,6 @@ async def async_main(test_mode: bool = False, root_path: Path = DEFAULT_ROOT_PAT
     fingerprint: Optional[int] = None
     farming_puzzle_hash: Optional[bytes32] = None
     plot_dir: str = "simulator-plots"
-    plots = 3  # 3 plots should be enough
-    plot_size = 19  # anything under k19 is a bit buggy
     if "simulator" in config:
         overrides = {}
         plot_dir = config["simulator"].get("plot_directory", "simulator-plots")
@@ -93,7 +93,7 @@ async def async_main(test_mode: bool = False, root_path: Path = DEFAULT_ROOT_PAT
         plot_dir=plot_dir,
     )
     await bt.setup_keys(fingerprint=fingerprint, reward_ph=farming_puzzle_hash)
-    await bt.setup_plots(num_og_plots=plots, num_pool_plots=0, num_non_keychain_plots=0, plot_size=plot_size)
+    await bt.setup_plots(num_og_plots=PLOTS, num_pool_plots=0, num_non_keychain_plots=0, plot_size=PLOT_SIZE)
     service = create_full_node_simulator_service(root_path, override_config(config, overrides), bt)
     if test_mode:
         return service

tests/util/ssl_certs_1.py renamed to chia/simulator/ssl_certs_1.py

@@ -1,10 +1,9 @@
-from typing import Dict, List, Optional, Tuple, Any
+from typing import Any, Dict, List, Optional, Tuple
 
-from blspy import AugSchemeMPL, G2Element, PrivateKey, G1Element
+from blspy import AugSchemeMPL, G1Element, G2Element, PrivateKey
 from clvm.casts import int_from_bytes, int_to_bytes
 
 from chia.consensus.constants import ConsensusConstants
-from chia.util.hash import std_hash
 from chia.types.announcement import Announcement
 from chia.types.blockchain_format.coin import Coin
 from chia.types.blockchain_format.program import Program, SerializedProgram
@@ -14,6 +13,7 @@
 from chia.types.condition_with_args import ConditionWithArgs
 from chia.types.spend_bundle import SpendBundle
 from chia.util.condition_tools import conditions_by_opcode, conditions_for_solution
+from chia.util.hash import std_hash
 from chia.util.ints import uint32, uint64
 from chia.wallet.derive_keys import master_sk_to_wallet_sk
 from chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle import (