From fa6c04fc1148e3ca2d0356e1067c82d003d9968a Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 11 Oct 2024 09:48:15 -0700 Subject: [PATCH 01/23] Working state --- btqs/btqs_cli.py | 1199 +++++++++++++++++++++++++++++++++++++++++ btqs/config.py | 11 + btqs/utils.py | 595 ++++++++++++++++++++ btqs_requirements.txt | 1 + setup.py | 3 + 5 files changed, 1809 insertions(+) create mode 100644 btqs/btqs_cli.py create mode 100644 btqs/config.py create mode 100644 btqs/utils.py create mode 100644 btqs_requirements.txt diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py new file mode 100644 index 00000000..eb432211 --- /dev/null +++ b/btqs/btqs_cli.py @@ -0,0 +1,1199 @@ +import os +import platform +import subprocess +import time +from time import sleep + +import psutil +import typer +import yaml +from bittensor_wallet import Keypair, Wallet +from git import GitCommandError, Repo +from rich.table import Table +from rich.text import Text + +from .config import ( + BTQS_DIRECTORY, + BTQS_WALLETS_DIRECTORY, + CONFIG_FILE_PATH, + EPILOG, + SUBNET_TEMPLATE_BRANCH, + SUBNET_TEMPLATE_REPO_URL, +) +from .utils import ( + console, + display_process_status_table, + exec_command, + get_bittensor_wallet_version, + get_btcli_version, + get_process_entries, + get_python_path, + get_python_version, + is_chain_running, + load_config, + remove_ansi_escape_sequences, + start_miner, + subnet_exists, + subnet_owner_exists, +) + + +class BTQSManager: + """ + Bittensor Quick Start (BTQS) Manager. + Handles CLI commands for managing the local chain and neurons. + """ + + def __init__(self): + self.app = typer.Typer( + rich_markup_mode="rich", + epilog=EPILOG, + no_args_is_help=True, + help="BTQS CLI - Bittensor Quickstart", + add_completion=False, + ) + + self.chain_app = typer.Typer(help="Subtensor Chain operations") + self.subnet_app = typer.Typer(help="Subnet setup") + self.neurons_app = typer.Typer(help="Neuron management") + + self.app.add_typer(self.chain_app, name="chain", no_args_is_help=True) + self.app.add_typer(self.subnet_app, name="subnet", no_args_is_help=True) + self.app.add_typer(self.neurons_app, name="neurons", no_args_is_help=True) + + # Chain commands + self.chain_app.command(name="start")(self.start_chain) + self.chain_app.command(name="stop")(self.stop_chain) + self.chain_app.command(name="reattach")(self.reattach_chain) + + # Setup commands + self.subnet_app.command(name="setup")(self.setup_subnet) + + # Neuron commands + self.neurons_app.command(name="setup")(self.setup_neurons) + self.neurons_app.command(name="run")(self.run_neurons) + self.neurons_app.command(name="stop")(self.stop_neurons) + self.neurons_app.command(name="reattach")(self.reattach_neurons) + self.neurons_app.command(name="status")(self.status_neurons) + self.neurons_app.command(name="start")(self.start_neurons) + + self.app.command(name="run-all", help="Create entire setup")(self.run_all) + + def run_all(self): + """ + Runs all commands in sequence to set up and start the local chain, subnet, and neurons. + + This command automates the entire setup process, including starting the local Subtensor chain, + setting up a subnet, creating and registering miner wallets, and running the miners. + + USAGE + + Run this command to perform all steps necessary to start the local chain and miners: + + [green]$[/green] btqs run-all + + [bold]Note[/bold]: This command is useful for quickly setting up the entire environment. + It will prompt for inputs as needed. + """ + text = Text("Starting Local Subtensor\n", style="bold light_goldenrod2") + sign = Text("πŸ”— ", style="bold yellow") + console.print(sign, text) + sleep(3) + + # Start the local chain + self.start_chain() + + text = Text("Checking chain status\n", style="bold light_goldenrod2") + sign = Text("\nπŸ”Ž ", style="bold yellow") + console.print(sign, text) + sleep(3) + + self.status_neurons() + + console.print( + "\nNext command will: 1. Create a subnet owner wallet 2. Create a Subnet 3. Register to the subnet" + ) + console.print("Press any key to continue..\n") + input() + + # Set up the subnet + text = Text("Setting up subnet\n", style="bold light_goldenrod2") + sign = Text("πŸ“‘ ", style="bold yellow") + console.print(sign, text) + self.setup_subnet() + + console.print( + "\nNext command will: 1. Create miner wallets 2. Register them to Netuid 1" + ) + console.print("Press any key to continue..\n") + input() + + text = Text("Setting up miners\n", style="bold light_goldenrod2") + sign = Text("\nβš’οΈ ", style="bold yellow") + console.print(sign, text) + + # Set up the neurons (miners) + self.setup_neurons() + + console.print("\nNext command will: 1. Start all miners processes") + console.print("Press any key to continue..\n") + input() + + text = Text("Running miners\n", style="bold light_goldenrod2") + sign = Text("πŸƒ ", style="bold yellow") + console.print(sign, text) + time.sleep(2) + + # Run the neurons + self.run_neurons() + + # Check status after running the neurons + self.status_neurons() + console.print("[dark_green]\nViewing Metagraph for Subnet 1") + subnets_list = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + + def start_neurons(self): + """ + Starts selected neurons. + + This command allows you to start specific miners that are not currently running. + + USAGE + + [green]$[/green] btqs neurons start + + [bold]Note[/bold]: You can select which miners to start or start all that are not running. + """ + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + if not config_data.get("Miners"): + console.print( + "[red]Miners not found. Please run `btqs neurons setup` first." + ) + return + + # Get process entries + process_entries, _, _ = get_process_entries(config_data) + display_process_status_table(process_entries, [], []) + + # Filter miners that are not running + miners_not_running = [] + for entry in process_entries: + if entry["process"].startswith("Miner") and entry["status"] != "Running": + miners_not_running.append(entry) + + if not miners_not_running: + console.print("[green]All miners are already running.") + return + + # Display the list of miners not running + console.print("\nMiners not running:") + for idx, miner in enumerate(miners_not_running, start=1): + console.print(f"{idx}. {miner['process']}") + + # Prompt user to select miners to start + selection = typer.prompt( + "Enter miner numbers to start (comma-separated), or 'all' to start all", + default="all", + ) + + if selection.lower() == "all": + selected_miners = miners_not_running + else: + selected_indices = [ + int(i.strip()) for i in selection.split(",") if i.strip().isdigit() + ] + selected_miners = [ + miners_not_running[i - 1] + for i in selected_indices + if 1 <= i <= len(miners_not_running) + ] + + if not selected_miners: + console.print("[red]No valid miners selected.") + return + + # TODO: Make this configurable + # Subnet template setup + subnet_template_path = os.path.join(BTQS_DIRECTORY, "subnet-template") + if not os.path.exists(subnet_template_path): + console.print("[green]Cloning subnet-template repository...") + repo = Repo.clone_from( + SUBNET_TEMPLATE_REPO_URL, + subnet_template_path, + ) + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + else: + console.print("[green]Using existing subnet-template repository.") + repo = Repo(subnet_template_path) + current_branch = repo.active_branch.name + if current_branch != SUBNET_TEMPLATE_BRANCH: + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + + # TODO: Add ability for users to define their own flags, entry point etc + # Start selected miners + for miner in selected_miners: + wallet_name = miner["process"].split("Miner: ")[-1] + wallet_info = config_data["Miners"][wallet_name] + success = start_miner( + wallet_name, wallet_info, subnet_template_path, config_data + ) + if success: + console.print(f"[green]Miner {wallet_name} started.") + else: + console.print(f"[red]Failed to start miner {wallet_name}.") + + # Update the config file + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + def start_chain(self): + """ + Starts the local Subtensor chain. + + This command initializes and starts a local instance of the Subtensor blockchain for development and testing. + + USAGE + + [green]$[/green] btqs chain start + + [bold]Note[/bold]: This command will clone or update the Subtensor repository if necessary and start the local chain. It may take several minutes to complete. + """ + console.print("[dark_orange]Starting the local chain...") + + if is_chain_running(CONFIG_FILE_PATH): + console.print( + "[red]The local chain is already running. Endpoint: ws://127.0.0.1:9945" + ) + return + + config_data = load_config(exit_if_missing=False) + if config_data: + console.print("[green] Refreshing config file") + config_data = {} + + directory = typer.prompt( + "Enter the directory to clone the subtensor repository", + default=os.path.expanduser("~/Desktop/Bittensor_quick_start"), + show_default=True, + ) + os.makedirs(directory, exist_ok=True) + + subtensor_path = os.path.join(directory, "subtensor") + repo_url = "https://github.com/opentensor/subtensor.git" + + # Clone or update the repository + if os.path.exists(subtensor_path) and os.listdir(subtensor_path): + update = typer.confirm( + "Subtensor is already cloned. Do you want to update it?" + ) + if update: + try: + repo = Repo(subtensor_path) + origin = repo.remotes.origin + origin.pull() + console.print("[green]Repository updated successfully.") + except GitCommandError as e: + console.print(f"[red]Error updating repository: {e}") + return + else: + console.print( + "[green]Using existing subtensor repository without updating." + ) + else: + try: + console.print("[green]Cloning subtensor repository...") + Repo.clone_from(repo_url, subtensor_path) + console.print("[green]Repository cloned successfully.") + except GitCommandError as e: + console.print(f"[red]Error cloning repository: {e}") + return + + localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") + + # Running localnet.sh + process = subprocess.Popen( + ["bash", localnet_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + cwd=subtensor_path, + start_new_session=True, + ) + + console.print("[green]Starting local chain. This may take a few minutes...") + + # Paths to subtensor log files + log_dir = os.path.join(subtensor_path, "logs") + alice_log = os.path.join(log_dir, "alice.log") + + # Waiting for chain compilation + timeout = 360 # 6 minutes + start_time = time.time() + while not os.path.exists(alice_log): + if time.time() - start_time > timeout: + console.print("[red]Timeout: Log files were not created.") + return + time.sleep(1) + + chain_ready = False + try: + with open(alice_log, "r") as log_file: + log_file.seek(0, os.SEEK_END) + while True: + line = log_file.readline() + if line: + console.print(line, end="") + if "Imported #" in line: + chain_ready = True + break + else: + if time.time() - start_time > timeout: + console.print( + "[red]Timeout: Chain did not compile in time." + ) + break + time.sleep(0.1) + except Exception as e: + console.print(f"[red]Error reading log files: {e}") + return + + if chain_ready: + text = Text( + "Local chain is running. You can now use it for development and testing.\n", + style="bold light_goldenrod2", + ) + sign = Text("\nβš™οΈ ", style="bold yellow") + console.print(sign, text) + + try: + # Fetch PIDs of 2 substrate nodes spawned + result = subprocess.run( + ["pgrep", "-f", "node-subtensor"], capture_output=True, text=True + ) + substrate_pids = [int(pid) for pid in result.stdout.strip().split()] + + config_data.update( + { + "pid": process.pid, + "substrate_pid": substrate_pids, + "subtensor_path": subtensor_path, + "base_path": directory, + } + ) + except ValueError: + console.print("[red]Failed to get the PID of the Subtensor process.") + return + + config_data.update( + { + "pid": process.pid, + "subtensor_path": subtensor_path, + "base_path": directory, + } + ) + + # Save config data + try: + os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True) + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + console.print( + "[green]Local chain started successfully and config file updated." + ) + except Exception as e: + console.print(f"[red]Failed to write to the config file: {e}") + else: + console.print("[red]Failed to start local chain.") + + def stop_chain(self): + """ + Stops the local Subtensor chain and any running miners. + + This command terminates the local Subtensor chain process and optionally cleans up configuration data. + + USAGE + + [green]$[/green] btqs chain stop + + [bold]Note[/bold]: Use this command to gracefully shut down the local chain. It will also stop any running miner processes. + """ + config_data = load_config( + "No running chain found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + pid = config_data.get("pid") + if not pid: + console.print("[red]No running chain found.") + return + + console.print("[red]Stopping the local chain...") + + try: + process = psutil.Process(pid) + process.terminate() + process.wait(timeout=10) + console.print("[green]Local chain stopped successfully.") + + except psutil.NoSuchProcess: + console.print( + "[red]Process not found. The chain may have already been stopped." + ) + + except psutil.TimeoutExpired: + console.print("[red]Timeout while stopping the chain. Forcing stop...") + process.kill() + + # Check for running miners + process_entries, _, _ = get_process_entries(config_data) + + # Filter running miners + running_miners = [] + for entry in process_entries: + if entry["process"].startswith("Miner") and entry["status"] == "Running": + running_miners.append(entry) + + if running_miners: + console.print( + "[yellow]\nSome miners are still running. Terminating them..." + ) + + for miner in running_miners: + pid = int(miner["pid"]) + wallet_name = miner["process"].split("Miner: ")[-1] + try: + miner_process = psutil.Process(pid) + miner_process.terminate() + miner_process.wait(timeout=10) + console.print(f"[green]Miner {wallet_name} stopped.") + + except psutil.NoSuchProcess: + console.print(f"[yellow]Miner {wallet_name} process not found.") + + except psutil.TimeoutExpired: + console.print( + f"[red]Timeout stopping miner {wallet_name}. Forcing stop." + ) + miner_process.kill() + + config_data["Miners"][wallet_name]["pid"] = None + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + else: + console.print("[green]No miners were running.") + + # Refresh data + refresh_config = typer.confirm( + "\nConfig data is outdated. Press Y to refresh it?" + ) + if refresh_config: + if os.path.exists(CONFIG_FILE_PATH): + os.remove(CONFIG_FILE_PATH) + console.print("[green]Configuration file removed.") + + def reattach_chain(self): + """ + Reattaches to the running local chain. + + This command allows you to view the logs of the running local Subtensor chain. + + USAGE + + [green]$[/green] btqs chain reattach + + [bold]Note[/bold]: Press Ctrl+C to detach from the chain logs. + """ + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + pid = config_data.get("pid") + subtensor_path = config_data.get("subtensor_path") + if not pid or not subtensor_path: + console.print("[red]No running chain found.") + return + + # Check if the process is still running + try: + process = psutil.Process(pid) + if not process.is_running(): + console.print( + "[red]Process not running. The chain may have been stopped." + ) + return + except psutil.NoSuchProcess: + console.print("[red]Process not found. The chain may have been stopped.") + return + + # Log file setup for Subtensor chain + log_dir = os.path.join(subtensor_path, "logs") + alice_log = os.path.join(log_dir, "alice.log") + + if not os.path.exists(alice_log): + console.print("[red]Log files not found.") + return + + try: + console.print("[green]Reattaching to the local chain...") + console.print("[green]Press Ctrl+C to detach.") + with open(alice_log, "r") as alice_file: + alice_file.seek(0, os.SEEK_END) + while True: + alice_line = alice_file.readline() + if not alice_line: + time.sleep(0.1) + continue + if alice_line: + print(f"[Alice] {alice_line}", end="") + except KeyboardInterrupt: + console.print("\n[green]Detached from the local chain.") + + def setup_subnet(self): + """ + Sets up a subnet on the local chain. + + This command creates a subnet owner wallet, creates a subnet with netuid 1, and registers the owner to the subnet. + + USAGE + + [green]$[/green] btqs subnet setup + + [bold]Note[/bold]: Ensure the local chain is running before executing this command. + """ + if not is_chain_running(CONFIG_FILE_PATH): + console.print( + "[red]Local chain is not running. Please start the chain first." + ) + return + + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + os.makedirs(BTQS_WALLETS_DIRECTORY, exist_ok=True) + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if subnet_owner: + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), + ) + + warning_text = Text( + "A Subnet Owner associated with this setup already exists", + style="bold light_goldenrod2", + ) + wallet_info = Text(f"\t{owner_wallet}\n", style="medium_purple") + warning_sign = Text("⚠️ ", style="bold yellow") + + console.print(warning_sign, warning_text) + console.print(wallet_info) + + else: + text = Text( + "Creating subnet owner wallet.\n", style="bold light_goldenrod2" + ) + sign = Text("πŸ‘‘ ", style="bold yellow") + console.print(sign, text) + + owner_wallet_name = typer.prompt( + "Enter subnet owner wallet name", default="owner", show_default=True + ) + owner_hotkey_name = typer.prompt( + "Enter subnet owner hotkey name", default="default", show_default=True + ) + + uri = "//Alice" + keypair = Keypair.create_from_uri(uri) + owner_wallet = Wallet( + path=BTQS_WALLETS_DIRECTORY, + name=owner_wallet_name, + hotkey=owner_hotkey_name, + ) + owner_wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + owner_wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + owner_wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + + console.print( + "Executed command: [dark_orange] btcli wallet create --wallet-name", + f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {BTQS_WALLETS_DIRECTORY}", + ) + + with open(CONFIG_FILE_PATH, "r") as config_file: + config_data = yaml.safe_load(config_file) + config_data["Owner"] = { + "wallet_name": owner_wallet_name, + "path": BTQS_WALLETS_DIRECTORY, + "hotkey": owner_hotkey_name, + "subtensor_pid": config_data["pid"], + } + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + if subnet_exists(owner_wallet.coldkeypub.ss58_address, 1): + warning_text = Text( + "A Subnet with netuid 1 already exists and is registered with the owner's wallet:", + style="bold light_goldenrod2", + ) + wallet_info = Text(f"\t{owner_wallet}", style="medium_purple") + sudo_info = Text( + f"\tSUDO (Coldkey of Subnet 1 owner): {owner_wallet.coldkeypub.ss58_address}", + style="medium_purple", + ) + warning_sign = Text("⚠️ ", style="bold yellow") + + console.print(warning_sign, warning_text) + console.print(wallet_info) + console.print(sudo_info) + else: + text = Text( + "Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2" + ) + sign = Text("\nπŸ’» ", style="bold yellow") + console.print(sign, text) + + create_subnet = exec_command( + command="subnets", + sub_command="create", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--chain", + "ws://127.0.0.1:9945", + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + ], + ) + clean_stdout = remove_ansi_escape_sequences(create_subnet.stdout) + if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: + console.print("[dark_green] Subnet created successfully with netuid 1") + + text = Text( + f"Registering Owner ({owner_wallet.name}) to Netuid 1\n", + style="bold light_goldenrod2", + ) + sign = Text("\nπŸ“ ", style="bold yellow") + console.print(sign, text) + + register_subnet = exec_command( + command="subnets", + sub_command="register", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--wallet-name", + owner_wallet.name, + "--wallet-hotkey", + owner_wallet.hotkey_str, + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + "--no-prompt", + ], + ) + clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) + if "βœ… Registered" in clean_stdout: + console.print("[green] Registered the owner to subnet 1") + + console.print("[dark_green]\nListing all subnets") + subnets_list = exec_command( + command="subnets", + sub_command="list", + extra_args=[ + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + + def setup_neurons(self): + """ + Sets up neurons (miners) for the subnet. + + This command creates miner wallets and registers them to the subnet. + + USAGE + + [green]$[/green] btqs neurons setup + + [bold]Note[/bold]: This command will prompt for wallet names and hotkey names for each miner. + """ + if not is_chain_running(CONFIG_FILE_PATH): + console.print( + "[red]Local chain is not running. Please start the chain first." + ) + return + + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if subnet_owner: + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), + ) + else: + console.print( + "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" + ) + return + + config_data.setdefault("Miners", {}) + miners = config_data.get("Miners", {}) + + if miners and all( + miner_info.get("subtensor_pid") == config_data.get("pid") + for miner_info in miners.values() + ): + console.print( + "[green]Miner wallets associated with this subtensor instance already present. Proceeding..." + ) + else: + uris = [ + "//Bob", + "//Charlie", + ] + ports = [8100, 8101, 8102, 8103] + for i, uri in enumerate(uris, start=0): + console.print(f"Miner {i+1}:") + wallet_name = typer.prompt( + f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" + ) + hotkey_name = typer.prompt( + f"Enter hotkey name for miner {i+1}", default="default" + ) + + keypair = Keypair.create_from_uri(uri) + wallet = Wallet( + path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=hotkey_name + ) + wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + + config_data["Miners"][wallet_name] = { + "path": BTQS_WALLETS_DIRECTORY, + "hotkey": hotkey_name, + "uri": uri, + "pid": None, + "subtensor_pid": config_data["pid"], + "port": ports[i], + } + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + console.print("[green]All wallets are created.") + + for wallet_name, wallet_info in config_data["Miners"].items(): + wallet = Wallet( + path=wallet_info["path"], + name=wallet_name, + hotkey=wallet_info["hotkey"], + ) + + text = Text( + f"Registering Miner ({wallet_name}) to Netuid 1\n", + style="bold light_goldenrod2", + ) + sign = Text("\nπŸ“ ", style="bold yellow") + console.print(sign, text) + + miner_registered = exec_command( + command="subnets", + sub_command="register", + extra_args=[ + "--wallet-path", + wallet.path, + "--wallet-name", + wallet.name, + "--hotkey", + wallet.hotkey_str, + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + "--no-prompt", + ], + ) + clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) + + if "βœ… Registered" in clean_stdout: + text = Text( + f"Registered miner ({wallet.name}) to Netuid 1\n", + style="bold light_goldenrod2", + ) + sign = Text("πŸ† ", style="bold yellow") + console.print(sign, text) + else: + print(clean_stdout) + console.print( + f"[red]Failed to register miner ({wallet.name}). Please register the miner manually using the following command:" + ) + command = f"btcli subnets register --wallet-path {wallet.path} --wallet-name {wallet.name} --hotkey {wallet.hotkey_str} --netuid 1 --chain ws://127.0.0.1:9945 --no-prompt" + console.print(f"[bold yellow]{command}\n") + + console.print("[dark_green]\nViewing Metagraph for Subnet 1") + subnets_list = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + + def run_neurons(self): + """ + Runs all neurons (miners). + + This command starts the miner processes for all configured miners, attaching to running miners if they are already running. + + USAGE + + [green]$[/green] btqs neurons run + + [bold]Note[/bold]: The command will attach to running miners or start new ones as necessary. Press Ctrl+C to detach from a miner and move to the next. + """ + if not os.path.exists(CONFIG_FILE_PATH): + console.print( + "[red]Config file not found. Please run `btqs chain start` first." + ) + return + + with open(CONFIG_FILE_PATH, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + + if not config_data.get("Miners"): + console.print( + "[red]Miners not found. Please run `btqs neurons setup` first." + ) + return + + # Subnet template setup + subnet_template_path = os.path.join(BTQS_DIRECTORY, "subnet-template") + if not os.path.exists(subnet_template_path): + console.print("[green]Cloning subnet-template repository...") + repo = Repo.clone_from( + SUBNET_TEMPLATE_REPO_URL, + subnet_template_path, + ) + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + else: + console.print("[green]Using existing subnet-template repository.") + repo = Repo(subnet_template_path) + current_branch = repo.active_branch.name + if current_branch != SUBNET_TEMPLATE_BRANCH: + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + + chain_pid = config_data.get("pid") + for wallet_name, wallet_info in config_data.get("Miners", {}).items(): + miner_pid = wallet_info.get("pid") + miner_subtensor_pid = wallet_info.get("subtensor_pid") + # Check if miner process is running and associated with the current chain + if ( + miner_pid + and psutil.pid_exists(miner_pid) + and miner_subtensor_pid == chain_pid + ): + console.print( + f"[green]Miner {wallet_name} is already running. Attaching to the process..." + ) + log_file_path = wallet_info.get("log_file") + if log_file_path and os.path.exists(log_file_path): + try: + with open(log_file_path, "r") as log_file: + # Move to the end of the file + log_file.seek(0, os.SEEK_END) + console.print( + f"[green]Attached to miner {wallet_name}. Press Ctrl+C to move to the next miner." + ) + while True: + line = log_file.readline() + if not line: + # Check if the process is still running + if not psutil.pid_exists(miner_pid): + console.print( + f"\n[red]Miner process {wallet_name} has terminated." + ) + break + time.sleep(0.1) + continue + print(line, end="") + except KeyboardInterrupt: + console.print(f"\n[green]Detached from miner {wallet_name}.") + except Exception as e: + console.print( + f"[red]Error attaching to miner {wallet_name}: {e}" + ) + else: + console.print( + f"[red]Log file not found for miner {wallet_name}. Cannot attach." + ) + else: + # Miner is not running, start it + success = start_miner( + wallet_name, wallet_info, subnet_template_path, config_data + ) + if not success: + console.print(f"[red]Failed to start miner {wallet_name}.") + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + def stop_neurons(self): + """ + Stops the running neurons. + + This command terminates the miner processes for the selected or all running miners. + + USAGE + + [green]$[/green] btqs neurons stop + + [bold]Note[/bold]: You can choose which miners to stop or stop all of them. + """ + if not os.path.exists(CONFIG_FILE_PATH): + console.print("[red]Config file not found.") + return + + with open(CONFIG_FILE_PATH, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + + # Get process entries + process_entries, _, _ = get_process_entries(config_data) + display_process_status_table(process_entries, [], []) + + # Filter running miners + running_miners = [] + for entry in process_entries: + if entry["process"].startswith("Miner") and entry["status"] == "Running": + running_miners.append(entry) + + if not running_miners: + console.print("[red]No running miners to stop.") + return + + console.print("\nSelect miners to stop:") + for idx, miner in enumerate(running_miners, start=1): + console.print(f"{idx}. {miner['process']} (PID: {miner['pid']})") + + selection = typer.prompt( + "Enter miner numbers to stop (comma-separated), or 'all' to stop all", + default="all", + ) + + if selection.lower() == "all": + selected_miners = running_miners + else: + selected_indices = [ + int(i.strip()) for i in selection.split(",") if i.strip().isdigit() + ] + selected_miners = [ + running_miners[i - 1] + for i in selected_indices + if 1 <= i <= len(running_miners) + ] + + if not selected_miners: + console.print("[red]No valid miners selected.") + return + + # Stop selected miners + for miner in selected_miners: + pid = int(miner["pid"]) + wallet_name = miner["process"].split("Miner: ")[-1] + try: + process = psutil.Process(pid) + process.terminate() + process.wait(timeout=10) + console.print(f"[green]Miner {wallet_name} stopped.") + + except psutil.NoSuchProcess: + console.print(f"[yellow]Miner {wallet_name} process not found.") + + except psutil.TimeoutExpired: + console.print( + f"[red]Timeout stopping miner {wallet_name}. Forcing stop." + ) + process.kill() + + config_data["Miners"][wallet_name]["pid"] = None + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + def reattach_neurons(self): + """ + Reattaches to a running neuron. + + This command allows you to view the logs of a running miner (neuron). + + USAGE + + [green]$[/green] btqs neurons reattach + + [bold]Note[/bold]: Press Ctrl+C to detach from the miner logs. + """ + if not os.path.exists(CONFIG_FILE_PATH): + console.print("[red]Config file not found.") + return + + with open(CONFIG_FILE_PATH, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + + # Choose which neuron to reattach to + all_neurons = { + **config_data.get("Validators", {}), + **config_data.get("Miners", {}), + } + neuron_names = list(all_neurons.keys()) + if not neuron_names: + console.print("[red]No neurons found.") + return + + neuron_choice = typer.prompt( + f"Which neuron do you want to reattach to? {neuron_names}", + default=neuron_names[0], + ) + if neuron_choice not in all_neurons: + console.print("[red]Invalid neuron name.") + return + + wallet_info = all_neurons[neuron_choice] + pid = wallet_info.get("pid") + log_file_path = wallet_info.get("log_file") + if not pid or not psutil.pid_exists(pid): + console.print("[red]Neuron process not running.") + return + + if not log_file_path or not os.path.exists(log_file_path): + console.print("[red]Log file not found for this neuron.") + return + + console.print( + f"[green]Reattaching to neuron {neuron_choice}. Press Ctrl+C to exit." + ) + + try: + with open(log_file_path, "r") as log_file: + # Move to the end of the file + log_file.seek(0, os.SEEK_END) + while True: + line = log_file.readline() + if not line: + if not psutil.pid_exists(pid): + console.print( + f"\n[red]Neuron process {neuron_choice} has terminated." + ) + break + time.sleep(0.1) + continue + print(line, end="") + + except KeyboardInterrupt: + console.print("\n[green]Detached from neuron logs.") + + except Exception as e: + console.print(f"[red]Error reattaching to neuron: {e}") + + def status_neurons(self): + """ + Shows the status of Subtensor and all neurons. + + This command displays the running status, CPU and memory usage of the local chain and all configured neurons. + + USAGE + + [green]$[/green] btqs neurons status + + [bold]Note[/bold]: Use this command to monitor the health and status of your local chain and miners. + """ + console.print("[green]Checking status of Subtensor and neurons...") + + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + # Get process data + process_entries, cpu_usage_list, memory_usage_list = get_process_entries( + config_data + ) + display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) + + spec_table = Table( + title="[underline dark_orange]Machine Specifications[/underline dark_orange]", + show_header=False, + border_style="bright_black", + ) + spec_table.add_column(style="cyan", justify="left") + spec_table.add_column(style="white") + + version_table = Table( + title="[underline dark_orange]Version Information[/underline dark_orange]", + show_header=False, + border_style="bright_black", + ) + version_table.add_column(style="cyan", justify="left") + version_table.add_column(style="white") + + # Add specs + spec_table.add_row( + "Operating System:", f"{platform.system()} {platform.release()}" + ) + spec_table.add_row("Processor:", platform.processor()) + spec_table.add_row( + "Total RAM:", + f"{psutil.virtual_memory().total / (1024 * 1024 * 1024):.2f} GB", + ) + spec_table.add_row( + "Available RAM:", + f"{psutil.virtual_memory().available / (1024 * 1024 * 1024):.2f} GB", + ) + + # Add version + version_table.add_row("btcli version:", get_btcli_version()) + version_table.add_row( + "bittensor-wallet version:", get_bittensor_wallet_version() + ) + version_table.add_row("Python version:", get_python_version()) + version_table.add_row("Python path:", get_python_path()) + + layout = Table.grid(expand=True) + layout.add_column(justify="left") + layout.add_column(justify="left") + layout.add_row(spec_table, version_table) + + console.print("\n") + console.print(layout) + + def run(self): + self.app() + + +def main(): + manager = BTQSManager() + manager.run() + + +if __name__ == "__main__": + main() diff --git a/btqs/config.py b/btqs/config.py new file mode 100644 index 00000000..046cd7f4 --- /dev/null +++ b/btqs/config.py @@ -0,0 +1,11 @@ +import os + +CONFIG_FILE_PATH = os.path.expanduser("~/.bittensor/btqs/btqs_config.yml") + +BTQS_DIRECTORY = os.path.expanduser("~/.bittensor/btqs") +BTQS_WALLETS_DIRECTORY = os.path.expanduser(os.path.join(BTQS_DIRECTORY, "wallets")) + +SUBNET_TEMPLATE_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" +SUBNET_TEMPLATE_BRANCH = "ench/abe/commented-info" + +EPILOG = "Made with [bold red]:heart:[/bold red] by The OpenΟ„ensor FoundaΟ„ion" diff --git a/btqs/utils.py b/btqs/utils.py new file mode 100644 index 00000000..7b2d9867 --- /dev/null +++ b/btqs/utils.py @@ -0,0 +1,595 @@ +import os +import re +import subprocess +import sys +import time +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +import psutil +import typer +import yaml +from bittensor_cli.cli import CLIManager +from bittensor_wallet import Wallet +from rich.console import Console +from rich.table import Table +from typer.testing import CliRunner + +from .config import ( + BTQS_DIRECTORY, + BTQS_WALLETS_DIRECTORY, + CONFIG_FILE_PATH, +) + +console = Console() + + +def load_config( + error_message: Optional[str] = None, exit_if_missing: bool = True +) -> Dict[str, Any]: + """ + Loads the configuration file. + + Args: + error_message (Optional[str]): Custom error message to display if config file is not found. + exit_if_missing (bool): If True, exits the program if the config file is missing. + + Returns: + Dict[str, Any]: Configuration data. + + Raises: + typer.Exit: If the config file is not found and exit_if_missing is True. + """ + if not os.path.exists(CONFIG_FILE_PATH): + if exit_if_missing: + if error_message: + console.print(f"[red]{error_message}") + else: + console.print("[red]Configuration file not found.") + raise typer.Exit() + else: + return {} # Return an empty config if not exiting + with open(CONFIG_FILE_PATH, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + return config_data + + +def load_config_data() -> Optional[Dict[str, Any]]: + """ + Load configuration data from the config file. + + Returns: + - dict: The loaded configuration data if successful. + - None: If the config file doesn't exist or can't be loaded. + """ + if not os.path.exists(CONFIG_FILE_PATH): + console.print( + "[red]Config file not found. Please run `btqs chain start` first." + ) + return None + + try: + with open(CONFIG_FILE_PATH, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + return config_data + except Exception as e: + console.print(f"[red]Error loading config file: {e}") + return None + + +def remove_ansi_escape_sequences(text: str) -> str: + """ + Removes ANSI escape sequences from the given text. + + Args: + text (str): The text from which to remove ANSI sequences. + + Returns: + str: The cleaned text. + """ + ansi_escape = re.compile( + r""" + \x1B # ESC character + (?: # Non-capturing group + [@-Z\\-_] # 7-bit C1 control codes + | # or + \[ # ESC[ + [0-?]* # Parameter bytes + [ -/]* # Intermediate bytes + [@-~] # Final byte + ) + """, + re.VERBOSE, + ) + return ansi_escape.sub("", text) + + +def exec_command( + command: str, + sub_command: str, + extra_args: Optional[List[str]] = None, + inputs: Optional[List[str]] = None, + internal_command: bool = False, +) -> typer.testing.Result: + """ + Executes a command using the CLIManager and returns the result. + + Args: + command (str): The main command to execute. + sub_command (str): The sub-command to execute. + extra_args (List[str], optional): Additional arguments for the command. + inputs (List[str], optional): Inputs for interactive prompts. + + Returns: + typer.testing.Result: The result of the command execution. + """ + extra_args = extra_args or [] + cli_manager = CLIManager() + runner = CliRunner() + + args = [command, sub_command] + extra_args + command_for_printing = ["btcli"] + [str(arg) for arg in args] + + if not internal_command: + console.print( + f"Executing command: [dark_orange]{' '.join(command_for_printing)}\n" + ) + + input_text = "\n".join(inputs) + "\n" if inputs else None + result = runner.invoke( + cli_manager.app, + args, + input=input_text, + env={"COLUMNS": "700"}, + catch_exceptions=False, + color=False, + ) + return result + + +def is_chain_running(config_file_path: str) -> bool: + """ + Checks if the local chain is running by verifying the PID in the config file. + + Args: + config_file_path (str): Path to the configuration file. + + Returns: + bool: True if the chain is running, False otherwise. + """ + if not os.path.exists(config_file_path): + return False + with open(config_file_path, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + pid = config_data.get("pid") + if not pid: + return False + try: + process = psutil.Process(pid) + return process.is_running() + except psutil.NoSuchProcess: + return False + + +def subnet_owner_exists(config_file_path: str) -> Tuple[bool, dict]: + """ + Checks if a subnet owner exists in the config file. + + Args: + config_file_path (str): Path to the configuration file. + + Returns: + Tuple[bool, dict]: (True, owner data) if exists, else (False, {}). + """ + if not os.path.exists(config_file_path): + return False, {} + with open(config_file_path, "r") as config_file: + config_data = yaml.safe_load(config_file) or {} + + owner_data = config_data.get("Owner") + pid = config_data.get("pid") + + if owner_data and pid: + if owner_data.get("subtensor_pid") == pid: + return True, owner_data + + return False, {} + + +def verify_subnet_entry(output_text: str, netuid: str, ss58_address: str) -> bool: + """ + Verifies the presence of a specific subnet entry in the subnets list output. + + Args: + output_text (str): Output of execution command. + netuid (str): The netuid to look for. + ss58_address (str): The SS58 address of the subnet owner. + + Returns: + bool: True if the entry is found, False otherwise. + """ + # Remove ANSI escape sequences + output_text = remove_ansi_escape_sequences(output_text) + lines = output_text.split("\n") + + # Flag to start processing data rows after the headers + data_started = False + + for line in lines: + line = line.strip() + # Skip empty lines + if not line: + continue + + # Identify the header line + if "NETUID" in line: + data_started = True + continue + + # Skip lines before the data starts + if not data_started: + continue + + # Skip separator lines + if set(line) <= {"━", "╇", "β”Ό", "─", "β•ˆ", "═", "β•‘", "╬", "β•£", "β• "}: + continue + + # Split the line into columns using the separator 'β”‚' or '|' + columns = re.split(r"β”‚|\|", line) + # Remove leading and trailing whitespace from each column + columns = [col.strip() for col in columns] + + # Check if columns have enough entries + if len(columns) < 8: + continue + + # Extract netuid and ss58_address from columns + netuid_col = columns[0] + ss58_address_col = columns[-1] + + # Compare with the given netuid and ss58_address + if netuid_col == str(netuid) and ss58_address_col == ss58_address: + return True + + return False + + +def subnet_exists(ss58_address: str, netuid: str) -> bool: + subnets_list = exec_command( + command="subnets", + sub_command="list", + extra_args=[ + "--chain", + "ws://127.0.0.1:9945", + ], + internal_command=True, + ) + exists = verify_subnet_entry( + remove_ansi_escape_sequences(subnets_list.stdout), netuid, ss58_address + ) + return exists + + +def get_btcli_version() -> str: + try: + result = subprocess.run(["btcli", "--version"], capture_output=True, text=True) + return result.stdout.strip() + except Exception: + return "Not installed or not found" + + +def get_bittensor_wallet_version() -> str: + try: + result = subprocess.run( + ["pip", "show", "bittensor-wallet"], capture_output=True, text=True + ) + for line in result.stdout.split("\n"): + if line.startswith("Version:"): + return line.split(":")[1].strip() + except Exception: + return "Not installed or not found" + + +def get_python_version() -> str: + return sys.version.split()[0] + + +def get_python_path() -> str: + return sys.executable + + +def get_process_info(pid: int) -> Tuple[str, str, str, str, float, float]: + if pid and psutil.pid_exists(pid): + try: + process = psutil.Process(pid) + status = "Running" + cpu_percent = process.cpu_percent(interval=0.1) + memory_percent = process.memory_percent() + cpu_usage = f"{cpu_percent:.1f}%" + memory_usage = f"{memory_percent:.1f}%" + create_time = datetime.fromtimestamp(process.create_time()) + uptime = datetime.now() - create_time + uptime_str = str(uptime).split(".")[0] # Remove microseconds + return ( + status, + cpu_usage, + memory_usage, + uptime_str, + cpu_percent, + memory_percent, + ) + except Exception as e: + console.print(f"[red]Error retrieving process info: {e}") + status = "Not Running" + cpu_usage = "N/A" + memory_usage = "N/A" + uptime_str = "N/A" + cpu_percent = 0.0 + memory_percent = 0.0 + pid = "N/A" + return status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent + + +def get_process_entries( + config_data: Dict[str, Any], +) -> Tuple[List[Dict[str, str]], List[float], List[float]]: + cpu_usage_list = [] + memory_usage_list = [] + process_entries = [] + + # Check Subtensor status + pid = config_data.get("pid") + subtensor_path = config_data.get("subtensor_path", "N/A") + status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( + get_process_info(pid) + ) + + status_style = "green" if status == "Running" else "red" + + if status == "Running": + cpu_usage_list.append(cpu_percent) + memory_usage_list.append(memory_percent) + + process_entries.append( + { + "process": "Subtensor", + "status": status, + "status_style": status_style, + "pid": str(pid), + "cpu_usage": cpu_usage, + "memory_usage": memory_usage, + "uptime_str": uptime_str, + "location": subtensor_path, + } + ) + + # Check status of Subtensor nodes (substrate_pids) + substrate_pids = config_data.get("substrate_pid", []) + for index, sub_pid in enumerate(substrate_pids, start=1): + status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( + get_process_info(sub_pid) + ) + status_style = "green" if status == "Running" else "red" + + if status == "Running": + cpu_usage_list.append(cpu_percent) + memory_usage_list.append(memory_percent) + + process_entries.append( + { + "process": f"Subtensor Node {index}", + "status": status, + "status_style": status_style, + "pid": str(sub_pid), + "cpu_usage": cpu_usage, + "memory_usage": memory_usage, + "uptime_str": uptime_str, + "location": subtensor_path, + } + ) + + # Check status of Miners + miners = config_data.get("Miners", {}) + for wallet_name, wallet_info in miners.items(): + pid = wallet_info.get("pid") + location = wallet_info.get("path") + status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( + get_process_info(pid) + ) + + status_style = "green" if status == "Running" else "red" + + if status == "Running": + cpu_usage_list.append(cpu_percent) + memory_usage_list.append(memory_percent) + + process_entries.append( + { + "process": f"Miner: {wallet_name}", + "status": status, + "status_style": status_style, + "pid": str(pid), + "cpu_usage": cpu_usage, + "memory_usage": memory_usage, + "uptime_str": uptime_str, + "location": location, + } + ) + + return process_entries, cpu_usage_list, memory_usage_list + + +def display_process_status_table( + process_entries: List[Dict[str, str]], + cpu_usage_list: List[float], + memory_usage_list: List[float], +) -> None: + table = Table( + title="[underline dark_orange]BTQS Process Manager[/underline dark_orange]\n", + show_footer=True, + show_edge=False, + header_style="bold white", + border_style="bright_black", + style="bold", + title_justify="center", + show_lines=False, + pad_edge=True, + ) + + table.add_column( + "[bold white]Process", + style="white", + no_wrap=True, + footer_style="bold white", + ) + + table.add_column( + "[bold white]Status", + style="bright_cyan", + justify="center", + footer_style="bold white", + ) + + table.add_column( + "[bold white]PID", + style="bright_magenta", + justify="right", + footer_style="bold white", + ) + + table.add_column( + "[bold white]CPU %", + style="light_goldenrod2", + justify="right", + footer_style="bold white", + ) + + table.add_column( + "[bold white]Memory %", + style="light_goldenrod2", + justify="right", + footer_style="bold white", + ) + + table.add_column( + "[bold white]Uptime", + style="dark_sea_green", + justify="right", + footer_style="bold white", + ) + + table.add_column( + "[bold white]Location", + style="white", + overflow="fold", + footer_style="bold white", + ) + + for entry in process_entries: + if entry["process"].startswith("Subtensor"): + process_style = "cyan" + elif entry["process"].startswith("Miner"): + process_style = "magenta" + else: + process_style = "white" + + table.add_row( + f"[{process_style}]{entry['process']}[/{process_style}]", + f"[{entry['status_style']}]{entry['status']}[/{entry['status_style']}]", + entry["pid"], + entry["cpu_usage"], + entry["memory_usage"], + entry["uptime_str"], + entry["location"], + ) + + # Compute total CPU and Memory usage + total_cpu = sum(cpu_usage_list) + total_memory = sum(memory_usage_list) + + # Set footers for columns + table.columns[0].footer = f"[white]{len(process_entries)} Processes[/white]" + table.columns[3].footer = f"[white]{total_cpu:.1f}%[/white]" + table.columns[4].footer = f"[white]{total_memory:.1f}%[/white]" + + # Display the table + console.print(table) + + +def start_miner( + wallet_name: str, + wallet_info: Dict[str, Any], + subnet_template_path: str, + config_data: Dict[str, Any], +) -> bool: + """Starts a single miner and displays logs until user presses Ctrl+C.""" + wallet = Wallet( + path=wallet_info["path"], + name=wallet_name, + hotkey=wallet_info["hotkey"], + ) + console.print(f"[green]Starting miner {wallet_name}...") + + env_variables = os.environ.copy() + env_variables["BT_AXON_PORT"] = str(wallet_info["port"]) + env_variables["PYTHONUNBUFFERED"] = "1" + + cmd = [ + sys.executable, + "-u", + "./neurons/miner.py", + "--wallet.name", + wallet.name, + "--wallet.hotkey", + wallet.hotkey_str, + "--wallet.path", + BTQS_WALLETS_DIRECTORY, + "--subtensor.chain_endpoint", + "ws://127.0.0.1:9945", + "--logging.trace", + ] + + # Create log file paths + logs_dir = os.path.join(BTQS_DIRECTORY, "logs") + os.makedirs(logs_dir, exist_ok=True) + log_file_path = os.path.join(logs_dir, f"miner_{wallet_name}.log") + + log_file = open(log_file_path, "a") + try: + # Start the subprocess, redirecting stdout and stderr to the log file + process = subprocess.Popen( + cmd, + cwd=subnet_template_path, + stdout=log_file, + stderr=subprocess.STDOUT, + env=env_variables, + start_new_session=True, + ) + wallet_info["pid"] = process.pid + wallet_info["log_file"] = log_file_path + log_file.close() + + # Update config_data + config_data["Miners"][wallet_name] = wallet_info + + console.print(f"[green]Miner {wallet_name} started. Press Ctrl+C to proceed.") + try: + with open(log_file_path, "r") as log_file: + # Move to the end of the file + log_file.seek(0, os.SEEK_END) + while True: + line = log_file.readline() + if not line: + if not psutil.pid_exists(process.pid): + console.print("\n[red]Miner process has terminated.") + break + time.sleep(0.1) + continue + print(line, end="") + except KeyboardInterrupt: + console.print("\n[green]Detached from miner logs.") + return True + except Exception as e: + console.print(f"[red]Error starting miner {wallet_name}: {e}") + log_file.close() + return False diff --git a/btqs_requirements.txt b/btqs_requirements.txt new file mode 100644 index 00000000..0b574b52 --- /dev/null +++ b/btqs_requirements.txt @@ -0,0 +1 @@ +psutil \ No newline at end of file diff --git a/setup.py b/setup.py index f9c92cda..ca49c9e3 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ def read_requirements(path): requirements = read_requirements("requirements.txt") cuda_requirements = read_requirements("cuda_requirements.txt") +btqs_requirements = read_requirements("btqs_requirements.txt") here = path.abspath(path.dirname(__file__)) @@ -76,10 +77,12 @@ def read_requirements(path): install_requires=requirements, extras_require={ "cuda": cuda_requirements, + "btqs": btqs_requirements, }, entry_points={ "console_scripts": [ "btcli=bittensor_cli.cli:main", + "btqs=btqs.btqs_cli:main", ], }, classifiers=[ From 19e17c3fbf777ac7f1feac69e9f975f2ab1278d4 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 11 Oct 2024 09:53:19 -0700 Subject: [PATCH 02/23] Adds git to requirements --- btqs_requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/btqs_requirements.txt b/btqs_requirements.txt index 0b574b52..4468d480 100644 --- a/btqs_requirements.txt +++ b/btqs_requirements.txt @@ -1 +1,2 @@ -psutil \ No newline at end of file +psutil +GitPython From c115012c42172f9326b9b65aba510bd4fbfacdf2 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 11 Oct 2024 15:31:45 -0700 Subject: [PATCH 03/23] WIP: lego blocks --- btqs/btqs_cli.py | 105 ++++++++++++++++--------------------- btqs/utils.py | 133 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 164 insertions(+), 74 deletions(-) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index eb432211..6724ed51 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -35,6 +35,8 @@ start_miner, subnet_exists, subnet_owner_exists, + attach_to_process_logs, + start_validator ) @@ -190,7 +192,7 @@ def start_neurons(self): # Filter miners that are not running miners_not_running = [] for entry in process_entries: - if entry["process"].startswith("Miner") and entry["status"] != "Running": + if (entry["process"].startswith("Miner") or entry["process"].startswith("Validator")) and entry["status"] != "Running": miners_not_running.append(entry) if not miners_not_running: @@ -536,28 +538,18 @@ def reattach_chain(self): console.print("[red]Process not found. The chain may have been stopped.") return - # Log file setup for Subtensor chain + # Paths to the log files log_dir = os.path.join(subtensor_path, "logs") alice_log = os.path.join(log_dir, "alice.log") + # Check if log file exists if not os.path.exists(alice_log): console.print("[red]Log files not found.") return - try: - console.print("[green]Reattaching to the local chain...") - console.print("[green]Press Ctrl+C to detach.") - with open(alice_log, "r") as alice_file: - alice_file.seek(0, os.SEEK_END) - while True: - alice_line = alice_file.readline() - if not alice_line: - time.sleep(0.1) - continue - if alice_line: - print(f"[Alice] {alice_line}", end="") - except KeyboardInterrupt: - console.print("\n[green]Detached from the local chain.") + # Reattach using attach_to_process_logs + attach_to_process_logs(alice_log, "Subtensor Chain (Alice)", pid) + def setup_subnet(self): """ @@ -865,32 +857,29 @@ def setup_neurons(self): def run_neurons(self): """ - Runs all neurons (miners). + Runs all neurons (miners and validators). - This command starts the miner processes for all configured miners, attaching to running miners if they are already running. + This command starts the processes for all configured neurons, attaching to + running processes if they are already running. USAGE [green]$[/green] btqs neurons run - [bold]Note[/bold]: The command will attach to running miners or start new ones as necessary. Press Ctrl+C to detach from a miner and move to the next. + [bold]Note[/bold]: The command will attach to running neurons or start new + ones as necessary. Press Ctrl+C to detach from a neuron and move to the next. """ - if not os.path.exists(CONFIG_FILE_PATH): - console.print( - "[red]Config file not found. Please run `btqs chain start` first." - ) - return - - with open(CONFIG_FILE_PATH, "r") as config_file: - config_data = yaml.safe_load(config_file) or {} + + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) - if not config_data.get("Miners"): - console.print( - "[red]Miners not found. Please run `btqs neurons setup` first." - ) + # Ensure neurons are configured + if not config_data.get("Miners") and not config_data.get("Owner"): + console.print("[red]No neurons found. Please run `btqs neurons setup` first.") return - # Subnet template setup + # Ensure subnet-template is available subnet_template_path = os.path.join(BTQS_DIRECTORY, "subnet-template") if not os.path.exists(subnet_template_path): console.print("[green]Cloning subnet-template repository...") @@ -907,6 +896,27 @@ def run_neurons(self): repo.git.checkout(SUBNET_TEMPLATE_BRANCH) chain_pid = config_data.get("pid") + + # Handle Validator + if config_data.get("Owner"): + owner_info = config_data["Owner"] + validator_pid = owner_info.get("pid") + validator_subtensor_pid = owner_info.get("subtensor_pid") + + if validator_pid and psutil.pid_exists(validator_pid) and validator_subtensor_pid == chain_pid: + console.print("[green]Validator is already running. Attaching to the process...") + log_file_path = owner_info.get("log_file") + if log_file_path and os.path.exists(log_file_path): + attach_to_process_logs(log_file_path, "Validator", validator_pid) + else: + console.print("[red]Log file not found for validator. Cannot attach.") + else: + # Validator is not running, start it + success = start_validator(owner_info, subnet_template_path, config_data) + if not success: + console.print("[red]Failed to start validator.") + + # Handle Miners for wallet_name, wallet_info in config_data.get("Miners", {}).items(): miner_pid = wallet_info.get("pid") miner_subtensor_pid = wallet_info.get("subtensor_pid") @@ -921,31 +931,7 @@ def run_neurons(self): ) log_file_path = wallet_info.get("log_file") if log_file_path and os.path.exists(log_file_path): - try: - with open(log_file_path, "r") as log_file: - # Move to the end of the file - log_file.seek(0, os.SEEK_END) - console.print( - f"[green]Attached to miner {wallet_name}. Press Ctrl+C to move to the next miner." - ) - while True: - line = log_file.readline() - if not line: - # Check if the process is still running - if not psutil.pid_exists(miner_pid): - console.print( - f"\n[red]Miner process {wallet_name} has terminated." - ) - break - time.sleep(0.1) - continue - print(line, end="") - except KeyboardInterrupt: - console.print(f"\n[green]Detached from miner {wallet_name}.") - except Exception as e: - console.print( - f"[red]Error attaching to miner {wallet_name}: {e}" - ) + attach_to_process_logs(log_file_path, f"Miner {wallet_name}", miner_pid) else: console.print( f"[red]Log file not found for miner {wallet_name}. Cannot attach." @@ -961,6 +947,7 @@ def run_neurons(self): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) + def stop_neurons(self): """ Stops the running neurons. @@ -987,7 +974,7 @@ def stop_neurons(self): # Filter running miners running_miners = [] for entry in process_entries: - if entry["process"].startswith("Miner") and entry["status"] == "Running": + if (entry["process"].startswith("Miner") or entry["process"].startswith("Validator")) and entry["status"] == "Running": running_miners.append(entry) if not running_miners: @@ -1022,7 +1009,7 @@ def stop_neurons(self): # Stop selected miners for miner in selected_miners: pid = int(miner["pid"]) - wallet_name = miner["process"].split("Miner: ")[-1] + wallet_name = miner["process"].split("Miner: ")[-1] if "Miner" in miner["process"] else miner["process"].split("Validator: ")[-1] try: process = psutil.Process(pid) process.terminate() diff --git a/btqs/utils.py b/btqs/utils.py index 7b2d9867..769357b9 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -416,6 +416,34 @@ def get_process_entries( } ) + # Check status of Validators + if config_data.get("Owner"): + owner_data = config_data.get("Owner") + pid = owner_data.get("pid") + location = owner_data.get("path") + status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( + get_process_info(pid) + ) + + status_style = "green" if status == "Running" else "red" + + if status == "Running": + cpu_usage_list.append(cpu_percent) + memory_usage_list.append(memory_percent) + + process_entries.append( + { + "process": f"Validator: {owner_data.get("wallet_name")}", + "status": status, + "status_style": status_style, + "pid": str(pid), + "cpu_usage": cpu_usage, + "memory_usage": memory_usage, + "uptime_str": uptime_str, + "location": location, + } + ) + return process_entries, cpu_usage_list, memory_usage_list @@ -573,23 +601,98 @@ def start_miner( config_data["Miners"][wallet_name] = wallet_info console.print(f"[green]Miner {wallet_name} started. Press Ctrl+C to proceed.") - try: - with open(log_file_path, "r") as log_file: - # Move to the end of the file - log_file.seek(0, os.SEEK_END) - while True: - line = log_file.readline() - if not line: - if not psutil.pid_exists(process.pid): - console.print("\n[red]Miner process has terminated.") - break - time.sleep(0.1) - continue - print(line, end="") - except KeyboardInterrupt: - console.print("\n[green]Detached from miner logs.") + attach_to_process_logs(log_file_path, f"Miner {wallet_name}", process.pid) return True except Exception as e: console.print(f"[red]Error starting miner {wallet_name}: {e}") log_file.close() return False + + +def start_validator( + owner_info: Dict[str, Any], subnet_template_path: str, config_data: Dict[str, Any] +) -> bool: + """Starts the validator process and displays logs until user presses Ctrl+C.""" + wallet = Wallet( + path=owner_info["path"], + name=owner_info["wallet_name"], + hotkey=owner_info["hotkey"], + ) + console.print("[green]Starting validator...") + + env_variables = os.environ.copy() + env_variables["PYTHONUNBUFFERED"] = "1" + env_variables["BT_AXON_PORT"] = str(8100) + + cmd = [ + sys.executable, + "-u", + "./neurons/validator.py", + "--wallet.name", + wallet.name, + "--wallet.hotkey", + wallet.hotkey_str, + "--wallet.path", + BTQS_WALLETS_DIRECTORY, + "--subtensor.chain_endpoint", + "ws://127.0.0.1:9945", + "--netuid", + "1", + "--logging.trace", + ] + + # Create log file paths + logs_dir = os.path.join(BTQS_DIRECTORY, "logs") + os.makedirs(logs_dir, exist_ok=True) + log_file_path = os.path.join(logs_dir, "validator.log") + + log_file = open(log_file_path, "a") + try: + # Start the subprocess, redirecting stdout and stderr to the log file + process = subprocess.Popen( + cmd, + cwd=subnet_template_path, + stdout=log_file, + stderr=subprocess.STDOUT, + env=env_variables, + start_new_session=True, + ) + owner_info["pid"] = process.pid + owner_info["log_file"] = log_file_path + log_file.close() + + # Update config_data + config_data["Owner"] = owner_info + + console.print("[green]Validator started. Press Ctrl+C to proceed.") + attach_to_process_logs(log_file_path, "Validator", process.pid) + return True + except Exception as e: + console.print(f"[red]Error starting validator: {e}") + log_file.close() + return False + + +def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = None): + """Attaches to the log file of a process and prints logs until user presses Ctrl+C or the process terminates.""" + try: + with open(log_file_path, "r") as log_file: + # Move to the end of the file + log_file.seek(0, os.SEEK_END) + console.print( + f"[green]Attached to {process_name}. Press Ctrl+C to move to the next process." + ) + while True: + line = log_file.readline() + if not line: + # Check if the process is still running + if pid and not psutil.pid_exists(pid): + console.print(f"\n[red]{process_name} process has terminated.") + break + time.sleep(0.1) + continue + print(line, end="") + except KeyboardInterrupt: + console.print(f"\n[green]Detached from {process_name}.") + except Exception as e: + console.print(f"[red]Error attaching to {process_name}: {e}") From 61277e95ba9733afcb8730b9af1b7dd39218e690 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Tue, 15 Oct 2024 20:37:28 -0700 Subject: [PATCH 04/23] Constructing --- btqs/__init__.py | 0 btqs/btqs_cli.py | 1317 +++++++++++++++------------------ btqs/src/__init__.py | 0 btqs/src/commands/__init__.py | 0 btqs/src/commands/chain.py | 261 +++++++ btqs/src/commands/neurons.py | 397 ++++++++++ btqs/utils.py | 245 +++--- btqs_requirements.txt | 1 + 8 files changed, 1363 insertions(+), 858 deletions(-) create mode 100644 btqs/__init__.py create mode 100644 btqs/src/__init__.py create mode 100644 btqs/src/commands/__init__.py create mode 100644 btqs/src/commands/chain.py create mode 100644 btqs/src/commands/neurons.py diff --git a/btqs/__init__.py b/btqs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 6724ed51..ffe428a7 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -2,8 +2,13 @@ import platform import subprocess import time -from time import sleep +import time +import threading +import sys +from tqdm import tqdm +from typing import Any, Dict, Optional +from time import sleep import psutil import typer import yaml @@ -33,12 +38,13 @@ load_config, remove_ansi_escape_sequences, start_miner, + start_validator, subnet_exists, subnet_owner_exists, attach_to_process_logs, - start_validator ) +from btqs.src.commands import chain, neurons class BTQSManager: """ @@ -68,7 +74,7 @@ def __init__(self): self.chain_app.command(name="stop")(self.stop_chain) self.chain_app.command(name="reattach")(self.reattach_chain) - # Setup commands + # Subnet commands self.subnet_app.command(name="setup")(self.setup_subnet) # Neuron commands @@ -77,25 +83,66 @@ def __init__(self): self.neurons_app.command(name="stop")(self.stop_neurons) self.neurons_app.command(name="reattach")(self.reattach_neurons) self.neurons_app.command(name="status")(self.status_neurons) + self.neurons_app.command(name="live")(self.display_live_metagraph) self.neurons_app.command(name="start")(self.start_neurons) + self.neurons_app.command(name="stake")(self.add_stake) self.app.command(name="run-all", help="Create entire setup")(self.run_all) + self.app.command(name="status", help="Current status of bittensor quick start")( + self.status_neurons + ) - def run_all(self): - """ - Runs all commands in sequence to set up and start the local chain, subnet, and neurons. + def display_live_metagraph(self): + def clear_screen(): + os.system("cls" if os.name == "nt" else "clear") - This command automates the entire setup process, including starting the local Subtensor chain, - setting up a subnet, creating and registering miner wallets, and running the miners. + def get_metagraph(): + result = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], + internal_command=True + ) + # clear_screen() + return result.stdout - USAGE + print("Starting live metagraph view. Press 'Ctrl + C' to exit.") + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + def input_thread(): + while True: + if input() == "q": + print("Exiting live view...") + sys.exit(0) - Run this command to perform all steps necessary to start the local chain and miners: + threading.Thread(target=input_thread, daemon=True).start() - [green]$[/green] btqs run-all + try: + while True: + metagraph = get_metagraph() + process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) + clear_screen() + print(metagraph) + display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) + + # Create a progress bar for 5 seconds + print("\n") + for _ in tqdm(range(5), desc="Refreshing", unit="s", total=5): + time.sleep(1) + + except KeyboardInterrupt: + print("Exiting live view...") - [bold]Note[/bold]: This command is useful for quickly setting up the entire environment. - It will prompt for inputs as needed. + def run_all(self): + """ + Runs all commands in sequence to set up and start the local chain, subnet, and neurons. """ text = Text("Starting Local Subtensor\n", style="bold light_goldenrod2") sign = Text("πŸ”— ", style="bold yellow") @@ -137,7 +184,7 @@ def run_all(self): # Set up the neurons (miners) self.setup_neurons() - console.print("\nNext command will: 1. Start all miners processes") + console.print("\nNext command will: 1. Start all miner processes") console.print("Press any key to continue..\n") input() @@ -164,101 +211,64 @@ def run_all(self): ) print(subnets_list.stdout, end="") - def start_neurons(self): - """ - Starts selected neurons. - - This command allows you to start specific miners that are not currently running. - - USAGE - - [green]$[/green] btqs neurons start - - [bold]Note[/bold]: You can select which miners to start or start all that are not running. - """ - config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." - ) - if not config_data.get("Miners"): - console.print( - "[red]Miners not found. Please run `btqs neurons setup` first." - ) - return - - # Get process entries - process_entries, _, _ = get_process_entries(config_data) - display_process_status_table(process_entries, [], []) - - # Filter miners that are not running - miners_not_running = [] - for entry in process_entries: - if (entry["process"].startswith("Miner") or entry["process"].startswith("Validator")) and entry["status"] != "Running": - miners_not_running.append(entry) - - if not miners_not_running: - console.print("[green]All miners are already running.") - return - - # Display the list of miners not running - console.print("\nMiners not running:") - for idx, miner in enumerate(miners_not_running, start=1): - console.print(f"{idx}. {miner['process']}") - - # Prompt user to select miners to start - selection = typer.prompt( - "Enter miner numbers to start (comma-separated), or 'all' to start all", - default="all", + self.add_stake() + console.print("[dark_green]\nViewing Metagraph for Subnet 1") + subnets_list = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], ) + print(subnets_list.stdout, end="") + self.display_live_metagraph() - if selection.lower() == "all": - selected_miners = miners_not_running - else: - selected_indices = [ - int(i.strip()) for i in selection.split(",") if i.strip().isdigit() - ] - selected_miners = [ - miners_not_running[i - 1] - for i in selected_indices - if 1 <= i <= len(miners_not_running) - ] - - if not selected_miners: - console.print("[red]No valid miners selected.") - return - - # TODO: Make this configurable - # Subnet template setup - subnet_template_path = os.path.join(BTQS_DIRECTORY, "subnet-template") - if not os.path.exists(subnet_template_path): - console.print("[green]Cloning subnet-template repository...") - repo = Repo.clone_from( - SUBNET_TEMPLATE_REPO_URL, - subnet_template_path, + def add_stake(self): + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if subnet_owner: + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), ) - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - else: - console.print("[green]Using existing subnet-template repository.") - repo = Repo(subnet_template_path) - current_branch = repo.active_branch.name - if current_branch != SUBNET_TEMPLATE_BRANCH: - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - - # TODO: Add ability for users to define their own flags, entry point etc - # Start selected miners - for miner in selected_miners: - wallet_name = miner["process"].split("Miner: ")[-1] - wallet_info = config_data["Miners"][wallet_name] - success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data + add_stake = exec_command( + command="stake", + sub_command="add", + extra_args=[ + "--amount", + 1000, + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--chain", + "ws://127.0.0.1:9945", + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + ], ) - if success: - console.print(f"[green]Miner {wallet_name} started.") + + clean_stdout = remove_ansi_escape_sequences(add_stake.stdout) + if "βœ… Finalized" in clean_stdout: + text = Text( + f"Stake added successfully by Validator ({owner_wallet})\n", + style="bold light_goldenrod2", + ) + sign = Text("πŸ“ˆ ", style="bold yellow") + console.print(sign, text) else: - console.print(f"[red]Failed to start miner {wallet_name}.") + console.print("\n[red] Failed to add stake. Command output:\n") + print(add_stake.stdout, end="") - # Update the config file - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) + else: + console.print( + "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" + ) + return def start_chain(self): """ @@ -280,143 +290,8 @@ def start_chain(self): ) return - config_data = load_config(exit_if_missing=False) - if config_data: - console.print("[green] Refreshing config file") - config_data = {} - - directory = typer.prompt( - "Enter the directory to clone the subtensor repository", - default=os.path.expanduser("~/Desktop/Bittensor_quick_start"), - show_default=True, - ) - os.makedirs(directory, exist_ok=True) - - subtensor_path = os.path.join(directory, "subtensor") - repo_url = "https://github.com/opentensor/subtensor.git" - - # Clone or update the repository - if os.path.exists(subtensor_path) and os.listdir(subtensor_path): - update = typer.confirm( - "Subtensor is already cloned. Do you want to update it?" - ) - if update: - try: - repo = Repo(subtensor_path) - origin = repo.remotes.origin - origin.pull() - console.print("[green]Repository updated successfully.") - except GitCommandError as e: - console.print(f"[red]Error updating repository: {e}") - return - else: - console.print( - "[green]Using existing subtensor repository without updating." - ) - else: - try: - console.print("[green]Cloning subtensor repository...") - Repo.clone_from(repo_url, subtensor_path) - console.print("[green]Repository cloned successfully.") - except GitCommandError as e: - console.print(f"[red]Error cloning repository: {e}") - return - - localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") - - # Running localnet.sh - process = subprocess.Popen( - ["bash", localnet_path], - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - cwd=subtensor_path, - start_new_session=True, - ) - - console.print("[green]Starting local chain. This may take a few minutes...") - - # Paths to subtensor log files - log_dir = os.path.join(subtensor_path, "logs") - alice_log = os.path.join(log_dir, "alice.log") - - # Waiting for chain compilation - timeout = 360 # 6 minutes - start_time = time.time() - while not os.path.exists(alice_log): - if time.time() - start_time > timeout: - console.print("[red]Timeout: Log files were not created.") - return - time.sleep(1) - - chain_ready = False - try: - with open(alice_log, "r") as log_file: - log_file.seek(0, os.SEEK_END) - while True: - line = log_file.readline() - if line: - console.print(line, end="") - if "Imported #" in line: - chain_ready = True - break - else: - if time.time() - start_time > timeout: - console.print( - "[red]Timeout: Chain did not compile in time." - ) - break - time.sleep(0.1) - except Exception as e: - console.print(f"[red]Error reading log files: {e}") - return - - if chain_ready: - text = Text( - "Local chain is running. You can now use it for development and testing.\n", - style="bold light_goldenrod2", - ) - sign = Text("\nβš™οΈ ", style="bold yellow") - console.print(sign, text) - - try: - # Fetch PIDs of 2 substrate nodes spawned - result = subprocess.run( - ["pgrep", "-f", "node-subtensor"], capture_output=True, text=True - ) - substrate_pids = [int(pid) for pid in result.stdout.strip().split()] - - config_data.update( - { - "pid": process.pid, - "substrate_pid": substrate_pids, - "subtensor_path": subtensor_path, - "base_path": directory, - } - ) - except ValueError: - console.print("[red]Failed to get the PID of the Subtensor process.") - return - - config_data.update( - { - "pid": process.pid, - "subtensor_path": subtensor_path, - "base_path": directory, - } - ) - - # Save config data - try: - os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True) - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - console.print( - "[green]Local chain started successfully and config file updated." - ) - except Exception as e: - console.print(f"[red]Failed to write to the config file: {e}") - else: - console.print("[red]Failed to start local chain.") + config_data = load_config(exit_if_missing=False) or {} + chain.start_chain(config_data) def stop_chain(self): """ @@ -433,76 +308,7 @@ def stop_chain(self): config_data = load_config( "No running chain found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - - pid = config_data.get("pid") - if not pid: - console.print("[red]No running chain found.") - return - - console.print("[red]Stopping the local chain...") - - try: - process = psutil.Process(pid) - process.terminate() - process.wait(timeout=10) - console.print("[green]Local chain stopped successfully.") - - except psutil.NoSuchProcess: - console.print( - "[red]Process not found. The chain may have already been stopped." - ) - - except psutil.TimeoutExpired: - console.print("[red]Timeout while stopping the chain. Forcing stop...") - process.kill() - - # Check for running miners - process_entries, _, _ = get_process_entries(config_data) - - # Filter running miners - running_miners = [] - for entry in process_entries: - if entry["process"].startswith("Miner") and entry["status"] == "Running": - running_miners.append(entry) - - if running_miners: - console.print( - "[yellow]\nSome miners are still running. Terminating them..." - ) - - for miner in running_miners: - pid = int(miner["pid"]) - wallet_name = miner["process"].split("Miner: ")[-1] - try: - miner_process = psutil.Process(pid) - miner_process.terminate() - miner_process.wait(timeout=10) - console.print(f"[green]Miner {wallet_name} stopped.") - - except psutil.NoSuchProcess: - console.print(f"[yellow]Miner {wallet_name} process not found.") - - except psutil.TimeoutExpired: - console.print( - f"[red]Timeout stopping miner {wallet_name}. Forcing stop." - ) - miner_process.kill() - - config_data["Miners"][wallet_name]["pid"] = None - - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - else: - console.print("[green]No miners were running.") - - # Refresh data - refresh_config = typer.confirm( - "\nConfig data is outdated. Press Y to refresh it?" - ) - if refresh_config: - if os.path.exists(CONFIG_FILE_PATH): - os.remove(CONFIG_FILE_PATH) - console.print("[green]Configuration file removed.") + chain.stop_chain(config_data) def reattach_chain(self): """ @@ -519,37 +325,7 @@ def reattach_chain(self): config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - - pid = config_data.get("pid") - subtensor_path = config_data.get("subtensor_path") - if not pid or not subtensor_path: - console.print("[red]No running chain found.") - return - - # Check if the process is still running - try: - process = psutil.Process(pid) - if not process.is_running(): - console.print( - "[red]Process not running. The chain may have been stopped." - ) - return - except psutil.NoSuchProcess: - console.print("[red]Process not found. The chain may have been stopped.") - return - - # Paths to the log files - log_dir = os.path.join(subtensor_path, "logs") - alice_log = os.path.join(log_dir, "alice.log") - - # Check if log file exists - if not os.path.exists(alice_log): - console.print("[red]Log files not found.") - return - - # Reattach using attach_to_process_logs - attach_to_process_logs(alice_log, "Subtensor Chain (Alice)", pid) - + chain.reattach_chain(config_data) def setup_subnet(self): """ @@ -591,47 +367,18 @@ def setup_subnet(self): console.print(warning_sign, warning_text) console.print(wallet_info) - else: - text = Text( - "Creating subnet owner wallet.\n", style="bold light_goldenrod2" - ) - sign = Text("πŸ‘‘ ", style="bold yellow") - console.print(sign, text) - - owner_wallet_name = typer.prompt( - "Enter subnet owner wallet name", default="owner", show_default=True - ) - owner_hotkey_name = typer.prompt( - "Enter subnet owner hotkey name", default="default", show_default=True + self._create_subnet_owner_wallet(config_data) + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - uri = "//Alice" - keypair = Keypair.create_from_uri(uri) - owner_wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, - name=owner_wallet_name, - hotkey=owner_hotkey_name, - ) - owner_wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) - owner_wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) - owner_wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) - - console.print( - "Executed command: [dark_orange] btcli wallet create --wallet-name", - f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {BTQS_WALLETS_DIRECTORY}", - ) - - with open(CONFIG_FILE_PATH, "r") as config_file: - config_data = yaml.safe_load(config_file) - config_data["Owner"] = { - "wallet_name": owner_wallet_name, - "path": BTQS_WALLETS_DIRECTORY, - "hotkey": owner_hotkey_name, - "subtensor_pid": config_data["pid"], - } - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) + owner_data = config_data["Owner"] + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), + ) if subnet_exists(owner_wallet.coldkeypub.ss58_address, 1): warning_text = Text( @@ -649,211 +396,33 @@ def setup_subnet(self): console.print(wallet_info) console.print(sudo_info) else: - text = Text( - "Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2" - ) - sign = Text("\nπŸ’» ", style="bold yellow") - console.print(sign, text) + self._create_subnet(owner_wallet) - create_subnet = exec_command( - command="subnets", - sub_command="create", - extra_args=[ - "--wallet-path", - BTQS_WALLETS_DIRECTORY, - "--chain", - "ws://127.0.0.1:9945", - "--wallet-name", - owner_wallet.name, - "--no-prompt", - "--wallet-hotkey", - owner_wallet.hotkey_str, - ], - ) - clean_stdout = remove_ansi_escape_sequences(create_subnet.stdout) - if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: - console.print("[dark_green] Subnet created successfully with netuid 1") + console.print("[dark_green]\nListing all subnets") + subnets_list = exec_command( + command="subnets", + sub_command="list", + extra_args=[ + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") - text = Text( - f"Registering Owner ({owner_wallet.name}) to Netuid 1\n", - style="bold light_goldenrod2", + def setup_neurons(self): + """ + Sets up neurons (miners) for the subnet. + """ + if not is_chain_running(CONFIG_FILE_PATH): + console.print( + "[red]Local chain is not running. Please start the chain first." ) - sign = Text("\nπŸ“ ", style="bold yellow") - console.print(sign, text) - - register_subnet = exec_command( - command="subnets", - sub_command="register", - extra_args=[ - "--wallet-path", - BTQS_WALLETS_DIRECTORY, - "--wallet-name", - owner_wallet.name, - "--wallet-hotkey", - owner_wallet.hotkey_str, - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - "--no-prompt", - ], - ) - clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) - if "βœ… Registered" in clean_stdout: - console.print("[green] Registered the owner to subnet 1") - - console.print("[dark_green]\nListing all subnets") - subnets_list = exec_command( - command="subnets", - sub_command="list", - extra_args=[ - "--chain", - "ws://127.0.0.1:9945", - ], - ) - print(subnets_list.stdout, end="") - - def setup_neurons(self): - """ - Sets up neurons (miners) for the subnet. - - This command creates miner wallets and registers them to the subnet. - - USAGE - - [green]$[/green] btqs neurons setup - - [bold]Note[/bold]: This command will prompt for wallet names and hotkey names for each miner. - """ - if not is_chain_running(CONFIG_FILE_PATH): - console.print( - "[red]Local chain is not running. Please start the chain first." - ) - return + return config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) - if subnet_owner: - owner_wallet = Wallet( - name=owner_data.get("wallet_name"), - path=owner_data.get("path"), - hotkey=owner_data.get("hotkey"), - ) - else: - console.print( - "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" - ) - return - - config_data.setdefault("Miners", {}) - miners = config_data.get("Miners", {}) - - if miners and all( - miner_info.get("subtensor_pid") == config_data.get("pid") - for miner_info in miners.values() - ): - console.print( - "[green]Miner wallets associated with this subtensor instance already present. Proceeding..." - ) - else: - uris = [ - "//Bob", - "//Charlie", - ] - ports = [8100, 8101, 8102, 8103] - for i, uri in enumerate(uris, start=0): - console.print(f"Miner {i+1}:") - wallet_name = typer.prompt( - f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" - ) - hotkey_name = typer.prompt( - f"Enter hotkey name for miner {i+1}", default="default" - ) - - keypair = Keypair.create_from_uri(uri) - wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=hotkey_name - ) - wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) - wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) - wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) - - config_data["Miners"][wallet_name] = { - "path": BTQS_WALLETS_DIRECTORY, - "hotkey": hotkey_name, - "uri": uri, - "pid": None, - "subtensor_pid": config_data["pid"], - "port": ports[i], - } - - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - - console.print("[green]All wallets are created.") - - for wallet_name, wallet_info in config_data["Miners"].items(): - wallet = Wallet( - path=wallet_info["path"], - name=wallet_name, - hotkey=wallet_info["hotkey"], - ) - - text = Text( - f"Registering Miner ({wallet_name}) to Netuid 1\n", - style="bold light_goldenrod2", - ) - sign = Text("\nπŸ“ ", style="bold yellow") - console.print(sign, text) - - miner_registered = exec_command( - command="subnets", - sub_command="register", - extra_args=[ - "--wallet-path", - wallet.path, - "--wallet-name", - wallet.name, - "--hotkey", - wallet.hotkey_str, - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - "--no-prompt", - ], - ) - clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) - - if "βœ… Registered" in clean_stdout: - text = Text( - f"Registered miner ({wallet.name}) to Netuid 1\n", - style="bold light_goldenrod2", - ) - sign = Text("πŸ† ", style="bold yellow") - console.print(sign, text) - else: - print(clean_stdout) - console.print( - f"[red]Failed to register miner ({wallet.name}). Please register the miner manually using the following command:" - ) - command = f"btcli subnets register --wallet-path {wallet.path} --wallet-name {wallet.name} --hotkey {wallet.hotkey_str} --netuid 1 --chain ws://127.0.0.1:9945 --no-prompt" - console.print(f"[bold yellow]{command}\n") - - console.print("[dark_green]\nViewing Metagraph for Subnet 1") - subnets_list = exec_command( - command="subnets", - sub_command="metagraph", - extra_args=[ - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - ], - ) - print(subnets_list.stdout, end="") + neurons.setup_neurons(config_data) def run_neurons(self): """ @@ -869,84 +438,18 @@ def run_neurons(self): [bold]Note[/bold]: The command will attach to running neurons or start new ones as necessary. Press Ctrl+C to detach from a neuron and move to the next. """ - config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) # Ensure neurons are configured if not config_data.get("Miners") and not config_data.get("Owner"): - console.print("[red]No neurons found. Please run `btqs neurons setup` first.") - return - - # Ensure subnet-template is available - subnet_template_path = os.path.join(BTQS_DIRECTORY, "subnet-template") - if not os.path.exists(subnet_template_path): - console.print("[green]Cloning subnet-template repository...") - repo = Repo.clone_from( - SUBNET_TEMPLATE_REPO_URL, - subnet_template_path, + console.print( + "[red]No neurons found. Please run `btqs neurons setup` first." ) - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - else: - console.print("[green]Using existing subnet-template repository.") - repo = Repo(subnet_template_path) - current_branch = repo.active_branch.name - if current_branch != SUBNET_TEMPLATE_BRANCH: - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - - chain_pid = config_data.get("pid") - - # Handle Validator - if config_data.get("Owner"): - owner_info = config_data["Owner"] - validator_pid = owner_info.get("pid") - validator_subtensor_pid = owner_info.get("subtensor_pid") - - if validator_pid and psutil.pid_exists(validator_pid) and validator_subtensor_pid == chain_pid: - console.print("[green]Validator is already running. Attaching to the process...") - log_file_path = owner_info.get("log_file") - if log_file_path and os.path.exists(log_file_path): - attach_to_process_logs(log_file_path, "Validator", validator_pid) - else: - console.print("[red]Log file not found for validator. Cannot attach.") - else: - # Validator is not running, start it - success = start_validator(owner_info, subnet_template_path, config_data) - if not success: - console.print("[red]Failed to start validator.") - - # Handle Miners - for wallet_name, wallet_info in config_data.get("Miners", {}).items(): - miner_pid = wallet_info.get("pid") - miner_subtensor_pid = wallet_info.get("subtensor_pid") - # Check if miner process is running and associated with the current chain - if ( - miner_pid - and psutil.pid_exists(miner_pid) - and miner_subtensor_pid == chain_pid - ): - console.print( - f"[green]Miner {wallet_name} is already running. Attaching to the process..." - ) - log_file_path = wallet_info.get("log_file") - if log_file_path and os.path.exists(log_file_path): - attach_to_process_logs(log_file_path, f"Miner {wallet_name}", miner_pid) - else: - console.print( - f"[red]Log file not found for miner {wallet_name}. Cannot attach." - ) - else: - # Miner is not running, start it - success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data - ) - if not success: - console.print(f"[red]Failed to start miner {wallet_name}.") - - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - + return + + neurons.run_neurons(config_data) def stop_neurons(self): """ @@ -964,70 +467,57 @@ def stop_neurons(self): console.print("[red]Config file not found.") return - with open(CONFIG_FILE_PATH, "r") as config_file: - config_data = yaml.safe_load(config_file) or {} + config_data = load_config() - # Get process entries - process_entries, _, _ = get_process_entries(config_data) - display_process_status_table(process_entries, [], []) + neurons.stop_neurons(config_data) - # Filter running miners - running_miners = [] - for entry in process_entries: - if (entry["process"].startswith("Miner") or entry["process"].startswith("Validator")) and entry["status"] == "Running": - running_miners.append(entry) + def start_neurons(self): + """ + Starts the stopped neurons. - if not running_miners: - console.print("[red]No running miners to stop.") - return + This command starts the miner processes for the selected or all stopped miners. - console.print("\nSelect miners to stop:") - for idx, miner in enumerate(running_miners, start=1): - console.print(f"{idx}. {miner['process']} (PID: {miner['pid']})") + USAGE - selection = typer.prompt( - "Enter miner numbers to stop (comma-separated), or 'all' to stop all", - default="all", - ) + [green]$[/green] btqs neurons start - if selection.lower() == "all": - selected_miners = running_miners - else: - selected_indices = [ - int(i.strip()) for i in selection.split(",") if i.strip().isdigit() - ] - selected_miners = [ - running_miners[i - 1] - for i in selected_indices - if 1 <= i <= len(running_miners) - ] - - if not selected_miners: - console.print("[red]No valid miners selected.") + [bold]Note[/bold]: You can choose which stopped miners to start or start all of them. + """ + if not os.path.exists(CONFIG_FILE_PATH): + console.print("[red]Config file not found.") return - # Stop selected miners - for miner in selected_miners: - pid = int(miner["pid"]) - wallet_name = miner["process"].split("Miner: ")[-1] if "Miner" in miner["process"] else miner["process"].split("Validator: ")[-1] - try: - process = psutil.Process(pid) - process.terminate() - process.wait(timeout=10) - console.print(f"[green]Miner {wallet_name} stopped.") + config_data = load_config() - except psutil.NoSuchProcess: - console.print(f"[yellow]Miner {wallet_name} process not found.") + neurons.start_neurons(config_data) - except psutil.TimeoutExpired: - console.print( - f"[red]Timeout stopping miner {wallet_name}. Forcing stop." + def _start_selected_neurons( + self, config_data: Dict[str, Any], selected_neurons: list[Dict[str, Any]] + ): + """Starts the selected neurons.""" + subnet_template_path = self._ensure_subnet_template(config_data) + + for neuron in selected_neurons: + neuron_name = neuron["process"] + if neuron_name.startswith("Validator"): + success = start_validator( + config_data["Owner"], subnet_template_path, config_data + ) + elif neuron_name.startswith("Miner"): + wallet_name = neuron_name.split("Miner: ")[-1] + wallet_info = config_data["Miners"][wallet_name] + success = start_miner( + wallet_name, wallet_info, subnet_template_path, config_data ) - process.kill() - config_data["Miners"][wallet_name]["pid"] = None - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) + if success: + console.print(f"[green]{neuron_name} started successfully.") + else: + console.print(f"[red]Failed to start {neuron_name}.") + + # Update the process entries after starting neurons + process_entries, _, _ = get_process_entries(config_data) + display_process_status_table(process_entries, [], []) def reattach_neurons(self): """ @@ -1045,13 +535,12 @@ def reattach_neurons(self): console.print("[red]Config file not found.") return - with open(CONFIG_FILE_PATH, "r") as config_file: - config_data = yaml.safe_load(config_file) or {} + config_data = load_config() # Choose which neuron to reattach to all_neurons = { - **config_data.get("Validators", {}), **config_data.get("Miners", {}), + "Validator": config_data.get("Owner", {}), } neuron_names = list(all_neurons.keys()) if not neuron_names: @@ -1081,27 +570,7 @@ def reattach_neurons(self): f"[green]Reattaching to neuron {neuron_choice}. Press Ctrl+C to exit." ) - try: - with open(log_file_path, "r") as log_file: - # Move to the end of the file - log_file.seek(0, os.SEEK_END) - while True: - line = log_file.readline() - if not line: - if not psutil.pid_exists(pid): - console.print( - f"\n[red]Neuron process {neuron_choice} has terminated." - ) - break - time.sleep(0.1) - continue - print(line, end="") - - except KeyboardInterrupt: - console.print("\n[green]Detached from neuron logs.") - - except Exception as e: - console.print(f"[red]Error reattaching to neuron: {e}") + attach_to_process_logs(log_file_path, neuron_choice, pid) def status_neurons(self): """ @@ -1172,10 +641,410 @@ def status_neurons(self): console.print("\n") console.print(layout) + return layout def run(self): self.app() + # TODO: See if we can further streamline these. Or change location if needed + # ------------------------ Helper Methods ------------------------ + + def _wait_for_chain_ready( + self, alice_log: str, start_time: float, timeout: int + ) -> bool: + """Waits for the chain to be ready by monitoring the alice.log file.""" + chain_ready = False + try: + with open(alice_log, "r") as log_file: + log_file.seek(0, os.SEEK_END) + while True: + line = log_file.readline() + if line: + console.print(line, end="") + if "Imported #" in line: + chain_ready = True + break + else: + if time.time() - start_time > timeout: + console.print( + "[red]Timeout: Chain did not compile in time." + ) + break + time.sleep(0.1) + except Exception as e: + console.print(f"[red]Error reading log files: {e}") + return chain_ready + + def _get_substrate_pids(self) -> Optional[list[int]]: + """Fetches the PIDs of the substrate nodes.""" + try: + result = subprocess.run( + ["pgrep", "-f", "node-subtensor"], capture_output=True, text=True + ) + substrate_pids = [int(pid) for pid in result.stdout.strip().split()] + return substrate_pids + except ValueError: + console.print("[red]Failed to get the PID of the Subtensor process.") + return None + + def _stop_running_neurons(self, config_data: Dict[str, Any]): + """Stops any running neurons.""" + process_entries, _, _ = get_process_entries(config_data) + + # Filter running neurons + running_neurons = [ + entry + for entry in process_entries + if ( + entry["process"].startswith("Miner") + or entry["process"].startswith("Validator") + ) + and entry["status"] == "Running" + ] + + if running_neurons: + console.print( + "[yellow]\nSome neurons are still running. Terminating them..." + ) + + for neuron in running_neurons: + pid = int(neuron["pid"]) + neuron_name = neuron["process"] + try: + neuron_process = psutil.Process(pid) + neuron_process.terminate() + neuron_process.wait(timeout=10) + console.print(f"[green]{neuron_name} stopped.") + except psutil.NoSuchProcess: + console.print(f"[yellow]{neuron_name} process not found.") + except psutil.TimeoutExpired: + console.print(f"[red]Timeout stopping {neuron_name}. Forcing stop.") + neuron_process.kill() + + if neuron["process"].startswith("Miner"): + wallet_name = neuron["process"].split("Miner: ")[-1] + config_data["Miners"][wallet_name]["pid"] = None + elif neuron["process"].startswith("Validator"): + config_data["Owner"]["pid"] = None + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + else: + console.print("[green]No neurons were running.") + + def _is_process_running(self, pid: int) -> bool: + """Checks if a process with the given PID is running.""" + try: + process = psutil.Process(pid) + if not process.is_running(): + console.print( + "[red]Process not running. The chain may have been stopped." + ) + return False + return True + except psutil.NoSuchProcess: + console.print("[red]Process not found. The chain may have been stopped.") + return False + + def _create_subnet_owner_wallet(self, config_data: Dict[str, Any]): + """Creates a subnet owner wallet.""" + console.print( + Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2"), + style="bold yellow", + ) + + owner_wallet_name = typer.prompt( + "Enter subnet owner wallet name", default="owner", show_default=True + ) + owner_hotkey_name = typer.prompt( + "Enter subnet owner hotkey name", default="default", show_default=True + ) + + uri = "//Alice" + keypair = Keypair.create_from_uri(uri) + owner_wallet = Wallet( + path=BTQS_WALLETS_DIRECTORY, + name=owner_wallet_name, + hotkey=owner_hotkey_name, + ) + owner_wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + owner_wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + owner_wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + + console.print( + "Executed command: [dark_orange] btcli wallet create --wallet-name", + f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {BTQS_WALLETS_DIRECTORY}", + ) + + config_data["Owner"] = { + "wallet_name": owner_wallet_name, + "path": BTQS_WALLETS_DIRECTORY, + "hotkey": owner_hotkey_name, + "subtensor_pid": config_data["pid"], + } + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + def _create_subnet(self, owner_wallet: Wallet): + """Creates a subnet with netuid 1 and registers the owner.""" + console.print( + Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2"), + style="bold yellow", + ) + + create_subnet = exec_command( + command="subnets", + sub_command="create", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--chain", + "ws://127.0.0.1:9945", + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + ], + ) + clean_stdout = remove_ansi_escape_sequences(create_subnet.stdout) + if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: + console.print("[dark_green] Subnet created successfully with netuid 1") + + console.print( + Text( + f"Registering Owner ({owner_wallet.name}) to Netuid 1\n", + style="bold light_goldenrod2", + ), + style="bold yellow", + ) + + register_subnet = exec_command( + command="subnets", + sub_command="register", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--wallet-name", + owner_wallet.name, + "--wallet-hotkey", + owner_wallet.hotkey_str, + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + "--no-prompt", + ], + ) + clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) + if "βœ… Registered" in clean_stdout: + console.print("[green] Registered the owner to subnet 1") + + def _create_miner_wallets(self, config_data: Dict[str, Any]): + """Creates miner wallets.""" + uris = ["//Bob", "//Charlie"] + ports = [8101, 8102, 8103] + for i, uri in enumerate(uris): + console.print(f"Miner {i+1}:") + wallet_name = typer.prompt( + f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" + ) + hotkey_name = typer.prompt( + f"Enter hotkey name for miner {i+1}", default="default" + ) + + keypair = Keypair.create_from_uri(uri) + wallet = Wallet( + path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=hotkey_name + ) + wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + + config_data["Miners"][wallet_name] = { + "path": BTQS_WALLETS_DIRECTORY, + "hotkey": hotkey_name, + "uri": uri, + "pid": None, + "subtensor_pid": config_data["pid"], + "port": ports[i], + } + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + console.print("[green]All wallets are created.") + + def _register_miners(self, config_data: Dict[str, Any]): + """Registers miners to the subnet.""" + for wallet_name, wallet_info in config_data["Miners"].items(): + wallet = Wallet( + path=wallet_info["path"], + name=wallet_name, + hotkey=wallet_info["hotkey"], + ) + + console.print( + Text( + f"Registering Miner ({wallet_name}) to Netuid 1\n", + style="bold light_goldenrod2", + ), + style="bold yellow", + ) + + miner_registered = exec_command( + command="subnets", + sub_command="register", + extra_args=[ + "--wallet-path", + wallet.path, + "--wallet-name", + wallet.name, + "--hotkey", + wallet.hotkey_str, + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + "--no-prompt", + ], + ) + clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) + + if "βœ… Registered" in clean_stdout: + console.print(f"[green]Registered miner ({wallet.name}) to Netuid 1") + else: + console.print( + f"[red]Failed to register miner ({wallet.name}). Please register the miner manually." + ) + command = ( + f"btcli subnets register --wallet-path {wallet.path} --wallet-name " + f"{wallet.name} --hotkey {wallet.hotkey_str} --netuid 1 --chain " + f"ws://127.0.0.1:9945 --no-prompt" + ) + console.print(f"[bold yellow]{command}\n") + + def _ensure_subnet_template(self, config_data: Dict[str, Any]): + """Ensures that the subnet-template repository is available.""" + base_path = config_data.get("base_path") + if not base_path: + console.print("[red]Base path not found in the configuration file.") + return + + subnet_template_path = os.path.join(base_path, "subnet-template") + + if not os.path.exists(subnet_template_path): + console.print("[green]Cloning subnet-template repository...") + try: + repo = Repo.clone_from( + SUBNET_TEMPLATE_REPO_URL, + subnet_template_path, + ) + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + console.print("[green]Cloned subnet-template repository successfully.") + except GitCommandError as e: + console.print(f"[red]Error cloning subnet-template repository: {e}") + else: + console.print("[green]Using existing subnet-template repository.") + repo = Repo(subnet_template_path) + current_branch = repo.active_branch.name + if current_branch != SUBNET_TEMPLATE_BRANCH: + try: + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + console.print( + f"[green]Switched to branch '{SUBNET_TEMPLATE_BRANCH}'." + ) + except GitCommandError as e: + console.print( + f"[red]Error switching to branch '{SUBNET_TEMPLATE_BRANCH}': {e}" + ) + + return subnet_template_path + + def _handle_validator( + self, config_data: Dict[str, Any], subnet_template_path: str, chain_pid: int + ): + """Handles the validator process.""" + owner_info = config_data["Owner"] + validator_pid = owner_info.get("pid") + validator_subtensor_pid = owner_info.get("subtensor_pid") + + if ( + validator_pid + and psutil.pid_exists(validator_pid) + and validator_subtensor_pid == chain_pid + ): + console.print( + "[green]Validator is already running. Attaching to the process..." + ) + log_file_path = owner_info.get("log_file") + if log_file_path and os.path.exists(log_file_path): + attach_to_process_logs(log_file_path, "Validator", validator_pid) + else: + console.print("[red]Log file not found for validator. Cannot attach.") + else: + # Validator is not running, start it + success = start_validator(owner_info, subnet_template_path, config_data) + if not success: + console.print("[red]Failed to start validator.") + + def _handle_miners( + self, config_data: Dict[str, Any], subnet_template_path: str, chain_pid: int + ): + """Handles the miner processes.""" + for wallet_name, wallet_info in config_data.get("Miners", {}).items(): + miner_pid = wallet_info.get("pid") + miner_subtensor_pid = wallet_info.get("subtensor_pid") + # Check if miner process is running and associated with the current chain + if ( + miner_pid + and psutil.pid_exists(miner_pid) + and miner_subtensor_pid == chain_pid + ): + console.print( + f"[green]Miner {wallet_name} is already running. Attaching to the process..." + ) + log_file_path = wallet_info.get("log_file") + if log_file_path and os.path.exists(log_file_path): + attach_to_process_logs( + log_file_path, f"Miner {wallet_name}", miner_pid + ) + else: + console.print( + f"[red]Log file not found for miner {wallet_name}. Cannot attach." + ) + else: + # Miner is not running, start it + success = start_miner( + wallet_name, wallet_info, subnet_template_path, config_data + ) + if not success: + console.print(f"[red]Failed to start miner {wallet_name}.") + + def _stop_selected_neurons( + self, config_data: Dict[str, Any], selected_neurons: list[Dict[str, Any]] + ): + """Stops the selected neurons.""" + for neuron in selected_neurons: + pid = int(neuron["pid"]) + neuron_name = neuron["process"] + try: + process = psutil.Process(pid) + process.terminate() + process.wait(timeout=10) + console.print(f"[green]{neuron_name} stopped.") + except psutil.NoSuchProcess: + console.print(f"[yellow]{neuron_name} process not found.") + except psutil.TimeoutExpired: + console.print(f"[red]Timeout stopping {neuron_name}. Forcing stop.") + process.kill() + + if neuron["process"].startswith("Miner"): + wallet_name = neuron["process"].split("Miner: ")[-1] + config_data["Miners"][wallet_name]["pid"] = None + elif neuron["process"].startswith("Validator"): + config_data["Owner"]["pid"] = None + def main(): manager = BTQSManager() diff --git a/btqs/src/__init__.py b/btqs/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/btqs/src/commands/__init__.py b/btqs/src/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/btqs/src/commands/chain.py b/btqs/src/commands/chain.py new file mode 100644 index 00000000..422dc08e --- /dev/null +++ b/btqs/src/commands/chain.py @@ -0,0 +1,261 @@ +import os +import subprocess +import time + +import psutil +import typer +import yaml +from git import GitCommandError, Repo +from rich.console import Console +from rich.text import Text + +from btqs.config import CONFIG_FILE_PATH +from btqs.utils import attach_to_process_logs, get_process_entries + +console = Console() + + +def start_chain(config_data): + directory = typer.prompt( + "Enter the directory to clone the subtensor repository", + default=os.path.expanduser("~/Desktop/Bittensor_quick_start"), + show_default=True, + ) + os.makedirs(directory, exist_ok=True) + + subtensor_path = os.path.join(directory, "subtensor") + repo_url = "https://github.com/opentensor/subtensor.git" + + # Clone or update the repository + if os.path.exists(subtensor_path) and os.listdir(subtensor_path): + update = typer.confirm("Subtensor is already cloned. Do you want to update it?") + if update: + try: + repo = Repo(subtensor_path) + origin = repo.remotes.origin + origin.pull() + console.print("[green]Repository updated successfully.") + except GitCommandError as e: + console.print(f"[red]Error updating repository: {e}") + return + else: + console.print( + "[green]Using existing subtensor repository without updating." + ) + else: + try: + console.print("[green]Cloning subtensor repository...") + Repo.clone_from(repo_url, subtensor_path) + console.print("[green]Repository cloned successfully.") + except GitCommandError as e: + console.print(f"[red]Error cloning repository: {e}") + return + + localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") + + # Running localnet.sh + process = subprocess.Popen( + ["bash", localnet_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + cwd=subtensor_path, + start_new_session=True, + ) + + console.print("[green]Starting local chain. This may take a few minutes...") + + # Paths to subtensor log files + log_dir = os.path.join(subtensor_path, "logs") + alice_log = os.path.join(log_dir, "alice.log") + + # Waiting for chain compilation + timeout = 360 # 6 minutes + start_time = time.time() + while not os.path.exists(alice_log): + if time.time() - start_time > timeout: + console.print("[red]Timeout: Log files were not created.") + return + time.sleep(1) + + chain_ready = wait_for_chain_ready(alice_log, start_time, timeout) + if chain_ready: + console.print( + Text( + "Local chain is running. You can now use it for development and testing.\n", + style="bold light_goldenrod2", + ), + style="bold yellow", + ) + + # Fetch PIDs of substrate nodes + substrate_pids = get_substrate_pids() + if substrate_pids is None: + return + + config_data.update( + { + "pid": process.pid, + "substrate_pid": substrate_pids, + "subtensor_path": subtensor_path, + "base_path": directory, + } + ) + + # Save config data + try: + os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True) + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + console.print( + "[green]Local chain started successfully and config file updated." + ) + except Exception as e: + console.print(f"[red]Failed to write to the config file: {e}") + else: + console.print("[red]Failed to start local chain.") + + +def stop_chain(config_data): + pid = config_data.get("pid") + if not pid: + console.print("[red]No running chain found.") + return + + console.print("[red]Stopping the local chain...") + + try: + process = psutil.Process(pid) + process.terminate() + process.wait(timeout=10) + console.print("[green]Local chain stopped successfully.") + except psutil.NoSuchProcess: + console.print( + "[red]Process not found. The chain may have already been stopped." + ) + except psutil.TimeoutExpired: + console.print("[red]Timeout while stopping the chain. Forcing stop...") + process.kill() + + # Stop running neurons + stop_running_neurons(config_data) + + # Refresh data + refresh_config = typer.confirm("\nConfig data is outdated. Press Y to refresh it?") + if refresh_config: + if os.path.exists(CONFIG_FILE_PATH): + os.remove(CONFIG_FILE_PATH) + console.print("[green]Configuration file removed.") + + +def reattach_chain(config_data): + pid = config_data.get("pid") + subtensor_path = config_data.get("subtensor_path") + if not pid or not subtensor_path: + console.print("[red]No running chain found.") + return + + # Check if the process is still running + if not is_process_running(pid): + return + + # Paths to the log files + log_dir = os.path.join(subtensor_path, "logs") + alice_log = os.path.join(log_dir, "alice.log") + + # Check if log file exists + if not os.path.exists(alice_log): + console.print("[red]Log files not found.") + return + + # Reattach using attach_to_process_logs + attach_to_process_logs(alice_log, "Subtensor Chain (Alice)", pid) + + +def wait_for_chain_ready(alice_log, start_time, timeout): + chain_ready = False + try: + with open(alice_log, "r") as log_file: + log_file.seek(0, os.SEEK_END) + while True: + line = log_file.readline() + if line: + console.print(line, end="") + if "Imported #" in line: + chain_ready = True + break + else: + if time.time() - start_time > timeout: + console.print("[red]Timeout: Chain did not compile in time.") + break + time.sleep(0.1) + except Exception as e: + console.print(f"[red]Error reading log files: {e}") + return chain_ready + + +def get_substrate_pids(): + try: + result = subprocess.run( + ["pgrep", "-f", "node-subtensor"], capture_output=True, text=True + ) + substrate_pids = [int(pid) for pid in result.stdout.strip().split()] + return substrate_pids + except ValueError: + console.print("[red]Failed to get the PID of the Subtensor process.") + return None + + +def stop_running_neurons(config_data): + """Stops any running neurons.""" + process_entries, _, _ = get_process_entries(config_data) + + # Filter running neurons + running_neurons = [ + entry + for entry in process_entries + if ( + entry["process"].startswith("Miner") + or entry["process"].startswith("Validator") + ) + and entry["status"] == "Running" + ] + + if running_neurons: + console.print("[yellow]\nSome neurons are still running. Terminating them...") + + for neuron in running_neurons: + pid = int(neuron["pid"]) + neuron_name = neuron["process"] + try: + neuron_process = psutil.Process(pid) + neuron_process.terminate() + neuron_process.wait(timeout=10) + console.print(f"[green]{neuron_name} stopped.") + except psutil.NoSuchProcess: + console.print(f"[yellow]{neuron_name} process not found.") + except psutil.TimeoutExpired: + console.print(f"[red]Timeout stopping {neuron_name}. Forcing stop.") + neuron_process.kill() + + if neuron["process"].startswith("Miner"): + wallet_name = neuron["process"].split("Miner: ")[-1] + config_data["Miners"][wallet_name]["pid"] = None + elif neuron["process"].startswith("Validator"): + config_data["Owner"]["pid"] = None + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + else: + console.print("[green]No neurons were running.") + + +def is_process_running(pid): + try: + process = psutil.Process(pid) + if not process.is_running(): + console.print("[red]Process not running. The chain may have been stopped.") + return False + return True + except psutil.NoSuchProcess: + console.print("[red]Process not found. The chain may have been stopped.") + return False diff --git a/btqs/src/commands/neurons.py b/btqs/src/commands/neurons.py new file mode 100644 index 00000000..82b3504b --- /dev/null +++ b/btqs/src/commands/neurons.py @@ -0,0 +1,397 @@ +import os +import psutil +import typer +import yaml +from rich.console import Console +from bittensor_wallet import Wallet, Keypair +from git import Repo, GitCommandError + +from btqs.config import ( + CONFIG_FILE_PATH, + BTQS_WALLETS_DIRECTORY, + SUBNET_TEMPLATE_REPO_URL, + SUBNET_TEMPLATE_BRANCH, +) +from btqs.utils import ( + console, + exec_command, + remove_ansi_escape_sequences, + get_process_entries, + display_process_status_table, + start_validator, + start_miner, + attach_to_process_logs, + subnet_owner_exists, +) + +def setup_neurons(config_data): + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if not subnet_owner: + console.print( + "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" + ) + return + + config_data.setdefault("Miners", {}) + miners = config_data.get("Miners", {}) + + if miners and all( + miner_info.get("subtensor_pid") == config_data.get("pid") + for miner_info in miners.values() + ): + console.print( + "[green]Miner wallets associated with this subtensor instance already present. Proceeding..." + ) + else: + _create_miner_wallets(config_data) + + _register_miners(config_data) + + console.print("[dark_green]\nViewing Metagraph for Subnet 1") + subnets_list = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + +def run_neurons(config_data): + # Ensure subnet-template is available + subnet_template_path = _ensure_subnet_template(config_data) + + chain_pid = config_data.get("pid") + config_data["subnet_path"] = subnet_template_path + + # Handle Validator + if config_data.get("Owner"): + _handle_validator(config_data, subnet_template_path, chain_pid) + + # Handle Miners + _handle_miners(config_data, subnet_template_path, chain_pid) + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + +def stop_neurons(config_data): + # Get process entries + process_entries, _, _ = get_process_entries(config_data) + display_process_status_table(process_entries, [], []) + + # Filter running neurons + running_neurons = [ + entry + for entry in process_entries + if ( + entry["process"].startswith("Miner") + or entry["process"].startswith("Validator") + ) + and entry["status"] == "Running" + ] + + if not running_neurons: + console.print("[red]No running neurons to stop.") + return + + console.print("\nSelect neurons to stop:") + for idx, neuron in enumerate(running_neurons, start=1): + console.print(f"{idx}. {neuron['process']} (PID: {neuron['pid']})") + + selection = typer.prompt( + "Enter neuron numbers to stop (comma-separated), or 'all' to stop all", + default="all", + ) + + if selection.lower() == "all": + selected_neurons = running_neurons + else: + selected_indices = [ + int(i.strip()) for i in selection.split(",") if i.strip().isdigit() + ] + selected_neurons = [ + running_neurons[i - 1] + for i in selected_indices + if 1 <= i <= len(running_neurons) + ] + + if not selected_neurons: + console.print("[red]No valid neurons selected.") + return + + # Stop selected neurons + _stop_selected_neurons(config_data, selected_neurons) + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + +def start_neurons(config_data): + # Get process entries + process_entries, _, _ = get_process_entries(config_data) + display_process_status_table(process_entries, [], []) + + # Filter stopped neurons + stopped_neurons = [ + entry + for entry in process_entries + if ( + entry["process"].startswith("Miner") + or entry["process"].startswith("Validator") + ) + and entry["status"] == "Not Running" + ] + + if not stopped_neurons: + console.print("[green]All neurons are already running.") + return + + console.print("\nSelect neurons to start:") + for idx, neuron in enumerate(stopped_neurons, start=1): + console.print(f"{idx}. {neuron['process']}") + + selection = typer.prompt( + "Enter neuron numbers to start (comma-separated), or 'all' to start all", + default="all", + ) + + if selection.lower() == "all": + selected_neurons = stopped_neurons + else: + selected_indices = [ + int(i.strip()) for i in selection.split(",") if i.strip().isdigit() + ] + selected_neurons = [ + stopped_neurons[i - 1] + for i in selected_indices + if 1 <= i <= len(stopped_neurons) + ] + + if not selected_neurons: + console.print("[red]No valid neurons selected.") + return + + # Start selected neurons + _start_selected_neurons(config_data, selected_neurons) + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + +# Helper functions + +def _create_miner_wallets(config_data): + uris = ["//Bob", "//Charlie"] + ports = [8101, 8102, 8103] + for i, uri in enumerate(uris): + console.print(f"Miner {i+1}:") + wallet_name = typer.prompt( + f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" + ) + hotkey_name = typer.prompt( + f"Enter hotkey name for miner {i+1}", default="default" + ) + + keypair = Keypair.create_from_uri(uri) + wallet = Wallet( + path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=hotkey_name + ) + wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + + config_data["Miners"][wallet_name] = { + "path": BTQS_WALLETS_DIRECTORY, + "hotkey": hotkey_name, + "uri": uri, + "pid": None, + "subtensor_pid": config_data["pid"], + "port": ports[i], + } + + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + + console.print("[green]All wallets are created.") + +def _register_miners(config_data): + for wallet_name, wallet_info in config_data["Miners"].items(): + wallet = Wallet( + path=wallet_info["path"], + name=wallet_name, + hotkey=wallet_info["hotkey"], + ) + + console.print( + f"Registering Miner ({wallet_name}) to Netuid 1\n", + style="bold light_goldenrod2", + ) + + miner_registered = exec_command( + command="subnets", + sub_command="register", + extra_args=[ + "--wallet-path", + wallet.path, + "--wallet-name", + wallet.name, + "--hotkey", + wallet.hotkey_str, + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + "--no-prompt", + ], + ) + clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) + + if "βœ… Registered" in clean_stdout: + console.print(f"[green]Registered miner ({wallet.name}) to Netuid 1") + else: + console.print( + f"[red]Failed to register miner ({wallet.name}). Please register the miner manually." + ) + command = ( + f"btcli subnets register --wallet-path {wallet.path} --wallet-name " + f"{wallet.name} --hotkey {wallet.hotkey_str} --netuid 1 --chain " + f"ws://127.0.0.1:9945 --no-prompt" + ) + console.print(f"[bold yellow]{command}\n") + +def _ensure_subnet_template(config_data): + base_path = config_data.get("base_path") + if not base_path: + console.print("[red]Base path not found in the configuration file.") + return + + subnet_template_path = os.path.join(base_path, "subnet-template") + + if not os.path.exists(subnet_template_path): + console.print("[green]Cloning subnet-template repository...") + try: + repo = Repo.clone_from( + SUBNET_TEMPLATE_REPO_URL, + subnet_template_path, + ) + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + console.print("[green]Cloned subnet-template repository successfully.") + except GitCommandError as e: + console.print(f"[red]Error cloning subnet-template repository: {e}") + else: + console.print("[green]Using existing subnet-template repository.") + repo = Repo(subnet_template_path) + current_branch = repo.active_branch.name + if current_branch != SUBNET_TEMPLATE_BRANCH: + try: + repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + console.print( + f"[green]Switched to branch '{SUBNET_TEMPLATE_BRANCH}'." + ) + except GitCommandError as e: + console.print( + f"[red]Error switching to branch '{SUBNET_TEMPLATE_BRANCH}': {e}" + ) + + return subnet_template_path + +def _handle_validator(config_data, subnet_template_path, chain_pid): + owner_info = config_data["Owner"] + validator_pid = owner_info.get("pid") + validator_subtensor_pid = owner_info.get("subtensor_pid") + + if ( + validator_pid + and psutil.pid_exists(validator_pid) + and validator_subtensor_pid == chain_pid + ): + console.print( + "[green]Validator is already running. Attaching to the process..." + ) + log_file_path = owner_info.get("log_file") + if log_file_path and os.path.exists(log_file_path): + attach_to_process_logs(log_file_path, "Validator", validator_pid) + else: + console.print("[red]Log file not found for validator. Cannot attach.") + else: + # Validator is not running, start it + success = start_validator(owner_info, subnet_template_path, config_data) + if not success: + console.print("[red]Failed to start validator.") + +def _handle_miners(config_data, subnet_template_path, chain_pid): + for wallet_name, wallet_info in config_data.get("Miners", {}).items(): + miner_pid = wallet_info.get("pid") + miner_subtensor_pid = wallet_info.get("subtensor_pid") + # Check if miner process is running and associated with the current chain + if ( + miner_pid + and psutil.pid_exists(miner_pid) + and miner_subtensor_pid == chain_pid + ): + console.print( + f"[green]Miner {wallet_name} is already running. Attaching to the process..." + ) + log_file_path = wallet_info.get("log_file") + if log_file_path and os.path.exists(log_file_path): + attach_to_process_logs( + log_file_path, f"Miner {wallet_name}", miner_pid + ) + else: + console.print( + f"[red]Log file not found for miner {wallet_name}. Cannot attach." + ) + else: + # Miner is not running, start it + success = start_miner( + wallet_name, wallet_info, subnet_template_path, config_data + ) + if not success: + console.print(f"[red]Failed to start miner {wallet_name}.") + +def _stop_selected_neurons(config_data, selected_neurons): + for neuron in selected_neurons: + pid = int(neuron["pid"]) + neuron_name = neuron["process"] + try: + process = psutil.Process(pid) + process.terminate() + process.wait(timeout=10) + console.print(f"[green]{neuron_name} stopped.") + except psutil.NoSuchProcess: + console.print(f"[yellow]{neuron_name} process not found.") + except psutil.TimeoutExpired: + console.print(f"[red]Timeout stopping {neuron_name}. Forcing stop.") + process.kill() + + if neuron["process"].startswith("Miner"): + wallet_name = neuron["process"].split("Miner: ")[-1] + config_data["Miners"][wallet_name]["pid"] = None + elif neuron["process"].startswith("Validator"): + config_data["Owner"]["pid"] = None + +def _start_selected_neurons(config_data, selected_neurons): + subnet_template_path = _ensure_subnet_template(config_data) + + for neuron in selected_neurons: + neuron_name = neuron["process"] + if neuron_name.startswith("Validator"): + success = start_validator( + config_data["Owner"], subnet_template_path, config_data + ) + elif neuron_name.startswith("Miner"): + wallet_name = neuron_name.split("Miner: ")[-1] + wallet_info = config_data["Miners"][wallet_name] + success = start_miner( + wallet_name, wallet_info, subnet_template_path, config_data + ) + + if success: + console.print(f"[green]{neuron_name} started successfully.") + else: + console.print(f"[red]Failed to start {neuron_name}.") + + # Update the process entries after starting neurons + process_entries, _, _ = get_process_entries(config_data) + display_process_status_table(process_entries, [], []) \ No newline at end of file diff --git a/btqs/utils.py b/btqs/utils.py index 769357b9..4ff44aa6 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -4,7 +4,7 @@ import sys import time from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Optional, Tuple import psutil import typer @@ -23,10 +23,9 @@ console = Console() - def load_config( error_message: Optional[str] = None, exit_if_missing: bool = True -) -> Dict[str, Any]: +) -> dict[str, Any]: """ Loads the configuration file. @@ -48,35 +47,11 @@ def load_config( console.print("[red]Configuration file not found.") raise typer.Exit() else: - return {} # Return an empty config if not exiting + return {} with open(CONFIG_FILE_PATH, "r") as config_file: config_data = yaml.safe_load(config_file) or {} return config_data - -def load_config_data() -> Optional[Dict[str, Any]]: - """ - Load configuration data from the config file. - - Returns: - - dict: The loaded configuration data if successful. - - None: If the config file doesn't exist or can't be loaded. - """ - if not os.path.exists(CONFIG_FILE_PATH): - console.print( - "[red]Config file not found. Please run `btqs chain start` first." - ) - return None - - try: - with open(CONFIG_FILE_PATH, "r") as config_file: - config_data = yaml.safe_load(config_file) or {} - return config_data - except Exception as e: - console.print(f"[red]Error loading config file: {e}") - return None - - def remove_ansi_escape_sequences(text: str) -> str: """ Removes ANSI escape sequences from the given text. @@ -103,12 +78,11 @@ def remove_ansi_escape_sequences(text: str) -> str: ) return ansi_escape.sub("", text) - def exec_command( command: str, sub_command: str, - extra_args: Optional[List[str]] = None, - inputs: Optional[List[str]] = None, + extra_args: Optional[list[str]] = None, + inputs: Optional[list[str]] = None, internal_command: bool = False, ) -> typer.testing.Result: """ @@ -146,7 +120,6 @@ def exec_command( ) return result - def is_chain_running(config_file_path: str) -> bool: """ Checks if the local chain is running by verifying the PID in the config file. @@ -170,7 +143,6 @@ def is_chain_running(config_file_path: str) -> bool: except psutil.NoSuchProcess: return False - def subnet_owner_exists(config_file_path: str) -> Tuple[bool, dict]: """ Checks if a subnet owner exists in the config file. @@ -195,8 +167,25 @@ def subnet_owner_exists(config_file_path: str) -> Tuple[bool, dict]: return False, {} +def subnet_exists(ss58_address: str, netuid: int) -> bool: + """ + Checks if a subnet exists by verifying the subnet list output. + """ + subnets_list = exec_command( + command="subnets", + sub_command="list", + extra_args=[ + "--chain", + "ws://127.0.0.1:9945", + ], + internal_command=True, + ) + exists = verify_subnet_entry( + remove_ansi_escape_sequences(subnets_list.stdout), netuid, ss58_address + ) + return exists -def verify_subnet_entry(output_text: str, netuid: str, ss58_address: str) -> bool: +def verify_subnet_entry(output_text: str, netuid: int, ss58_address: str) -> bool: """ Verifies the presence of a specific subnet entry in the subnets list output. @@ -208,77 +197,54 @@ def verify_subnet_entry(output_text: str, netuid: str, ss58_address: str) -> boo Returns: bool: True if the entry is found, False otherwise. """ - # Remove ANSI escape sequences output_text = remove_ansi_escape_sequences(output_text) lines = output_text.split("\n") - # Flag to start processing data rows after the headers data_started = False for line in lines: line = line.strip() - # Skip empty lines if not line: continue - # Identify the header line if "NETUID" in line: data_started = True continue - # Skip lines before the data starts if not data_started: continue - # Skip separator lines if set(line) <= {"━", "╇", "β”Ό", "─", "β•ˆ", "═", "β•‘", "╬", "β•£", "β• "}: continue - # Split the line into columns using the separator 'β”‚' or '|' columns = re.split(r"β”‚|\|", line) - # Remove leading and trailing whitespace from each column columns = [col.strip() for col in columns] - # Check if columns have enough entries if len(columns) < 8: continue - # Extract netuid and ss58_address from columns netuid_col = columns[0] ss58_address_col = columns[-1] - # Compare with the given netuid and ss58_address if netuid_col == str(netuid) and ss58_address_col == ss58_address: return True return False - -def subnet_exists(ss58_address: str, netuid: str) -> bool: - subnets_list = exec_command( - command="subnets", - sub_command="list", - extra_args=[ - "--chain", - "ws://127.0.0.1:9945", - ], - internal_command=True, - ) - exists = verify_subnet_entry( - remove_ansi_escape_sequences(subnets_list.stdout), netuid, ss58_address - ) - return exists - - def get_btcli_version() -> str: + """ + Gets the version of btcli. + """ try: result = subprocess.run(["btcli", "--version"], capture_output=True, text=True) return result.stdout.strip() except Exception: return "Not installed or not found" - def get_bittensor_wallet_version() -> str: + """ + Gets the version of bittensor-wallet. + """ try: result = subprocess.run( ["pip", "show", "bittensor-wallet"], capture_output=True, text=True @@ -289,16 +255,22 @@ def get_bittensor_wallet_version() -> str: except Exception: return "Not installed or not found" - def get_python_version() -> str: + """ + Gets the Python version. + """ return sys.version.split()[0] - def get_python_path() -> str: + """ + Gets the Python executable path. + """ return sys.executable - def get_process_info(pid: int) -> Tuple[str, str, str, str, float, float]: + """ + Retrieves process information. + """ if pid and psutil.pid_exists(pid): try: process = psutil.Process(pid) @@ -309,7 +281,7 @@ def get_process_info(pid: int) -> Tuple[str, str, str, str, float, float]: memory_usage = f"{memory_percent:.1f}%" create_time = datetime.fromtimestamp(process.create_time()) uptime = datetime.now() - create_time - uptime_str = str(uptime).split(".")[0] # Remove microseconds + uptime_str = str(uptime).split(".")[0] return ( status, cpu_usage, @@ -329,10 +301,12 @@ def get_process_info(pid: int) -> Tuple[str, str, str, str, float, float]: pid = "N/A" return status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent - def get_process_entries( config_data: Dict[str, Any], -) -> Tuple[List[Dict[str, str]], List[float], List[float]]: +) -> Tuple[list[Dict[str, str]], list[float], list[float]]: + """ + Gets process entries for display. + """ cpu_usage_list = [] memory_usage_list = [] process_entries = [] @@ -416,9 +390,9 @@ def get_process_entries( } ) - # Check status of Validators - if config_data.get("Owner"): - owner_data = config_data.get("Owner") + # Check status of Validator + owner_data = config_data.get("Owner") + if owner_data: pid = owner_data.get("pid") location = owner_data.get("path") status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( @@ -433,7 +407,7 @@ def get_process_entries( process_entries.append( { - "process": f"Validator: {owner_data.get("wallet_name")}", + "process": "Validator", "status": status, "status_style": status_style, "pid": str(pid), @@ -446,12 +420,14 @@ def get_process_entries( return process_entries, cpu_usage_list, memory_usage_list - def display_process_status_table( - process_entries: List[Dict[str, str]], - cpu_usage_list: List[float], - memory_usage_list: List[float], + process_entries: list[Dict[str, str]], + cpu_usage_list: list[float], + memory_usage_list: list[float], ) -> None: + """ + Displays the process status table. + """ table = Table( title="[underline dark_orange]BTQS Process Manager[/underline dark_orange]\n", show_footer=True, @@ -518,6 +494,8 @@ def display_process_status_table( process_style = "cyan" elif entry["process"].startswith("Miner"): process_style = "magenta" + elif entry["process"].startswith("Validator"): + process_style = "yellow" else: process_style = "white" @@ -543,14 +521,15 @@ def display_process_status_table( # Display the table console.print(table) - def start_miner( wallet_name: str, wallet_info: Dict[str, Any], subnet_template_path: str, config_data: Dict[str, Any], ) -> bool: - """Starts a single miner and displays logs until user presses Ctrl+C.""" + """ + Starts a single miner and displays logs until user presses Ctrl+C. + """ wallet = Wallet( path=wallet_info["path"], name=wallet_name, @@ -578,41 +557,41 @@ def start_miner( ] # Create log file paths - logs_dir = os.path.join(BTQS_DIRECTORY, "logs") + logs_dir = os.path.join(config_data["base_path"], "logs") os.makedirs(logs_dir, exist_ok=True) log_file_path = os.path.join(logs_dir, f"miner_{wallet_name}.log") - log_file = open(log_file_path, "a") - try: - # Start the subprocess, redirecting stdout and stderr to the log file - process = subprocess.Popen( - cmd, - cwd=subnet_template_path, - stdout=log_file, - stderr=subprocess.STDOUT, - env=env_variables, - start_new_session=True, - ) - wallet_info["pid"] = process.pid - wallet_info["log_file"] = log_file_path - log_file.close() - - # Update config_data - config_data["Miners"][wallet_name] = wallet_info - - console.print(f"[green]Miner {wallet_name} started. Press Ctrl+C to proceed.") - attach_to_process_logs(log_file_path, f"Miner {wallet_name}", process.pid) - return True - except Exception as e: - console.print(f"[red]Error starting miner {wallet_name}: {e}") - log_file.close() - return False + with open(log_file_path, "a") as log_file: + try: + # Start the subprocess, redirecting stdout and stderr to the log file + process = subprocess.Popen( + cmd, + cwd=subnet_template_path, + stdout=log_file, + stderr=subprocess.STDOUT, + env=env_variables, + start_new_session=True, + ) + wallet_info["pid"] = process.pid + wallet_info["log_file"] = log_file_path + # Update config_data + config_data["Miners"][wallet_name] = wallet_info + console.print(f"[green]Miner {wallet_name} started. Press Ctrl+C to proceed.") + attach_to_process_logs(log_file_path, f"Miner {wallet_name}", process.pid) + return True + except Exception as e: + console.print(f"[red]Error starting miner {wallet_name}: {e}") + return False def start_validator( - owner_info: Dict[str, Any], subnet_template_path: str, config_data: Dict[str, Any] + owner_info: Dict[str, Any], + subnet_template_path: str, + config_data: Dict[str, Any], ) -> bool: - """Starts the validator process and displays logs until user presses Ctrl+C.""" + """ + Starts the validator process and displays logs until user presses Ctrl+C. + """ wallet = Wallet( path=owner_info["path"], name=owner_info["wallet_name"], @@ -642,39 +621,37 @@ def start_validator( ] # Create log file paths - logs_dir = os.path.join(BTQS_DIRECTORY, "logs") + logs_dir = os.path.join(config_data["base_path"], "logs") os.makedirs(logs_dir, exist_ok=True) log_file_path = os.path.join(logs_dir, "validator.log") - log_file = open(log_file_path, "a") - try: - # Start the subprocess, redirecting stdout and stderr to the log file - process = subprocess.Popen( - cmd, - cwd=subnet_template_path, - stdout=log_file, - stderr=subprocess.STDOUT, - env=env_variables, - start_new_session=True, - ) - owner_info["pid"] = process.pid - owner_info["log_file"] = log_file_path - log_file.close() - - # Update config_data - config_data["Owner"] = owner_info - - console.print("[green]Validator started. Press Ctrl+C to proceed.") - attach_to_process_logs(log_file_path, "Validator", process.pid) - return True - except Exception as e: - console.print(f"[red]Error starting validator: {e}") - log_file.close() - return False + with open(log_file_path, "a") as log_file: + try: + # Start the subprocess, redirecting stdout and stderr to the log file + process = subprocess.Popen( + cmd, + cwd=subnet_template_path, + stdout=log_file, + stderr=subprocess.STDOUT, + env=env_variables, + start_new_session=True, + ) + owner_info["pid"] = process.pid + owner_info["log_file"] = log_file_path + # Update config_data + config_data["Owner"] = owner_info + console.print("[green]Validator started. Press Ctrl+C to proceed.") + attach_to_process_logs(log_file_path, "Validator", process.pid) + return True + except Exception as e: + console.print(f"[red]Error starting validator: {e}") + return False def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = None): - """Attaches to the log file of a process and prints logs until user presses Ctrl+C or the process terminates.""" + """ + Attaches to the log file of a process and prints logs until user presses Ctrl+C or the process terminates. + """ try: with open(log_file_path, "r") as log_file: # Move to the end of the file diff --git a/btqs_requirements.txt b/btqs_requirements.txt index 4468d480..14cb04c2 100644 --- a/btqs_requirements.txt +++ b/btqs_requirements.txt @@ -1,2 +1,3 @@ psutil GitPython +tdqm From d87c2e4a983301bf0f047eac3f58d3dcf717f9b4 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 01:06:29 -0700 Subject: [PATCH 05/23] Getting together --- btqs/btqs_cli.py | 731 +++-------------------------- btqs/{src => commands}/__init__.py | 0 btqs/{src => }/commands/chain.py | 58 +-- btqs/{src => }/commands/neurons.py | 91 +++- btqs/commands/subnet.py | 327 +++++++++++++ btqs/config.py | 10 +- btqs/src/commands/__init__.py | 0 btqs/utils.py | 7 +- 8 files changed, 512 insertions(+), 712 deletions(-) rename btqs/{src => commands}/__init__.py (100%) rename btqs/{src => }/commands/chain.py (85%) rename btqs/{src => }/commands/neurons.py (85%) create mode 100644 btqs/commands/subnet.py delete mode 100644 btqs/src/commands/__init__.py diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index ffe428a7..32cb55ad 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -1,29 +1,22 @@ import os import platform -import subprocess import time -import time -import threading -import sys -from tqdm import tqdm - -from typing import Any, Dict, Optional from time import sleep +from typing import Optional + + import psutil +from rich.prompt import Confirm import typer -import yaml -from bittensor_wallet import Keypair, Wallet -from git import GitCommandError, Repo +from btqs.commands import chain, neurons, subnet from rich.table import Table from rich.text import Text from .config import ( - BTQS_DIRECTORY, - BTQS_WALLETS_DIRECTORY, CONFIG_FILE_PATH, EPILOG, - SUBNET_TEMPLATE_BRANCH, - SUBNET_TEMPLATE_REPO_URL, + LOCALNET_ENDPOINT, + DEFAULT_WORKSPACE_DIRECTORY, ) from .utils import ( console, @@ -36,15 +29,8 @@ get_python_version, is_chain_running, load_config, - remove_ansi_escape_sequences, - start_miner, - start_validator, - subnet_exists, - subnet_owner_exists, - attach_to_process_logs, ) -from btqs.src.commands import chain, neurons class BTQSManager: """ @@ -69,6 +55,15 @@ def __init__(self): self.app.add_typer(self.subnet_app, name="subnet", no_args_is_help=True) self.app.add_typer(self.neurons_app, name="neurons", no_args_is_help=True) + # Core commands + self.app.command(name="run-all", help="Create entire setup")(self.run_all) + self.app.command(name="status", help="Current status of bittensor quick start")( + self.status_neurons + ) + self.app.command(name="steps", help="Display steps for subnet setup")( + self.setup_steps + ) + # Chain commands self.chain_app.command(name="start")(self.start_chain) self.chain_app.command(name="stop")(self.stop_chain) @@ -76,6 +71,8 @@ def __init__(self): # Subnet commands self.subnet_app.command(name="setup")(self.setup_subnet) + self.subnet_app.command(name="live")(self.display_live_metagraph) + self.subnet_app.command(name="stake")(self.add_stake) # Neuron commands self.neurons_app.command(name="setup")(self.setup_neurons) @@ -83,62 +80,13 @@ def __init__(self): self.neurons_app.command(name="stop")(self.stop_neurons) self.neurons_app.command(name="reattach")(self.reattach_neurons) self.neurons_app.command(name="status")(self.status_neurons) - self.neurons_app.command(name="live")(self.display_live_metagraph) self.neurons_app.command(name="start")(self.start_neurons) - self.neurons_app.command(name="stake")(self.add_stake) - - self.app.command(name="run-all", help="Create entire setup")(self.run_all) - self.app.command(name="status", help="Current status of bittensor quick start")( - self.status_neurons - ) def display_live_metagraph(self): - def clear_screen(): - os.system("cls" if os.name == "nt" else "clear") - - def get_metagraph(): - result = exec_command( - command="subnets", - sub_command="metagraph", - extra_args=[ - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - ], - internal_command=True - ) - # clear_screen() - return result.stdout - - print("Starting live metagraph view. Press 'Ctrl + C' to exit.") - config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." - ) + subnet.display_live_metagraph() - def input_thread(): - while True: - if input() == "q": - print("Exiting live view...") - sys.exit(0) - - threading.Thread(target=input_thread, daemon=True).start() - - try: - while True: - metagraph = get_metagraph() - process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) - clear_screen() - print(metagraph) - display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) - - # Create a progress bar for 5 seconds - print("\n") - for _ in tqdm(range(5), desc="Refreshing", unit="s", total=5): - time.sleep(1) - - except KeyboardInterrupt: - print("Exiting live view...") + def setup_steps(self): + subnet.steps() def run_all(self): """ @@ -150,7 +98,7 @@ def run_all(self): sleep(3) # Start the local chain - self.start_chain() + self.start_chain(workspace_path=None, branch=None) text = Text("Checking chain status\n", style="bold light_goldenrod2") sign = Text("\nπŸ”Ž ", style="bold yellow") @@ -211,6 +159,11 @@ def run_all(self): ) print(subnets_list.stdout, end="") + text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") + sign = Text("πŸͺ™ ", style="bold yellow") + console.print(sign, text) + time.sleep(2) + self.add_stake() console.print("[dark_green]\nViewing Metagraph for Subnet 1") subnets_list = exec_command( @@ -226,78 +179,67 @@ def run_all(self): print(subnets_list.stdout, end="") self.display_live_metagraph() - def add_stake(self): - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) - if subnet_owner: - owner_wallet = Wallet( - name=owner_data.get("wallet_name"), - path=owner_data.get("path"), - hotkey=owner_data.get("hotkey"), - ) - add_stake = exec_command( - command="stake", - sub_command="add", - extra_args=[ - "--amount", - 1000, - "--wallet-path", - BTQS_WALLETS_DIRECTORY, - "--chain", - "ws://127.0.0.1:9945", - "--wallet-name", - owner_wallet.name, - "--no-prompt", - "--wallet-hotkey", - owner_wallet.hotkey_str, - ], - ) - - clean_stdout = remove_ansi_escape_sequences(add_stake.stdout) - if "βœ… Finalized" in clean_stdout: - text = Text( - f"Stake added successfully by Validator ({owner_wallet})\n", - style="bold light_goldenrod2", - ) - sign = Text("πŸ“ˆ ", style="bold yellow") - console.print(sign, text) - else: - console.print("\n[red] Failed to add stake. Command output:\n") - print(add_stake.stdout, end="") - - else: - console.print( - "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" - ) - return - - def start_chain(self): + def start_chain( + self, + workspace_path: Optional[str] = typer.Option( + None, + "--path", + "--workspace", + help="Path to Bittensor's quick start workspace", + ), + branch: Optional[str] = typer.Option( + None, "--subtensor_branch", help="Subtensor branch to checkout" + ), + ): """ Starts the local Subtensor chain. - This command initializes and starts a local instance of the Subtensor blockchain for development and testing. + This command initializes and starts a local instance of the Subtensor blockchain. USAGE [green]$[/green] btqs chain start - [bold]Note[/bold]: This command will clone or update the Subtensor repository if necessary and start the local chain. It may take several minutes to complete. + [bold]Note[/bold]: This command may take several minutes to complete during the Subtensor compilation process. """ - console.print("[dark_orange]Starting the local chain...") - - if is_chain_running(CONFIG_FILE_PATH): + config_data = load_config(exit_if_missing=False) + if is_chain_running(): console.print( - "[red]The local chain is already running. Endpoint: ws://127.0.0.1:9945" + f"[red]The local chain is already running. Endpoint: {LOCALNET_ENDPOINT}" ) return + if not workspace_path: + if config_data.get("workspace_path"): + reuse_config_path = Confirm.ask( + f"[blue]Previously saved workspace_path found: [green]({config_data.get("workspace_path")}) \n[blue]Do you want to re-use it?", + default=True, + show_default=True, + ) + if reuse_config_path: + workspace_path = config_data.get("workspace_path") + else: + workspace_path = typer.prompt( + typer.style( + "Enter path to create Bittensor development workspace", + fg="blue", + ), + default=DEFAULT_WORKSPACE_DIRECTORY, + ) + + if not branch: + branch = typer.prompt( + typer.style("Enter Subtensor branch", fg="blue"), + default="testnet", + ) - config_data = load_config(exit_if_missing=False) or {} - chain.start_chain(config_data) + console.print("[dark_orange]Starting the local chain...") + chain.start(config_data, workspace_path, branch) def stop_chain(self): """ Stops the local Subtensor chain and any running miners. - This command terminates the local Subtensor chain process and optionally cleans up configuration data. + This command terminates the local Subtensor chain process, miner and validator processes, and optionally cleans up configuration data. USAGE @@ -308,7 +250,7 @@ def stop_chain(self): config_data = load_config( "No running chain found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - chain.stop_chain(config_data) + chain.stop(config_data) def reattach_chain(self): """ @@ -325,7 +267,10 @@ def reattach_chain(self): config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - chain.reattach_chain(config_data) + chain.reattach(config_data) + + def add_stake(self): + subnet.add_stake() def setup_subnet(self): """ @@ -348,66 +293,7 @@ def setup_subnet(self): config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - - os.makedirs(BTQS_WALLETS_DIRECTORY, exist_ok=True) - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) - if subnet_owner: - owner_wallet = Wallet( - name=owner_data.get("wallet_name"), - path=owner_data.get("path"), - hotkey=owner_data.get("hotkey"), - ) - - warning_text = Text( - "A Subnet Owner associated with this setup already exists", - style="bold light_goldenrod2", - ) - wallet_info = Text(f"\t{owner_wallet}\n", style="medium_purple") - warning_sign = Text("⚠️ ", style="bold yellow") - - console.print(warning_sign, warning_text) - console.print(wallet_info) - else: - self._create_subnet_owner_wallet(config_data) - config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." - ) - - owner_data = config_data["Owner"] - owner_wallet = Wallet( - name=owner_data.get("wallet_name"), - path=owner_data.get("path"), - hotkey=owner_data.get("hotkey"), - ) - - if subnet_exists(owner_wallet.coldkeypub.ss58_address, 1): - warning_text = Text( - "A Subnet with netuid 1 already exists and is registered with the owner's wallet:", - style="bold light_goldenrod2", - ) - wallet_info = Text(f"\t{owner_wallet}", style="medium_purple") - sudo_info = Text( - f"\tSUDO (Coldkey of Subnet 1 owner): {owner_wallet.coldkeypub.ss58_address}", - style="medium_purple", - ) - warning_sign = Text("⚠️ ", style="bold yellow") - - console.print(warning_sign, warning_text) - console.print(wallet_info) - console.print(sudo_info) - else: - self._create_subnet(owner_wallet) - - console.print("[dark_green]\nListing all subnets") - subnets_list = exec_command( - command="subnets", - sub_command="list", - extra_args=[ - "--chain", - "ws://127.0.0.1:9945", - ], - ) - print(subnets_list.stdout, end="") + subnet.setup_subnet(config_data) def setup_neurons(self): """ @@ -448,7 +334,7 @@ def run_neurons(self): "[red]No neurons found. Please run `btqs neurons setup` first." ) return - + neurons.run_neurons(config_data) def stop_neurons(self): @@ -491,34 +377,6 @@ def start_neurons(self): neurons.start_neurons(config_data) - def _start_selected_neurons( - self, config_data: Dict[str, Any], selected_neurons: list[Dict[str, Any]] - ): - """Starts the selected neurons.""" - subnet_template_path = self._ensure_subnet_template(config_data) - - for neuron in selected_neurons: - neuron_name = neuron["process"] - if neuron_name.startswith("Validator"): - success = start_validator( - config_data["Owner"], subnet_template_path, config_data - ) - elif neuron_name.startswith("Miner"): - wallet_name = neuron_name.split("Miner: ")[-1] - wallet_info = config_data["Miners"][wallet_name] - success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data - ) - - if success: - console.print(f"[green]{neuron_name} started successfully.") - else: - console.print(f"[red]Failed to start {neuron_name}.") - - # Update the process entries after starting neurons - process_entries, _, _ = get_process_entries(config_data) - display_process_status_table(process_entries, [], []) - def reattach_neurons(self): """ Reattaches to a running neuron. @@ -536,41 +394,7 @@ def reattach_neurons(self): return config_data = load_config() - - # Choose which neuron to reattach to - all_neurons = { - **config_data.get("Miners", {}), - "Validator": config_data.get("Owner", {}), - } - neuron_names = list(all_neurons.keys()) - if not neuron_names: - console.print("[red]No neurons found.") - return - - neuron_choice = typer.prompt( - f"Which neuron do you want to reattach to? {neuron_names}", - default=neuron_names[0], - ) - if neuron_choice not in all_neurons: - console.print("[red]Invalid neuron name.") - return - - wallet_info = all_neurons[neuron_choice] - pid = wallet_info.get("pid") - log_file_path = wallet_info.get("log_file") - if not pid or not psutil.pid_exists(pid): - console.print("[red]Neuron process not running.") - return - - if not log_file_path or not os.path.exists(log_file_path): - console.print("[red]Log file not found for this neuron.") - return - - console.print( - f"[green]Reattaching to neuron {neuron_choice}. Press Ctrl+C to exit." - ) - - attach_to_process_logs(log_file_path, neuron_choice, pid) + neurons.reattach_neurons(config_data) def status_neurons(self): """ @@ -646,405 +470,6 @@ def status_neurons(self): def run(self): self.app() - # TODO: See if we can further streamline these. Or change location if needed - # ------------------------ Helper Methods ------------------------ - - def _wait_for_chain_ready( - self, alice_log: str, start_time: float, timeout: int - ) -> bool: - """Waits for the chain to be ready by monitoring the alice.log file.""" - chain_ready = False - try: - with open(alice_log, "r") as log_file: - log_file.seek(0, os.SEEK_END) - while True: - line = log_file.readline() - if line: - console.print(line, end="") - if "Imported #" in line: - chain_ready = True - break - else: - if time.time() - start_time > timeout: - console.print( - "[red]Timeout: Chain did not compile in time." - ) - break - time.sleep(0.1) - except Exception as e: - console.print(f"[red]Error reading log files: {e}") - return chain_ready - - def _get_substrate_pids(self) -> Optional[list[int]]: - """Fetches the PIDs of the substrate nodes.""" - try: - result = subprocess.run( - ["pgrep", "-f", "node-subtensor"], capture_output=True, text=True - ) - substrate_pids = [int(pid) for pid in result.stdout.strip().split()] - return substrate_pids - except ValueError: - console.print("[red]Failed to get the PID of the Subtensor process.") - return None - - def _stop_running_neurons(self, config_data: Dict[str, Any]): - """Stops any running neurons.""" - process_entries, _, _ = get_process_entries(config_data) - - # Filter running neurons - running_neurons = [ - entry - for entry in process_entries - if ( - entry["process"].startswith("Miner") - or entry["process"].startswith("Validator") - ) - and entry["status"] == "Running" - ] - - if running_neurons: - console.print( - "[yellow]\nSome neurons are still running. Terminating them..." - ) - - for neuron in running_neurons: - pid = int(neuron["pid"]) - neuron_name = neuron["process"] - try: - neuron_process = psutil.Process(pid) - neuron_process.terminate() - neuron_process.wait(timeout=10) - console.print(f"[green]{neuron_name} stopped.") - except psutil.NoSuchProcess: - console.print(f"[yellow]{neuron_name} process not found.") - except psutil.TimeoutExpired: - console.print(f"[red]Timeout stopping {neuron_name}. Forcing stop.") - neuron_process.kill() - - if neuron["process"].startswith("Miner"): - wallet_name = neuron["process"].split("Miner: ")[-1] - config_data["Miners"][wallet_name]["pid"] = None - elif neuron["process"].startswith("Validator"): - config_data["Owner"]["pid"] = None - - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - else: - console.print("[green]No neurons were running.") - - def _is_process_running(self, pid: int) -> bool: - """Checks if a process with the given PID is running.""" - try: - process = psutil.Process(pid) - if not process.is_running(): - console.print( - "[red]Process not running. The chain may have been stopped." - ) - return False - return True - except psutil.NoSuchProcess: - console.print("[red]Process not found. The chain may have been stopped.") - return False - - def _create_subnet_owner_wallet(self, config_data: Dict[str, Any]): - """Creates a subnet owner wallet.""" - console.print( - Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2"), - style="bold yellow", - ) - - owner_wallet_name = typer.prompt( - "Enter subnet owner wallet name", default="owner", show_default=True - ) - owner_hotkey_name = typer.prompt( - "Enter subnet owner hotkey name", default="default", show_default=True - ) - - uri = "//Alice" - keypair = Keypair.create_from_uri(uri) - owner_wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, - name=owner_wallet_name, - hotkey=owner_hotkey_name, - ) - owner_wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) - owner_wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) - owner_wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) - - console.print( - "Executed command: [dark_orange] btcli wallet create --wallet-name", - f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {BTQS_WALLETS_DIRECTORY}", - ) - - config_data["Owner"] = { - "wallet_name": owner_wallet_name, - "path": BTQS_WALLETS_DIRECTORY, - "hotkey": owner_hotkey_name, - "subtensor_pid": config_data["pid"], - } - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - - def _create_subnet(self, owner_wallet: Wallet): - """Creates a subnet with netuid 1 and registers the owner.""" - console.print( - Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2"), - style="bold yellow", - ) - - create_subnet = exec_command( - command="subnets", - sub_command="create", - extra_args=[ - "--wallet-path", - BTQS_WALLETS_DIRECTORY, - "--chain", - "ws://127.0.0.1:9945", - "--wallet-name", - owner_wallet.name, - "--no-prompt", - "--wallet-hotkey", - owner_wallet.hotkey_str, - ], - ) - clean_stdout = remove_ansi_escape_sequences(create_subnet.stdout) - if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: - console.print("[dark_green] Subnet created successfully with netuid 1") - - console.print( - Text( - f"Registering Owner ({owner_wallet.name}) to Netuid 1\n", - style="bold light_goldenrod2", - ), - style="bold yellow", - ) - - register_subnet = exec_command( - command="subnets", - sub_command="register", - extra_args=[ - "--wallet-path", - BTQS_WALLETS_DIRECTORY, - "--wallet-name", - owner_wallet.name, - "--wallet-hotkey", - owner_wallet.hotkey_str, - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - "--no-prompt", - ], - ) - clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) - if "βœ… Registered" in clean_stdout: - console.print("[green] Registered the owner to subnet 1") - - def _create_miner_wallets(self, config_data: Dict[str, Any]): - """Creates miner wallets.""" - uris = ["//Bob", "//Charlie"] - ports = [8101, 8102, 8103] - for i, uri in enumerate(uris): - console.print(f"Miner {i+1}:") - wallet_name = typer.prompt( - f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" - ) - hotkey_name = typer.prompt( - f"Enter hotkey name for miner {i+1}", default="default" - ) - - keypair = Keypair.create_from_uri(uri) - wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=hotkey_name - ) - wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) - wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) - wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) - - config_data["Miners"][wallet_name] = { - "path": BTQS_WALLETS_DIRECTORY, - "hotkey": hotkey_name, - "uri": uri, - "pid": None, - "subtensor_pid": config_data["pid"], - "port": ports[i], - } - - with open(CONFIG_FILE_PATH, "w") as config_file: - yaml.safe_dump(config_data, config_file) - - console.print("[green]All wallets are created.") - - def _register_miners(self, config_data: Dict[str, Any]): - """Registers miners to the subnet.""" - for wallet_name, wallet_info in config_data["Miners"].items(): - wallet = Wallet( - path=wallet_info["path"], - name=wallet_name, - hotkey=wallet_info["hotkey"], - ) - - console.print( - Text( - f"Registering Miner ({wallet_name}) to Netuid 1\n", - style="bold light_goldenrod2", - ), - style="bold yellow", - ) - - miner_registered = exec_command( - command="subnets", - sub_command="register", - extra_args=[ - "--wallet-path", - wallet.path, - "--wallet-name", - wallet.name, - "--hotkey", - wallet.hotkey_str, - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - "--no-prompt", - ], - ) - clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) - - if "βœ… Registered" in clean_stdout: - console.print(f"[green]Registered miner ({wallet.name}) to Netuid 1") - else: - console.print( - f"[red]Failed to register miner ({wallet.name}). Please register the miner manually." - ) - command = ( - f"btcli subnets register --wallet-path {wallet.path} --wallet-name " - f"{wallet.name} --hotkey {wallet.hotkey_str} --netuid 1 --chain " - f"ws://127.0.0.1:9945 --no-prompt" - ) - console.print(f"[bold yellow]{command}\n") - - def _ensure_subnet_template(self, config_data: Dict[str, Any]): - """Ensures that the subnet-template repository is available.""" - base_path = config_data.get("base_path") - if not base_path: - console.print("[red]Base path not found in the configuration file.") - return - - subnet_template_path = os.path.join(base_path, "subnet-template") - - if not os.path.exists(subnet_template_path): - console.print("[green]Cloning subnet-template repository...") - try: - repo = Repo.clone_from( - SUBNET_TEMPLATE_REPO_URL, - subnet_template_path, - ) - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - console.print("[green]Cloned subnet-template repository successfully.") - except GitCommandError as e: - console.print(f"[red]Error cloning subnet-template repository: {e}") - else: - console.print("[green]Using existing subnet-template repository.") - repo = Repo(subnet_template_path) - current_branch = repo.active_branch.name - if current_branch != SUBNET_TEMPLATE_BRANCH: - try: - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - console.print( - f"[green]Switched to branch '{SUBNET_TEMPLATE_BRANCH}'." - ) - except GitCommandError as e: - console.print( - f"[red]Error switching to branch '{SUBNET_TEMPLATE_BRANCH}': {e}" - ) - - return subnet_template_path - - def _handle_validator( - self, config_data: Dict[str, Any], subnet_template_path: str, chain_pid: int - ): - """Handles the validator process.""" - owner_info = config_data["Owner"] - validator_pid = owner_info.get("pid") - validator_subtensor_pid = owner_info.get("subtensor_pid") - - if ( - validator_pid - and psutil.pid_exists(validator_pid) - and validator_subtensor_pid == chain_pid - ): - console.print( - "[green]Validator is already running. Attaching to the process..." - ) - log_file_path = owner_info.get("log_file") - if log_file_path and os.path.exists(log_file_path): - attach_to_process_logs(log_file_path, "Validator", validator_pid) - else: - console.print("[red]Log file not found for validator. Cannot attach.") - else: - # Validator is not running, start it - success = start_validator(owner_info, subnet_template_path, config_data) - if not success: - console.print("[red]Failed to start validator.") - - def _handle_miners( - self, config_data: Dict[str, Any], subnet_template_path: str, chain_pid: int - ): - """Handles the miner processes.""" - for wallet_name, wallet_info in config_data.get("Miners", {}).items(): - miner_pid = wallet_info.get("pid") - miner_subtensor_pid = wallet_info.get("subtensor_pid") - # Check if miner process is running and associated with the current chain - if ( - miner_pid - and psutil.pid_exists(miner_pid) - and miner_subtensor_pid == chain_pid - ): - console.print( - f"[green]Miner {wallet_name} is already running. Attaching to the process..." - ) - log_file_path = wallet_info.get("log_file") - if log_file_path and os.path.exists(log_file_path): - attach_to_process_logs( - log_file_path, f"Miner {wallet_name}", miner_pid - ) - else: - console.print( - f"[red]Log file not found for miner {wallet_name}. Cannot attach." - ) - else: - # Miner is not running, start it - success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data - ) - if not success: - console.print(f"[red]Failed to start miner {wallet_name}.") - - def _stop_selected_neurons( - self, config_data: Dict[str, Any], selected_neurons: list[Dict[str, Any]] - ): - """Stops the selected neurons.""" - for neuron in selected_neurons: - pid = int(neuron["pid"]) - neuron_name = neuron["process"] - try: - process = psutil.Process(pid) - process.terminate() - process.wait(timeout=10) - console.print(f"[green]{neuron_name} stopped.") - except psutil.NoSuchProcess: - console.print(f"[yellow]{neuron_name} process not found.") - except psutil.TimeoutExpired: - console.print(f"[red]Timeout stopping {neuron_name}. Forcing stop.") - process.kill() - - if neuron["process"].startswith("Miner"): - wallet_name = neuron["process"].split("Miner: ")[-1] - config_data["Miners"][wallet_name]["pid"] = None - elif neuron["process"].startswith("Validator"): - config_data["Owner"]["pid"] = None - def main(): manager = BTQSManager() diff --git a/btqs/src/__init__.py b/btqs/commands/__init__.py similarity index 100% rename from btqs/src/__init__.py rename to btqs/commands/__init__.py diff --git a/btqs/src/commands/chain.py b/btqs/commands/chain.py similarity index 85% rename from btqs/src/commands/chain.py rename to btqs/commands/chain.py index 422dc08e..79ff4a77 100644 --- a/btqs/src/commands/chain.py +++ b/btqs/commands/chain.py @@ -9,30 +9,29 @@ from rich.console import Console from rich.text import Text -from btqs.config import CONFIG_FILE_PATH +from btqs.config import CONFIG_FILE_PATH, SUBTENSOR_REPO_URL from btqs.utils import attach_to_process_logs, get_process_entries +from rich.prompt import Confirm console = Console() -def start_chain(config_data): - directory = typer.prompt( - "Enter the directory to clone the subtensor repository", - default=os.path.expanduser("~/Desktop/Bittensor_quick_start"), - show_default=True, - ) - os.makedirs(directory, exist_ok=True) - - subtensor_path = os.path.join(directory, "subtensor") - repo_url = "https://github.com/opentensor/subtensor.git" +def start(config_data, workspace_path, branch): + os.makedirs(workspace_path, exist_ok=True) + subtensor_path = os.path.join(workspace_path, "subtensor") # Clone or update the repository if os.path.exists(subtensor_path) and os.listdir(subtensor_path): - update = typer.confirm("Subtensor is already cloned. Do you want to update it?") + update = Confirm.ask( + "[blue]Subtensor is already cloned. Do you want to update it?", + default=False, + show_default=True, + ) if update: try: repo = Repo(subtensor_path) origin = repo.remotes.origin + repo.git.checkout(branch) origin.pull() console.print("[green]Repository updated successfully.") except GitCommandError as e: @@ -45,14 +44,15 @@ def start_chain(config_data): else: try: console.print("[green]Cloning subtensor repository...") - Repo.clone_from(repo_url, subtensor_path) + repo = Repo.clone_from(SUBTENSOR_REPO_URL, subtensor_path) + if branch: + repo.git.checkout(branch) console.print("[green]Repository cloned successfully.") except GitCommandError as e: console.print(f"[red]Error cloning repository: {e}") return localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") - # Running localnet.sh process = subprocess.Popen( ["bash", localnet_path], @@ -77,15 +77,11 @@ def start_chain(config_data): return time.sleep(1) - chain_ready = wait_for_chain_ready(alice_log, start_time, timeout) + chain_ready = wait_for_chain_compilation(alice_log, start_time, timeout) if chain_ready: - console.print( - Text( - "Local chain is running. You can now use it for development and testing.\n", - style="bold light_goldenrod2", - ), - style="bold yellow", - ) + text = Text("Local chain is running. You can now use it for development and testing.\n", style="bold light_goldenrod2") + sign = Text("\nℹ️ ", style="bold yellow") + console.print(sign, text) # Fetch PIDs of substrate nodes substrate_pids = get_substrate_pids() @@ -97,7 +93,7 @@ def start_chain(config_data): "pid": process.pid, "substrate_pid": substrate_pids, "subtensor_path": subtensor_path, - "base_path": directory, + "workspace_path": workspace_path, } ) @@ -107,7 +103,7 @@ def start_chain(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) console.print( - "[green]Local chain started successfully and config file updated." + "[green]Config file updated." ) except Exception as e: console.print(f"[red]Failed to write to the config file: {e}") @@ -115,7 +111,7 @@ def start_chain(config_data): console.print("[red]Failed to start local chain.") -def stop_chain(config_data): +def stop(config_data): pid = config_data.get("pid") if not pid: console.print("[red]No running chain found.") @@ -140,14 +136,18 @@ def stop_chain(config_data): stop_running_neurons(config_data) # Refresh data - refresh_config = typer.confirm("\nConfig data is outdated. Press Y to refresh it?") + refresh_config = Confirm.ask( + "\n[blue]Config data is outdated. Do you want to refresh it?", + default=True, + show_default=True, + ) if refresh_config: if os.path.exists(CONFIG_FILE_PATH): os.remove(CONFIG_FILE_PATH) - console.print("[green]Configuration file removed.") + console.print("[green]Configuration file refreshed.") -def reattach_chain(config_data): +def reattach(config_data): pid = config_data.get("pid") subtensor_path = config_data.get("subtensor_path") if not pid or not subtensor_path: @@ -171,7 +171,7 @@ def reattach_chain(config_data): attach_to_process_logs(alice_log, "Subtensor Chain (Alice)", pid) -def wait_for_chain_ready(alice_log, start_time, timeout): +def wait_for_chain_compilation(alice_log, start_time, timeout): chain_ready = False try: with open(alice_log, "r") as log_file: diff --git a/btqs/src/commands/neurons.py b/btqs/commands/neurons.py similarity index 85% rename from btqs/src/commands/neurons.py rename to btqs/commands/neurons.py index 82b3504b..82c15690 100644 --- a/btqs/src/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -11,6 +11,8 @@ BTQS_WALLETS_DIRECTORY, SUBNET_TEMPLATE_REPO_URL, SUBNET_TEMPLATE_BRANCH, + WALLET_URIS, + MINER_PORTS ) from btqs.utils import ( console, @@ -24,6 +26,7 @@ subnet_owner_exists, ) + def setup_neurons(config_data): subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) if not subnet_owner: @@ -60,19 +63,19 @@ def setup_neurons(config_data): ) print(subnets_list.stdout, end="") + def run_neurons(config_data): - # Ensure subnet-template is available - subnet_template_path = _ensure_subnet_template(config_data) + subnet_template_path = _add_subnet_template(config_data) chain_pid = config_data.get("pid") config_data["subnet_path"] = subnet_template_path # Handle Validator if config_data.get("Owner"): - _handle_validator(config_data, subnet_template_path, chain_pid) + _run_validator(config_data, subnet_template_path, chain_pid) # Handle Miners - _handle_miners(config_data, subnet_template_path, chain_pid) + _run_miners(config_data, subnet_template_path, chain_pid) with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) @@ -128,6 +131,7 @@ def stop_neurons(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) + def start_neurons(config_data): # Get process entries process_entries, _, _ = get_process_entries(config_data) @@ -179,12 +183,49 @@ def start_neurons(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) + +def reattach_neurons(config_data): + + # Choose which neuron to reattach to + all_neurons = { + **config_data.get("Miners", {}), + "Validator": config_data.get("Owner", {}), + } + neuron_names = list(all_neurons.keys()) + if not neuron_names: + console.print("[red]No neurons found.") + return + + neuron_choice = typer.prompt( + f"Which neuron do you want to reattach to? {neuron_names}", + default=neuron_names[0], + ) + if neuron_choice not in all_neurons: + console.print("[red]Invalid neuron name.") + return + + wallet_info = all_neurons[neuron_choice] + pid = wallet_info.get("pid") + log_file_path = wallet_info.get("log_file") + if not pid or not psutil.pid_exists(pid): + console.print("[red]Neuron process not running.") + return + + if not log_file_path or not os.path.exists(log_file_path): + console.print("[red]Log file not found for this neuron.") + return + + console.print( + f"[green]Reattaching to neuron {neuron_choice}. Press Ctrl+C to exit." + ) + + attach_to_process_logs(log_file_path, neuron_choice, pid) + + # Helper functions def _create_miner_wallets(config_data): - uris = ["//Bob", "//Charlie"] - ports = [8101, 8102, 8103] - for i, uri in enumerate(uris): + for i, uri in enumerate(WALLET_URIS): console.print(f"Miner {i+1}:") wallet_name = typer.prompt( f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" @@ -207,13 +248,13 @@ def _create_miner_wallets(config_data): "uri": uri, "pid": None, "subtensor_pid": config_data["pid"], - "port": ports[i], + "port": MINER_PORTS[i], } with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) - console.print("[green]All wallets are created.") + console.print("[green]Miner wallets are created.") def _register_miners(config_data): for wallet_name, wallet_info in config_data["Miners"].items(): @@ -248,10 +289,10 @@ def _register_miners(config_data): clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) if "βœ… Registered" in clean_stdout: - console.print(f"[green]Registered miner ({wallet.name}) to Netuid 1") + console.print(f"[green]Registered miner ({wallet.name}) to Netuid 1\n") else: console.print( - f"[red]Failed to register miner ({wallet.name}). Please register the miner manually." + f"[red]Failed to register miner ({wallet.name}). You can register the miner manually using:" ) command = ( f"btcli subnets register --wallet-path {wallet.path} --wallet-name " @@ -260,13 +301,14 @@ def _register_miners(config_data): ) console.print(f"[bold yellow]{command}\n") -def _ensure_subnet_template(config_data): - base_path = config_data.get("base_path") - if not base_path: + +def _add_subnet_template(config_data): + workspace_path = config_data.get("workspace_path") + if not workspace_path: console.print("[red]Base path not found in the configuration file.") return - subnet_template_path = os.path.join(base_path, "subnet-template") + subnet_template_path = os.path.join(workspace_path, "subnet-template") if not os.path.exists(subnet_template_path): console.print("[green]Cloning subnet-template repository...") @@ -286,9 +328,6 @@ def _ensure_subnet_template(config_data): if current_branch != SUBNET_TEMPLATE_BRANCH: try: repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - console.print( - f"[green]Switched to branch '{SUBNET_TEMPLATE_BRANCH}'." - ) except GitCommandError as e: console.print( f"[red]Error switching to branch '{SUBNET_TEMPLATE_BRANCH}': {e}" @@ -296,7 +335,8 @@ def _ensure_subnet_template(config_data): return subnet_template_path -def _handle_validator(config_data, subnet_template_path, chain_pid): + +def _run_validator(config_data, subnet_template_path, chain_pid): owner_info = config_data["Owner"] validator_pid = owner_info.get("pid") validator_subtensor_pid = owner_info.get("subtensor_pid") @@ -320,7 +360,8 @@ def _handle_validator(config_data, subnet_template_path, chain_pid): if not success: console.print("[red]Failed to start validator.") -def _handle_miners(config_data, subnet_template_path, chain_pid): + +def _run_miners(config_data, subnet_template_path, chain_pid): for wallet_name, wallet_info in config_data.get("Miners", {}).items(): miner_pid = wallet_info.get("pid") miner_subtensor_pid = wallet_info.get("subtensor_pid") @@ -335,9 +376,7 @@ def _handle_miners(config_data, subnet_template_path, chain_pid): ) log_file_path = wallet_info.get("log_file") if log_file_path and os.path.exists(log_file_path): - attach_to_process_logs( - log_file_path, f"Miner {wallet_name}", miner_pid - ) + attach_to_process_logs(log_file_path, f"Miner {wallet_name}", miner_pid) else: console.print( f"[red]Log file not found for miner {wallet_name}. Cannot attach." @@ -350,6 +389,7 @@ def _handle_miners(config_data, subnet_template_path, chain_pid): if not success: console.print(f"[red]Failed to start miner {wallet_name}.") + def _stop_selected_neurons(config_data, selected_neurons): for neuron in selected_neurons: pid = int(neuron["pid"]) @@ -371,8 +411,9 @@ def _stop_selected_neurons(config_data, selected_neurons): elif neuron["process"].startswith("Validator"): config_data["Owner"]["pid"] = None + def _start_selected_neurons(config_data, selected_neurons): - subnet_template_path = _ensure_subnet_template(config_data) + subnet_template_path = _add_subnet_template(config_data) for neuron in selected_neurons: neuron_name = neuron["process"] @@ -394,4 +435,4 @@ def _start_selected_neurons(config_data, selected_neurons): # Update the process entries after starting neurons process_entries, _, _ = get_process_entries(config_data) - display_process_status_table(process_entries, [], []) \ No newline at end of file + display_process_status_table(process_entries, [], []) diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py new file mode 100644 index 00000000..6ead60ad --- /dev/null +++ b/btqs/commands/subnet.py @@ -0,0 +1,327 @@ +import os +import sys +import time +import threading +from tqdm import tqdm +import typer +import yaml +from rich.text import Text +from rich.table import Table +from bittensor_wallet import Wallet, Keypair +from btqs.config import ( + BTQS_WALLETS_DIRECTORY, + CONFIG_FILE_PATH, + VALIDATOR_URI, +) +from btqs.utils import ( + console, + exec_command, + remove_ansi_escape_sequences, + subnet_exists, + subnet_owner_exists, + get_process_entries, + display_process_status_table, + load_config, +) + +def add_stake(config_data = None): + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if subnet_owner: + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), + ) + add_stake = exec_command( + command="stake", + sub_command="add", + extra_args=[ + "--amount", + 1000, + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--chain", + "ws://127.0.0.1:9945", + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + ], + ) + + clean_stdout = remove_ansi_escape_sequences(add_stake.stdout) + if "βœ… Finalized" in clean_stdout: + text = Text( + f"Stake added successfully by Validator ({owner_wallet})\n", + style="bold light_goldenrod2", + ) + sign = Text("πŸ“ˆ ", style="bold yellow") + console.print(sign, text) + else: + console.print("\n[red] Failed to add stake. Command output:\n") + print(add_stake.stdout, end="") + + else: + console.print( + "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" + ) + return + + +def display_live_metagraph(): + """ + Displays a live view of the metagraph for subnet 1. + + This command shows real-time updates of the metagraph and neuron statuses. + + USAGE + + [green]$[/green] btqs subnet live + + [bold]Note[/bold]: Press Ctrl+C to exit the live view. + """ + def clear_screen(): + os.system("cls" if os.name == "nt" else "clear") + + def get_metagraph(): + result = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], + internal_command=True + ) + return result.stdout + + print("Starting live metagraph view. Press 'Ctrl + C' to exit.") + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + def input_thread(): + while True: + if input() == "q": + print("Exiting live view...") + sys.exit(0) + + threading.Thread(target=input_thread, daemon=True).start() + + try: + while True: + metagraph = get_metagraph() + process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) + clear_screen() + print(metagraph) + display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) + + # Create a progress bar for 5 seconds + print("\n") + for _ in tqdm(range(5), desc="Refreshing", unit="s", total=5): + time.sleep(1) + + except KeyboardInterrupt: + print("Exiting live view...") + +def setup_subnet(config_data): + os.makedirs(BTQS_WALLETS_DIRECTORY, exist_ok=True) + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if subnet_owner: + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), + ) + + warning_text = Text( + "A Subnet Owner associated with this setup already exists", + style="bold light_goldenrod2", + ) + wallet_info = Text(f"\t{owner_wallet}\n", style="medium_purple") + warning_sign = Text("⚠️ ", style="bold yellow") + + console.print(warning_sign, warning_text) + console.print(wallet_info) + else: + create_subnet_owner_wallet(config_data) + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + + owner_data = config_data["Owner"] + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=owner_data.get("path"), + hotkey=owner_data.get("hotkey"), + ) + + if subnet_exists(owner_wallet.coldkeypub.ss58_address, 1): + warning_text = Text( + "A Subnet with netuid 1 already exists and is registered with the owner's wallet:", + style="bold light_goldenrod2", + ) + wallet_info = Text(f"\t{owner_wallet}", style="medium_purple") + sudo_info = Text( + f"\tSUDO (Coldkey of Subnet 1 owner): {owner_wallet.coldkeypub.ss58_address}", + style="medium_purple", + ) + warning_sign = Text("⚠️ ", style="bold yellow") + + console.print(warning_sign, warning_text) + console.print(wallet_info) + console.print(sudo_info) + else: + create_subnet(owner_wallet) + + console.print("[dark_green]\nListing all subnets") + subnets_list = exec_command( + command="subnets", + sub_command="list", + extra_args=[ + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + +def create_subnet_owner_wallet(config_data): + console.print( + Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2"), + ) + + owner_wallet_name = typer.prompt( + "Enter subnet owner wallet name", default="owner", show_default=True + ) + owner_hotkey_name = typer.prompt( + "Enter subnet owner hotkey name", default="default", show_default=True + ) + + keypair = Keypair.create_from_uri(VALIDATOR_URI) + owner_wallet = Wallet( + path=BTQS_WALLETS_DIRECTORY, + name=owner_wallet_name, + hotkey=owner_hotkey_name, + ) + owner_wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + owner_wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + owner_wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + + console.print( + "Executed command: [dark_orange] btcli wallet create --wallet-name", + f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {BTQS_WALLETS_DIRECTORY}", + ) + + config_data["Owner"] = { + "wallet_name": owner_wallet_name, + "path": BTQS_WALLETS_DIRECTORY, + "hotkey": owner_hotkey_name, + "subtensor_pid": config_data["pid"], + } + with open(CONFIG_FILE_PATH, "w") as config_file: + yaml.safe_dump(config_data, config_file) + +def create_subnet(owner_wallet): + console.print( + Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2"), + style="bold yellow", + ) + + create_subnet = exec_command( + command="subnets", + sub_command="create", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--chain", + "ws://127.0.0.1:9945", + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + ], + ) + clean_stdout = remove_ansi_escape_sequences(create_subnet.stdout) + if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: + console.print("[dark_green] Subnet created successfully with netuid 1") + + console.print( + Text( + f"Registering Owner ({owner_wallet.name}) to Netuid 1\n", + style="bold light_goldenrod2", + ), + style="bold yellow", + ) + + register_subnet = exec_command( + command="subnets", + sub_command="register", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--wallet-name", + owner_wallet.name, + "--wallet-hotkey", + owner_wallet.hotkey_str, + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + "--no-prompt", + ], + ) + clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) + if "βœ… Registered" in clean_stdout: + console.print("[green] Registered the owner to subnet 1") + + +def steps(): + steps = [ + { + "command": "btqs chain start", + "description": "Start and initialize a local Subtensor blockchain. It may take several minutes to complete during the Subtensor compilation process. This is the entry point of the tutorial", + }, + { + "command": "btqs subnet setup", + "description": "This command creates a subnet owner's wallet, creates a new subnet, and registers the subnet owner to the subnet. Ensure the local chain is running before executing this command.", + }, + { + "command": "btqs neurons setup", + "description": "This command creates miner wallets and registers them to the subnet.", + }, + { + "command": "btqs neurons run", + "description": "Run all neurons (miners and validators). This command starts the processes for all configured neurons, attaching to running processes if they are already running.", + }, + { + "command": "btqs subnet stake", + "description": "Add stake to the subnet. This command allows the subnet owner to stake tokens to the subnet.", + }, + { + "command": "btqs subnet live", + "description": "Display the live metagraph of the subnet. This is used to monitor neuron performance and changing variables.", + }, + ] + + table = Table( + title="[bold dark_orange]Subnet Setup Steps", + header_style="dark_orange", + leading=True, + show_edge=False, + border_style="bright_black", + ) + table.add_column("Step", style="cyan", width=12, justify="center") + table.add_column("Command", justify="left", style="green") + table.add_column("Description", justify="left", style="white") + + for index, step in enumerate(steps, start=1): + table.add_row(str(index), step["command"], step["description"]) + + console.print(table) + + console.print("\n[dark_orange] You can run an automated script covering all the steps using:\n") + console.print("[blue]$ [green]btqs run-all") \ No newline at end of file diff --git a/btqs/config.py b/btqs/config.py index 046cd7f4..21f15fd8 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -3,9 +3,17 @@ CONFIG_FILE_PATH = os.path.expanduser("~/.bittensor/btqs/btqs_config.yml") BTQS_DIRECTORY = os.path.expanduser("~/.bittensor/btqs") -BTQS_WALLETS_DIRECTORY = os.path.expanduser(os.path.join(BTQS_DIRECTORY, "wallets")) +DEFAULT_WORKSPACE_DIRECTORY = os.path.expanduser("~/Desktop/Bittensor_quick_start") +BTQS_WALLETS_DIRECTORY = os.path.expanduser(os.path.join(DEFAULT_WORKSPACE_DIRECTORY, "wallets")) SUBNET_TEMPLATE_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" SUBNET_TEMPLATE_BRANCH = "ench/abe/commented-info" +SUBTENSOR_REPO_URL = "https://github.com/opentensor/subtensor.git" +WALLET_URIS = ["//Bob", "//Charlie"] +VALIDATOR_URI = "//Alice" +MINER_PORTS = [8101, 8102, 8103] + EPILOG = "Made with [bold red]:heart:[/bold red] by The OpenΟ„ensor FoundaΟ„ion" + +LOCALNET_ENDPOINT = "ws://127.0.0.1:9945" diff --git a/btqs/src/commands/__init__.py b/btqs/src/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/btqs/utils.py b/btqs/utils.py index 4ff44aa6..3eb8e911 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -16,7 +16,6 @@ from typer.testing import CliRunner from .config import ( - BTQS_DIRECTORY, BTQS_WALLETS_DIRECTORY, CONFIG_FILE_PATH, ) @@ -120,7 +119,7 @@ def exec_command( ) return result -def is_chain_running(config_file_path: str) -> bool: +def is_chain_running(config_file_path: str = CONFIG_FILE_PATH) -> bool: """ Checks if the local chain is running by verifying the PID in the config file. @@ -557,7 +556,7 @@ def start_miner( ] # Create log file paths - logs_dir = os.path.join(config_data["base_path"], "logs") + logs_dir = os.path.join(config_data["workspace_path"], "logs") os.makedirs(logs_dir, exist_ok=True) log_file_path = os.path.join(logs_dir, f"miner_{wallet_name}.log") @@ -621,7 +620,7 @@ def start_validator( ] # Create log file paths - logs_dir = os.path.join(config_data["base_path"], "logs") + logs_dir = os.path.join(config_data["workspace_path"], "logs") os.makedirs(logs_dir, exist_ok=True) log_file_path = os.path.join(logs_dir, "validator.log") From 8c2b1e11ef17e0ffb8d2fd5c9517e80be343a532 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 01:31:27 -0700 Subject: [PATCH 06/23] Polishing --- btqs/btqs_cli.py | 4 +++- btqs/commands/chain.py | 1 - btqs/commands/neurons.py | 46 ++++++++++++++++++++++++++++------------ btqs/commands/subnet.py | 31 ++++++++++++++------------- btqs/utils.py | 2 +- 5 files changed, 53 insertions(+), 31 deletions(-) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 32cb55ad..293a1e4e 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -160,7 +160,7 @@ def run_all(self): print(subnets_list.stdout, end="") text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") - sign = Text("πŸͺ™ ", style="bold yellow") + sign = Text("\nπŸͺ™ ", style="bold yellow") console.print(sign, text) time.sleep(2) @@ -394,6 +394,8 @@ def reattach_neurons(self): return config_data = load_config() + process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) + display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) neurons.reattach_neurons(config_data) def status_neurons(self): diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 79ff4a77..826a7265 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -3,7 +3,6 @@ import time import psutil -import typer import yaml from git import GitCommandError, Repo from rich.console import Console diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index 82c15690..b1cc1f40 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -2,7 +2,6 @@ import psutil import typer import yaml -from rich.console import Console from bittensor_wallet import Wallet, Keypair from git import Repo, GitCommandError @@ -185,28 +184,48 @@ def start_neurons(config_data): def reattach_neurons(config_data): - - # Choose which neuron to reattach to + # Fetch all available neurons all_neurons = { **config_data.get("Miners", {}), "Validator": config_data.get("Owner", {}), } - neuron_names = list(all_neurons.keys()) - if not neuron_names: - console.print("[red]No neurons found.") + + neuron_entries = [ + {"name": name, "info": info} + for name, info in all_neurons.items() + if info and psutil.pid_exists(info.get("pid", 0)) and info.get("log_file") + ] + + if not neuron_entries: + console.print("[red]No neurons found or none are running.") return - neuron_choice = typer.prompt( - f"Which neuron do you want to reattach to? {neuron_names}", - default=neuron_names[0], + # Display a list of neurons for the user to choose from + console.print("\nSelect neuron to reattach to:") + for idx, neuron in enumerate(neuron_entries, start=1): + console.print(f"{idx}. {neuron['name']} (PID: {neuron['info']['pid']})") + + selection = typer.prompt( + "Enter neuron number to reattach to, or 'q' to quit", + default="1", ) - if neuron_choice not in all_neurons: - console.print("[red]Invalid neuron name.") + + if selection.lower() == 'q': + console.print("[yellow]Reattach aborted.") return - wallet_info = all_neurons[neuron_choice] + if not selection.isdigit() or int(selection) < 1 or int(selection) > len(neuron_entries): + console.print("[red]Invalid selection.") + return + + # Get the selected neuron based on user input + selected_neuron = neuron_entries[int(selection) - 1] + neuron_choice = selected_neuron['name'] + wallet_info = selected_neuron['info'] pid = wallet_info.get("pid") log_file_path = wallet_info.get("log_file") + + # Ensure the neuron process is running if not pid or not psutil.pid_exists(pid): console.print("[red]Neuron process not running.") return @@ -216,9 +235,10 @@ def reattach_neurons(config_data): return console.print( - f"[green]Reattaching to neuron {neuron_choice}. Press Ctrl+C to exit." + f"[green]Reattaching to neuron {neuron_choice}." ) + # Attach to the process logs attach_to_process_logs(log_file_path, neuron_choice, pid) diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index 6ead60ad..1e32939f 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -121,7 +121,13 @@ def input_thread(): # Create a progress bar for 5 seconds print("\n") - for _ in tqdm(range(5), desc="Refreshing", unit="s", total=5): + console.print("[green] Live view active: Press Ctrl + C to exit\n") + for _ in tqdm(range(5), + desc="Refreshing", + bar_format="{desc}: {bar}", + ascii=" β––β–˜β–β–—β–šβ–ž", + ncols=20, + colour="green"): time.sleep(1) except KeyboardInterrupt: @@ -189,9 +195,9 @@ def setup_subnet(config_data): print(subnets_list.stdout, end="") def create_subnet_owner_wallet(config_data): - console.print( - Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2"), - ) + text = Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2") + sign = Text("\nℹ️ ", style="bold yellow") + console.print(sign, text) owner_wallet_name = typer.prompt( "Enter subnet owner wallet name", default="owner", show_default=True @@ -225,10 +231,9 @@ def create_subnet_owner_wallet(config_data): yaml.safe_dump(config_data, config_file) def create_subnet(owner_wallet): - console.print( - Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2"), - style="bold yellow", - ) + text = Text("\nCreating a subnet with Netuid 1.\n", style="bold light_goldenrod2") + sign = Text("\nℹ️ ", style="bold yellow") + console.print(sign, text) create_subnet = exec_command( command="subnets", @@ -249,13 +254,9 @@ def create_subnet(owner_wallet): if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: console.print("[dark_green] Subnet created successfully with netuid 1") - console.print( - Text( - f"Registering Owner ({owner_wallet.name}) to Netuid 1\n", - style="bold light_goldenrod2", - ), - style="bold yellow", - ) + text = Text(f"\nRegistering Owner ({owner_wallet}) to Netuid 1\n", style="bold light_goldenrod2") + sign = Text("\nℹ️ ", style="bold yellow") + console.print(sign, text) register_subnet = exec_command( command="subnets", diff --git a/btqs/utils.py b/btqs/utils.py index 3eb8e911..b24571e5 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -656,7 +656,7 @@ def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = Non # Move to the end of the file log_file.seek(0, os.SEEK_END) console.print( - f"[green]Attached to {process_name}. Press Ctrl+C to move to the next process." + f"[green]Attached to {process_name}. Press Ctrl+C to move on." ) while True: line = log_file.readline() From d38f7b2f860ed726d56b69909fa34578928d2692 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 15:03:59 -0700 Subject: [PATCH 07/23] Wreck --- bittensor_cli/src/commands/subnets.py | 3 +- btqs/btqs_cli.py | 53 +++--- btqs/commands/chain.py | 17 +- btqs/commands/neurons.py | 35 ++-- btqs/commands/subnet.py | 97 +++++++--- btqs/config.py | 6 +- btqs/utils.py | 258 ++++++++++++++++++++++++-- 7 files changed, 383 insertions(+), 86 deletions(-) diff --git a/bittensor_cli/src/commands/subnets.py b/bittensor_cli/src/commands/subnets.py index 122fec6a..6c4faee4 100644 --- a/bittensor_cli/src/commands/subnets.py +++ b/bittensor_cli/src/commands/subnets.py @@ -86,6 +86,7 @@ async def _find_event_attributes_in_extrinsic_receipt( print_verbose("Fetching lock_cost") burn_cost = await lock_cost(subtensor) + print(burn_cost) if burn_cost > your_balance: err_console.print( f"Your balance of: [green]{your_balance}[/green] is not enough to pay the subnet lock cost of: " @@ -363,7 +364,7 @@ async def lock_cost(subtensor: "SubtensorInterface") -> Optional[Balance]: method="get_network_registration_cost", params=[], ) - if lc: + if lc is not None: lock_cost_ = Balance(lc) console.print(f"Subnet lock cost: [green]{lock_cost_}[/green]") return lock_cost_ diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 293a1e4e..00379570 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -17,6 +17,7 @@ EPILOG, LOCALNET_ENDPOINT, DEFAULT_WORKSPACE_DIRECTORY, + DEFAULT_SUBTENSOR_BRANCH, ) from .utils import ( console, @@ -25,10 +26,9 @@ get_bittensor_wallet_version, get_btcli_version, get_process_entries, - get_python_path, - get_python_version, is_chain_running, load_config, + get_bittensor_version, ) @@ -83,7 +83,10 @@ def __init__(self): self.neurons_app.command(name="start")(self.start_neurons) def display_live_metagraph(self): - subnet.display_live_metagraph() + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + subnet.display_live_metagraph(config_data) def setup_steps(self): subnet.steps() @@ -119,6 +122,13 @@ def run_all(self): console.print(sign, text) self.setup_subnet() + text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") + sign = Text("\nπŸͺ™ ", style="bold yellow") + console.print(sign, text) + time.sleep(2) + + self.add_stake() + console.print( "\nNext command will: 1. Create miner wallets 2. Register them to Netuid 1" ) @@ -159,24 +169,6 @@ def run_all(self): ) print(subnets_list.stdout, end="") - text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") - sign = Text("\nπŸͺ™ ", style="bold yellow") - console.print(sign, text) - time.sleep(2) - - self.add_stake() - console.print("[dark_green]\nViewing Metagraph for Subnet 1") - subnets_list = exec_command( - command="subnets", - sub_command="metagraph", - extra_args=[ - "--netuid", - "1", - "--chain", - "ws://127.0.0.1:9945", - ], - ) - print(subnets_list.stdout, end="") self.display_live_metagraph() def start_chain( @@ -229,7 +221,7 @@ def start_chain( if not branch: branch = typer.prompt( typer.style("Enter Subtensor branch", fg="blue"), - default="testnet", + default=DEFAULT_SUBTENSOR_BRANCH, ) console.print("[dark_orange]Starting the local chain...") @@ -394,8 +386,12 @@ def reattach_neurons(self): return config_data = load_config() - process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) - display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) + process_entries, cpu_usage_list, memory_usage_list = get_process_entries( + config_data + ) + display_process_status_table( + process_entries, cpu_usage_list, memory_usage_list, config_data + ) neurons.reattach_neurons(config_data) def status_neurons(self): @@ -420,7 +416,9 @@ def status_neurons(self): process_entries, cpu_usage_list, memory_usage_list = get_process_entries( config_data ) - display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) + display_process_status_table( + process_entries, cpu_usage_list, memory_usage_list, config_data + ) spec_table = Table( title="[underline dark_orange]Machine Specifications[/underline dark_orange]", @@ -457,8 +455,9 @@ def status_neurons(self): version_table.add_row( "bittensor-wallet version:", get_bittensor_wallet_version() ) - version_table.add_row("Python version:", get_python_version()) - version_table.add_row("Python path:", get_python_path()) + version_table.add_row( + "bittensor version:", get_bittensor_version() + ) layout = Table.grid(expand=True) layout.add_column(justify="left") diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 826a7265..fb7b98c3 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -3,13 +3,14 @@ import time import psutil + import yaml from git import GitCommandError, Repo from rich.console import Console from rich.text import Text from btqs.config import CONFIG_FILE_PATH, SUBTENSOR_REPO_URL -from btqs.utils import attach_to_process_logs, get_process_entries +from btqs.utils import attach_to_process_logs, get_process_entries, create_virtualenv, install_neuron_dependencies, install_subtensor_dependencies from rich.prompt import Confirm console = Console() @@ -50,15 +51,25 @@ def start(config_data, workspace_path, branch): except GitCommandError as e: console.print(f"[red]Error cloning repository: {e}") return + + venv_subtensor_path = os.path.join(workspace_path, 'venv_subtensor') + venv_python = create_virtualenv(venv_subtensor_path) + install_subtensor_dependencies() + + config_data['venv_subtensor'] = venv_subtensor_path + # Running localnet.sh using the virtual environment's Python localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") - # Running localnet.sh + + env_variables = os.environ.copy() + env_variables["PATH"] = os.path.dirname(venv_python) + os.pathsep + env_variables["PATH"] process = subprocess.Popen( - ["bash", localnet_path], + [localnet_path], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, cwd=subtensor_path, start_new_session=True, + env=env_variables, ) console.print("[green]Starting local chain. This may take a few minutes...") diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index b1cc1f40..b6d5a3cf 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -11,7 +11,8 @@ SUBNET_TEMPLATE_REPO_URL, SUBNET_TEMPLATE_BRANCH, WALLET_URIS, - MINER_PORTS + MINER_PORTS, + DEFAULT_SUBNET_PATH, ) from btqs.utils import ( console, @@ -23,6 +24,8 @@ start_miner, attach_to_process_logs, subnet_owner_exists, + create_virtualenv, + install_neuron_dependencies, ) @@ -69,12 +72,20 @@ def run_neurons(config_data): chain_pid = config_data.get("pid") config_data["subnet_path"] = subnet_template_path + venv_neurons_path = os.path.join(config_data["workspace_path"], 'venv_neurons') + venv_python = create_virtualenv(venv_neurons_path) + install_neuron_dependencies(venv_python, subnet_template_path) + # Handle Validator if config_data.get("Owner"): - _run_validator(config_data, subnet_template_path, chain_pid) + config_data["Owner"]["venv"] = venv_python + _run_validator(config_data, subnet_template_path, chain_pid, venv_python) # Handle Miners - _run_miners(config_data, subnet_template_path, chain_pid) + for wallet_name, wallet_info in config_data.get("Miners", {}).items(): + config_data["Miners"][wallet_name]["venv"] = venv_python + + _run_miners(config_data, subnet_template_path, chain_pid, venv_python) with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) @@ -263,7 +274,6 @@ def _create_miner_wallets(config_data): wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) config_data["Miners"][wallet_name] = { - "path": BTQS_WALLETS_DIRECTORY, "hotkey": hotkey_name, "uri": uri, "pid": None, @@ -279,7 +289,7 @@ def _create_miner_wallets(config_data): def _register_miners(config_data): for wallet_name, wallet_info in config_data["Miners"].items(): wallet = Wallet( - path=wallet_info["path"], + path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=wallet_info["hotkey"], ) @@ -328,9 +338,8 @@ def _add_subnet_template(config_data): console.print("[red]Base path not found in the configuration file.") return - subnet_template_path = os.path.join(workspace_path, "subnet-template") - - if not os.path.exists(subnet_template_path): + subnet_template_path = DEFAULT_SUBNET_PATH + if not os.path.exists(DEFAULT_SUBNET_PATH): console.print("[green]Cloning subnet-template repository...") try: repo = Repo.clone_from( @@ -356,7 +365,7 @@ def _add_subnet_template(config_data): return subnet_template_path -def _run_validator(config_data, subnet_template_path, chain_pid): +def _run_validator(config_data, subnet_template_path, chain_pid, venv_python): owner_info = config_data["Owner"] validator_pid = owner_info.get("pid") validator_subtensor_pid = owner_info.get("subtensor_pid") @@ -376,12 +385,12 @@ def _run_validator(config_data, subnet_template_path, chain_pid): console.print("[red]Log file not found for validator. Cannot attach.") else: # Validator is not running, start it - success = start_validator(owner_info, subnet_template_path, config_data) + success = start_validator(owner_info, subnet_template_path, config_data, venv_python) if not success: console.print("[red]Failed to start validator.") -def _run_miners(config_data, subnet_template_path, chain_pid): +def _run_miners(config_data, subnet_template_path, chain_pid, venv_python): for wallet_name, wallet_info in config_data.get("Miners", {}).items(): miner_pid = wallet_info.get("pid") miner_subtensor_pid = wallet_info.get("subtensor_pid") @@ -404,7 +413,7 @@ def _run_miners(config_data, subnet_template_path, chain_pid): else: # Miner is not running, start it success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data + wallet_name, wallet_info, subnet_template_path, config_data, venv_python ) if not success: console.print(f"[red]Failed to start miner {wallet_name}.") @@ -439,7 +448,7 @@ def _start_selected_neurons(config_data, selected_neurons): neuron_name = neuron["process"] if neuron_name.startswith("Validator"): success = start_validator( - config_data["Owner"], subnet_template_path, config_data + config_data["Owner"], subnet_template_path, config_data, config_data["Owner"]["venv"] ) elif neuron_name.startswith("Miner"): wallet_name = neuron_name.split("Miner: ")[-1] diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index 1e32939f..41c741c9 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -1,7 +1,5 @@ import os -import sys import time -import threading from tqdm import tqdm import typer import yaml @@ -29,9 +27,17 @@ def add_stake(config_data = None): if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), - path=owner_data.get("path"), + path=BTQS_WALLETS_DIRECTORY, hotkey=owner_data.get("hotkey"), ) + + text = Text( + f"Validator is adding stake to own hotkey ({owner_wallet})\n", + style="bold light_goldenrod2", + ) + sign = Text("πŸ”–", style="bold yellow") + console.print(sign, text) + add_stake = exec_command( command="stake", sub_command="add", @@ -58,10 +64,71 @@ def add_stake(config_data = None): ) sign = Text("πŸ“ˆ ", style="bold yellow") console.print(sign, text) + + console.print("[dark_green]\nViewing Metagraph for Subnet 1") + subnets_list = exec_command( + command="subnets", + sub_command="metagraph", + extra_args=[ + "--netuid", + "1", + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + else: console.print("\n[red] Failed to add stake. Command output:\n") print(add_stake.stdout, end="") + text = Text( + f"Validator is registering to root ({owner_wallet})\n", + style="bold light_goldenrod2", + ) + sign = Text("\nπŸ€ ", style="bold yellow") + console.print(sign, text) + + register_root = exec_command( + command="root", + sub_command="register", + extra_args=[ + "--wallet-path", + BTQS_WALLETS_DIRECTORY, + "--chain", + "ws://127.0.0.1:9945", + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + ], + ) + clean_stdout = remove_ansi_escape_sequences(register_root.stdout) + if "βœ… Registered" in clean_stdout: + text = Text( + "Successfully registered to root (Netuid 0)\n", + style="bold light_goldenrod2", + ) + sign = Text("🌟 ", style="bold yellow") + console.print(sign, text) + elif "βœ… Already registered on root network" in clean_stdout: + console.print("[bold light_goldenrod2]βœ… Validator is already registered to Root network") + else: + console.print("\n[red] Failed to register to root. Command output:\n") + print(register_root.stdout, end="") + + console.print("[dark_green]\nViewing Root list") + subnets_list = exec_command( + command="root", + sub_command="list", + extra_args=[ + "--chain", + "ws://127.0.0.1:9945", + ], + ) + print(subnets_list.stdout, end="") + else: console.print( "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" @@ -69,7 +136,7 @@ def add_stake(config_data = None): return -def display_live_metagraph(): +def display_live_metagraph(config_data): """ Displays a live view of the metagraph for subnet 1. @@ -99,17 +166,6 @@ def get_metagraph(): return result.stdout print("Starting live metagraph view. Press 'Ctrl + C' to exit.") - config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." - ) - - def input_thread(): - while True: - if input() == "q": - print("Exiting live view...") - sys.exit(0) - - threading.Thread(target=input_thread, daemon=True).start() try: while True: @@ -117,7 +173,7 @@ def input_thread(): process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) clear_screen() print(metagraph) - display_process_status_table(process_entries, cpu_usage_list, memory_usage_list) + display_process_status_table(process_entries, cpu_usage_list, memory_usage_list, config_data) # Create a progress bar for 5 seconds print("\n") @@ -139,7 +195,7 @@ def setup_subnet(config_data): if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), - path=owner_data.get("path"), + path=BTQS_WALLETS_DIRECTORY, hotkey=owner_data.get("hotkey"), ) @@ -161,7 +217,7 @@ def setup_subnet(config_data): owner_data = config_data["Owner"] owner_wallet = Wallet( name=owner_data.get("wallet_name"), - path=owner_data.get("path"), + path=BTQS_WALLETS_DIRECTORY, hotkey=owner_data.get("hotkey"), ) @@ -223,7 +279,6 @@ def create_subnet_owner_wallet(config_data): config_data["Owner"] = { "wallet_name": owner_wallet_name, - "path": BTQS_WALLETS_DIRECTORY, "hotkey": owner_hotkey_name, "subtensor_pid": config_data["pid"], } @@ -231,7 +286,7 @@ def create_subnet_owner_wallet(config_data): yaml.safe_dump(config_data, config_file) def create_subnet(owner_wallet): - text = Text("\nCreating a subnet with Netuid 1.\n", style="bold light_goldenrod2") + text = Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2") sign = Text("\nℹ️ ", style="bold yellow") console.print(sign, text) @@ -254,7 +309,7 @@ def create_subnet(owner_wallet): if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: console.print("[dark_green] Subnet created successfully with netuid 1") - text = Text(f"\nRegistering Owner ({owner_wallet}) to Netuid 1\n", style="bold light_goldenrod2") + text = Text(f"Registering Owner ({owner_wallet}) to Netuid 1\n", style="bold light_goldenrod2") sign = Text("\nℹ️ ", style="bold yellow") console.print(sign, text) diff --git a/btqs/config.py b/btqs/config.py index 21f15fd8..fa7ff593 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -4,12 +4,14 @@ BTQS_DIRECTORY = os.path.expanduser("~/.bittensor/btqs") DEFAULT_WORKSPACE_DIRECTORY = os.path.expanduser("~/Desktop/Bittensor_quick_start") -BTQS_WALLETS_DIRECTORY = os.path.expanduser(os.path.join(DEFAULT_WORKSPACE_DIRECTORY, "wallets")) +BTQS_WALLETS_DIRECTORY = os.path.join(DEFAULT_WORKSPACE_DIRECTORY, "wallets") +DEFAULT_SUBNET_PATH = os.path.join(DEFAULT_WORKSPACE_DIRECTORY, "subnet-template") SUBNET_TEMPLATE_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" SUBNET_TEMPLATE_BRANCH = "ench/abe/commented-info" - SUBTENSOR_REPO_URL = "https://github.com/opentensor/subtensor.git" +DEFAULT_SUBTENSOR_BRANCH = "testnet" + WALLET_URIS = ["//Bob", "//Charlie"] VALIDATOR_URI = "//Alice" MINER_PORTS = [8101, 8102, 8103] diff --git a/btqs/utils.py b/btqs/utils.py index b24571e5..472f897b 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -1,8 +1,9 @@ import os import re -import subprocess import sys +import subprocess import time +import platform from datetime import datetime from typing import Any, Dict, Optional, Tuple @@ -51,6 +52,157 @@ def load_config( config_data = yaml.safe_load(config_file) or {} return config_data +def is_package_installed(package_name: str) -> bool: + """ + Checks if a system package is installed. + + Args: + package_name (str): The name of the package to check. + + Returns: + bool: True if installed, False otherwise. + """ + try: + if platform.system() == 'Linux': + result = subprocess.run(['dpkg', '-l', package_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return package_name in result.stdout + elif platform.system() == 'Darwin': # macOS check + result = subprocess.run(['brew', 'list', package_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return package_name in result.stdout + else: + console.print(f"[red]Unsupported operating system: {platform.system()}") + return False + except Exception as e: + console.print(f"[red]Error checking package {package_name}: {e}") + return False + + +def is_rust_installed(required_version: str) -> bool: + """ + Checks if Rust is installed and matches the required version. + + Args: + required_version (str): The required Rust version. + + Returns: + bool: True if Rust is installed and matches the required version, False otherwise. + """ + try: + result = subprocess.run(['rustc', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + installed_version = result.stdout.strip().split()[1] + print(installed_version, required_version, result.stdout.strip().split()[1]) + return installed_version == required_version + except Exception: + return False + + +def create_virtualenv(venv_path: str) -> str: + """ + Creates a virtual environment at the specified path. + + Args: + venv_path (str): The path where the virtual environment should be created. + + Returns: + str: The path to the Python executable within the virtual environment. + """ + if not os.path.exists(venv_path): + console.print(f"[green]Creating virtual environment at {venv_path}...") + subprocess.run([sys.executable, '-m', 'venv', venv_path], check=True) + console.print("[green]Virtual environment created.") + # Print activation snippet + activate_command = f"source {os.path.join(venv_path, 'bin', 'activate')}" + console.print( + f"[yellow]To activate the virtual environment manually, run:\n[bold cyan]{activate_command}\n" + ) + else: + console.print(f"[green]Using existing virtual environment at {venv_path}.") + + # Get the path to the Python executable in the virtual environment + venv_python = os.path.join(venv_path, 'bin', 'python') + return venv_python + +def install_subtensor_dependencies() -> None: + """ + Installs subtensor dependencies, including system-level dependencies and Rust. + """ + console.print("[green]Installing subtensor system dependencies...") + + # Install required system dependencies + system_dependencies = [ + 'clang', + 'curl', + 'libssl-dev', + 'llvm', + 'libudev-dev', + 'protobuf-compiler', + ] + missing_packages = [] + + if platform.system() == 'Linux': + for package in system_dependencies: + if not is_package_installed(package): + missing_packages.append(package) + + if missing_packages: + console.print(f"[yellow]Installing missing system packages: {', '.join(missing_packages)}") + subprocess.run(['sudo', 'apt-get', 'update'], check=True) + subprocess.run(['sudo', 'apt-get', 'install', '-y'] + missing_packages, check=True) + else: + console.print("[green]All required system packages are already installed.") + + elif platform.system() == 'Darwin': # macOS check + macos_dependencies = ['protobuf'] + missing_packages = [] + for package in macos_dependencies: + if not is_package_installed(package): + missing_packages.append(package) + + if missing_packages: + console.print(f"[yellow]Installing missing macOS system packages: {', '.join(missing_packages)}") + subprocess.run(['brew', 'update'], check=True) + subprocess.run(['brew', 'install'] + missing_packages, check=True) + else: + console.print("[green]All required macOS system packages are already installed.") + + else: + console.print("[red]Unsupported operating system for automatic system dependency installation.") + return + + # Install Rust globally + console.print("[green]Checking Rust installation...") + + installation_version = 'nightly-2024-03-05' + check_version = 'rustc 1.78.0-nightly' + + if not is_rust_installed(check_version): + console.print(f"[yellow]Installing Rust {installation_version} globally...") + subprocess.run(['curl', '--proto', '=https', '--tlsv1.2', '-sSf', 'https://sh.rustup.rs', '-o', 'rustup.sh'], check=True) + subprocess.run(['sh', 'rustup.sh', '-y', '--default-toolchain', installation_version], check=True) + else: + console.print(f"[green]Required Rust version {check_version} is already installed.") + + # Add necessary Rust targets + console.print("[green]Configuring Rust toolchain...") + subprocess.run(['rustup', 'target', 'add', 'wasm32-unknown-unknown', '--toolchain', 'stable'], check=True) + subprocess.run(['rustup', 'component', 'add', 'rust-src', '--toolchain', 'stable'], check=True) + + console.print("[green]Subtensor dependencies installed.") + + +def install_neuron_dependencies(venv_python: str, cwd: str) -> None: + """ + Installs neuron dependencies into the virtual environment. + + Args: + venv_python (str): Path to the Python executable in the virtual environment. + cwd (str): Current working directory where the setup should run. + """ + console.print("[green]Installing neuron dependencies...") + subprocess.run([venv_python, '-m', 'pip', 'install', '--upgrade', 'pip'], cwd=cwd, check=True) + subprocess.run([venv_python, '-m', 'pip', 'install', '-e', '.'], cwd=cwd, check=True) + console.print("[green]Neuron dependencies installed.") + def remove_ansi_escape_sequences(text: str) -> str: """ Removes ANSI escape sequences from the given text. @@ -254,6 +406,20 @@ def get_bittensor_wallet_version() -> str: except Exception: return "Not installed or not found" +def get_bittensor_version() -> str: + """ + Gets the version of bittensor-wallet. + """ + try: + result = subprocess.run( + ["pip", "show", "bittensor"], capture_output=True, text=True + ) + for line in result.stdout.split("\n"): + if line.startswith("Version:"): + return line.split(":")[1].strip() + except Exception: + return "Not installed or not found" + def get_python_version() -> str: """ Gets the Python version. @@ -333,6 +499,7 @@ def get_process_entries( "memory_usage": memory_usage, "uptime_str": uptime_str, "location": subtensor_path, + "venv_path": config_data.get("venv_subtensor") } ) @@ -358,6 +525,7 @@ def get_process_entries( "memory_usage": memory_usage, "uptime_str": uptime_str, "location": subtensor_path, + "venv_path": "~" } ) @@ -365,7 +533,7 @@ def get_process_entries( miners = config_data.get("Miners", {}) for wallet_name, wallet_info in miners.items(): pid = wallet_info.get("pid") - location = wallet_info.get("path") + location = BTQS_WALLETS_DIRECTORY status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( get_process_info(pid) ) @@ -385,7 +553,8 @@ def get_process_entries( "cpu_usage": cpu_usage, "memory_usage": memory_usage, "uptime_str": uptime_str, - "location": location, + "location": config_data.get("subnet_path"), + "venv_path": wallet_info.get("venv") } ) @@ -393,7 +562,7 @@ def get_process_entries( owner_data = config_data.get("Owner") if owner_data: pid = owner_data.get("pid") - location = owner_data.get("path") + location = BTQS_WALLETS_DIRECTORY status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( get_process_info(pid) ) @@ -406,14 +575,15 @@ def get_process_entries( process_entries.append( { - "process": "Validator", + "process": f"Validator: {owner_data.get("wallet_name")}", "status": status, "status_style": status_style, "pid": str(pid), "cpu_usage": cpu_usage, "memory_usage": memory_usage, "uptime_str": uptime_str, - "location": location, + "location": config_data.get("subnet_path"), + "venv_path": owner_data.get("venv") } ) @@ -423,12 +593,13 @@ def display_process_status_table( process_entries: list[Dict[str, str]], cpu_usage_list: list[float], memory_usage_list: list[float], + config_data = None ) -> None: """ Displays the process status table. """ table = Table( - title="[underline dark_orange]BTQS Process Manager[/underline dark_orange]\n", + title="\n[underline dark_orange]BTQS Process Manager[/underline dark_orange]\n", show_footer=True, show_edge=False, header_style="bold white", @@ -481,18 +652,14 @@ def display_process_status_table( footer_style="bold white", ) - table.add_column( - "[bold white]Location", - style="white", - overflow="fold", - footer_style="bold white", - ) - for entry in process_entries: + if entry["process"] == "Subtensor": + subtensor_venv = entry["venv_path"] if entry["process"].startswith("Subtensor"): process_style = "cyan" elif entry["process"].startswith("Miner"): process_style = "magenta" + neurons_venv = entry["venv_path"] elif entry["process"].startswith("Validator"): process_style = "yellow" else: @@ -505,7 +672,6 @@ def display_process_status_table( entry["cpu_usage"], entry["memory_usage"], entry["uptime_str"], - entry["location"], ) # Compute total CPU and Memory usage @@ -520,17 +686,35 @@ def display_process_status_table( # Display the table console.print(table) + if config_data: + print("\n") + wallet_path = config_data.get("wallet_path", "") + if wallet_path: + console.print("[dark_orange]Wallet Path", wallet_path) + subnet_path = config_data.get("subnet_path", "") + if subnet_path: + console.print("[dark_orange]Subnet Path", subnet_path) + + workspace_path = config_data.get("workspace_path", "") + if workspace_path: + console.print("[dark_orange]Workspace Path", workspace_path) + if subtensor_venv: + console.print("[dark_orange]Subtensor virtual environment", subtensor_venv) + if neurons_venv: + console.print("[dark_orange]Neurons virtual environment", neurons_venv) + def start_miner( wallet_name: str, wallet_info: Dict[str, Any], subnet_template_path: str, config_data: Dict[str, Any], + venv_python: str, ) -> bool: """ Starts a single miner and displays logs until user presses Ctrl+C. """ wallet = Wallet( - path=wallet_info["path"], + path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=wallet_info["hotkey"], ) @@ -541,7 +725,7 @@ def start_miner( env_variables["PYTHONUNBUFFERED"] = "1" cmd = [ - sys.executable, + venv_python, "-u", "./neurons/miner.py", "--wallet.name", @@ -583,16 +767,18 @@ def start_miner( except Exception as e: console.print(f"[red]Error starting miner {wallet_name}: {e}") return False + def start_validator( owner_info: Dict[str, Any], subnet_template_path: str, config_data: Dict[str, Any], + venv_python: str, ) -> bool: """ Starts the validator process and displays logs until user presses Ctrl+C. """ wallet = Wallet( - path=owner_info["path"], + path=BTQS_WALLETS_DIRECTORY, name=owner_info["wallet_name"], hotkey=owner_info["hotkey"], ) @@ -603,7 +789,7 @@ def start_validator( env_variables["BT_AXON_PORT"] = str(8100) cmd = [ - sys.executable, + venv_python, "-u", "./neurons/validator.py", "--wallet.name", @@ -672,3 +858,37 @@ def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = Non console.print(f"\n[green]Detached from {process_name}.") except Exception as e: console.print(f"[red]Error attaching to {process_name}: {e}") + + +# def activate_venv(workspace_path): +# venv_path = os.path.join(workspace_path, 'venv') +# if not os.path.exists(venv_path): +# console.print("[green]Creating virtual environment for subnet-template...") +# subprocess.run([sys.executable, '-m', 'venv', 'venv'], cwd=workspace_path) +# console.print("[green]Virtual environment created.") +# # Print activation snippet +# activate_command = ( +# f"source {os.path.join(venv_path, 'bin', 'activate')}" +# if os.name != 'nt' +# else f"{os.path.join(venv_path, 'Scripts', 'activate')}" +# ) +# console.print( +# f"[yellow]To activate the virtual environment manually, run:\n[bold cyan]{activate_command}\n" +# ) +# # Install dependencies +# venv_python = ( +# os.path.join(venv_path, 'bin', 'python') +# if os.name != 'nt' +# else os.path.join(venv_path, 'Scripts', 'python.exe') +# ) +# console.print("[green]Installing subnet-template dependencies...") +# subprocess.run([venv_python, '-m', 'pip', 'install', '--upgrade', 'pip'], cwd=workspace_path) +# subprocess.run([venv_python, '-m', 'pip', 'install', '-e', '.'], cwd=workspace_path) +# console.print("[green]Dependencies installed.") +# else: +# console.print("[green]Using existing virtual environment for subnet-template.") +# venv_python = ( +# os.path.join(venv_path, 'bin', 'python') +# if os.name != 'nt' +# else os.path.join(venv_path, 'Scripts', 'python.exe') +# ) \ No newline at end of file From c0853c1113803fb83fed019f8c62c080c53c5470 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 16:04:01 -0700 Subject: [PATCH 08/23] Coffee --- btqs/btqs_cli.py | 188 ++++++++++++++++++++------------------- btqs/commands/chain.py | 3 + btqs/commands/neurons.py | 10 +-- btqs/commands/subnet.py | 35 ++++---- btqs/config.py | 5 +- btqs/utils.py | 15 ++-- 6 files changed, 130 insertions(+), 126 deletions(-) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 00379570..6c48f470 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -63,11 +63,13 @@ def __init__(self): self.app.command(name="steps", help="Display steps for subnet setup")( self.setup_steps ) + self.app.command(name="live")(self.display_live_metagraph) # Chain commands self.chain_app.command(name="start")(self.start_chain) self.chain_app.command(name="stop")(self.stop_chain) self.chain_app.command(name="reattach")(self.reattach_chain) + self.chain_app.command(name="status")(self.status_neurons) # Subnet commands self.subnet_app.command(name="setup")(self.setup_subnet) @@ -82,14 +84,87 @@ def __init__(self): self.neurons_app.command(name="status")(self.status_neurons) self.neurons_app.command(name="start")(self.start_neurons) - def display_live_metagraph(self): + def start_chain( + self, + workspace_path: Optional[str] = typer.Option( + None, + "--path", + "--workspace", + help="Path to Bittensor's quick start workspace", + ), + branch: Optional[str] = typer.Option( + None, "--subtensor_branch", help="Subtensor branch to checkout" + ), + ): + """ + Starts the local Subtensor chain. + """ + config_data = load_config(exit_if_missing=False) + if is_chain_running(): + console.print( + f"[red]The local chain is already running. Endpoint: {LOCALNET_ENDPOINT}" + ) + return + if not workspace_path: + if config_data.get("workspace_path"): + reuse_config_path = Confirm.ask( + f"[blue]Previously saved workspace_path found: [green]({config_data.get('workspace_path')}) \n[blue]Do you want to re-use it?", + default=True, + show_default=True, + ) + if reuse_config_path: + workspace_path = config_data.get("workspace_path") + else: + workspace_path = typer.prompt( + typer.style( + "Enter path to create Bittensor development workspace", + fg="blue", + ), + default=DEFAULT_WORKSPACE_DIRECTORY, + ) + + if not branch: + branch = typer.prompt( + typer.style("Enter Subtensor branch", fg="blue"), + default=DEFAULT_SUBTENSOR_BRANCH, + ) + + console.print("[dark_orange]Starting the local chain...") + chain.start(config_data, workspace_path, branch) + + def stop_chain(self): + """ + Stops the local Subtensor chain and any running miners. + + This command terminates the local Subtensor chain process, miner and validator processes, and optionally cleans up configuration data. + + USAGE + + [green]$[/green] btqs chain stop + + [bold]Note[/bold]: Use this command to gracefully shut down the local chain. It will also stop any running miner processes. + """ config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "No running chain found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - subnet.display_live_metagraph(config_data) + chain.stop(config_data) - def setup_steps(self): - subnet.steps() + def reattach_chain(self): + """ + Reattaches to the running local chain. + + This command allows you to view the logs of the running local Subtensor chain. + + USAGE + + [green]$[/green] btqs chain reattach + + [bold]Note[/bold]: Press Ctrl+C to detach from the chain logs. + """ + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + chain.reattach(config_data) def run_all(self): """ @@ -127,6 +202,12 @@ def run_all(self): console.print(sign, text) time.sleep(2) + console.print( + "\nNext command will: 1. Add stake to the validator 2. Register the validator to the root network (netuid 0)" + ) + console.print("Press any key to continue..\n") + input() + self.add_stake() console.print( @@ -169,100 +250,27 @@ def run_all(self): ) print(subnets_list.stdout, end="") + console.print( + "\nNext command will start a live view of the metagraph to monitor the subnet and its status\nPress Ctrl + C to exit the live view" + ) + console.print("Press any key to continue..\n") + input() self.display_live_metagraph() - def start_chain( - self, - workspace_path: Optional[str] = typer.Option( - None, - "--path", - "--workspace", - help="Path to Bittensor's quick start workspace", - ), - branch: Optional[str] = typer.Option( - None, "--subtensor_branch", help="Subtensor branch to checkout" - ), - ): - """ - Starts the local Subtensor chain. - - This command initializes and starts a local instance of the Subtensor blockchain. - - USAGE - - [green]$[/green] btqs chain start - - [bold]Note[/bold]: This command may take several minutes to complete during the Subtensor compilation process. - """ - config_data = load_config(exit_if_missing=False) - if is_chain_running(): - console.print( - f"[red]The local chain is already running. Endpoint: {LOCALNET_ENDPOINT}" - ) - return - if not workspace_path: - if config_data.get("workspace_path"): - reuse_config_path = Confirm.ask( - f"[blue]Previously saved workspace_path found: [green]({config_data.get("workspace_path")}) \n[blue]Do you want to re-use it?", - default=True, - show_default=True, - ) - if reuse_config_path: - workspace_path = config_data.get("workspace_path") - else: - workspace_path = typer.prompt( - typer.style( - "Enter path to create Bittensor development workspace", - fg="blue", - ), - default=DEFAULT_WORKSPACE_DIRECTORY, - ) - - if not branch: - branch = typer.prompt( - typer.style("Enter Subtensor branch", fg="blue"), - default=DEFAULT_SUBTENSOR_BRANCH, - ) - - console.print("[dark_orange]Starting the local chain...") - chain.start(config_data, workspace_path, branch) - - def stop_chain(self): - """ - Stops the local Subtensor chain and any running miners. - - This command terminates the local Subtensor chain process, miner and validator processes, and optionally cleans up configuration data. - - USAGE - - [green]$[/green] btqs chain stop - - [bold]Note[/bold]: Use this command to gracefully shut down the local chain. It will also stop any running miner processes. - """ + def display_live_metagraph(self): config_data = load_config( - "No running chain found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - chain.stop(config_data) - - def reattach_chain(self): - """ - Reattaches to the running local chain. - - This command allows you to view the logs of the running local Subtensor chain. - - USAGE + subnet.display_live_metagraph(config_data) - [green]$[/green] btqs chain reattach + def setup_steps(self): + subnet.steps() - [bold]Note[/bold]: Press Ctrl+C to detach from the chain logs. - """ + def add_stake(self): config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." ) - chain.reattach(config_data) - - def add_stake(self): - subnet.add_stake() + subnet.add_stake(config_data) def setup_subnet(self): """ diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index fb7b98c3..f6fe9e69 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -104,6 +104,9 @@ def start(config_data, workspace_path, branch): "substrate_pid": substrate_pids, "subtensor_path": subtensor_path, "workspace_path": workspace_path, + "venv_subtensor": venv_subtensor_path, + "wallets_path": os.path.join(workspace_path, "wallets"), + "subnet_path": os.path.join(workspace_path, "subnet-template") } ) diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index b6d5a3cf..42aa2153 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -7,12 +7,10 @@ from btqs.config import ( CONFIG_FILE_PATH, - BTQS_WALLETS_DIRECTORY, SUBNET_TEMPLATE_REPO_URL, SUBNET_TEMPLATE_BRANCH, WALLET_URIS, MINER_PORTS, - DEFAULT_SUBNET_PATH, ) from btqs.utils import ( console, @@ -267,7 +265,7 @@ def _create_miner_wallets(config_data): keypair = Keypair.create_from_uri(uri) wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, name=wallet_name, hotkey=hotkey_name + path=config_data["wallets_path"], name=wallet_name, hotkey=hotkey_name ) wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) @@ -289,7 +287,7 @@ def _create_miner_wallets(config_data): def _register_miners(config_data): for wallet_name, wallet_info in config_data["Miners"].items(): wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], name=wallet_name, hotkey=wallet_info["hotkey"], ) @@ -338,8 +336,8 @@ def _add_subnet_template(config_data): console.print("[red]Base path not found in the configuration file.") return - subnet_template_path = DEFAULT_SUBNET_PATH - if not os.path.exists(DEFAULT_SUBNET_PATH): + subnet_template_path = config_data["subnet_path"] + if not os.path.exists(subnet_template_path): console.print("[green]Cloning subnet-template repository...") try: repo = Repo.clone_from( diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index 41c741c9..e1640fc5 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -7,7 +7,6 @@ from rich.table import Table from bittensor_wallet import Wallet, Keypair from btqs.config import ( - BTQS_WALLETS_DIRECTORY, CONFIG_FILE_PATH, VALIDATOR_URI, ) @@ -22,12 +21,12 @@ load_config, ) -def add_stake(config_data = None): +def add_stake(config_data): subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], hotkey=owner_data.get("hotkey"), ) @@ -45,7 +44,7 @@ def add_stake(config_data = None): "--amount", 1000, "--wallet-path", - BTQS_WALLETS_DIRECTORY, + config_data["wallets_path"], "--chain", "ws://127.0.0.1:9945", "--wallet-name", @@ -94,7 +93,7 @@ def add_stake(config_data = None): sub_command="register", extra_args=[ "--wallet-path", - BTQS_WALLETS_DIRECTORY, + config_data["wallets_path"], "--chain", "ws://127.0.0.1:9945", "--wallet-name", @@ -190,12 +189,12 @@ def get_metagraph(): print("Exiting live view...") def setup_subnet(config_data): - os.makedirs(BTQS_WALLETS_DIRECTORY, exist_ok=True) + os.makedirs(config_data["wallets_path"], exist_ok=True) subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], hotkey=owner_data.get("hotkey"), ) @@ -217,7 +216,7 @@ def setup_subnet(config_data): owner_data = config_data["Owner"] owner_wallet = Wallet( name=owner_data.get("wallet_name"), - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], hotkey=owner_data.get("hotkey"), ) @@ -237,7 +236,7 @@ def setup_subnet(config_data): console.print(wallet_info) console.print(sudo_info) else: - create_subnet(owner_wallet) + create_subnet(owner_wallet, config_data) console.print("[dark_green]\nListing all subnets") subnets_list = exec_command( @@ -264,7 +263,7 @@ def create_subnet_owner_wallet(config_data): keypair = Keypair.create_from_uri(VALIDATOR_URI) owner_wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], name=owner_wallet_name, hotkey=owner_hotkey_name, ) @@ -274,7 +273,7 @@ def create_subnet_owner_wallet(config_data): console.print( "Executed command: [dark_orange] btcli wallet create --wallet-name", - f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {BTQS_WALLETS_DIRECTORY}", + f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {config_data["wallets_path"]}", ) config_data["Owner"] = { @@ -285,7 +284,7 @@ def create_subnet_owner_wallet(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) -def create_subnet(owner_wallet): +def create_subnet(owner_wallet, config_data): text = Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2") sign = Text("\nℹ️ ", style="bold yellow") console.print(sign, text) @@ -295,7 +294,7 @@ def create_subnet(owner_wallet): sub_command="create", extra_args=[ "--wallet-path", - BTQS_WALLETS_DIRECTORY, + config_data["wallets_path"], "--chain", "ws://127.0.0.1:9945", "--wallet-name", @@ -318,7 +317,7 @@ def create_subnet(owner_wallet): sub_command="register", extra_args=[ "--wallet-path", - BTQS_WALLETS_DIRECTORY, + config_data["wallets_path"], "--wallet-name", owner_wallet.name, "--wallet-hotkey", @@ -345,6 +344,10 @@ def steps(): "command": "btqs subnet setup", "description": "This command creates a subnet owner's wallet, creates a new subnet, and registers the subnet owner to the subnet. Ensure the local chain is running before executing this command.", }, + { + "command": "btqs subnet stake", + "description": "Add stake to the subnet and register to root. This command stakes Tao to the validator's own hotkey and registers to root network", + }, { "command": "btqs neurons setup", "description": "This command creates miner wallets and registers them to the subnet.", @@ -353,10 +356,6 @@ def steps(): "command": "btqs neurons run", "description": "Run all neurons (miners and validators). This command starts the processes for all configured neurons, attaching to running processes if they are already running.", }, - { - "command": "btqs subnet stake", - "description": "Add stake to the subnet. This command allows the subnet owner to stake tokens to the subnet.", - }, { "command": "btqs subnet live", "description": "Display the live metagraph of the subnet. This is used to monitor neuron performance and changing variables.", diff --git a/btqs/config.py b/btqs/config.py index fa7ff593..0e34f15a 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -4,13 +4,12 @@ BTQS_DIRECTORY = os.path.expanduser("~/.bittensor/btqs") DEFAULT_WORKSPACE_DIRECTORY = os.path.expanduser("~/Desktop/Bittensor_quick_start") -BTQS_WALLETS_DIRECTORY = os.path.join(DEFAULT_WORKSPACE_DIRECTORY, "wallets") -DEFAULT_SUBNET_PATH = os.path.join(DEFAULT_WORKSPACE_DIRECTORY, "subnet-template") SUBNET_TEMPLATE_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" SUBNET_TEMPLATE_BRANCH = "ench/abe/commented-info" + SUBTENSOR_REPO_URL = "https://github.com/opentensor/subtensor.git" -DEFAULT_SUBTENSOR_BRANCH = "testnet" +DEFAULT_SUBTENSOR_BRANCH = "abe/temp/logging-dirs-for-nodes" WALLET_URIS = ["//Bob", "//Charlie"] VALIDATOR_URI = "//Alice" diff --git a/btqs/utils.py b/btqs/utils.py index 472f897b..82c152a6 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -17,7 +17,6 @@ from typer.testing import CliRunner from .config import ( - BTQS_WALLETS_DIRECTORY, CONFIG_FILE_PATH, ) @@ -533,7 +532,6 @@ def get_process_entries( miners = config_data.get("Miners", {}) for wallet_name, wallet_info in miners.items(): pid = wallet_info.get("pid") - location = BTQS_WALLETS_DIRECTORY status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( get_process_info(pid) ) @@ -562,7 +560,6 @@ def get_process_entries( owner_data = config_data.get("Owner") if owner_data: pid = owner_data.get("pid") - location = BTQS_WALLETS_DIRECTORY status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( get_process_info(pid) ) @@ -651,7 +648,7 @@ def display_process_status_table( justify="right", footer_style="bold white", ) - + subtensor_venv, neurons_venv = None, None for entry in process_entries: if entry["process"] == "Subtensor": subtensor_venv = entry["venv_path"] @@ -688,7 +685,7 @@ def display_process_status_table( if config_data: print("\n") - wallet_path = config_data.get("wallet_path", "") + wallet_path = config_data.get("wallets_path", "") if wallet_path: console.print("[dark_orange]Wallet Path", wallet_path) subnet_path = config_data.get("subnet_path", "") @@ -714,7 +711,7 @@ def start_miner( Starts a single miner and displays logs until user presses Ctrl+C. """ wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], name=wallet_name, hotkey=wallet_info["hotkey"], ) @@ -733,7 +730,7 @@ def start_miner( "--wallet.hotkey", wallet.hotkey_str, "--wallet.path", - BTQS_WALLETS_DIRECTORY, + config_data["wallets_path"], "--subtensor.chain_endpoint", "ws://127.0.0.1:9945", "--logging.trace", @@ -778,7 +775,7 @@ def start_validator( Starts the validator process and displays logs until user presses Ctrl+C. """ wallet = Wallet( - path=BTQS_WALLETS_DIRECTORY, + path=config_data["wallets_path"], name=owner_info["wallet_name"], hotkey=owner_info["hotkey"], ) @@ -797,7 +794,7 @@ def start_validator( "--wallet.hotkey", wallet.hotkey_str, "--wallet.path", - BTQS_WALLETS_DIRECTORY, + config_data["wallets_path"], "--subtensor.chain_endpoint", "ws://127.0.0.1:9945", "--netuid", From 558e28bde3a5311239673a0336ecd02c14e3546a Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 16:08:04 -0700 Subject: [PATCH 09/23] Localnet url added --- btqs/btqs_cli.py | 2 +- btqs/commands/neurons.py | 7 ++++--- btqs/config.py | 1 + btqs/utils.py | 10 ++++++---- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 6c48f470..6ed3066a 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -245,7 +245,7 @@ def run_all(self): "--netuid", "1", "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, ], ) print(subnets_list.stdout, end="") diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index 42aa2153..5c9344c0 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -11,6 +11,7 @@ SUBNET_TEMPLATE_BRANCH, WALLET_URIS, MINER_PORTS, + LOCALNET_ENDPOINT ) from btqs.utils import ( console, @@ -58,7 +59,7 @@ def setup_neurons(config_data): "--netuid", "1", "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, ], ) print(subnets_list.stdout, end="") @@ -310,7 +311,7 @@ def _register_miners(config_data): "--netuid", "1", "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, "--no-prompt", ], ) @@ -325,7 +326,7 @@ def _register_miners(config_data): command = ( f"btcli subnets register --wallet-path {wallet.path} --wallet-name " f"{wallet.name} --hotkey {wallet.hotkey_str} --netuid 1 --chain " - f"ws://127.0.0.1:9945 --no-prompt" + f"{LOCALNET_ENDPOINT} --no-prompt" ) console.print(f"[bold yellow]{command}\n") diff --git a/btqs/config.py b/btqs/config.py index 0e34f15a..64823cd0 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -14,6 +14,7 @@ WALLET_URIS = ["//Bob", "//Charlie"] VALIDATOR_URI = "//Alice" MINER_PORTS = [8101, 8102, 8103] +VALIDATOR_PORT = 8100 EPILOG = "Made with [bold red]:heart:[/bold red] by The OpenΟ„ensor FoundaΟ„ion" diff --git a/btqs/utils.py b/btqs/utils.py index 82c152a6..7f9cafe1 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -18,6 +18,8 @@ from .config import ( CONFIG_FILE_PATH, + VALIDATOR_PORT, + LOCALNET_ENDPOINT ) console = Console() @@ -326,7 +328,7 @@ def subnet_exists(ss58_address: str, netuid: int) -> bool: sub_command="list", extra_args=[ "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, ], internal_command=True, ) @@ -732,7 +734,7 @@ def start_miner( "--wallet.path", config_data["wallets_path"], "--subtensor.chain_endpoint", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, "--logging.trace", ] @@ -783,7 +785,7 @@ def start_validator( env_variables = os.environ.copy() env_variables["PYTHONUNBUFFERED"] = "1" - env_variables["BT_AXON_PORT"] = str(8100) + env_variables["BT_AXON_PORT"] = str(VALIDATOR_PORT) cmd = [ venv_python, @@ -796,7 +798,7 @@ def start_validator( "--wallet.path", config_data["wallets_path"], "--subtensor.chain_endpoint", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, "--netuid", "1", "--logging.trace", From 9e07675f7ce412f0fa709f019e63ca144ac03d32 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 16:13:03 -0700 Subject: [PATCH 10/23] Fixes str --- btqs/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btqs/utils.py b/btqs/utils.py index 7f9cafe1..794a3415 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -574,7 +574,7 @@ def get_process_entries( process_entries.append( { - "process": f"Validator: {owner_data.get("wallet_name")}", + "process": f"Validator: {owner_data.get('wallet_name')}", "status": status, "status_style": status_style, "pid": str(pid), From 19d3e5ec7be4a1089d79a7d2faa78969263ba306 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 16:14:09 -0700 Subject: [PATCH 11/23] Fixes string --- btqs/commands/subnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index e1640fc5..efeed8ef 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -273,7 +273,7 @@ def create_subnet_owner_wallet(config_data): console.print( "Executed command: [dark_orange] btcli wallet create --wallet-name", - f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {config_data["wallets_path"]}", + f"[dark_orange]{owner_hotkey_name} --wallet-hotkey {owner_wallet_name} --wallet-path {config_data['wallets_path']}", ) config_data["Owner"] = { From b8ba6495503574b09c325c966c6b21e2a58af65b Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 16:23:04 -0700 Subject: [PATCH 12/23] Increases compilation time --- btqs/commands/chain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index f6fe9e69..25fc49bc 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -72,14 +72,14 @@ def start(config_data, workspace_path, branch): env=env_variables, ) - console.print("[green]Starting local chain. This may take a few minutes...") + console.print("[green]Compiling and starting local chain. This may take a few minutes... (Timeout at 20 minutes)") # Paths to subtensor log files log_dir = os.path.join(subtensor_path, "logs") alice_log = os.path.join(log_dir, "alice.log") # Waiting for chain compilation - timeout = 360 # 6 minutes + timeout = 1200 # 17 minutes start_time = time.time() while not os.path.exists(alice_log): if time.time() - start_time > timeout: From c957d0c0bd0d806f6cccd6925c1dbc93b20157a6 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 17:06:19 -0700 Subject: [PATCH 13/23] Debugging --- btqs/commands/chain.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 25fc49bc..253496fb 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -65,15 +65,23 @@ def start(config_data, workspace_path, branch): env_variables["PATH"] = os.path.dirname(venv_python) + os.pathsep + env_variables["PATH"] process = subprocess.Popen( [localnet_path], - stdout=subprocess.DEVNULL, + # stdout=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=subtensor_path, start_new_session=True, env=env_variables, + universal_newlines=True, ) console.print("[green]Compiling and starting local chain. This may take a few minutes... (Timeout at 20 minutes)") + for line in process.stdout: + console.print(line, end="") + if "Imported #" in line: + console.print("[green] Chain comp") + continue + # Paths to subtensor log files log_dir = os.path.join(subtensor_path, "logs") alice_log = os.path.join(log_dir, "alice.log") From be2dbc416726534da054bef218e6b480b3fc1d98 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 17:24:30 -0700 Subject: [PATCH 14/23] Debugging --- btqs/commands/chain.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 253496fb..a9501926 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -65,22 +65,22 @@ def start(config_data, workspace_path, branch): env_variables["PATH"] = os.path.dirname(venv_python) + os.pathsep + env_variables["PATH"] process = subprocess.Popen( [localnet_path], - # stdout=subprocess.DEVNULL, - stdout=subprocess.PIPE, + stdout=subprocess.DEVNULL, + # stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=subtensor_path, start_new_session=True, env=env_variables, - universal_newlines=True, + # universal_newlines=True, ) console.print("[green]Compiling and starting local chain. This may take a few minutes... (Timeout at 20 minutes)") - for line in process.stdout: - console.print(line, end="") - if "Imported #" in line: - console.print("[green] Chain comp") - continue + # for line in process.stdout: + # console.print(line, end="") + # if "Imported #" in line: + # console.print("[green] Chain comp") + # continue # Paths to subtensor log files log_dir = os.path.join(subtensor_path, "logs") @@ -189,7 +189,7 @@ def reattach(config_data): return # Reattach using attach_to_process_logs - attach_to_process_logs(alice_log, "Subtensor Chain (Alice)", pid) + attach_to_process_logs(alice_log, "Subtensor Chain", pid) def wait_for_chain_compilation(alice_log, start_time, timeout): From 67ca53bdabf3403e4a0bea76ce4de810959f0f94 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Wed, 16 Oct 2024 18:37:01 -0700 Subject: [PATCH 15/23] Nits --- btqs/btqs_cli.py | 28 ++++-- btqs/commands/subnet.py | 186 ++++++++++++++++++++++++++++++++-------- 2 files changed, 168 insertions(+), 46 deletions(-) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 6ed3066a..c2d686bd 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -75,6 +75,7 @@ def __init__(self): self.subnet_app.command(name="setup")(self.setup_subnet) self.subnet_app.command(name="live")(self.display_live_metagraph) self.subnet_app.command(name="stake")(self.add_stake) + self.subnet_app.command(name="add-weights")(self.add_weights) # Neuron commands self.neurons_app.command(name="setup")(self.setup_neurons) @@ -197,17 +198,15 @@ def run_all(self): console.print(sign, text) self.setup_subnet() - text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") - sign = Text("\nπŸͺ™ ", style="bold yellow") - console.print(sign, text) - time.sleep(2) - console.print( "\nNext command will: 1. Add stake to the validator 2. Register the validator to the root network (netuid 0)" ) console.print("Press any key to continue..\n") input() - + text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") + sign = Text("\nπŸͺ™ ", style="bold yellow") + console.print(sign, text) + time.sleep(2) self.add_stake() console.print( @@ -250,6 +249,13 @@ def run_all(self): ) print(subnets_list.stdout, end="") + console.print( + "\nNext command will set weights to Netuid 1 through the Validator" + ) + console.print("Press any key to continue..\n") + input() + self.add_weights() + console.print( "\nNext command will start a live view of the metagraph to monitor the subnet and its status\nPress Ctrl + C to exit the live view" ) @@ -272,6 +278,12 @@ def add_stake(self): ) subnet.add_stake(config_data) + def add_weights(self): + config_data = load_config( + "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + ) + subnet.add_weights(config_data) + def setup_subnet(self): """ Sets up a subnet on the local chain. @@ -463,9 +475,7 @@ def status_neurons(self): version_table.add_row( "bittensor-wallet version:", get_bittensor_wallet_version() ) - version_table.add_row( - "bittensor version:", get_bittensor_version() - ) + version_table.add_row("bittensor version:", get_bittensor_version()) layout = Table.grid(expand=True) layout.add_column(justify="left") diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index efeed8ef..af84f35a 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -6,10 +6,8 @@ from rich.text import Text from rich.table import Table from bittensor_wallet import Wallet, Keypair -from btqs.config import ( - CONFIG_FILE_PATH, - VALIDATOR_URI, -) +from rich.progress import Progress, BarColumn, TimeRemainingColumn +from btqs.config import CONFIG_FILE_PATH, VALIDATOR_URI, LOCALNET_ENDPOINT from btqs.utils import ( console, exec_command, @@ -21,6 +19,7 @@ load_config, ) + def add_stake(config_data): subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) if subnet_owner: @@ -31,9 +30,9 @@ def add_stake(config_data): ) text = Text( - f"Validator is adding stake to own hotkey ({owner_wallet})\n", - style="bold light_goldenrod2", - ) + f"Validator is adding stake to own hotkey ({owner_wallet})\n", + style="bold light_goldenrod2", + ) sign = Text("πŸ”–", style="bold yellow") console.print(sign, text) @@ -46,7 +45,7 @@ def add_stake(config_data): "--wallet-path", config_data["wallets_path"], "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, "--wallet-name", owner_wallet.name, "--no-prompt", @@ -72,22 +71,22 @@ def add_stake(config_data): "--netuid", "1", "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, ], ) print(subnets_list.stdout, end="") - + else: console.print("\n[red] Failed to add stake. Command output:\n") print(add_stake.stdout, end="") text = Text( - f"Validator is registering to root ({owner_wallet})\n", - style="bold light_goldenrod2", - ) + f"Validator is registering to root ({owner_wallet})\n", + style="bold light_goldenrod2", + ) sign = Text("\nπŸ€ ", style="bold yellow") console.print(sign, text) - + register_root = exec_command( command="root", sub_command="register", @@ -95,7 +94,7 @@ def add_stake(config_data): "--wallet-path", config_data["wallets_path"], "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, "--wallet-name", owner_wallet.name, "--no-prompt", @@ -112,7 +111,9 @@ def add_stake(config_data): sign = Text("🌟 ", style="bold yellow") console.print(sign, text) elif "βœ… Already registered on root network" in clean_stdout: - console.print("[bold light_goldenrod2]βœ… Validator is already registered to Root network") + console.print( + "[bold light_goldenrod2]βœ… Validator is already registered to Root network" + ) else: console.print("\n[red] Failed to register to root. Command output:\n") print(register_root.stdout, end="") @@ -123,7 +124,7 @@ def add_stake(config_data): sub_command="list", extra_args=[ "--chain", - "ws://127.0.0.1:9945", + LOCALNET_ENDPOINT, ], ) print(subnets_list.stdout, end="") @@ -133,20 +134,113 @@ def add_stake(config_data): "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" ) return - -def display_live_metagraph(config_data): - """ - Displays a live view of the metagraph for subnet 1. - This command shows real-time updates of the metagraph and neuron statuses. +def add_weights(config_data): + subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + if subnet_owner: + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=config_data["wallets_path"], + hotkey=owner_data.get("hotkey"), + ) + text = Text( + "Validator is now setting weights on root network\n", + style="bold light_goldenrod2", + ) + sign = Text("\nπŸ‹οΈ ", style="bold yellow") + console.print(sign, text) + + max_retries = 5 + attempt = 0 + wait_time = 30 + + while attempt < max_retries: + try: + set_weights = exec_command( + command="root", + sub_command="set-weights", + extra_args=[ + "--wallet-path", + config_data["wallets_path"], + "--chain", + LOCALNET_ENDPOINT, + "--wallet-name", + owner_wallet.name, + "--no-prompt", + "--wallet-hotkey", + owner_wallet.hotkey_str, + "--netuid", + 1, + "--weights", + 1, + ], + ) + clean_stdout = remove_ansi_escape_sequences(set_weights.stdout) + + if "βœ… Finalized" in clean_stdout: + text = Text( + "Successfully set weights to Netuid 1\n", + style="bold light_goldenrod2", + ) + sign = Text("🌟 ", style="bold yellow") + console.print(sign, text) + break + + elif "Transaction has a bad signature" in clean_stdout: + attempt += 1 + if attempt < max_retries: + console.print( + f"[red]Attempt {attempt}/{max_retries}: Transaction has a bad signature. Retrying in {wait_time} seconds..." + ) + + with Progress( + "[progress.percentage]{task.percentage:>3.0f}%", + BarColumn(), + TimeRemainingColumn(), + console=console, + transient=True, + ) as progress: + task = progress.add_task("Waiting...", total=wait_time) + while not progress.finished: + progress.update(task, advance=1) + time.sleep(1) + else: + console.print( + f"[red]Attempt {attempt}/{max_retries}: Transaction has a bad signature. Maximum retries reached." + ) + console.print( + "\n[red]Failed to set weights after multiple attempts. Please try again later\n" + ) + else: + console.print("\n[red]Failed to set weights. Command output:\n") + print(set_weights.stdout, end="") + break + + except KeyboardInterrupt: + console.print("\n[yellow]Process interrupted by user. Exiting...") + return + + except Exception as e: + console.print(f"[red]An unexpected error occurred: {e}") + break - USAGE + else: + console.print( + "[red]All retry attempts exhausted. Unable to set weights on the root network." + ) + else: + console.print( + "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" + ) + return - [green]$[/green] btqs subnet live - [bold]Note[/bold]: Press Ctrl+C to exit the live view. +def display_live_metagraph(config_data): """ + Displays a live view of the metagraph. + """ + def clear_screen(): os.system("cls" if os.name == "nt" else "clear") @@ -160,7 +254,7 @@ def get_metagraph(): "--chain", "ws://127.0.0.1:9945", ], - internal_command=True + internal_command=True, ) return result.stdout @@ -169,25 +263,32 @@ def get_metagraph(): try: while True: metagraph = get_metagraph() - process_entries, cpu_usage_list, memory_usage_list = get_process_entries(config_data) + process_entries, cpu_usage_list, memory_usage_list = get_process_entries( + config_data + ) clear_screen() print(metagraph) - display_process_status_table(process_entries, cpu_usage_list, memory_usage_list, config_data) + display_process_status_table( + process_entries, cpu_usage_list, memory_usage_list, config_data + ) # Create a progress bar for 5 seconds print("\n") console.print("[green] Live view active: Press Ctrl + C to exit\n") - for _ in tqdm(range(5), - desc="Refreshing", - bar_format="{desc}: {bar}", - ascii=" β––β–˜β–β–—β–šβ–ž", - ncols=20, - colour="green"): + for _ in tqdm( + range(5), + desc="Refreshing", + bar_format="{desc}: {bar}", + ascii=" β––β–˜β–β–—β–šβ–ž", + ncols=20, + colour="green", + ): time.sleep(1) except KeyboardInterrupt: print("Exiting live view...") + def setup_subnet(config_data): os.makedirs(config_data["wallets_path"], exist_ok=True) subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) @@ -249,6 +350,7 @@ def setup_subnet(config_data): ) print(subnets_list.stdout, end="") + def create_subnet_owner_wallet(config_data): text = Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2") sign = Text("\nℹ️ ", style="bold yellow") @@ -284,6 +386,7 @@ def create_subnet_owner_wallet(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) + def create_subnet(owner_wallet, config_data): text = Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2") sign = Text("\nℹ️ ", style="bold yellow") @@ -308,7 +411,10 @@ def create_subnet(owner_wallet, config_data): if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: console.print("[dark_green] Subnet created successfully with netuid 1") - text = Text(f"Registering Owner ({owner_wallet}) to Netuid 1\n", style="bold light_goldenrod2") + text = Text( + f"Registering Owner ({owner_wallet}) to Netuid 1\n", + style="bold light_goldenrod2", + ) sign = Text("\nℹ️ ", style="bold yellow") console.print(sign, text) @@ -356,6 +462,10 @@ def steps(): "command": "btqs neurons run", "description": "Run all neurons (miners and validators). This command starts the processes for all configured neurons, attaching to running processes if they are already running.", }, + { + "command": "btqs subnet add-weight", + "description": "Add weight to netuid 1 through the validator", + }, { "command": "btqs subnet live", "description": "Display the live metagraph of the subnet. This is used to monitor neuron performance and changing variables.", @@ -378,5 +488,7 @@ def steps(): console.print(table) - console.print("\n[dark_orange] You can run an automated script covering all the steps using:\n") - console.print("[blue]$ [green]btqs run-all") \ No newline at end of file + console.print( + "\n[dark_orange] You can run an automated script covering all the steps using:\n" + ) + console.print("[blue]$ [green]btqs run-all") From f43499fa6445a77bf7dcd311a51c954e52e2b7c2 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Thu, 17 Oct 2024 02:30:22 -0700 Subject: [PATCH 16/23] Coming together --- btqs/btqs_cli.py | 166 ++++---- btqs/commands/chain.py | 73 ++-- btqs/commands/neurons.py | 63 +-- btqs/commands/subnet.py | 85 ++-- btqs/config.py | 31 +- btqs/rustup.sh | 811 +++++++++++++++++++++++++++++++++++++++ btqs/utils.py | 373 +++++++++++------- 7 files changed, 1256 insertions(+), 346 deletions(-) create mode 100644 btqs/rustup.sh diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index c2d686bd..61804677 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -1,34 +1,34 @@ import os import platform -import time -from time import sleep from typing import Optional - import psutil from rich.prompt import Confirm import typer from btqs.commands import chain, neurons, subnet from rich.table import Table -from rich.text import Text from .config import ( CONFIG_FILE_PATH, EPILOG, LOCALNET_ENDPOINT, DEFAULT_WORKSPACE_DIRECTORY, - DEFAULT_SUBTENSOR_BRANCH, + SUBTENSOR_BRANCH, ) from .utils import ( console, display_process_status_table, - exec_command, get_bittensor_wallet_version, get_btcli_version, get_process_entries, is_chain_running, load_config, get_bittensor_version, + print_info, + print_step, + print_success, + print_warning, + print_info_box, ) @@ -85,6 +85,51 @@ def __init__(self): self.neurons_app.command(name="status")(self.status_neurons) self.neurons_app.command(name="start")(self.start_neurons) + self.steps = [ + { + "title": "Start Local Subtensor", + "description": "Initialize and start the local Subtensor blockchain. This may take several minutes due to the compilation process.", + "info": "πŸ”— **Subtensor** is the underlying blockchain network that facilitates decentralized activities. Starting the local chain sets up your personal development environment for experimenting with Bittensor.", + "action": lambda: self.start_chain(workspace_path=None, branch=None), + }, + { + "title": "Check Chain Status", + "description": "Verify that the local chain is running correctly.", + "info": "πŸ“Š **Chain Nodes**: During your local Subtensor setup, two blockchain nodes are initiated. These nodes communicate and collaborate to reach consensus, ensuring the integrity and synchronization of your blockchain network.", + "action": lambda: self.status_neurons(), + }, + { + "title": "Set Up Subnet", + "description": "Create a subnet owner wallet, establish a new subnet, and register the owner to the subnet.", + "info": "πŸ”‘ **Wallets** in Bittensor are essential for managing your stake, interacting with the network, and running validators or miners. Each wallet has a unique name and associated hotkey that serves as your identity within the network.", + "action": lambda: self.setup_subnet(), + }, + { + "title": "Add Stake by Validator", + "description": "Stake Tao to the validator's hotkey and register to the root network.", + "info": "πŸ’° **Staking Tao** to your hotkey is the process of committing your tokens to support your role as a validator in the subnet. Your wallet can potentially earn rewards based on your performance and the subnet's incentive mechanism.", + "action": lambda: self.add_stake(), + }, + { + "title": "Set Up Miners", + "description": "Create miner wallets and register them to the subnet.", + "info": "βš’οΈ **Miners** are responsible for performing computations and contributing to the subnet's tasks. Setting up miners involves creating dedicated wallets for each miner entity and registering them to the subnet.", + "action": lambda: self.setup_neurons(), + }, + { + "title": "Run Miners", + "description": "Start all miner processes.", + "info": "πŸƒ This step starts and runs your miner processes so they can start contributing to the network", + "action": lambda: self.run_neurons(), + }, + { + "title": "Add Weights to Netuid 1", + "description": "Configure weights for Netuid 1 through the validator.", + "info": "πŸ‹οΈ Setting **Root weights** in Bittensor means assigning relative importance to different subnets within the network, which directly influences their share of network rewards and resources.", + "action": lambda: self.add_weights(), + }, + ] + def start_chain( self, workspace_path: Optional[str] = typer.Option( @@ -127,10 +172,10 @@ def start_chain( if not branch: branch = typer.prompt( typer.style("Enter Subtensor branch", fg="blue"), - default=DEFAULT_SUBTENSOR_BRANCH, + default=SUBTENSOR_BRANCH, ) - console.print("[dark_orange]Starting the local chain...") + console.print("πŸ”— [dark_orange]Starting the local chain...") chain.start(config_data, workspace_path, branch) def stop_chain(self): @@ -171,97 +216,36 @@ def run_all(self): """ Runs all commands in sequence to set up and start the local chain, subnet, and neurons. """ - text = Text("Starting Local Subtensor\n", style="bold light_goldenrod2") - sign = Text("πŸ”— ", style="bold yellow") - console.print(sign, text) - sleep(3) - - # Start the local chain - self.start_chain(workspace_path=None, branch=None) - - text = Text("Checking chain status\n", style="bold light_goldenrod2") - sign = Text("\nπŸ”Ž ", style="bold yellow") - console.print(sign, text) - sleep(3) - - self.status_neurons() - + console.clear() + print_info("Welcome to the Bittensor Quick Start Tutorial", emoji="πŸš€") console.print( - "\nNext command will: 1. Create a subnet owner wallet 2. Create a Subnet 3. Register to the subnet" + "\nThis tutorial will guide you through setting up the local chain, subnet, and neurons (miners + validators).\n", + style="magenta", ) - console.print("Press any key to continue..\n") - input() - # Set up the subnet - text = Text("Setting up subnet\n", style="bold light_goldenrod2") - sign = Text("πŸ“‘ ", style="bold yellow") - console.print(sign, text) - self.setup_subnet() + for idx, step in enumerate(self.steps, start=1): + if "info" in step: + print_info_box(step["info"], title="Info") + print_step(step["title"], step["description"], idx) - console.print( - "\nNext command will: 1. Add stake to the validator 2. Register the validator to the root network (netuid 0)" - ) - console.print("Press any key to continue..\n") - input() - text = Text("Adding stake by Validator\n", style="bold light_goldenrod2") - sign = Text("\nπŸͺ™ ", style="bold yellow") - console.print(sign, text) - time.sleep(2) - self.add_stake() + console.print( + "[bold blue]Press [yellow]Enter[/yellow] to continue to the next step or [yellow]Ctrl+C[/yellow] to exit.\n" + ) + try: + input() + except KeyboardInterrupt: + print_warning("Tutorial interrupted by user. Exiting...") + return - console.print( - "\nNext command will: 1. Create miner wallets 2. Register them to Netuid 1" - ) - console.print("Press any key to continue..\n") - input() - - text = Text("Setting up miners\n", style="bold light_goldenrod2") - sign = Text("\nβš’οΈ ", style="bold yellow") - console.print(sign, text) - - # Set up the neurons (miners) - self.setup_neurons() - - console.print("\nNext command will: 1. Start all miner processes") - console.print("Press any key to continue..\n") - input() - - text = Text("Running miners\n", style="bold light_goldenrod2") - sign = Text("πŸƒ ", style="bold yellow") - console.print(sign, text) - time.sleep(2) - - # Run the neurons - self.run_neurons() - - # Check status after running the neurons - self.status_neurons() - console.print("[dark_green]\nViewing Metagraph for Subnet 1") - subnets_list = exec_command( - command="subnets", - sub_command="metagraph", - extra_args=[ - "--netuid", - "1", - "--chain", - LOCALNET_ENDPOINT, - ], - ) - print(subnets_list.stdout, end="") + # Execute the action + step["action"]() - console.print( - "\nNext command will set weights to Netuid 1 through the Validator" + print_success( + "Your local chain, subnet, and neurons are up and running", emoji="πŸŽ‰" ) - console.print("Press any key to continue..\n") - input() - self.add_weights() - console.print( - "\nNext command will start a live view of the metagraph to monitor the subnet and its status\nPress Ctrl + C to exit the live view" + "[green]Next, execute the following command to get a live view of all the progress through the metagraph: [dark_green]$ [dark_orange]btqs live" ) - console.print("Press any key to continue..\n") - input() - self.display_live_metagraph() def display_live_metagraph(self): config_data = load_config( @@ -426,7 +410,7 @@ def status_neurons(self): [bold]Note[/bold]: Use this command to monitor the health and status of your local chain and miners. """ - console.print("[green]Checking status of Subtensor and neurons...") + print_info("Checking status of Subtensor and neurons...", emoji="πŸ” ") config_data = load_config( "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index a9501926..1968c81e 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -3,14 +3,19 @@ import time import psutil - import yaml +from btqs.config import CONFIG_FILE_PATH, SUBTENSOR_REPO_URL +from btqs.utils import ( + attach_to_process_logs, + create_virtualenv, + get_process_entries, + install_subtensor_dependencies, + print_info, + print_success, + print_error, +) from git import GitCommandError, Repo from rich.console import Console -from rich.text import Text - -from btqs.config import CONFIG_FILE_PATH, SUBTENSOR_REPO_URL -from btqs.utils import attach_to_process_logs, get_process_entries, create_virtualenv, install_neuron_dependencies, install_subtensor_dependencies from rich.prompt import Confirm console = Console() @@ -33,36 +38,38 @@ def start(config_data, workspace_path, branch): origin = repo.remotes.origin repo.git.checkout(branch) origin.pull() - console.print("[green]Repository updated successfully.") + print_info("Repository updated successfully.", emoji="πŸ“¦") except GitCommandError as e: - console.print(f"[red]Error updating repository: {e}") + print_error(f"Error updating repository: {e}") return else: - console.print( - "[green]Using existing subtensor repository without updating." + print_info( + "Using existing subtensor repository without updating.", emoji="πŸ“¦" ) else: try: - console.print("[green]Cloning subtensor repository...") + print_info("Cloning subtensor repository...", emoji="πŸ“¦") repo = Repo.clone_from(SUBTENSOR_REPO_URL, subtensor_path) if branch: repo.git.checkout(branch) - console.print("[green]Repository cloned successfully.") + print_success("Repository cloned successfully.", emoji="🏷") except GitCommandError as e: - console.print(f"[red]Error cloning repository: {e}") + print_error(f"Error cloning repository: {e}") return - - venv_subtensor_path = os.path.join(workspace_path, 'venv_subtensor') + + venv_subtensor_path = os.path.join(workspace_path, "venv_subtensor") venv_python = create_virtualenv(venv_subtensor_path) install_subtensor_dependencies() - config_data['venv_subtensor'] = venv_subtensor_path + config_data["venv_subtensor"] = venv_python # Running localnet.sh using the virtual environment's Python localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") env_variables = os.environ.copy() - env_variables["PATH"] = os.path.dirname(venv_python) + os.pathsep + env_variables["PATH"] + env_variables["PATH"] = ( + os.path.dirname(venv_python) + os.pathsep + env_variables["PATH"] + ) process = subprocess.Popen( [localnet_path], stdout=subprocess.DEVNULL, @@ -74,7 +81,10 @@ def start(config_data, workspace_path, branch): # universal_newlines=True, ) - console.print("[green]Compiling and starting local chain. This may take a few minutes... (Timeout at 20 minutes)") + print_info( + "Compiling and starting local chain. This may take a few minutes... (Timeout at 20 minutes)", + emoji="πŸ› οΈ ", + ) # for line in process.stdout: # console.print(line, end="") @@ -91,15 +101,16 @@ def start(config_data, workspace_path, branch): start_time = time.time() while not os.path.exists(alice_log): if time.time() - start_time > timeout: - console.print("[red]Timeout: Log files were not created.") + print_error("Timeout: Log files were not created.") return time.sleep(1) chain_ready = wait_for_chain_compilation(alice_log, start_time, timeout) if chain_ready: - text = Text("Local chain is running. You can now use it for development and testing.\n", style="bold light_goldenrod2") - sign = Text("\nℹ️ ", style="bold yellow") - console.print(sign, text) + print_info( + "Local chain is running. You can now use it for development and testing.\n", + emoji="\nπŸš€", + ) # Fetch PIDs of substrate nodes substrate_pids = get_substrate_pids() @@ -114,7 +125,7 @@ def start(config_data, workspace_path, branch): "workspace_path": workspace_path, "venv_subtensor": venv_subtensor_path, "wallets_path": os.path.join(workspace_path, "wallets"), - "subnet_path": os.path.join(workspace_path, "subnet-template") + "subnet_path": os.path.join(workspace_path, "subnet-template"), } ) @@ -123,13 +134,11 @@ def start(config_data, workspace_path, branch): os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True) with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) - console.print( - "[green]Config file updated." - ) + print_info("Config file updated.", emoji="πŸ“ ") except Exception as e: - console.print(f"[red]Failed to write to the config file: {e}") + print_error(f"Failed to write to the config file: {e}") else: - console.print("[red]Failed to start local chain.") + print_error("Failed to start local chain.") def stop(config_data): @@ -144,7 +153,7 @@ def stop(config_data): process = psutil.Process(pid) process.terminate() process.wait(timeout=10) - console.print("[green]Local chain stopped successfully.") + print_info("Local chain stopped successfully.", emoji="πŸ›‘ ") except psutil.NoSuchProcess: console.print( "[red]Process not found. The chain may have already been stopped." @@ -165,7 +174,7 @@ def stop(config_data): if refresh_config: if os.path.exists(CONFIG_FILE_PATH): os.remove(CONFIG_FILE_PATH) - console.print("[green]Configuration file refreshed.") + print_info("Configuration file refreshed.", emoji="πŸ”„ ") def reattach(config_data): @@ -242,7 +251,7 @@ def stop_running_neurons(config_data): ] if running_neurons: - console.print("[yellow]\nSome neurons are still running. Terminating them...") + print_info("Some neurons are still running. Terminating them...", emoji="\n🧨 ") for neuron in running_neurons: pid = int(neuron["pid"]) @@ -251,7 +260,7 @@ def stop_running_neurons(config_data): neuron_process = psutil.Process(pid) neuron_process.terminate() neuron_process.wait(timeout=10) - console.print(f"[green]{neuron_name} stopped.") + print_info(f"{neuron_name} stopped.", emoji="πŸ›‘ ") except psutil.NoSuchProcess: console.print(f"[yellow]{neuron_name} process not found.") except psutil.TimeoutExpired: @@ -267,7 +276,7 @@ def stop_running_neurons(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) else: - console.print("[green]No neurons were running.") + print_info("No neurons were running.", emoji="βœ… ") def is_process_running(pid): diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index 5c9344c0..917241f5 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -7,11 +7,11 @@ from btqs.config import ( CONFIG_FILE_PATH, - SUBNET_TEMPLATE_REPO_URL, - SUBNET_TEMPLATE_BRANCH, + SUBNET_REPO_URL, + SUBNET_REPO_BRANCH, WALLET_URIS, MINER_PORTS, - LOCALNET_ENDPOINT + LOCALNET_ENDPOINT, ) from btqs.utils import ( console, @@ -25,6 +25,7 @@ subnet_owner_exists, create_virtualenv, install_neuron_dependencies, + print_info, ) @@ -51,7 +52,7 @@ def setup_neurons(config_data): _register_miners(config_data) - console.print("[dark_green]\nViewing Metagraph for Subnet 1") + print_info("Viewing Metagraph for Subnet 1\n", emoji="πŸ“Š ") subnets_list = exec_command( command="subnets", sub_command="metagraph", @@ -71,7 +72,7 @@ def run_neurons(config_data): chain_pid = config_data.get("pid") config_data["subnet_path"] = subnet_template_path - venv_neurons_path = os.path.join(config_data["workspace_path"], 'venv_neurons') + venv_neurons_path = os.path.join(config_data["workspace_path"], "venv_neurons") venv_python = create_virtualenv(venv_neurons_path) install_neuron_dependencies(venv_python, subnet_template_path) @@ -89,6 +90,7 @@ def run_neurons(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) + def stop_neurons(config_data): # Get process entries process_entries, _, _ = get_process_entries(config_data) @@ -220,18 +222,22 @@ def reattach_neurons(config_data): default="1", ) - if selection.lower() == 'q': + if selection.lower() == "q": console.print("[yellow]Reattach aborted.") return - if not selection.isdigit() or int(selection) < 1 or int(selection) > len(neuron_entries): + if ( + not selection.isdigit() + or int(selection) < 1 + or int(selection) > len(neuron_entries) + ): console.print("[red]Invalid selection.") return # Get the selected neuron based on user input selected_neuron = neuron_entries[int(selection) - 1] - neuron_choice = selected_neuron['name'] - wallet_info = selected_neuron['info'] + neuron_choice = selected_neuron["name"] + wallet_info = selected_neuron["info"] pid = wallet_info.get("pid") log_file_path = wallet_info.get("log_file") @@ -244,9 +250,7 @@ def reattach_neurons(config_data): console.print("[red]Log file not found for this neuron.") return - console.print( - f"[green]Reattaching to neuron {neuron_choice}." - ) + console.print(f"[green]Reattaching to neuron {neuron_choice}.") # Attach to the process logs attach_to_process_logs(log_file_path, neuron_choice, pid) @@ -254,6 +258,7 @@ def reattach_neurons(config_data): # Helper functions + def _create_miner_wallets(config_data): for i, uri in enumerate(WALLET_URIS): console.print(f"Miner {i+1}:") @@ -283,7 +288,8 @@ def _create_miner_wallets(config_data): with open(CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) - console.print("[green]Miner wallets are created.") + print_info("Miner wallets are created.\n", emoji="πŸ—‚οΈ ") + def _register_miners(config_data): for wallet_name, wallet_info in config_data["Miners"].items(): @@ -292,11 +298,7 @@ def _register_miners(config_data): name=wallet_name, hotkey=wallet_info["hotkey"], ) - - console.print( - f"Registering Miner ({wallet_name}) to Netuid 1\n", - style="bold light_goldenrod2", - ) + print_info(f"Registering Miner ({wallet_name}) to Netuid 1\n", emoji="πŸ”§ ") miner_registered = exec_command( command="subnets", @@ -318,7 +320,7 @@ def _register_miners(config_data): clean_stdout = remove_ansi_escape_sequences(miner_registered.stdout) if "βœ… Registered" in clean_stdout: - console.print(f"[green]Registered miner ({wallet.name}) to Netuid 1\n") + print_info(f"Registered miner: ({wallet.name}) to Netuid 1\n", emoji="βœ… ") else: console.print( f"[red]Failed to register miner ({wallet.name}). You can register the miner manually using:" @@ -342,23 +344,23 @@ def _add_subnet_template(config_data): console.print("[green]Cloning subnet-template repository...") try: repo = Repo.clone_from( - SUBNET_TEMPLATE_REPO_URL, + SUBNET_REPO_URL, subnet_template_path, ) - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) - console.print("[green]Cloned subnet-template repository successfully.") + repo.git.checkout(SUBNET_REPO_BRANCH) + print_info("Cloned subnet-template repository successfully.", emoji="πŸ“¦ ") except GitCommandError as e: console.print(f"[red]Error cloning subnet-template repository: {e}") else: - console.print("[green]Using existing subnet-template repository.") + print_info("Using existing subnet-template repository.", emoji="πŸ“¦ ") repo = Repo(subnet_template_path) current_branch = repo.active_branch.name - if current_branch != SUBNET_TEMPLATE_BRANCH: + if current_branch != SUBNET_REPO_BRANCH: try: - repo.git.checkout(SUBNET_TEMPLATE_BRANCH) + repo.git.checkout(SUBNET_REPO_BRANCH) except GitCommandError as e: console.print( - f"[red]Error switching to branch '{SUBNET_TEMPLATE_BRANCH}': {e}" + f"[red]Error switching to branch '{SUBNET_REPO_BRANCH}': {e}" ) return subnet_template_path @@ -384,7 +386,9 @@ def _run_validator(config_data, subnet_template_path, chain_pid, venv_python): console.print("[red]Log file not found for validator. Cannot attach.") else: # Validator is not running, start it - success = start_validator(owner_info, subnet_template_path, config_data, venv_python) + success = start_validator( + owner_info, subnet_template_path, config_data, venv_python + ) if not success: console.print("[red]Failed to start validator.") @@ -447,7 +451,10 @@ def _start_selected_neurons(config_data, selected_neurons): neuron_name = neuron["process"] if neuron_name.startswith("Validator"): success = start_validator( - config_data["Owner"], subnet_template_path, config_data, config_data["Owner"]["venv"] + config_data["Owner"], + subnet_template_path, + config_data, + config_data["Owner"]["venv"], ) elif neuron_name.startswith("Miner"): wallet_name = neuron_name.split("Miner: ")[-1] diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index af84f35a..b119598b 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -17,6 +17,8 @@ get_process_entries, display_process_status_table, load_config, + print_info, + print_error, ) @@ -28,13 +30,10 @@ def add_stake(config_data): path=config_data["wallets_path"], hotkey=owner_data.get("hotkey"), ) - - text = Text( - f"Validator is adding stake to own hotkey ({owner_wallet})\n", - style="bold light_goldenrod2", + print_info( + f"Validator is adding stake to its own hotkey: {owner_wallet}\n", + emoji="πŸ”– ", ) - sign = Text("πŸ”–", style="bold yellow") - console.print(sign, text) add_stake = exec_command( command="stake", @@ -56,14 +55,9 @@ def add_stake(config_data): clean_stdout = remove_ansi_escape_sequences(add_stake.stdout) if "βœ… Finalized" in clean_stdout: - text = Text( - f"Stake added successfully by Validator ({owner_wallet})\n", - style="bold light_goldenrod2", - ) - sign = Text("πŸ“ˆ ", style="bold yellow") - console.print(sign, text) + print_info("Stake added by Validator", emoji="πŸ“ˆ ") - console.print("[dark_green]\nViewing Metagraph for Subnet 1") + print_info("Viewing Metagraph for Subnet 1", emoji="\nπŸ”Ž ") subnets_list = exec_command( command="subnets", sub_command="metagraph", @@ -77,15 +71,13 @@ def add_stake(config_data): print(subnets_list.stdout, end="") else: - console.print("\n[red] Failed to add stake. Command output:\n") + print_error("\nFailed to add stake. Command output:\n") print(add_stake.stdout, end="") - text = Text( - f"Validator is registering to root ({owner_wallet})\n", - style="bold light_goldenrod2", + print_info( + f"Validator is registering to root network (netuid 0) ({owner_wallet})\n", + emoji="\n🫚 ", ) - sign = Text("\nπŸ€ ", style="bold yellow") - console.print(sign, text) register_root = exec_command( command="root", @@ -104,21 +96,14 @@ def add_stake(config_data): ) clean_stdout = remove_ansi_escape_sequences(register_root.stdout) if "βœ… Registered" in clean_stdout: - text = Text( - "Successfully registered to root (Netuid 0)\n", - style="bold light_goldenrod2", - ) - sign = Text("🌟 ", style="bold yellow") - console.print(sign, text) + print_info("Successfully registered to the root network\n", emoji="βœ… ") elif "βœ… Already registered on root network" in clean_stdout: - console.print( - "[bold light_goldenrod2]βœ… Validator is already registered to Root network" - ) + print_info("Validator is already registered to Root network\n", emoji="βœ… ") else: - console.print("\n[red] Failed to register to root. Command output:\n") + print_error("\nFailed to register to root. Command output:\n") print(register_root.stdout, end="") - console.print("[dark_green]\nViewing Root list") + print_info("Viewing Root list\n", emoji="πŸ”Ž ") subnets_list = exec_command( command="root", sub_command="list", @@ -130,8 +115,8 @@ def add_stake(config_data): print(subnets_list.stdout, end="") else: - console.print( - "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" + print_error( + "Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" ) return @@ -144,16 +129,15 @@ def add_weights(config_data): path=config_data["wallets_path"], hotkey=owner_data.get("hotkey"), ) - text = Text( - "Validator is now setting weights on root network\n", - style="bold light_goldenrod2", + print_info( + "Validator is now setting weights of subnet 1 on the root network\n", + emoji="πŸ‹οΈ ", ) - sign = Text("\nπŸ‹οΈ ", style="bold yellow") - console.print(sign, text) max_retries = 5 attempt = 0 wait_time = 30 + retry_patterns = ["ancient birth block", "Transaction has a bad signature"] while attempt < max_retries: try: @@ -187,11 +171,11 @@ def add_weights(config_data): console.print(sign, text) break - elif "Transaction has a bad signature" in clean_stdout: + elif any(pattern in clean_stdout for pattern in retry_patterns): attempt += 1 if attempt < max_retries: console.print( - f"[red]Attempt {attempt}/{max_retries}: Transaction has a bad signature. Retrying in {wait_time} seconds..." + f"[red]Attempt {attempt}/{max_retries}: Failed to set weights. \nError: {clean_stdout} Retrying in {wait_time} seconds..." ) with Progress( @@ -207,7 +191,7 @@ def add_weights(config_data): time.sleep(1) else: console.print( - f"[red]Attempt {attempt}/{max_retries}: Transaction has a bad signature. Maximum retries reached." + f"[red]Attempt {attempt}/{max_retries}: Maximum retries reached." ) console.print( "\n[red]Failed to set weights after multiple attempts. Please try again later\n" @@ -339,7 +323,7 @@ def setup_subnet(config_data): else: create_subnet(owner_wallet, config_data) - console.print("[dark_green]\nListing all subnets") + print_info("\nListing all subnets\n", emoji="πŸ“‹ ") subnets_list = exec_command( command="subnets", sub_command="list", @@ -352,9 +336,7 @@ def setup_subnet(config_data): def create_subnet_owner_wallet(config_data): - text = Text("Creating subnet owner wallet.\n", style="bold light_goldenrod2") - sign = Text("\nℹ️ ", style="bold yellow") - console.print(sign, text) + print_info("Creating subnet owner wallet.\n", emoji="πŸ—‚οΈ ") owner_wallet_name = typer.prompt( "Enter subnet owner wallet name", default="owner", show_default=True @@ -388,9 +370,7 @@ def create_subnet_owner_wallet(config_data): def create_subnet(owner_wallet, config_data): - text = Text("Creating a subnet with Netuid 1.\n", style="bold light_goldenrod2") - sign = Text("\nℹ️ ", style="bold yellow") - console.print(sign, text) + print_info("Creating a subnet with Netuid 1\n", emoji="\n🌎 ") create_subnet = exec_command( command="subnets", @@ -409,14 +389,9 @@ def create_subnet(owner_wallet, config_data): ) clean_stdout = remove_ansi_escape_sequences(create_subnet.stdout) if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: - console.print("[dark_green] Subnet created successfully with netuid 1") + print_info("Subnet created successfully with netuid 1\n", emoji="πŸ₯‡ ") - text = Text( - f"Registering Owner ({owner_wallet}) to Netuid 1\n", - style="bold light_goldenrod2", - ) - sign = Text("\nℹ️ ", style="bold yellow") - console.print(sign, text) + print_info(f"Registering Owner ({owner_wallet}) to Netuid 1\n", emoji="πŸ“ ") register_subnet = exec_command( command="subnets", @@ -437,7 +412,7 @@ def create_subnet(owner_wallet, config_data): ) clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) if "βœ… Registered" in clean_stdout: - console.print("[green] Registered the owner to subnet 1") + print_info("Registered the owner's wallet to subnet 1", emoji="βœ… ") def steps(): diff --git a/btqs/config.py b/btqs/config.py index 64823cd0..9c649906 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -1,21 +1,38 @@ import os CONFIG_FILE_PATH = os.path.expanduser("~/.bittensor/btqs/btqs_config.yml") +DEFAULT_WORKSPACE_DIRECTORY = os.path.expanduser("~/Desktop/bittensor_quick_start") -BTQS_DIRECTORY = os.path.expanduser("~/.bittensor/btqs") -DEFAULT_WORKSPACE_DIRECTORY = os.path.expanduser("~/Desktop/Bittensor_quick_start") +SUBNET_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" +SUBNET_REPO_BRANCH = "ench/abe/commented-info" -SUBNET_TEMPLATE_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" -SUBNET_TEMPLATE_BRANCH = "ench/abe/commented-info" +# You can add commands with args. Eg: "./neurons/miner.py --model="openai" --safe-mode" +DEFAULT_MINER_COMMAND = "./neurons/miner.py" +DEFAULT_VALIDATOR_COMMAND = "./neurons/validator.py" SUBTENSOR_REPO_URL = "https://github.com/opentensor/subtensor.git" -DEFAULT_SUBTENSOR_BRANCH = "abe/temp/logging-dirs-for-nodes" +SUBTENSOR_BRANCH = "abe/temp/logging-dirs-for-nodes" +RUST_INSTALLATION_VERSION = "nightly-2024-03-05" +RUST_CHECK_VERSION = "rustc 1.78.0-nightly" +RUST_TARGETS = [ + ["rustup", "target", "add", "wasm32-unknown-unknown", "--toolchain", "stable"], + ["rustup", "component", "add", "rust-src", "--toolchain", "stable"], +] +SUBTENSOR_MACOS_DEPS = ["protobuf"] +SUBTENSOR_LINUX_DEPS = [ + "clang", + "curl", + "libssl-dev", + "llvm", + "libudev-dev", + "protobuf-compiler", +] + WALLET_URIS = ["//Bob", "//Charlie"] VALIDATOR_URI = "//Alice" MINER_PORTS = [8101, 8102, 8103] VALIDATOR_PORT = 8100 +LOCALNET_ENDPOINT = "ws://127.0.0.1:9945" EPILOG = "Made with [bold red]:heart:[/bold red] by The OpenΟ„ensor FoundaΟ„ion" - -LOCALNET_ENDPOINT = "ws://127.0.0.1:9945" diff --git a/btqs/rustup.sh b/btqs/rustup.sh new file mode 100644 index 00000000..c49b6ab4 --- /dev/null +++ b/btqs/rustup.sh @@ -0,0 +1,811 @@ +#!/bin/sh +# shellcheck shell=dash +# shellcheck disable=SC2039 # local is non-POSIX + +# This is just a little script that can be downloaded from the internet to +# install rustup. It just does platform detection, downloads the installer +# and runs it. + +# It runs on Unix shells like {a,ba,da,k,z}sh. It uses the common `local` +# extension. Note: Most shells limit `local` to 1 var per line, contra bash. + +# Some versions of ksh have no `local` keyword. Alias it to `typeset`, but +# beware this makes variables global with f()-style function syntax in ksh93. +# mksh has this alias by default. +has_local() { + # shellcheck disable=SC2034 # deliberately unused + local _has_local +} + +has_local 2>/dev/null || alias local=typeset + +is_zsh() { + [ -n "${ZSH_VERSION-}" ] +} + +set -u + +# If RUSTUP_UPDATE_ROOT is unset or empty, default it. +RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-https://static.rust-lang.org/rustup}" + +# NOTICE: If you change anything here, please make the same changes in setup_mode.rs +usage() { + cat < + Choose a default host triple + --default-toolchain + Choose a default toolchain to install. Use 'none' to not install any toolchains at all + --profile + [default: default] [possible values: minimal, default, complete] + -c, --component ... + Component name to also install + -t, --target ... + Target name to also install + --no-update-default-toolchain + Don't update any existing default toolchain after install + --no-modify-path + Don't configure the PATH environment variable + -h, --help + Print help + -V, --version + Print version +EOF +} + +main() { + downloader --check + need_cmd uname + need_cmd mktemp + need_cmd chmod + need_cmd mkdir + need_cmd rm + need_cmd rmdir + + get_architecture || return 1 + local _arch="$RETVAL" + assert_nz "$_arch" "arch" + + local _ext="" + case "$_arch" in + *windows*) + _ext=".exe" + ;; + esac + + local _url="${RUSTUP_UPDATE_ROOT}/dist/${_arch}/rustup-init${_ext}" + + local _dir + if ! _dir="$(ensure mktemp -d)"; then + # Because the previous command ran in a subshell, we must manually + # propagate exit status. + exit 1 + fi + local _file="${_dir}/rustup-init${_ext}" + + local _ansi_escapes_are_valid=false + if [ -t 2 ]; then + if [ "${TERM+set}" = 'set' ]; then + case "$TERM" in + xterm*|rxvt*|urxvt*|linux*|vt*) + _ansi_escapes_are_valid=true + ;; + esac + fi + fi + + # check if we have to use /dev/tty to prompt the user + local need_tty=yes + for arg in "$@"; do + case "$arg" in + --help) + usage + exit 0 + ;; + *) + OPTIND=1 + if [ "${arg%%--*}" = "" ]; then + # Long option (other than --help); + # don't attempt to interpret it. + continue + fi + while getopts :hy sub_arg "$arg"; do + case "$sub_arg" in + h) + usage + exit 0 + ;; + y) + # user wants to skip the prompt -- + # we don't need /dev/tty + need_tty=no + ;; + *) + ;; + esac + done + ;; + esac + done + + if $_ansi_escapes_are_valid; then + printf "\33[1minfo:\33[0m downloading installer\n" 1>&2 + else + printf '%s\n' 'info: downloading installer' 1>&2 + fi + + ensure mkdir -p "$_dir" + ensure downloader "$_url" "$_file" "$_arch" + ensure chmod u+x "$_file" + if [ ! -x "$_file" ]; then + printf '%s\n' "Cannot execute $_file (likely because of mounting /tmp as noexec)." 1>&2 + printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./rustup-init${_ext}." 1>&2 + exit 1 + fi + + if [ "$need_tty" = "yes" ] && [ ! -t 0 ]; then + # The installer is going to want to ask for confirmation by + # reading stdin. This script was piped into `sh` though and + # doesn't have stdin to pass to its children. Instead we're going + # to explicitly connect /dev/tty to the installer's stdin. + if [ ! -t 1 ]; then + err "Unable to run interactively. Run with -y to accept defaults, --help for additional options" + fi + + ignore "$_file" "$@" < /dev/tty + else + ignore "$_file" "$@" + fi + + local _retval=$? + + ignore rm "$_file" + ignore rmdir "$_dir" + + return "$_retval" +} + +check_proc() { + # Check for /proc by looking for the /proc/self/exe link + # This is only run on Linux + if ! test -L /proc/self/exe ; then + err "fatal: Unable to find /proc/self/exe. Is /proc mounted? Installation cannot proceed without /proc." + fi +} + +get_bitness() { + need_cmd head + # Architecture detection without dependencies beyond coreutils. + # ELF files start out "\x7fELF", and the following byte is + # 0x01 for 32-bit and + # 0x02 for 64-bit. + # The printf builtin on some shells like dash only supports octal + # escape sequences, so we use those. + local _current_exe_head + _current_exe_head=$(head -c 5 /proc/self/exe ) + if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then + echo 32 + elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then + echo 64 + else + err "unknown platform bitness" + fi +} + +is_host_amd64_elf() { + need_cmd head + need_cmd tail + # ELF e_machine detection without dependencies beyond coreutils. + # Two-byte field at offset 0x12 indicates the CPU, + # but we're interested in it being 0x3E to indicate amd64, or not that. + local _current_exe_machine + _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1) + [ "$_current_exe_machine" = "$(printf '\076')" ] +} + +get_endianness() { + local cputype=$1 + local suffix_eb=$2 + local suffix_el=$3 + + # detect endianness without od/hexdump, like get_bitness() does. + need_cmd head + need_cmd tail + + local _current_exe_endianness + _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" + if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then + echo "${cputype}${suffix_el}" + elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then + echo "${cputype}${suffix_eb}" + else + err "unknown platform endianness" + fi +} + +# Detect the Linux/LoongArch UAPI flavor, with all errors being non-fatal. +# Returns 0 or 234 in case of successful detection, 1 otherwise (/tmp being +# noexec, or other causes). +check_loongarch_uapi() { + need_cmd base64 + + local _tmp + if ! _tmp="$(ensure mktemp)"; then + return 1 + fi + + # Minimal Linux/LoongArch UAPI detection, exiting with 0 in case of + # upstream ("new world") UAPI, and 234 (-EINVAL truncated) in case of + # old-world (as deployed on several early commercial Linux distributions + # for LoongArch). + # + # See https://gist.github.com/xen0n/5ee04aaa6cecc5c7794b9a0c3b65fc7f for + # source to this helper binary. + ignore base64 -d > "$_tmp" <&2 + echo 'Your Linux kernel does not provide the ABI required by this Rust' >&2 + echo 'distribution. Please check with your OS provider for how to obtain a' >&2 + echo 'compatible Rust package for your system.' >&2 + echo >&2 + exit 1 + ;; + *) + echo "Warning: Cannot determine current system's ABI flavor, continuing anyway." >&2 + echo >&2 + echo 'Note that the official Rust distribution only works with the upstream' >&2 + echo 'kernel ABI. Installation will fail if your running kernel happens to be' >&2 + echo 'incompatible.' >&2 + ;; + esac +} + +get_architecture() { + local _ostype _cputype _bitness _arch _clibtype + _ostype="$(uname -s)" + _cputype="$(uname -m)" + _clibtype="gnu" + + if [ "$_ostype" = Linux ]; then + if [ "$(uname -o)" = Android ]; then + _ostype=Android + fi + if ldd --version 2>&1 | grep -q 'musl'; then + _clibtype="musl" + fi + fi + + if [ "$_ostype" = Darwin ]; then + # Darwin `uname -m` can lie due to Rosetta shenanigans. If you manage to + # invoke a native shell binary and then a native uname binary, you can + # get the real answer, but that's hard to ensure, so instead we use + # `sysctl` (which doesn't lie) to check for the actual architecture. + if [ "$_cputype" = i386 ]; then + # Handling i386 compatibility mode in older macOS versions (<10.15) + # running on x86_64-based Macs. + # Starting from 10.15, macOS explicitly bans all i386 binaries from running. + # See: + + # Avoid `sysctl: unknown oid` stderr output and/or non-zero exit code. + if sysctl hw.optional.x86_64 2> /dev/null || true | grep -q ': 1'; then + _cputype=x86_64 + fi + elif [ "$_cputype" = x86_64 ]; then + # Handling x86-64 compatibility mode (a.k.a. Rosetta 2) + # in newer macOS versions (>=11) running on arm64-based Macs. + # Rosetta 2 is built exclusively for x86-64 and cannot run i386 binaries. + + # Avoid `sysctl: unknown oid` stderr output and/or non-zero exit code. + if sysctl hw.optional.arm64 2> /dev/null || true | grep -q ': 1'; then + _cputype=arm64 + fi + fi + fi + + if [ "$_ostype" = SunOS ]; then + # Both Solaris and illumos presently announce as "SunOS" in "uname -s" + # so use "uname -o" to disambiguate. We use the full path to the + # system uname in case the user has coreutils uname first in PATH, + # which has historically sometimes printed the wrong value here. + if [ "$(/usr/bin/uname -o)" = illumos ]; then + _ostype=illumos + fi + + # illumos systems have multi-arch userlands, and "uname -m" reports the + # machine hardware name; e.g., "i86pc" on both 32- and 64-bit x86 + # systems. Check for the native (widest) instruction set on the + # running kernel: + if [ "$_cputype" = i86pc ]; then + _cputype="$(isainfo -n)" + fi + fi + + case "$_ostype" in + + Android) + _ostype=linux-android + ;; + + Linux) + check_proc + _ostype=unknown-linux-$_clibtype + _bitness=$(get_bitness) + ;; + + FreeBSD) + _ostype=unknown-freebsd + ;; + + NetBSD) + _ostype=unknown-netbsd + ;; + + DragonFly) + _ostype=unknown-dragonfly + ;; + + Darwin) + _ostype=apple-darwin + ;; + + illumos) + _ostype=unknown-illumos + ;; + + MINGW* | MSYS* | CYGWIN* | Windows_NT) + _ostype=pc-windows-gnu + ;; + + *) + err "unrecognized OS type: $_ostype" + ;; + + esac + + case "$_cputype" in + + i386 | i486 | i686 | i786 | x86) + _cputype=i686 + ;; + + xscale | arm) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + fi + ;; + + armv6l) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + armv7l | armv8l) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + aarch64 | arm64) + _cputype=aarch64 + ;; + + x86_64 | x86-64 | x64 | amd64) + _cputype=x86_64 + ;; + + mips) + _cputype=$(get_endianness mips '' el) + ;; + + mips64) + if [ "$_bitness" -eq 64 ]; then + # only n64 ABI is supported for now + _ostype="${_ostype}abi64" + _cputype=$(get_endianness mips64 '' el) + fi + ;; + + ppc) + _cputype=powerpc + ;; + + ppc64) + _cputype=powerpc64 + ;; + + ppc64le) + _cputype=powerpc64le + ;; + + s390x) + _cputype=s390x + ;; + riscv64) + _cputype=riscv64gc + ;; + loongarch64) + _cputype=loongarch64 + ensure_loongarch_uapi + ;; + *) + err "unknown CPU type: $_cputype" + + esac + + # Detect 64-bit linux with 32-bit userland + if [ "${_ostype}" = unknown-linux-gnu ] && [ "${_bitness}" -eq 32 ]; then + case $_cputype in + x86_64) + if [ -n "${RUSTUP_CPUTYPE:-}" ]; then + _cputype="$RUSTUP_CPUTYPE" + else { + # 32-bit executable for amd64 = x32 + if is_host_amd64_elf; then { + echo "This host is running an x32 userland; as it stands, x32 support is poor," 1>&2 + echo "and there isn't a native toolchain -- you will have to install" 1>&2 + echo "multiarch compatibility with i686 and/or amd64, then select one" 1>&2 + echo "by re-running this script with the RUSTUP_CPUTYPE environment variable" 1>&2 + echo "set to i686 or x86_64, respectively." 1>&2 + echo 1>&2 + echo "You will be able to add an x32 target after installation by running" 1>&2 + echo " rustup target add x86_64-unknown-linux-gnux32" 1>&2 + exit 1 + }; else + _cputype=i686 + fi + }; fi + ;; + mips64) + _cputype=$(get_endianness mips '' el) + ;; + powerpc64) + _cputype=powerpc + ;; + aarch64) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + riscv64gc) + err "riscv64 with 32-bit userland unsupported" + ;; + esac + fi + + # Detect armv7 but without the CPU features Rust needs in that build, + # and fall back to arm. + # See https://github.com/rust-lang/rustup.rs/issues/587. + if [ "$_ostype" = "unknown-linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then + if ensure grep '^Features' /proc/cpuinfo | grep -E -q -v 'neon|simd'; then + # At least one processor does not have NEON (which is asimd on armv8+). + _cputype=arm + fi + fi + + _arch="${_cputype}-${_ostype}" + + RETVAL="$_arch" +} + +say() { + printf 'rustup: %s\n' "$1" +} + +err() { + say "$1" >&2 + exit 1 +} + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (command not found)" + fi +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 +} + +assert_nz() { + if [ -z "$1" ]; then err "assert_nz $2"; fi +} + +# Run a command that should never fail. If the command fails execution +# will immediately terminate with an error showing the failing +# command. +ensure() { + if ! "$@"; then err "command failed: $*"; fi +} + +# This is just for indicating that commands' results are being +# intentionally ignored. Usually, because it's being executed +# as part of error handling. +ignore() { + "$@" +} + +# This wraps curl or wget. Try curl first, if not installed, +# use wget instead. +downloader() { + # zsh does not split words by default, Required for curl retry arguments below. + is_zsh && setopt local_options shwordsplit + + local _dld + local _ciphersuites + local _err + local _status + local _retry + if check_cmd curl; then + _dld=curl + elif check_cmd wget; then + _dld=wget + else + _dld='curl or wget' # to be used in error message of need_cmd + fi + + if [ "$1" = --check ]; then + need_cmd "$_dld" + elif [ "$_dld" = curl ]; then + check_curl_for_retry_support + _retry="$RETVAL" + get_ciphersuites_for_curl + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" curl --proto --tlsv1.2; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + fi + fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q 404$; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status + elif [ "$_dld" = wget ]; then + if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then + echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + get_ciphersuites_for_wget + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" wget --https-only --secure-protocol; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) + _status=$? + fi + fi + fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q ' 404 Not Found$'; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status + else + err "Unknown downloader" # should not reach here + fi +} + +check_help_for() { + local _arch + local _cmd + local _arg + _arch="$1" + shift + _cmd="$1" + shift + + local _category + if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then + _category="all" + else + _category="" + fi + + case "$_arch" in + + *darwin*) + if check_cmd sw_vers; then + case $(sw_vers -productVersion) in + 10.*) + # If we're running on macOS, older than 10.13, then we always + # fail to find these options to force fallback + if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then + # Older than 10.13 + echo "Warning: Detected macOS platform older than 10.13" + return 1 + fi + ;; + 11.*) + # We assume Big Sur will be OK for now + ;; + *) + # Unknown product version, warn and continue + echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" + echo "Warning TLS capabilities detection may fail" + ;; + esac + fi + ;; + + esac + + for _arg in "$@"; do + if ! "$_cmd" --help "$_category" | grep -q -- "$_arg"; then + return 1 + fi + done + + true # not strictly needed +} + +# Check if curl supports the --retry flag, then pass it to the curl invocation. +check_curl_for_retry_support() { + local _retry_supported="" + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--retry"; then + _retry_supported="--retry 3" + if check_help_for "notspecified" "curl" "--continue-at"; then + # "-C -" tells curl to automatically find where to resume the download when retrying. + _retry_supported="--retry 3 -C -" + fi + fi + + RETVAL="$_retry_supported" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these curl backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_curl() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _openssl_syntax="no" + local _gnutls_syntax="no" + local _backend_supported="yes" + if curl -V | grep -q ' OpenSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' LibreSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' BoringSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' GnuTLS/'; then + _gnutls_syntax="yes" + else + _backend_supported="no" + fi + + local _args_supported="no" + if [ "$_backend_supported" = "yes" ]; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then + _args_supported="yes" + fi + fi + + local _cs="" + if [ "$_args_supported" = "yes" ]; then + if [ "$_openssl_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "openssl") + elif [ "$_gnutls_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these wget backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_wget() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _cs="" + if wget -V | grep -q '\-DHAVE_LIBSSL'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "openssl") + fi + elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 +# excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad +# DH params often found on servers (see RFC 7919). Sequence matches or is +# similar to Firefox 68 ESR with weak cipher suites disabled via about:config. +# $1 must be openssl or gnutls. +get_strong_ciphersuites_for() { + if [ "$1" = "openssl" ]; then + # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. + echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" + elif [ "$1" = "gnutls" ]; then + # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. + # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. + echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" + fi +} + +main "$@" || exit 1 diff --git a/btqs/utils.py b/btqs/utils.py index 794a3415..e2263c90 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -4,6 +4,7 @@ import subprocess import time import platform +import shlex from datetime import datetime from typing import Any, Dict, Optional, Tuple @@ -14,16 +15,71 @@ from bittensor_wallet import Wallet from rich.console import Console from rich.table import Table +from rich.text import Text +from rich.markdown import Markdown +from rich.panel import Panel +from rich.align import Align from typer.testing import CliRunner from .config import ( CONFIG_FILE_PATH, VALIDATOR_PORT, - LOCALNET_ENDPOINT + LOCALNET_ENDPOINT, + DEFAULT_MINER_COMMAND, + DEFAULT_VALIDATOR_COMMAND, + RUST_CHECK_VERSION, + RUST_INSTALLATION_VERSION, + RUST_TARGETS, + SUBTENSOR_MACOS_DEPS, + SUBTENSOR_LINUX_DEPS, ) console = Console() + +def print_info(message: str, emoji: str = "", style: str = "yellow"): + styled_text = Text(f"{emoji} {message}", style=style) + console.print(styled_text) + + +def print_success(message: str, emoji: str = "βœ…"): + styled_text = Text(f"{emoji} {message}", style="bold green") + console.print(styled_text) + + +def print_warning(message: str, emoji: str = "⚠️"): + styled_text = Text(f"{emoji} {message}", style="bold yellow") + console.print(styled_text) + + +def print_error(message: str, emoji: str = "❌"): + styled_text = Text(f"{emoji} {message}", style="bold red") + console.print(styled_text) + + +def print_step(title: str, description: str, step_number: int): + panel = Panel.fit( + Align.left(f"[bold]{step_number}. {title}[/bold]\n{description}"), + title=f"Step {step_number}", + border_style="bright_black", + padding=(1, 2), + ) + console.print(panel) + + +def print_info_box(message: str, title: str = "Info", style: str = ""): + markdown_message = Markdown(message, style=style) + panel = Panel( + Align.left(markdown_message), + title=Text(title, style="bold yellow"), + border_style="bold yellow", + expand=False, + padding=(1, 1), + width=console.width - 20, + ) + console.print(panel) + + def load_config( error_message: Optional[str] = None, exit_if_missing: bool = True ) -> dict[str, Any]: @@ -43,9 +99,9 @@ def load_config( if not os.path.exists(CONFIG_FILE_PATH): if exit_if_missing: if error_message: - console.print(f"[red]{error_message}") + print_error(f"{error_message}") else: - console.print("[red]Configuration file not found.") + print_error("Configuration file not found.") raise typer.Exit() else: return {} @@ -53,6 +109,7 @@ def load_config( config_data = yaml.safe_load(config_file) or {} return config_data + def is_package_installed(package_name: str) -> bool: """ Checks if a system package is installed. @@ -64,17 +121,27 @@ def is_package_installed(package_name: str) -> bool: bool: True if installed, False otherwise. """ try: - if platform.system() == 'Linux': - result = subprocess.run(['dpkg', '-l', package_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if platform.system() == "Linux": + result = subprocess.run( + ["dpkg", "-l", package_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) return package_name in result.stdout - elif platform.system() == 'Darwin': # macOS check - result = subprocess.run(['brew', 'list', package_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + elif platform.system() == "Darwin": + result = subprocess.run( + ["brew", "list", package_name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) return package_name in result.stdout else: - console.print(f"[red]Unsupported operating system: {platform.system()}") + print_error(f"Unsupported operating system: {platform.system()}") return False except Exception as e: - console.print(f"[red]Error checking package {package_name}: {e}") + print_error(f"Error checking package {package_name}: {e}") return False @@ -89,13 +156,20 @@ def is_rust_installed(required_version: str) -> bool: bool: True if Rust is installed and matches the required version, False otherwise. """ try: - result = subprocess.run(['rustc', '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run( + ["rustc", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) installed_version = result.stdout.strip().split()[1] - print(installed_version, required_version, result.stdout.strip().split()[1]) + print_info( + installed_version, required_version, result.stdout.strip().split()[1] + ) return installed_version == required_version except Exception: return False - + def create_virtualenv(venv_path: str) -> str: """ @@ -108,87 +182,102 @@ def create_virtualenv(venv_path: str) -> str: str: The path to the Python executable within the virtual environment. """ if not os.path.exists(venv_path): - console.print(f"[green]Creating virtual environment at {venv_path}...") - subprocess.run([sys.executable, '-m', 'venv', venv_path], check=True) - console.print("[green]Virtual environment created.") - # Print activation snippet + print_info(f"Creating virtual environment at {venv_path}...") + subprocess.run([sys.executable, "-m", "venv", venv_path], check=True) + print_success("Virtual environment created.") + activate_command = f"source {os.path.join(venv_path, 'bin', 'activate')}" console.print( f"[yellow]To activate the virtual environment manually, run:\n[bold cyan]{activate_command}\n" ) else: - console.print(f"[green]Using existing virtual environment at {venv_path}.") + print_info(f"Using existing virtual environment at {venv_path}.", emoji="πŸ–₯️ ") # Get the path to the Python executable in the virtual environment - venv_python = os.path.join(venv_path, 'bin', 'python') + venv_python = os.path.join(venv_path, "bin", "python") return venv_python + def install_subtensor_dependencies() -> None: """ Installs subtensor dependencies, including system-level dependencies and Rust. """ - console.print("[green]Installing subtensor system dependencies...") + print_info("Installing subtensor system dependencies...", emoji="βš™οΈ ") # Install required system dependencies - system_dependencies = [ - 'clang', - 'curl', - 'libssl-dev', - 'llvm', - 'libudev-dev', - 'protobuf-compiler', - ] missing_packages = [] - - if platform.system() == 'Linux': - for package in system_dependencies: + if platform.system() == "Linux": + for package in SUBTENSOR_LINUX_DEPS: if not is_package_installed(package): missing_packages.append(package) if missing_packages: - console.print(f"[yellow]Installing missing system packages: {', '.join(missing_packages)}") - subprocess.run(['sudo', 'apt-get', 'update'], check=True) - subprocess.run(['sudo', 'apt-get', 'install', '-y'] + missing_packages, check=True) + console.print( + f"[yellow]Installing missing system packages: {', '.join(missing_packages)}" + ) + subprocess.run(["sudo", "apt-get", "update"], check=True) + subprocess.run( + ["sudo", "apt-get", "install", "-y"] + missing_packages, check=True + ) else: - console.print("[green]All required system packages are already installed.") - - elif platform.system() == 'Darwin': # macOS check - macos_dependencies = ['protobuf'] + print_info("All required system packages are already installed.") + + elif platform.system() == "Darwin": missing_packages = [] - for package in macos_dependencies: + for package in SUBTENSOR_MACOS_DEPS: if not is_package_installed(package): missing_packages.append(package) if missing_packages: - console.print(f"[yellow]Installing missing macOS system packages: {', '.join(missing_packages)}") - subprocess.run(['brew', 'update'], check=True) - subprocess.run(['brew', 'install'] + missing_packages, check=True) + console.print( + f"[yellow]Installing missing macOS system packages: {', '.join(missing_packages)}" + ) + subprocess.run(["brew", "update"], check=True) + subprocess.run(["brew", "install"] + missing_packages, check=True) else: - console.print("[green]All required macOS system packages are already installed.") + print_info( + "All required macOS system packages are already installed.", emoji="πŸ””" + ) else: - console.print("[red]Unsupported operating system for automatic system dependency installation.") + print_error( + "[Unsupported operating system for automatic system dependency installation." + ) return - # Install Rust globally - console.print("[green]Checking Rust installation...") - - installation_version = 'nightly-2024-03-05' - check_version = 'rustc 1.78.0-nightly' - - if not is_rust_installed(check_version): - console.print(f"[yellow]Installing Rust {installation_version} globally...") - subprocess.run(['curl', '--proto', '=https', '--tlsv1.2', '-sSf', 'https://sh.rustup.rs', '-o', 'rustup.sh'], check=True) - subprocess.run(['sh', 'rustup.sh', '-y', '--default-toolchain', installation_version], check=True) + # Install Rust + print_info("Checking Rust installation...", emoji="πŸ” ") + + if not is_rust_installed(RUST_CHECK_VERSION): + print_info(f"Installing Rust {RUST_INSTALLATION_VERSION}...", emoji="πŸ§ͺ ") + subprocess.run( + [ + "curl", + "--proto", + "=https", + "--tlsv1.2", + "-sSf", + "https://sh.rustup.rs", + "-o", + "rustup.sh", + ], + check=True, + ) + subprocess.run( + ["sh", "rustup.sh", "-y", "--default-toolchain", RUST_INSTALLATION_VERSION], + check=True, + ) else: - console.print(f"[green]Required Rust version {check_version} is already installed.") + console.print( + f"[green]Required Rust version {RUST_CHECK_VERSION} is already installed." + ) # Add necessary Rust targets - console.print("[green]Configuring Rust toolchain...") - subprocess.run(['rustup', 'target', 'add', 'wasm32-unknown-unknown', '--toolchain', 'stable'], check=True) - subprocess.run(['rustup', 'component', 'add', 'rust-src', '--toolchain', 'stable'], check=True) + print_info("Configuring Rust toolchain...", emoji="πŸ› οΈ ") + for target in RUST_TARGETS: + subprocess.run(target, check=True) - console.print("[green]Subtensor dependencies installed.") + print_info("Subtensor dependencies installed.", emoji="🧰 ") def install_neuron_dependencies(venv_python: str, cwd: str) -> None: @@ -199,10 +288,15 @@ def install_neuron_dependencies(venv_python: str, cwd: str) -> None: venv_python (str): Path to the Python executable in the virtual environment. cwd (str): Current working directory where the setup should run. """ - console.print("[green]Installing neuron dependencies...") - subprocess.run([venv_python, '-m', 'pip', 'install', '--upgrade', 'pip'], cwd=cwd, check=True) - subprocess.run([venv_python, '-m', 'pip', 'install', '-e', '.'], cwd=cwd, check=True) - console.print("[green]Neuron dependencies installed.") + print_info("Installing neuron dependencies...", emoji="βš™οΈ ") + subprocess.run( + [venv_python, "-m", "pip", "install", "--upgrade", "pip"], cwd=cwd, check=True + ) + subprocess.run( + [venv_python, "-m", "pip", "install", "-e", "."], cwd=cwd, check=True + ) + print_info("Neuron dependencies installed.", emoji="πŸ–₯️ ") + def remove_ansi_escape_sequences(text: str) -> str: """ @@ -230,6 +324,7 @@ def remove_ansi_escape_sequences(text: str) -> str: ) return ansi_escape.sub("", text) + def exec_command( command: str, sub_command: str, @@ -272,6 +367,7 @@ def exec_command( ) return result + def is_chain_running(config_file_path: str = CONFIG_FILE_PATH) -> bool: """ Checks if the local chain is running by verifying the PID in the config file. @@ -295,6 +391,7 @@ def is_chain_running(config_file_path: str = CONFIG_FILE_PATH) -> bool: except psutil.NoSuchProcess: return False + def subnet_owner_exists(config_file_path: str) -> Tuple[bool, dict]: """ Checks if a subnet owner exists in the config file. @@ -319,6 +416,7 @@ def subnet_owner_exists(config_file_path: str) -> Tuple[bool, dict]: return False, {} + def subnet_exists(ss58_address: str, netuid: int) -> bool: """ Checks if a subnet exists by verifying the subnet list output. @@ -337,6 +435,7 @@ def subnet_exists(ss58_address: str, netuid: int) -> bool: ) return exists + def verify_subnet_entry(output_text: str, netuid: int, ss58_address: str) -> bool: """ Verifies the presence of a specific subnet entry in the subnets list output. @@ -383,6 +482,7 @@ def verify_subnet_entry(output_text: str, netuid: int, ss58_address: str) -> boo return False + def get_btcli_version() -> str: """ Gets the version of btcli. @@ -393,6 +493,7 @@ def get_btcli_version() -> str: except Exception: return "Not installed or not found" + def get_bittensor_wallet_version() -> str: """ Gets the version of bittensor-wallet. @@ -407,6 +508,7 @@ def get_bittensor_wallet_version() -> str: except Exception: return "Not installed or not found" + def get_bittensor_version() -> str: """ Gets the version of bittensor-wallet. @@ -421,18 +523,21 @@ def get_bittensor_version() -> str: except Exception: return "Not installed or not found" + def get_python_version() -> str: """ Gets the Python version. """ return sys.version.split()[0] + def get_python_path() -> str: """ Gets the Python executable path. """ return sys.executable + def get_process_info(pid: int) -> Tuple[str, str, str, str, float, float]: """ Retrieves process information. @@ -467,6 +572,7 @@ def get_process_info(pid: int) -> Tuple[str, str, str, str, float, float]: pid = "N/A" return status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent + def get_process_entries( config_data: Dict[str, Any], ) -> Tuple[list[Dict[str, str]], list[float], list[float]]: @@ -500,7 +606,7 @@ def get_process_entries( "memory_usage": memory_usage, "uptime_str": uptime_str, "location": subtensor_path, - "venv_path": config_data.get("venv_subtensor") + "venv_path": config_data.get("venv_subtensor"), } ) @@ -526,7 +632,7 @@ def get_process_entries( "memory_usage": memory_usage, "uptime_str": uptime_str, "location": subtensor_path, - "venv_path": "~" + "venv_path": "~", } ) @@ -554,7 +660,7 @@ def get_process_entries( "memory_usage": memory_usage, "uptime_str": uptime_str, "location": config_data.get("subnet_path"), - "venv_path": wallet_info.get("venv") + "venv_path": wallet_info.get("venv"), } ) @@ -565,7 +671,6 @@ def get_process_entries( status, cpu_usage, memory_usage, uptime_str, cpu_percent, memory_percent = ( get_process_info(pid) ) - status_style = "green" if status == "Running" else "red" if status == "Running": @@ -582,17 +687,18 @@ def get_process_entries( "memory_usage": memory_usage, "uptime_str": uptime_str, "location": config_data.get("subnet_path"), - "venv_path": owner_data.get("venv") + "venv_path": owner_data.get("venv"), } ) return process_entries, cpu_usage_list, memory_usage_list + def display_process_status_table( process_entries: list[Dict[str, str]], cpu_usage_list: list[float], memory_usage_list: list[float], - config_data = None + config_data=None, ) -> None: """ Displays the process status table. @@ -693,7 +799,6 @@ def display_process_status_table( subnet_path = config_data.get("subnet_path", "") if subnet_path: console.print("[dark_orange]Subnet Path", subnet_path) - workspace_path = config_data.get("workspace_path", "") if workspace_path: console.print("[dark_orange]Workspace Path", workspace_path) @@ -702,6 +807,7 @@ def display_process_status_table( if neurons_venv: console.print("[dark_orange]Neurons virtual environment", neurons_venv) + def start_miner( wallet_name: str, wallet_info: Dict[str, Any], @@ -717,16 +823,11 @@ def start_miner( name=wallet_name, hotkey=wallet_info["hotkey"], ) - console.print(f"[green]Starting miner {wallet_name}...") - - env_variables = os.environ.copy() - env_variables["BT_AXON_PORT"] = str(wallet_info["port"]) - env_variables["PYTHONUNBUFFERED"] = "1" + print_info(f"Starting miner: {wallet}...\nPress any key to continue.", emoji="🏁 ") + input() - cmd = [ - venv_python, - "-u", - "./neurons/miner.py", + miner_command = get_miner_command(wallet_name, config_data) + base_args = [ "--wallet.name", wallet.name, "--wallet.hotkey", @@ -737,15 +838,21 @@ def start_miner( LOCALNET_ENDPOINT, "--logging.trace", ] + cmd = [venv_python, "-u"] + miner_command + base_args + + env_variables = os.environ.copy() + env_variables["BT_AXON_PORT"] = str(wallet_info["port"]) + env_variables["PYTHONUNBUFFERED"] = "1" - # Create log file paths + # Setup log files for miner logs_dir = os.path.join(config_data["workspace_path"], "logs") os.makedirs(logs_dir, exist_ok=True) - log_file_path = os.path.join(logs_dir, f"miner_{wallet_name}.log") + log_file_path = os.path.join( + logs_dir, f"miner_{wallet_name}_{config_data['pid']}.log" + ) with open(log_file_path, "a") as log_file: try: - # Start the subprocess, redirecting stdout and stderr to the log file process = subprocess.Popen( cmd, cwd=subnet_template_path, @@ -759,14 +866,13 @@ def start_miner( # Update config_data config_data["Miners"][wallet_name] = wallet_info - - console.print(f"[green]Miner {wallet_name} started. Press Ctrl+C to proceed.") attach_to_process_logs(log_file_path, f"Miner {wallet_name}", process.pid) return True except Exception as e: console.print(f"[red]Error starting miner {wallet_name}: {e}") return False - + + def start_validator( owner_info: Dict[str, Any], subnet_template_path: str, @@ -781,16 +887,11 @@ def start_validator( name=owner_info["wallet_name"], hotkey=owner_info["hotkey"], ) - console.print("[green]Starting validator...") - - env_variables = os.environ.copy() - env_variables["PYTHONUNBUFFERED"] = "1" - env_variables["BT_AXON_PORT"] = str(VALIDATOR_PORT) + print_info("Starting validator...\nPress any key to continue.", emoji="🏁 ") + input() - cmd = [ - venv_python, - "-u", - "./neurons/validator.py", + validator_command = get_validator_command(config_data) + base_args = [ "--wallet.name", wallet.name, "--wallet.hotkey", @@ -799,19 +900,21 @@ def start_validator( config_data["wallets_path"], "--subtensor.chain_endpoint", LOCALNET_ENDPOINT, - "--netuid", - "1", "--logging.trace", ] + cmd = [venv_python, "-u"] + validator_command + base_args + + env_variables = os.environ.copy() + env_variables["PYTHONUNBUFFERED"] = "1" + env_variables["BT_AXON_PORT"] = str(VALIDATOR_PORT) - # Create log file paths + # Setup log files for validator logs_dir = os.path.join(config_data["workspace_path"], "logs") os.makedirs(logs_dir, exist_ok=True) log_file_path = os.path.join(logs_dir, "validator.log") with open(log_file_path, "a") as log_file: try: - # Start the subprocess, redirecting stdout and stderr to the log file process = subprocess.Popen( cmd, cwd=subnet_template_path, @@ -826,12 +929,13 @@ def start_validator( # Update config_data config_data["Owner"] = owner_info - console.print("[green]Validator started. Press Ctrl+C to proceed.") attach_to_process_logs(log_file_path, "Validator", process.pid) return True except Exception as e: console.print(f"[red]Error starting validator: {e}") return False + + def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = None): """ Attaches to the log file of a process and prints logs until user presses Ctrl+C or the process terminates. @@ -840,8 +944,8 @@ def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = Non with open(log_file_path, "r") as log_file: # Move to the end of the file log_file.seek(0, os.SEEK_END) - console.print( - f"[green]Attached to {process_name}. Press Ctrl+C to move on." + print_info( + f"Attached to {process_name}. Press Ctrl+C to move on.", emoji="πŸ“Ž " ) while True: line = log_file.readline() @@ -854,40 +958,43 @@ def attach_to_process_logs(log_file_path: str, process_name: str, pid: int = Non continue print(line, end="") except KeyboardInterrupt: - console.print(f"\n[green]Detached from {process_name}.") + print_info(f"Detached from {process_name}.", emoji="\nπŸ”Œ ") except Exception as e: console.print(f"[red]Error attaching to {process_name}: {e}") -# def activate_venv(workspace_path): -# venv_path = os.path.join(workspace_path, 'venv') -# if not os.path.exists(venv_path): -# console.print("[green]Creating virtual environment for subnet-template...") -# subprocess.run([sys.executable, '-m', 'venv', 'venv'], cwd=workspace_path) -# console.print("[green]Virtual environment created.") -# # Print activation snippet -# activate_command = ( -# f"source {os.path.join(venv_path, 'bin', 'activate')}" -# if os.name != 'nt' -# else f"{os.path.join(venv_path, 'Scripts', 'activate')}" -# ) -# console.print( -# f"[yellow]To activate the virtual environment manually, run:\n[bold cyan]{activate_command}\n" -# ) -# # Install dependencies -# venv_python = ( -# os.path.join(venv_path, 'bin', 'python') -# if os.name != 'nt' -# else os.path.join(venv_path, 'Scripts', 'python.exe') -# ) -# console.print("[green]Installing subnet-template dependencies...") -# subprocess.run([venv_python, '-m', 'pip', 'install', '--upgrade', 'pip'], cwd=workspace_path) -# subprocess.run([venv_python, '-m', 'pip', 'install', '-e', '.'], cwd=workspace_path) -# console.print("[green]Dependencies installed.") -# else: -# console.print("[green]Using existing virtual environment for subnet-template.") -# venv_python = ( -# os.path.join(venv_path, 'bin', 'python') -# if os.name != 'nt' -# else os.path.join(venv_path, 'Scripts', 'python.exe') -# ) \ No newline at end of file +def get_miner_command(wallet_name: str, config_data: dict) -> list[str]: + """ + Retrieves the command and arguments to start a miner. + + Args: + wallet_name (str): The name of the miner wallet. + config_data (dict): The loaded configuration data. + + Returns: + List[str]: The command split into a list suitable for subprocess. + """ + miner_info = config_data.get("Miners", {}).get(wallet_name, {}) + custom_command = miner_info.get("miner_command") + if custom_command: + return shlex.split(custom_command) + else: + return shlex.split(DEFAULT_MINER_COMMAND) + + +def get_validator_command(config_data: dict) -> list[str]: + """ + Retrieves the command and arguments to start the validator. + + Args: + config_data (dict): The loaded configuration data. + + Returns: + List[str]: The command split into a list suitable for subprocess. + """ + owner_info = config_data.get("Owner", {}) + custom_command = owner_info.get("validator_command") + if custom_command: + return shlex.split(custom_command) + else: + return shlex.split(DEFAULT_VALIDATOR_COMMAND) From b7286fe66bdb23fe80cc5a2b3409495b7d664354 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Thu, 17 Oct 2024 02:31:38 -0700 Subject: [PATCH 17/23] Removed rust file --- btqs/rustup.sh | 811 ------------------------------------------------- 1 file changed, 811 deletions(-) delete mode 100644 btqs/rustup.sh diff --git a/btqs/rustup.sh b/btqs/rustup.sh deleted file mode 100644 index c49b6ab4..00000000 --- a/btqs/rustup.sh +++ /dev/null @@ -1,811 +0,0 @@ -#!/bin/sh -# shellcheck shell=dash -# shellcheck disable=SC2039 # local is non-POSIX - -# This is just a little script that can be downloaded from the internet to -# install rustup. It just does platform detection, downloads the installer -# and runs it. - -# It runs on Unix shells like {a,ba,da,k,z}sh. It uses the common `local` -# extension. Note: Most shells limit `local` to 1 var per line, contra bash. - -# Some versions of ksh have no `local` keyword. Alias it to `typeset`, but -# beware this makes variables global with f()-style function syntax in ksh93. -# mksh has this alias by default. -has_local() { - # shellcheck disable=SC2034 # deliberately unused - local _has_local -} - -has_local 2>/dev/null || alias local=typeset - -is_zsh() { - [ -n "${ZSH_VERSION-}" ] -} - -set -u - -# If RUSTUP_UPDATE_ROOT is unset or empty, default it. -RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-https://static.rust-lang.org/rustup}" - -# NOTICE: If you change anything here, please make the same changes in setup_mode.rs -usage() { - cat < - Choose a default host triple - --default-toolchain - Choose a default toolchain to install. Use 'none' to not install any toolchains at all - --profile - [default: default] [possible values: minimal, default, complete] - -c, --component ... - Component name to also install - -t, --target ... - Target name to also install - --no-update-default-toolchain - Don't update any existing default toolchain after install - --no-modify-path - Don't configure the PATH environment variable - -h, --help - Print help - -V, --version - Print version -EOF -} - -main() { - downloader --check - need_cmd uname - need_cmd mktemp - need_cmd chmod - need_cmd mkdir - need_cmd rm - need_cmd rmdir - - get_architecture || return 1 - local _arch="$RETVAL" - assert_nz "$_arch" "arch" - - local _ext="" - case "$_arch" in - *windows*) - _ext=".exe" - ;; - esac - - local _url="${RUSTUP_UPDATE_ROOT}/dist/${_arch}/rustup-init${_ext}" - - local _dir - if ! _dir="$(ensure mktemp -d)"; then - # Because the previous command ran in a subshell, we must manually - # propagate exit status. - exit 1 - fi - local _file="${_dir}/rustup-init${_ext}" - - local _ansi_escapes_are_valid=false - if [ -t 2 ]; then - if [ "${TERM+set}" = 'set' ]; then - case "$TERM" in - xterm*|rxvt*|urxvt*|linux*|vt*) - _ansi_escapes_are_valid=true - ;; - esac - fi - fi - - # check if we have to use /dev/tty to prompt the user - local need_tty=yes - for arg in "$@"; do - case "$arg" in - --help) - usage - exit 0 - ;; - *) - OPTIND=1 - if [ "${arg%%--*}" = "" ]; then - # Long option (other than --help); - # don't attempt to interpret it. - continue - fi - while getopts :hy sub_arg "$arg"; do - case "$sub_arg" in - h) - usage - exit 0 - ;; - y) - # user wants to skip the prompt -- - # we don't need /dev/tty - need_tty=no - ;; - *) - ;; - esac - done - ;; - esac - done - - if $_ansi_escapes_are_valid; then - printf "\33[1minfo:\33[0m downloading installer\n" 1>&2 - else - printf '%s\n' 'info: downloading installer' 1>&2 - fi - - ensure mkdir -p "$_dir" - ensure downloader "$_url" "$_file" "$_arch" - ensure chmod u+x "$_file" - if [ ! -x "$_file" ]; then - printf '%s\n' "Cannot execute $_file (likely because of mounting /tmp as noexec)." 1>&2 - printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./rustup-init${_ext}." 1>&2 - exit 1 - fi - - if [ "$need_tty" = "yes" ] && [ ! -t 0 ]; then - # The installer is going to want to ask for confirmation by - # reading stdin. This script was piped into `sh` though and - # doesn't have stdin to pass to its children. Instead we're going - # to explicitly connect /dev/tty to the installer's stdin. - if [ ! -t 1 ]; then - err "Unable to run interactively. Run with -y to accept defaults, --help for additional options" - fi - - ignore "$_file" "$@" < /dev/tty - else - ignore "$_file" "$@" - fi - - local _retval=$? - - ignore rm "$_file" - ignore rmdir "$_dir" - - return "$_retval" -} - -check_proc() { - # Check for /proc by looking for the /proc/self/exe link - # This is only run on Linux - if ! test -L /proc/self/exe ; then - err "fatal: Unable to find /proc/self/exe. Is /proc mounted? Installation cannot proceed without /proc." - fi -} - -get_bitness() { - need_cmd head - # Architecture detection without dependencies beyond coreutils. - # ELF files start out "\x7fELF", and the following byte is - # 0x01 for 32-bit and - # 0x02 for 64-bit. - # The printf builtin on some shells like dash only supports octal - # escape sequences, so we use those. - local _current_exe_head - _current_exe_head=$(head -c 5 /proc/self/exe ) - if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then - echo 32 - elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then - echo 64 - else - err "unknown platform bitness" - fi -} - -is_host_amd64_elf() { - need_cmd head - need_cmd tail - # ELF e_machine detection without dependencies beyond coreutils. - # Two-byte field at offset 0x12 indicates the CPU, - # but we're interested in it being 0x3E to indicate amd64, or not that. - local _current_exe_machine - _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1) - [ "$_current_exe_machine" = "$(printf '\076')" ] -} - -get_endianness() { - local cputype=$1 - local suffix_eb=$2 - local suffix_el=$3 - - # detect endianness without od/hexdump, like get_bitness() does. - need_cmd head - need_cmd tail - - local _current_exe_endianness - _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" - if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then - echo "${cputype}${suffix_el}" - elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then - echo "${cputype}${suffix_eb}" - else - err "unknown platform endianness" - fi -} - -# Detect the Linux/LoongArch UAPI flavor, with all errors being non-fatal. -# Returns 0 or 234 in case of successful detection, 1 otherwise (/tmp being -# noexec, or other causes). -check_loongarch_uapi() { - need_cmd base64 - - local _tmp - if ! _tmp="$(ensure mktemp)"; then - return 1 - fi - - # Minimal Linux/LoongArch UAPI detection, exiting with 0 in case of - # upstream ("new world") UAPI, and 234 (-EINVAL truncated) in case of - # old-world (as deployed on several early commercial Linux distributions - # for LoongArch). - # - # See https://gist.github.com/xen0n/5ee04aaa6cecc5c7794b9a0c3b65fc7f for - # source to this helper binary. - ignore base64 -d > "$_tmp" <&2 - echo 'Your Linux kernel does not provide the ABI required by this Rust' >&2 - echo 'distribution. Please check with your OS provider for how to obtain a' >&2 - echo 'compatible Rust package for your system.' >&2 - echo >&2 - exit 1 - ;; - *) - echo "Warning: Cannot determine current system's ABI flavor, continuing anyway." >&2 - echo >&2 - echo 'Note that the official Rust distribution only works with the upstream' >&2 - echo 'kernel ABI. Installation will fail if your running kernel happens to be' >&2 - echo 'incompatible.' >&2 - ;; - esac -} - -get_architecture() { - local _ostype _cputype _bitness _arch _clibtype - _ostype="$(uname -s)" - _cputype="$(uname -m)" - _clibtype="gnu" - - if [ "$_ostype" = Linux ]; then - if [ "$(uname -o)" = Android ]; then - _ostype=Android - fi - if ldd --version 2>&1 | grep -q 'musl'; then - _clibtype="musl" - fi - fi - - if [ "$_ostype" = Darwin ]; then - # Darwin `uname -m` can lie due to Rosetta shenanigans. If you manage to - # invoke a native shell binary and then a native uname binary, you can - # get the real answer, but that's hard to ensure, so instead we use - # `sysctl` (which doesn't lie) to check for the actual architecture. - if [ "$_cputype" = i386 ]; then - # Handling i386 compatibility mode in older macOS versions (<10.15) - # running on x86_64-based Macs. - # Starting from 10.15, macOS explicitly bans all i386 binaries from running. - # See: - - # Avoid `sysctl: unknown oid` stderr output and/or non-zero exit code. - if sysctl hw.optional.x86_64 2> /dev/null || true | grep -q ': 1'; then - _cputype=x86_64 - fi - elif [ "$_cputype" = x86_64 ]; then - # Handling x86-64 compatibility mode (a.k.a. Rosetta 2) - # in newer macOS versions (>=11) running on arm64-based Macs. - # Rosetta 2 is built exclusively for x86-64 and cannot run i386 binaries. - - # Avoid `sysctl: unknown oid` stderr output and/or non-zero exit code. - if sysctl hw.optional.arm64 2> /dev/null || true | grep -q ': 1'; then - _cputype=arm64 - fi - fi - fi - - if [ "$_ostype" = SunOS ]; then - # Both Solaris and illumos presently announce as "SunOS" in "uname -s" - # so use "uname -o" to disambiguate. We use the full path to the - # system uname in case the user has coreutils uname first in PATH, - # which has historically sometimes printed the wrong value here. - if [ "$(/usr/bin/uname -o)" = illumos ]; then - _ostype=illumos - fi - - # illumos systems have multi-arch userlands, and "uname -m" reports the - # machine hardware name; e.g., "i86pc" on both 32- and 64-bit x86 - # systems. Check for the native (widest) instruction set on the - # running kernel: - if [ "$_cputype" = i86pc ]; then - _cputype="$(isainfo -n)" - fi - fi - - case "$_ostype" in - - Android) - _ostype=linux-android - ;; - - Linux) - check_proc - _ostype=unknown-linux-$_clibtype - _bitness=$(get_bitness) - ;; - - FreeBSD) - _ostype=unknown-freebsd - ;; - - NetBSD) - _ostype=unknown-netbsd - ;; - - DragonFly) - _ostype=unknown-dragonfly - ;; - - Darwin) - _ostype=apple-darwin - ;; - - illumos) - _ostype=unknown-illumos - ;; - - MINGW* | MSYS* | CYGWIN* | Windows_NT) - _ostype=pc-windows-gnu - ;; - - *) - err "unrecognized OS type: $_ostype" - ;; - - esac - - case "$_cputype" in - - i386 | i486 | i686 | i786 | x86) - _cputype=i686 - ;; - - xscale | arm) - _cputype=arm - if [ "$_ostype" = "linux-android" ]; then - _ostype=linux-androideabi - fi - ;; - - armv6l) - _cputype=arm - if [ "$_ostype" = "linux-android" ]; then - _ostype=linux-androideabi - else - _ostype="${_ostype}eabihf" - fi - ;; - - armv7l | armv8l) - _cputype=armv7 - if [ "$_ostype" = "linux-android" ]; then - _ostype=linux-androideabi - else - _ostype="${_ostype}eabihf" - fi - ;; - - aarch64 | arm64) - _cputype=aarch64 - ;; - - x86_64 | x86-64 | x64 | amd64) - _cputype=x86_64 - ;; - - mips) - _cputype=$(get_endianness mips '' el) - ;; - - mips64) - if [ "$_bitness" -eq 64 ]; then - # only n64 ABI is supported for now - _ostype="${_ostype}abi64" - _cputype=$(get_endianness mips64 '' el) - fi - ;; - - ppc) - _cputype=powerpc - ;; - - ppc64) - _cputype=powerpc64 - ;; - - ppc64le) - _cputype=powerpc64le - ;; - - s390x) - _cputype=s390x - ;; - riscv64) - _cputype=riscv64gc - ;; - loongarch64) - _cputype=loongarch64 - ensure_loongarch_uapi - ;; - *) - err "unknown CPU type: $_cputype" - - esac - - # Detect 64-bit linux with 32-bit userland - if [ "${_ostype}" = unknown-linux-gnu ] && [ "${_bitness}" -eq 32 ]; then - case $_cputype in - x86_64) - if [ -n "${RUSTUP_CPUTYPE:-}" ]; then - _cputype="$RUSTUP_CPUTYPE" - else { - # 32-bit executable for amd64 = x32 - if is_host_amd64_elf; then { - echo "This host is running an x32 userland; as it stands, x32 support is poor," 1>&2 - echo "and there isn't a native toolchain -- you will have to install" 1>&2 - echo "multiarch compatibility with i686 and/or amd64, then select one" 1>&2 - echo "by re-running this script with the RUSTUP_CPUTYPE environment variable" 1>&2 - echo "set to i686 or x86_64, respectively." 1>&2 - echo 1>&2 - echo "You will be able to add an x32 target after installation by running" 1>&2 - echo " rustup target add x86_64-unknown-linux-gnux32" 1>&2 - exit 1 - }; else - _cputype=i686 - fi - }; fi - ;; - mips64) - _cputype=$(get_endianness mips '' el) - ;; - powerpc64) - _cputype=powerpc - ;; - aarch64) - _cputype=armv7 - if [ "$_ostype" = "linux-android" ]; then - _ostype=linux-androideabi - else - _ostype="${_ostype}eabihf" - fi - ;; - riscv64gc) - err "riscv64 with 32-bit userland unsupported" - ;; - esac - fi - - # Detect armv7 but without the CPU features Rust needs in that build, - # and fall back to arm. - # See https://github.com/rust-lang/rustup.rs/issues/587. - if [ "$_ostype" = "unknown-linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then - if ensure grep '^Features' /proc/cpuinfo | grep -E -q -v 'neon|simd'; then - # At least one processor does not have NEON (which is asimd on armv8+). - _cputype=arm - fi - fi - - _arch="${_cputype}-${_ostype}" - - RETVAL="$_arch" -} - -say() { - printf 'rustup: %s\n' "$1" -} - -err() { - say "$1" >&2 - exit 1 -} - -need_cmd() { - if ! check_cmd "$1"; then - err "need '$1' (command not found)" - fi -} - -check_cmd() { - command -v "$1" > /dev/null 2>&1 -} - -assert_nz() { - if [ -z "$1" ]; then err "assert_nz $2"; fi -} - -# Run a command that should never fail. If the command fails execution -# will immediately terminate with an error showing the failing -# command. -ensure() { - if ! "$@"; then err "command failed: $*"; fi -} - -# This is just for indicating that commands' results are being -# intentionally ignored. Usually, because it's being executed -# as part of error handling. -ignore() { - "$@" -} - -# This wraps curl or wget. Try curl first, if not installed, -# use wget instead. -downloader() { - # zsh does not split words by default, Required for curl retry arguments below. - is_zsh && setopt local_options shwordsplit - - local _dld - local _ciphersuites - local _err - local _status - local _retry - if check_cmd curl; then - _dld=curl - elif check_cmd wget; then - _dld=wget - else - _dld='curl or wget' # to be used in error message of need_cmd - fi - - if [ "$1" = --check ]; then - need_cmd "$_dld" - elif [ "$_dld" = curl ]; then - check_curl_for_retry_support - _retry="$RETVAL" - get_ciphersuites_for_curl - _ciphersuites="$RETVAL" - if [ -n "$_ciphersuites" ]; then - _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) - _status=$? - else - echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" - if ! check_help_for "$3" curl --proto --tlsv1.2; then - echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" - _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) - _status=$? - else - _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) - _status=$? - fi - fi - if [ -n "$_err" ]; then - echo "$_err" >&2 - if echo "$_err" | grep -q 404$; then - err "installer for platform '$3' not found, this may be unsupported" - fi - fi - return $_status - elif [ "$_dld" = wget ]; then - if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then - echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" - _err=$(wget "$1" -O "$2" 2>&1) - _status=$? - else - get_ciphersuites_for_wget - _ciphersuites="$RETVAL" - if [ -n "$_ciphersuites" ]; then - _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) - _status=$? - else - echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" - if ! check_help_for "$3" wget --https-only --secure-protocol; then - echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" - _err=$(wget "$1" -O "$2" 2>&1) - _status=$? - else - _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) - _status=$? - fi - fi - fi - if [ -n "$_err" ]; then - echo "$_err" >&2 - if echo "$_err" | grep -q ' 404 Not Found$'; then - err "installer for platform '$3' not found, this may be unsupported" - fi - fi - return $_status - else - err "Unknown downloader" # should not reach here - fi -} - -check_help_for() { - local _arch - local _cmd - local _arg - _arch="$1" - shift - _cmd="$1" - shift - - local _category - if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then - _category="all" - else - _category="" - fi - - case "$_arch" in - - *darwin*) - if check_cmd sw_vers; then - case $(sw_vers -productVersion) in - 10.*) - # If we're running on macOS, older than 10.13, then we always - # fail to find these options to force fallback - if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then - # Older than 10.13 - echo "Warning: Detected macOS platform older than 10.13" - return 1 - fi - ;; - 11.*) - # We assume Big Sur will be OK for now - ;; - *) - # Unknown product version, warn and continue - echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" - echo "Warning TLS capabilities detection may fail" - ;; - esac - fi - ;; - - esac - - for _arg in "$@"; do - if ! "$_cmd" --help "$_category" | grep -q -- "$_arg"; then - return 1 - fi - done - - true # not strictly needed -} - -# Check if curl supports the --retry flag, then pass it to the curl invocation. -check_curl_for_retry_support() { - local _retry_supported="" - # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. - if check_help_for "notspecified" "curl" "--retry"; then - _retry_supported="--retry 3" - if check_help_for "notspecified" "curl" "--continue-at"; then - # "-C -" tells curl to automatically find where to resume the download when retrying. - _retry_supported="--retry 3 -C -" - fi - fi - - RETVAL="$_retry_supported" -} - -# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites -# if support by local tools is detected. Detection currently supports these curl backends: -# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. -get_ciphersuites_for_curl() { - if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then - # user specified custom cipher suites, assume they know what they're doing - RETVAL="$RUSTUP_TLS_CIPHERSUITES" - return - fi - - local _openssl_syntax="no" - local _gnutls_syntax="no" - local _backend_supported="yes" - if curl -V | grep -q ' OpenSSL/'; then - _openssl_syntax="yes" - elif curl -V | grep -iq ' LibreSSL/'; then - _openssl_syntax="yes" - elif curl -V | grep -iq ' BoringSSL/'; then - _openssl_syntax="yes" - elif curl -V | grep -iq ' GnuTLS/'; then - _gnutls_syntax="yes" - else - _backend_supported="no" - fi - - local _args_supported="no" - if [ "$_backend_supported" = "yes" ]; then - # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. - if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then - _args_supported="yes" - fi - fi - - local _cs="" - if [ "$_args_supported" = "yes" ]; then - if [ "$_openssl_syntax" = "yes" ]; then - _cs=$(get_strong_ciphersuites_for "openssl") - elif [ "$_gnutls_syntax" = "yes" ]; then - _cs=$(get_strong_ciphersuites_for "gnutls") - fi - fi - - RETVAL="$_cs" -} - -# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites -# if support by local tools is detected. Detection currently supports these wget backends: -# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. -get_ciphersuites_for_wget() { - if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then - # user specified custom cipher suites, assume they know what they're doing - RETVAL="$RUSTUP_TLS_CIPHERSUITES" - return - fi - - local _cs="" - if wget -V | grep -q '\-DHAVE_LIBSSL'; then - # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. - if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then - _cs=$(get_strong_ciphersuites_for "openssl") - fi - elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then - # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. - if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then - _cs=$(get_strong_ciphersuites_for "gnutls") - fi - fi - - RETVAL="$_cs" -} - -# Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 -# excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad -# DH params often found on servers (see RFC 7919). Sequence matches or is -# similar to Firefox 68 ESR with weak cipher suites disabled via about:config. -# $1 must be openssl or gnutls. -get_strong_ciphersuites_for() { - if [ "$1" = "openssl" ]; then - # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. - echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" - elif [ "$1" = "gnutls" ]; then - # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. - # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. - echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" - fi -} - -main "$@" || exit 1 From ca91cc72289a47cd2955fe1ffe8c02190b73ded2 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 18 Oct 2024 03:27:16 -0700 Subject: [PATCH 18/23] Final approach --- bittensor_cli/cli.py | 4 +- btqs/btqs_cli.py | 199 +++++++++++++++++++++--------- btqs/commands/chain.py | 256 +++++++++++++++++++++++++++++++++++---- btqs/commands/neurons.py | 134 ++++++++++++++++---- btqs/commands/subnet.py | 148 +++++++++------------- btqs/config.py | 42 ++++++- btqs/utils.py | 119 +++++++++++++----- 7 files changed, 669 insertions(+), 233 deletions(-) diff --git a/bittensor_cli/cli.py b/bittensor_cli/cli.py index b012c989..09001cab 100755 --- a/bittensor_cli/cli.py +++ b/bittensor_cli/cli.py @@ -3977,8 +3977,8 @@ def subnets_create( wallet_name, wallet_path, wallet_hotkey, - ask_for=[WO.NAME, WO.PATH, WO.HOTKEY], - validate=WV.WALLET_AND_HOTKEY, + ask_for=[WO.NAME, WO.PATH], + validate=WV.WALLET, ) return self._run_command( subnets.create(wallet, self.initialize_chain(network), prompt) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index 61804677..a2ba7940 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -9,7 +9,7 @@ from rich.table import Table from .config import ( - CONFIG_FILE_PATH, + BTQS_LOCK_CONFIG_FILE_PATH, EPILOG, LOCALNET_ENDPOINT, DEFAULT_WORKSPACE_DIRECTORY, @@ -74,8 +74,8 @@ def __init__(self): # Subnet commands self.subnet_app.command(name="setup")(self.setup_subnet) self.subnet_app.command(name="live")(self.display_live_metagraph) - self.subnet_app.command(name="stake")(self.add_stake) - self.subnet_app.command(name="add-weights")(self.add_weights) + self.subnet_app.command(name="stake", hidden=True)(self.add_stake) + self.subnet_app.command(name="add-weights", hidden=True)(self.add_weights) # Neuron commands self.neurons_app.command(name="setup")(self.setup_neurons) @@ -85,48 +85,43 @@ def __init__(self): self.neurons_app.command(name="status")(self.status_neurons) self.neurons_app.command(name="start")(self.start_neurons) + self.verbose = False + self.fast_blocks = True + self.workspace_path = None + self.subtensor_branch = None self.steps = [ { "title": "Start Local Subtensor", + "command": "btqs chain start", "description": "Initialize and start the local Subtensor blockchain. This may take several minutes due to the compilation process.", "info": "πŸ”— **Subtensor** is the underlying blockchain network that facilitates decentralized activities. Starting the local chain sets up your personal development environment for experimenting with Bittensor.", - "action": lambda: self.start_chain(workspace_path=None, branch=None), - }, - { - "title": "Check Chain Status", - "description": "Verify that the local chain is running correctly.", - "info": "πŸ“Š **Chain Nodes**: During your local Subtensor setup, two blockchain nodes are initiated. These nodes communicate and collaborate to reach consensus, ensuring the integrity and synchronization of your blockchain network.", - "action": lambda: self.status_neurons(), + "action": lambda: self.start_chain( + workspace_path=self.workspace_path, + branch=self.subtensor_branch, + fast_blocks=self.fast_blocks, + verbose=self.verbose, + ), }, { "title": "Set Up Subnet", - "description": "Create a subnet owner wallet, establish a new subnet, and register the owner to the subnet.", - "info": "πŸ”‘ **Wallets** in Bittensor are essential for managing your stake, interacting with the network, and running validators or miners. Each wallet has a unique name and associated hotkey that serves as your identity within the network.", + "command": "btqs subnet setup", + "description": "Create a subnet owner wallet, establish a new subnet, and register the owner to the subnet. Register to root, add stake, and set weights.", + "info": "πŸ”‘ **Wallets** (coldkeys) in Bittensor are essential for managing your stake, interacting with the network, and running validators or miners. Each wallet (coldkey) has a unique name and an associated hotkey that serves as your identity within the network.", "action": lambda: self.setup_subnet(), }, { - "title": "Add Stake by Validator", - "description": "Stake Tao to the validator's hotkey and register to the root network.", - "info": "πŸ’° **Staking Tao** to your hotkey is the process of committing your tokens to support your role as a validator in the subnet. Your wallet can potentially earn rewards based on your performance and the subnet's incentive mechanism.", - "action": lambda: self.add_stake(), - }, - { - "title": "Set Up Miners", + "title": "Set Up Neurons (Miners)", + "command": "btqs neurons setup", "description": "Create miner wallets and register them to the subnet.", - "info": "βš’οΈ **Miners** are responsible for performing computations and contributing to the subnet's tasks. Setting up miners involves creating dedicated wallets for each miner entity and registering them to the subnet.", + "info": "βš’οΈ **Miners** perform tasks that are given to them by validators. A miner can be registered into a subnet with a coldkey and a hotkey pair.", "action": lambda: self.setup_neurons(), }, { - "title": "Run Miners", - "description": "Start all miner processes.", - "info": "πŸƒ This step starts and runs your miner processes so they can start contributing to the network", - "action": lambda: self.run_neurons(), - }, - { - "title": "Add Weights to Netuid 1", - "description": "Configure weights for Netuid 1 through the validator.", - "info": "πŸ‹οΈ Setting **Root weights** in Bittensor means assigning relative importance to different subnets within the network, which directly influences their share of network rewards and resources.", - "action": lambda: self.add_weights(), + "title": "Run Neurons", + "command": "btqs neurons run", + "description": "Start all neuron (miner & validator) processes.", + "info": "πŸƒ Running neurons means starting one or more validator processes and one or more miner processes. In practice validators and miners are subnet-specific.\nHowever, in this tutorial we will use simple validator and miner modules. Configure your own setup using btqs_config.py.", # Add path + "action": lambda: self.run_neurons(verbose=self.verbose), }, ] @@ -141,6 +136,12 @@ def start_chain( branch: Optional[str] = typer.Option( None, "--subtensor_branch", help="Subtensor branch to checkout" ), + fast_blocks: bool = typer.Option( + True, "--fast/--slow", help="Enable or disable fast blocks" + ), + verbose: bool = typer.Option( + False, "--verbose", "-v", help="Enable verbose output" + ), ): """ Starts the local Subtensor chain. @@ -161,9 +162,13 @@ def start_chain( if reuse_config_path: workspace_path = config_data.get("workspace_path") else: + print_info( + "The development working directory will host subnet and subtensor repos, logs, and wallets created during the quick start.", + emoji="πŸ’‘ ", + ) workspace_path = typer.prompt( typer.style( - "Enter path to create Bittensor development workspace", + "Enter path to the development working directory (Press Enter for default)", fg="blue", ), default=DEFAULT_WORKSPACE_DIRECTORY, @@ -171,12 +176,18 @@ def start_chain( if not branch: branch = typer.prompt( - typer.style("Enter Subtensor branch", fg="blue"), + typer.style( + "Enter Subtensor branch (press Enter for default)", fg="blue" + ), default=SUBTENSOR_BRANCH, ) - - console.print("πŸ”— [dark_orange]Starting the local chain...") - chain.start(config_data, workspace_path, branch) + chain.start( + config_data, + workspace_path, + branch, + fast_blocks=fast_blocks, + verbose=verbose, + ) def stop_chain(self): """ @@ -191,7 +202,7 @@ def stop_chain(self): [bold]Note[/bold]: Use this command to gracefully shut down the local chain. It will also stop any running miner processes. """ config_data = load_config( - "No running chain found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "No running chain found. Please run `btqs chain start` first." ) chain.stop(config_data) @@ -208,14 +219,33 @@ def reattach_chain(self): [bold]Note[/bold]: Press Ctrl+C to detach from the chain logs. """ config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) chain.reattach(config_data) - def run_all(self): + def run_all( + self, + workspace_path: Optional[str] = typer.Option( + None, + "--path", + "--workspace", + help="Path to Bittensor's quick start workspace", + ), + fast_blocks: bool = typer.Option( + True, "--fast/--slow", help="Enable or disable fast blocks" + ), + verbose: bool = typer.Option( + False, "--verbose", "-v", help="Enable verbose output" + ), + ): """ Runs all commands in sequence to set up and start the local chain, subnet, and neurons. """ + + self.workspace_path = workspace_path + self.fast_blocks = fast_blocks + self.verbose = verbose + console.clear() print_info("Welcome to the Bittensor Quick Start Tutorial", emoji="πŸš€") console.print( @@ -224,12 +254,12 @@ def run_all(self): ) for idx, step in enumerate(self.steps, start=1): + print_step(step["title"], step["description"], idx) if "info" in step: print_info_box(step["info"], title="Info") - print_step(step["title"], step["description"], idx) console.print( - "[bold blue]Press [yellow]Enter[/yellow] to continue to the next step or [yellow]Ctrl+C[/yellow] to exit.\n" + f"[blue]Press [yellow]Enter[/yellow] to continue to the [dark_orange]Step {idx}[/dark_orange] or [yellow]Ctrl+C[/yellow] to exit.\n" ) try: input() @@ -239,6 +269,7 @@ def run_all(self): # Execute the action step["action"]() + console.print(f"\n🏁 Step {idx} has finished!\n") print_success( "Your local chain, subnet, and neurons are up and running", emoji="πŸŽ‰" @@ -249,22 +280,54 @@ def run_all(self): def display_live_metagraph(self): config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) subnet.display_live_metagraph(config_data) def setup_steps(self): - subnet.steps() + """ + Display the steps for subnet setup. + """ + table = Table( + title="[bold dark_orange]Setup Steps", + header_style="dark_orange", + leading=True, + show_edge=False, + border_style="bright_black", + ) + table.add_column("Step", style="cyan", width=5, justify="center") + table.add_column( + "Command", justify="left" + ) # Removed 'style' to allow inline styling + table.add_column("Title", justify="left", style="white") + table.add_column("Description", justify="left", style="white") + + for idx, step in enumerate(self.steps, start=1): + command_with_prefix = ( + f"[blue]$[/blue] [green]{step.get('command', '')}[/green]" + ) + table.add_row( + str(idx), + command_with_prefix, + step["title"], + step["description"], + ) + + console.print(table) + console.print( + "\n[dark_orange]You can run an automated script covering all the steps using:\n" + ) + console.print("[blue]$ [green]btqs run-all") def add_stake(self): config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) subnet.add_stake(config_data) def add_weights(self): config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) subnet.add_weights(config_data) @@ -280,33 +343,46 @@ def setup_subnet(self): [bold]Note[/bold]: Ensure the local chain is running before executing this command. """ - if not is_chain_running(CONFIG_FILE_PATH): + if not is_chain_running(BTQS_LOCK_CONFIG_FILE_PATH): console.print( "[red]Local chain is not running. Please start the chain first." ) return config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) subnet.setup_subnet(config_data) + console.print( + "🎊 Subnet setup is complete! Press any key to continue adding stake...\n" + ) + input() + subnet.add_stake(config_data) + console.print("\nπŸ‘ Added stake! Press any key to continue adding weights...\n") + input() + subnet.add_weights(config_data) def setup_neurons(self): """ Sets up neurons (miners) for the subnet. """ - if not is_chain_running(CONFIG_FILE_PATH): + if not is_chain_running(BTQS_LOCK_CONFIG_FILE_PATH): console.print( "[red]Local chain is not running. Please start the chain first." ) return config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) neurons.setup_neurons(config_data) - def run_neurons(self): + def run_neurons( + self, + verbose: bool = typer.Option( + False, "--verbose", "-v", help="Enable verbose output" + ), + ): """ Runs all neurons (miners and validators). @@ -321,7 +397,7 @@ def run_neurons(self): ones as necessary. Press Ctrl+C to detach from a neuron and move to the next. """ config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) # Ensure neurons are configured @@ -331,7 +407,7 @@ def run_neurons(self): ) return - neurons.run_neurons(config_data) + neurons.run_neurons(config_data, verbose) def stop_neurons(self): """ @@ -345,7 +421,7 @@ def stop_neurons(self): [bold]Note[/bold]: You can choose which miners to stop or stop all of them. """ - if not os.path.exists(CONFIG_FILE_PATH): + if not os.path.exists(BTQS_LOCK_CONFIG_FILE_PATH): console.print("[red]Config file not found.") return @@ -353,7 +429,12 @@ def stop_neurons(self): neurons.stop_neurons(config_data) - def start_neurons(self): + def start_neurons( + self, + verbose: bool = typer.Option( + False, "--verbose", "-v", help="Enable verbose output" + ), + ): """ Starts the stopped neurons. @@ -365,13 +446,13 @@ def start_neurons(self): [bold]Note[/bold]: You can choose which stopped miners to start or start all of them. """ - if not os.path.exists(CONFIG_FILE_PATH): + if not os.path.exists(BTQS_LOCK_CONFIG_FILE_PATH): console.print("[red]Config file not found.") return config_data = load_config() - neurons.start_neurons(config_data) + neurons.start_neurons(config_data, verbose) def reattach_neurons(self): """ @@ -385,7 +466,7 @@ def reattach_neurons(self): [bold]Note[/bold]: Press Ctrl+C to detach from the miner logs. """ - if not os.path.exists(CONFIG_FILE_PATH): + if not os.path.exists(BTQS_LOCK_CONFIG_FILE_PATH): console.print("[red]Config file not found.") return @@ -413,7 +494,7 @@ def status_neurons(self): print_info("Checking status of Subtensor and neurons...", emoji="πŸ” ") config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) # Get process data @@ -457,9 +538,9 @@ def status_neurons(self): # Add version version_table.add_row("btcli version:", get_btcli_version()) version_table.add_row( - "bittensor-wallet version:", get_bittensor_wallet_version() + "bittensor-wallet sdk version:", get_bittensor_wallet_version() ) - version_table.add_row("bittensor version:", get_bittensor_version()) + version_table.add_row("bittensor-sdk version:", get_bittensor_version()) layout = Table.grid(expand=True) layout.add_column(justify="left") diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 1968c81e..f0310f2c 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -1,10 +1,27 @@ import os import subprocess import time +from threading import Thread +from rich.console import Console +from rich.progress import ( + Progress, + SpinnerColumn, + BarColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.text import Text +import asyncio import psutil import yaml -from btqs.config import CONFIG_FILE_PATH, SUBTENSOR_REPO_URL +from btqs.config import ( + BTQS_LOCK_CONFIG_FILE_PATH, + SUBTENSOR_REPO_URL, + LOCALNET_ENDPOINT, + SUDO_URI, +) +from bittensor_wallet import Wallet, Keypair from btqs.utils import ( attach_to_process_logs, create_virtualenv, @@ -13,15 +30,186 @@ print_info, print_success, print_error, + print_info_box, + messages, + display_process_status_table, + console, ) from git import GitCommandError, Repo -from rich.console import Console from rich.prompt import Confirm +from bittensor_cli.src.bittensor.async_substrate_interface import ( + AsyncSubstrateInterface, +) + + +def update_chain_tempos( + config_data: dict, + netuid_tempo_list: list[tuple[int, int]], + add_emission_tempo: bool = True, + emission_tempo: int = 10, +): + """ + Update tempos on the chain for given netuids and optionally update emission tempo. + + Parameters: + - config_data: Configuration data containing paths and other settings. + - netuid_tempo_list: List of tuples where each tuple contains (netuid, tempo). + - add_emission_tempo: Boolean flag to indicate if emission tempo should be updated. + - emission_tempo: The value to set for emission tempo if `add_emission_tempo` is True. + """ + # Initialize the Rich console + console = Console() + + keypair = Keypair.create_from_uri(SUDO_URI) + sudo_wallet = Wallet( + path=config_data["wallets_path"], + name="sudo", + hotkey="default", + ) + sudo_wallet.set_coldkey(keypair=keypair, encrypt=False, overwrite=True) + sudo_wallet.set_coldkeypub(keypair=keypair, encrypt=False, overwrite=True) + sudo_wallet.set_hotkey(keypair=keypair, encrypt=False, overwrite=True) + print_info_box( + "**Emissions Tempo** control the frequency at which rewards (emissions) are distributed to hotkeys. On Finney (mainnet), the setting is every 7200 blocks, which equates to approximately 1 day.", + title="Info: Emission and Subnet Tempos", + ) + total_steps = len(netuid_tempo_list) + (1 if add_emission_tempo else 0) + + desc_width = 30 + progress = Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}", justify="left"), + BarColumn(), + TextColumn("{task.completed}/{task.total}", justify="right"), + TimeElapsedColumn(), + console=console, + ) + + console.print("\n[bold yellow]Getting the chain ready...\n") + with progress: + task = progress.add_task("Initializing...", total=total_steps) + + def update_description(): + i = 0 + while not progress.finished: + message = messages[i % len(messages)] + printable_length = len(Text.from_markup(message).plain) + if printable_length < desc_width: + padded_message = message + " " * (desc_width - printable_length) + else: + padded_message = message[:desc_width] + progress.update(task, description=padded_message) + time.sleep(3) # Update every 3 seconds + i += 1 + + # Start the thread to update the description + desc_thread = Thread(target=update_description) + desc_thread.start() + local_chain = AsyncSubstrateInterface(chain_endpoint=LOCALNET_ENDPOINT) + try: + if add_emission_tempo: + local_chain = AsyncSubstrateInterface(chain_endpoint=LOCALNET_ENDPOINT) + success = asyncio.run( + sudo_set_emission_tempo(local_chain, sudo_wallet, emission_tempo) + ) + progress.advance(task) + + for netuid, tempo in netuid_tempo_list: + local_chain = AsyncSubstrateInterface(chain_endpoint=LOCALNET_ENDPOINT) + success = asyncio.run( + sudo_set_tempo(local_chain, sudo_wallet, netuid, tempo) + ) + progress.advance(task) + finally: + desc_thread.join() + + +async def sudo_set_emission_tempo( + substrate: "AsyncSubstrateInterface", wallet: Wallet, emission_tempo: int +) -> bool: + async with substrate: + emission_call = await substrate.compose_call( + call_module="AdminUtils", + call_function="sudo_set_hotkey_emission_tempo", + call_params={"emission_tempo": emission_tempo}, + ) + sudo_call = await substrate.compose_call( + call_module="Sudo", + call_function="sudo", + call_params={"call": emission_call}, + ) + extrinsic = await substrate.create_signed_extrinsic( + call=sudo_call, keypair=wallet.coldkey + ) + response = await substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + await response.process_events() + return await response.is_success + + +async def sudo_set_target_registrations_per_interval( + substrate: "AsyncSubstrateInterface", + wallet: Wallet, + netuid: int, + target_registrations_per_interval: int, +) -> bool: + async with substrate: + registration_call = await substrate.compose_call( + call_module="AdminUtils", + call_function="sudo_set_target_registrations_per_interval", + call_params={ + "netuid": netuid, + "target_registrations_per_interval": target_registrations_per_interval, + }, + ) + sudo_call = await substrate.compose_call( + call_module="Sudo", + call_function="sudo", + call_params={"call": registration_call}, + ) + extrinsic = await substrate.create_signed_extrinsic( + call=sudo_call, keypair=wallet.coldkey + ) + response = await substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + await response.process_events() + return await response.is_success + -console = Console() +async def sudo_set_tempo( + substrate: "AsyncSubstrateInterface", wallet: Wallet, netuid: int, tempo: int +) -> bool: + async with substrate: + set_tempo_call = await substrate.compose_call( + call_module="AdminUtils", + call_function="sudo_set_tempo", + call_params={"netuid": netuid, "tempo": tempo}, + ) + sudo_call = await substrate.compose_call( + call_module="Sudo", + call_function="sudo", + call_params={"call": set_tempo_call}, + ) + extrinsic = await substrate.create_signed_extrinsic( + call=sudo_call, keypair=wallet.coldkey + ) + response = await substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + await response.process_events() + return await response.is_success -def start(config_data, workspace_path, branch): +def start(config_data, workspace_path, branch, fast_blocks=True, verbose=False): os.makedirs(workspace_path, exist_ok=True) subtensor_path = os.path.join(workspace_path, "subtensor") @@ -57,9 +245,14 @@ def start(config_data, workspace_path, branch): print_error(f"Error cloning repository: {e}") return + if fast_blocks: + print_info("Fast blocks are On", emoji="🏎️ ") + else: + print_info("Fast blocks are Off", emoji="🐌 ") + venv_subtensor_path = os.path.join(workspace_path, "venv_subtensor") venv_python = create_virtualenv(venv_subtensor_path) - install_subtensor_dependencies() + install_subtensor_dependencies(verbose) config_data["venv_subtensor"] = venv_python @@ -71,14 +264,12 @@ def start(config_data, workspace_path, branch): os.path.dirname(venv_python) + os.pathsep + env_variables["PATH"] ) process = subprocess.Popen( - [localnet_path], + [localnet_path, str(fast_blocks)], stdout=subprocess.DEVNULL, - # stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=subtensor_path, start_new_session=True, env=env_variables, - # universal_newlines=True, ) print_info( @@ -86,12 +277,6 @@ def start(config_data, workspace_path, branch): emoji="πŸ› οΈ ", ) - # for line in process.stdout: - # console.print(line, end="") - # if "Imported #" in line: - # console.print("[green] Chain comp") - # continue - # Paths to subtensor log files log_dir = os.path.join(subtensor_path, "logs") alice_log = os.path.join(log_dir, "alice.log") @@ -105,11 +290,11 @@ def start(config_data, workspace_path, branch): return time.sleep(1) - chain_ready = wait_for_chain_compilation(alice_log, start_time, timeout) + chain_ready = wait_for_chain_compilation(alice_log, start_time, timeout, verbose) if chain_ready: print_info( - "Local chain is running. You can now use it for development and testing.\n", - emoji="\nπŸš€", + "Local chain is running.\n", + emoji="\nπŸ”—", ) # Fetch PIDs of substrate nodes @@ -131,10 +316,32 @@ def start(config_data, workspace_path, branch): # Save config data try: - os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True) - with open(CONFIG_FILE_PATH, "w") as config_file: + print_info("Updating config file.\n", emoji="πŸ–‹οΈ ") + os.makedirs(os.path.dirname(BTQS_LOCK_CONFIG_FILE_PATH), exist_ok=True) + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) print_info("Config file updated.", emoji="πŸ“ ") + update_chain_tempos( + netuid_tempo_list=[(0, 9), (1, 10)], + config_data=config_data, + add_emission_tempo=True, + emission_tempo=20, + ) + print_info( + "Local chain is now set up. You can now use it for development and testing.", + emoji="\nπŸš€", + ) + console.print(f"[dark_orange]Endpoint: {LOCALNET_ENDPOINT}\n") + print_info_box( + "πŸ“Š **Subtensor Nodes**: During your local Subtensor setup, two blockchain nodes are initiated. These nodes communicate and collaborate to reach consensus, ensuring the integrity and synchronization of your blockchain network.", + title="Info: Subtensor Nodes", + ) + process_entries, cpu_usage_list, memory_usage_list = get_process_entries( + config_data + ) + display_process_status_table( + process_entries, cpu_usage_list, memory_usage_list + ) except Exception as e: print_error(f"Failed to write to the config file: {e}") else: @@ -172,8 +379,8 @@ def stop(config_data): show_default=True, ) if refresh_config: - if os.path.exists(CONFIG_FILE_PATH): - os.remove(CONFIG_FILE_PATH) + if os.path.exists(BTQS_LOCK_CONFIG_FILE_PATH): + os.remove(BTQS_LOCK_CONFIG_FILE_PATH) print_info("Configuration file refreshed.", emoji="πŸ”„ ") @@ -201,7 +408,7 @@ def reattach(config_data): attach_to_process_logs(alice_log, "Subtensor Chain", pid) -def wait_for_chain_compilation(alice_log, start_time, timeout): +def wait_for_chain_compilation(alice_log, start_time, timeout, verbose): chain_ready = False try: with open(alice_log, "r") as log_file: @@ -209,7 +416,8 @@ def wait_for_chain_compilation(alice_log, start_time, timeout): while True: line = log_file.readline() if line: - console.print(line, end="") + if verbose: + console.print(line, end="") if "Imported #" in line: chain_ready = True break @@ -273,7 +481,7 @@ def stop_running_neurons(config_data): elif neuron["process"].startswith("Validator"): config_data["Owner"]["pid"] = None - with open(CONFIG_FILE_PATH, "w") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) else: print_info("No neurons were running.", emoji="βœ… ") diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index 917241f5..2e6ec82a 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -1,15 +1,17 @@ import os import psutil import typer +import click +import asyncio import yaml from bittensor_wallet import Wallet, Keypair from git import Repo, GitCommandError from btqs.config import ( - CONFIG_FILE_PATH, + BTQS_LOCK_CONFIG_FILE_PATH, SUBNET_REPO_URL, SUBNET_REPO_BRANCH, - WALLET_URIS, + MINER_URIS, MINER_PORTS, LOCALNET_ENDPOINT, ) @@ -26,16 +28,25 @@ create_virtualenv, install_neuron_dependencies, print_info, + print_warning, +) +from bittensor_cli.src.bittensor.async_substrate_interface import ( + AsyncSubstrateInterface, ) def setup_neurons(config_data): - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + subnet_owner, owner_data = subnet_owner_exists(BTQS_LOCK_CONFIG_FILE_PATH) if not subnet_owner: console.print( "[red]Subnet netuid 1 registered to the owner not found. Run `btqs subnet setup` first" ) return + owner_wallet = Wallet( + name=owner_data.get("wallet_name"), + path=config_data["wallets_path"], + hotkey=owner_data.get("hotkey"), + ) config_data.setdefault("Miners", {}) miners = config_data.get("Miners", {}) @@ -50,9 +61,21 @@ def setup_neurons(config_data): else: _create_miner_wallets(config_data) + print_info("Preparing for miner registrations. Please wait...\n", emoji="⏱️ ") + local_chain = AsyncSubstrateInterface(chain_endpoint=LOCALNET_ENDPOINT) + success = asyncio.run( + sudo_set_target_registrations_per_interval(local_chain, owner_wallet, 1, 1000) + ) + if success: + print_info("Proceeding with the miner registrations.\n", emoji="πŸ›£οΈ ") + else: + print_warning("All neurons might not be able to register at once.") + _register_miners(config_data) - print_info("Viewing Metagraph for Subnet 1\n", emoji="πŸ“Š ") + print_info("All registrations are complete.\n", emoji="πŸ“š ") + + print_info("Viewing Metagraph for Subnet 1.\n", emoji="πŸ“Š ") subnets_list = exec_command( command="subnets", sub_command="metagraph", @@ -66,7 +89,7 @@ def setup_neurons(config_data): print(subnets_list.stdout, end="") -def run_neurons(config_data): +def run_neurons(config_data, verbose=False): subnet_template_path = _add_subnet_template(config_data) chain_pid = config_data.get("pid") @@ -74,20 +97,22 @@ def run_neurons(config_data): venv_neurons_path = os.path.join(config_data["workspace_path"], "venv_neurons") venv_python = create_virtualenv(venv_neurons_path) - install_neuron_dependencies(venv_python, subnet_template_path) + install_neuron_dependencies(venv_python, subnet_template_path, verbose) # Handle Validator if config_data.get("Owner"): config_data["Owner"]["venv"] = venv_python - _run_validator(config_data, subnet_template_path, chain_pid, venv_python) + _run_validator( + config_data, subnet_template_path, chain_pid, venv_python, verbose + ) # Handle Miners for wallet_name, wallet_info in config_data.get("Miners", {}).items(): config_data["Miners"][wallet_name]["venv"] = venv_python - _run_miners(config_data, subnet_template_path, chain_pid, venv_python) + _run_miners(config_data, subnet_template_path, chain_pid, venv_python, verbose) - with open(CONFIG_FILE_PATH, "w") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) @@ -139,11 +164,11 @@ def stop_neurons(config_data): # Stop selected neurons _stop_selected_neurons(config_data, selected_neurons) - with open(CONFIG_FILE_PATH, "w") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) -def start_neurons(config_data): +def start_neurons(config_data, verbose=False): # Get process entries process_entries, _, _ = get_process_entries(config_data) display_process_status_table(process_entries, [], []) @@ -189,9 +214,9 @@ def start_neurons(config_data): return # Start selected neurons - _start_selected_neurons(config_data, selected_neurons) + _start_selected_neurons(config_data, selected_neurons, verbose) - with open(CONFIG_FILE_PATH, "w") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) @@ -260,7 +285,21 @@ def reattach_neurons(config_data): def _create_miner_wallets(config_data): - for i, uri in enumerate(WALLET_URIS): + max_miners = len(MINER_URIS) + + total_miners = typer.prompt( + f"How many miners do you want to run? (Choose between 1 and {max_miners})", + type=click.IntRange(1, max_miners), + default=3, + show_default=True, + ) + + # Ensure that the total number of miners doesn't exceed the available URIs + if total_miners > max_miners: + total_miners = max_miners + console.print(f"Limiting the number of miners to {max_miners}.") + + for i, uri in enumerate(MINER_URIS[:total_miners]): console.print(f"Miner {i+1}:") wallet_name = typer.prompt( f"Enter wallet name for miner {i+1}", default=f"{uri.strip('//')}" @@ -285,10 +324,10 @@ def _create_miner_wallets(config_data): "port": MINER_PORTS[i], } - with open(CONFIG_FILE_PATH, "w") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) - print_info("Miner wallets are created.\n", emoji="πŸ—‚οΈ ") + print_info("Miner wallets are created.\n", emoji="\nπŸ—‚οΈ ") def _register_miners(config_data): @@ -298,7 +337,7 @@ def _register_miners(config_data): name=wallet_name, hotkey=wallet_info["hotkey"], ) - print_info(f"Registering Miner ({wallet_name}) to Netuid 1\n", emoji="πŸ”§ ") + print_info(f"Registering Miner ({wallet_name}) to Netuid 1\n", emoji="βš’οΈ ") miner_registered = exec_command( command="subnets", @@ -366,11 +405,12 @@ def _add_subnet_template(config_data): return subnet_template_path -def _run_validator(config_data, subnet_template_path, chain_pid, venv_python): +def _run_validator( + config_data, subnet_template_path, chain_pid, venv_python, verbose=False +): owner_info = config_data["Owner"] validator_pid = owner_info.get("pid") validator_subtensor_pid = owner_info.get("subtensor_pid") - if ( validator_pid and psutil.pid_exists(validator_pid) @@ -387,13 +427,15 @@ def _run_validator(config_data, subnet_template_path, chain_pid, venv_python): else: # Validator is not running, start it success = start_validator( - owner_info, subnet_template_path, config_data, venv_python + owner_info, subnet_template_path, config_data, venv_python, verbose ) if not success: console.print("[red]Failed to start validator.") -def _run_miners(config_data, subnet_template_path, chain_pid, venv_python): +def _run_miners( + config_data, subnet_template_path, chain_pid, venv_python, verbose=False +): for wallet_name, wallet_info in config_data.get("Miners", {}).items(): miner_pid = wallet_info.get("pid") miner_subtensor_pid = wallet_info.get("subtensor_pid") @@ -416,7 +458,12 @@ def _run_miners(config_data, subnet_template_path, chain_pid, venv_python): else: # Miner is not running, start it success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data, venv_python + wallet_name, + wallet_info, + subnet_template_path, + config_data, + venv_python, + verbose, ) if not success: console.print(f"[red]Failed to start miner {wallet_name}.") @@ -444,7 +491,7 @@ def _stop_selected_neurons(config_data, selected_neurons): config_data["Owner"]["pid"] = None -def _start_selected_neurons(config_data, selected_neurons): +def _start_selected_neurons(config_data, selected_neurons, verbose): subnet_template_path = _add_subnet_template(config_data) for neuron in selected_neurons: @@ -455,12 +502,18 @@ def _start_selected_neurons(config_data, selected_neurons): subnet_template_path, config_data, config_data["Owner"]["venv"], + verbose, ) elif neuron_name.startswith("Miner"): wallet_name = neuron_name.split("Miner: ")[-1] wallet_info = config_data["Miners"][wallet_name] success = start_miner( - wallet_name, wallet_info, subnet_template_path, config_data + wallet_name, + wallet_info, + subnet_template_path, + config_data, + config_data["Miners"][wallet_name]["venv"], + verbose, ) if success: @@ -471,3 +524,36 @@ def _start_selected_neurons(config_data, selected_neurons): # Update the process entries after starting neurons process_entries, _, _ = get_process_entries(config_data) display_process_status_table(process_entries, [], []) + + +async def sudo_set_target_registrations_per_interval( + substrate: "AsyncSubstrateInterface", + wallet: Wallet, + netuid: int, + target_registrations_per_interval: int, +) -> bool: + async with substrate: + registration_call = await substrate.compose_call( + call_module="AdminUtils", + call_function="sudo_set_target_registrations_per_interval", + call_params={ + "netuid": netuid, + "target_registrations_per_interval": target_registrations_per_interval, + }, + ) + sudo_call = await substrate.compose_call( + call_module="Sudo", + call_function="sudo", + call_params={"call": registration_call}, + ) + extrinsic = await substrate.create_signed_extrinsic( + call=sudo_call, keypair=wallet.coldkey + ) + response = await substrate.submit_extrinsic( + extrinsic, + wait_for_inclusion=True, + wait_for_finalization=True, + ) + + await response.process_events() + return await response.is_success diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index b119598b..16d73a6f 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -6,8 +6,7 @@ from rich.text import Text from rich.table import Table from bittensor_wallet import Wallet, Keypair -from rich.progress import Progress, BarColumn, TimeRemainingColumn -from btqs.config import CONFIG_FILE_PATH, VALIDATOR_URI, LOCALNET_ENDPOINT +from btqs.config import BTQS_LOCK_CONFIG_FILE_PATH, VALIDATOR_URI, LOCALNET_ENDPOINT from btqs.utils import ( console, exec_command, @@ -19,11 +18,18 @@ load_config, print_info, print_error, + print_info_box, ) def add_stake(config_data): - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + print_info_box( + "πŸ’° You stake **TAO** to your hotkey to become a validator in a subnet. Your hotkey can potentially earn rewards based on your validating performance.", + title="Info: Staking to become a Validator", + ) + print("\n") + + subnet_owner, owner_data = subnet_owner_exists(BTQS_LOCK_CONFIG_FILE_PATH) if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), @@ -31,7 +37,7 @@ def add_stake(config_data): hotkey=owner_data.get("hotkey"), ) print_info( - f"Validator is adding stake to its own hotkey: {owner_wallet}\n", + f"Validator is adding stake to its own hotkey\n{owner_wallet}\n", emoji="πŸ”– ", ) @@ -56,7 +62,10 @@ def add_stake(config_data): clean_stdout = remove_ansi_escape_sequences(add_stake.stdout) if "βœ… Finalized" in clean_stdout: print_info("Stake added by Validator", emoji="πŸ“ˆ ") - + print_info_box( + "Metagraph contains important information on a subnet. Displaying a metagraph is a useful way to see a snapshot of a subnet.", + title="Info: Metagraph", + ) print_info("Viewing Metagraph for Subnet 1", emoji="\nπŸ”Ž ") subnets_list = exec_command( command="subnets", @@ -74,6 +83,12 @@ def add_stake(config_data): print_error("\nFailed to add stake. Command output:\n") print(add_stake.stdout, end="") + print("\n") + print_info_box( + "The validator's hotkey must be registered in the root network (netuid 0) to set root weights.", + title="Root network registration", + ) + print_info( f"Validator is registering to root network (netuid 0) ({owner_wallet})\n", emoji="\n🫚 ", @@ -122,7 +137,11 @@ def add_stake(config_data): def add_weights(config_data): - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + print_info_box( + "πŸ‹οΈ Setting **Root weights** in Bittensor means assigning relative importance to different subnets within the network, which directly influences their share of network rewards and resources.", + title="Info: Setting Root weights", + ) + subnet_owner, owner_data = subnet_owner_exists(BTQS_LOCK_CONFIG_FILE_PATH) if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), @@ -130,13 +149,11 @@ def add_weights(config_data): hotkey=owner_data.get("hotkey"), ) print_info( - "Validator is now setting weights of subnet 1 on the root network\n", + "Validator is now setting weights of subnet 1 on the root network.\n Please wait... (Timeout: 60 seconds)", emoji="πŸ‹οΈ ", ) - - max_retries = 5 + max_retries = 30 attempt = 0 - wait_time = 30 retry_patterns = ["ancient birth block", "Transaction has a bad signature"] while attempt < max_retries: @@ -159,6 +176,7 @@ def add_weights(config_data): "--weights", 1, ], + internal_command=True, ) clean_stdout = remove_ansi_escape_sequences(set_weights.stdout) @@ -174,25 +192,8 @@ def add_weights(config_data): elif any(pattern in clean_stdout for pattern in retry_patterns): attempt += 1 if attempt < max_retries: - console.print( - f"[red]Attempt {attempt}/{max_retries}: Failed to set weights. \nError: {clean_stdout} Retrying in {wait_time} seconds..." - ) - - with Progress( - "[progress.percentage]{task.percentage:>3.0f}%", - BarColumn(), - TimeRemainingColumn(), - console=console, - transient=True, - ) as progress: - task = progress.add_task("Waiting...", total=wait_time) - while not progress.finished: - progress.update(task, advance=1) - time.sleep(1) + time.sleep(1) else: - console.print( - f"[red]Attempt {attempt}/{max_retries}: Maximum retries reached." - ) console.print( "\n[red]Failed to set weights after multiple attempts. Please try again later\n" ) @@ -253,7 +254,7 @@ def get_metagraph(): clear_screen() print(metagraph) display_process_status_table( - process_entries, cpu_usage_list, memory_usage_list, config_data + process_entries, cpu_usage_list, memory_usage_list ) # Create a progress bar for 5 seconds @@ -275,13 +276,14 @@ def get_metagraph(): def setup_subnet(config_data): os.makedirs(config_data["wallets_path"], exist_ok=True) - subnet_owner, owner_data = subnet_owner_exists(CONFIG_FILE_PATH) + subnet_owner, owner_data = subnet_owner_exists(BTQS_LOCK_CONFIG_FILE_PATH) if subnet_owner: owner_wallet = Wallet( name=owner_data.get("wallet_name"), path=config_data["wallets_path"], hotkey=owner_data.get("hotkey"), ) + input() warning_text = Text( "A Subnet Owner associated with this setup already exists", @@ -295,7 +297,7 @@ def setup_subnet(config_data): else: create_subnet_owner_wallet(config_data) config_data = load_config( - "A running Subtensor not found. Please run [dark_orange]`btqs chain start`[/dark_orange] first." + "A running Subtensor not found. Please run `btqs chain start` first." ) owner_data = config_data["Owner"] @@ -323,7 +325,12 @@ def setup_subnet(config_data): else: create_subnet(owner_wallet, config_data) - print_info("\nListing all subnets\n", emoji="πŸ“‹ ") + print_info("Listing all subnets\n", emoji="\nπŸ“‹ ") + print_info_box( + "In the below table, the netuid 0 shown is root network that is automatically created when the local blockchain starts", + title="Info: Root network (netuid 0)", + ) + print("\n") subnets_list = exec_command( command="subnets", sub_command="list", @@ -336,10 +343,12 @@ def setup_subnet(config_data): def create_subnet_owner_wallet(config_data): - print_info("Creating subnet owner wallet.\n", emoji="πŸ—‚οΈ ") + print_info( + "Creating a wallet (coldkey) and a hotkey to create a subnet.\n", emoji="πŸ—‚οΈ " + ) owner_wallet_name = typer.prompt( - "Enter subnet owner wallet name", default="owner", show_default=True + "Enter subnet owner wallet name", default="Alice", show_default=True ) owner_hotkey_name = typer.prompt( "Enter subnet owner hotkey name", default="default", show_default=True @@ -365,12 +374,12 @@ def create_subnet_owner_wallet(config_data): "hotkey": owner_hotkey_name, "subtensor_pid": config_data["pid"], } - with open(CONFIG_FILE_PATH, "w") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "w") as config_file: yaml.safe_dump(config_data, config_file) def create_subnet(owner_wallet, config_data): - print_info("Creating a subnet with Netuid 1\n", emoji="\n🌎 ") + print_info("Creating a subnet.\n", emoji="\n🌎 ") create_subnet = exec_command( command="subnets", @@ -391,7 +400,16 @@ def create_subnet(owner_wallet, config_data): if "βœ… Registered subnetwork with netuid: 1" in clean_stdout: print_info("Subnet created successfully with netuid 1\n", emoji="πŸ₯‡ ") - print_info(f"Registering Owner ({owner_wallet}) to Netuid 1\n", emoji="πŸ“ ") + print_info_box( + "πŸ”‘ Creating a subnet involves signing with your *coldkey*, which is the permanent keypair associated with your account. " + "It is typically used for long-term ownership and higher-value operations.\n\n" + "πŸ”₯ When registering to a subnet, your *hotkey* is used. The hotkey is a more frequently used keypair " + "designed for day-to-day interactions with the network." + ) + + print_info( + f"Registering the subnet owner to Netuid 1\n{owner_wallet}\n", emoji="\nπŸ“ " + ) register_subnet = exec_command( command="subnets", @@ -412,58 +430,4 @@ def create_subnet(owner_wallet, config_data): ) clean_stdout = remove_ansi_escape_sequences(register_subnet.stdout) if "βœ… Registered" in clean_stdout: - print_info("Registered the owner's wallet to subnet 1", emoji="βœ… ") - - -def steps(): - steps = [ - { - "command": "btqs chain start", - "description": "Start and initialize a local Subtensor blockchain. It may take several minutes to complete during the Subtensor compilation process. This is the entry point of the tutorial", - }, - { - "command": "btqs subnet setup", - "description": "This command creates a subnet owner's wallet, creates a new subnet, and registers the subnet owner to the subnet. Ensure the local chain is running before executing this command.", - }, - { - "command": "btqs subnet stake", - "description": "Add stake to the subnet and register to root. This command stakes Tao to the validator's own hotkey and registers to root network", - }, - { - "command": "btqs neurons setup", - "description": "This command creates miner wallets and registers them to the subnet.", - }, - { - "command": "btqs neurons run", - "description": "Run all neurons (miners and validators). This command starts the processes for all configured neurons, attaching to running processes if they are already running.", - }, - { - "command": "btqs subnet add-weight", - "description": "Add weight to netuid 1 through the validator", - }, - { - "command": "btqs subnet live", - "description": "Display the live metagraph of the subnet. This is used to monitor neuron performance and changing variables.", - }, - ] - - table = Table( - title="[bold dark_orange]Subnet Setup Steps", - header_style="dark_orange", - leading=True, - show_edge=False, - border_style="bright_black", - ) - table.add_column("Step", style="cyan", width=12, justify="center") - table.add_column("Command", justify="left", style="green") - table.add_column("Description", justify="left", style="white") - - for index, step in enumerate(steps, start=1): - table.add_row(str(index), step["command"], step["description"]) - - console.print(table) - - console.print( - "\n[dark_orange] You can run an automated script covering all the steps using:\n" - ) - console.print("[blue]$ [green]btqs run-all") + print_info("Registered the owner's hotkey to subnet 1", emoji="βœ… ") diff --git a/btqs/config.py b/btqs/config.py index 9c649906..85af78f8 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -1,6 +1,6 @@ import os -CONFIG_FILE_PATH = os.path.expanduser("~/.bittensor/btqs/btqs_config.yml") +BTQS_LOCK_CONFIG_FILE_PATH = os.path.expanduser("~/.bittensor/btqs/btqs-lock.yml") DEFAULT_WORKSPACE_DIRECTORY = os.path.expanduser("~/Desktop/bittensor_quick_start") SUBNET_REPO_URL = "https://github.com/opentensor/bittensor-subnet-template.git" @@ -11,7 +11,7 @@ DEFAULT_VALIDATOR_COMMAND = "./neurons/validator.py" SUBTENSOR_REPO_URL = "https://github.com/opentensor/subtensor.git" -SUBTENSOR_BRANCH = "abe/temp/logging-dirs-for-nodes" +SUBTENSOR_BRANCH = "cam/junius/feat-localnet-improve" RUST_INSTALLATION_VERSION = "nightly-2024-03-05" RUST_CHECK_VERSION = "rustc 1.78.0-nightly" RUST_TARGETS = [ @@ -28,10 +28,42 @@ "protobuf-compiler", ] - -WALLET_URIS = ["//Bob", "//Charlie"] +MINER_URIS = [ + "//Bob", + "//Charlie", + "//Dave", + "//Eve", + "//Ferdie", + "//Grace", + "//Tom", + "//Ivy", + "//Judy", + "//Jerry", + "//Harry", + "//Oscar", + "//Trent", + "//Victor", + "//Wendy", +] VALIDATOR_URI = "//Alice" -MINER_PORTS = [8101, 8102, 8103] +SUDO_URI = "//Alice" +MINER_PORTS = [ + 8101, + 8102, + 8103, + 8104, + 8105, + 8106, + 8107, + 8108, + 8109, + 8110, + 8111, + 8112, + 8113, + 8114, + 8115, +] VALIDATOR_PORT = 8100 LOCALNET_ENDPOINT = "ws://127.0.0.1:9945" diff --git a/btqs/utils.py b/btqs/utils.py index e2263c90..71f5b8cc 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -22,7 +22,7 @@ from typer.testing import CliRunner from .config import ( - CONFIG_FILE_PATH, + BTQS_LOCK_CONFIG_FILE_PATH, VALIDATOR_PORT, LOCALNET_ENDPOINT, DEFAULT_MINER_COMMAND, @@ -36,6 +36,20 @@ console = Console() +messages = [ + "πŸ”§ [cyan]Polishing the blocks[/cyan]", + "πŸ›’οΈ [yellow]Oiling up the chain[/yellow]", + "βš™οΈ [green]Calibrating gears[/green]", + "πŸ”Œ [magenta]Plugging in nodes[/magenta]", + "🧰 [blue]Gathering tools[/blue]", + "πŸ“° [cyan]Mapping the network[/cyan]", + "πŸ”­ [yellow]Aligning dependencies[/yellow]", + "πŸ§ͺ [green]Mixing tempos[/green]", + "πŸ“‘ [magenta]Tuning frequencies[/magenta]", + "πŸš€ [red]Preparing for launch[/red]", + "πŸ™Œ [dark_orange]Bittensor on 3! 1, 2, 3!", +] + def print_info(message: str, emoji: str = "", style: str = "yellow"): styled_text = Text(f"{emoji} {message}", style=style) @@ -60,7 +74,7 @@ def print_error(message: str, emoji: str = "❌"): def print_step(title: str, description: str, step_number: int): panel = Panel.fit( Align.left(f"[bold]{step_number}. {title}[/bold]\n{description}"), - title=f"Step {step_number}", + title=f"[dark_orange]Step {step_number}", border_style="bright_black", padding=(1, 2), ) @@ -96,7 +110,7 @@ def load_config( Raises: typer.Exit: If the config file is not found and exit_if_missing is True. """ - if not os.path.exists(CONFIG_FILE_PATH): + if not os.path.exists(BTQS_LOCK_CONFIG_FILE_PATH): if exit_if_missing: if error_message: print_error(f"{error_message}") @@ -105,7 +119,7 @@ def load_config( raise typer.Exit() else: return {} - with open(CONFIG_FILE_PATH, "r") as config_file: + with open(BTQS_LOCK_CONFIG_FILE_PATH, "r") as config_file: config_data = yaml.safe_load(config_file) or {} return config_data @@ -163,9 +177,6 @@ def is_rust_installed(required_version: str) -> bool: text=True, ) installed_version = result.stdout.strip().split()[1] - print_info( - installed_version, required_version, result.stdout.strip().split()[1] - ) return installed_version == required_version except Exception: return False @@ -198,12 +209,14 @@ def create_virtualenv(venv_path: str) -> str: return venv_python -def install_subtensor_dependencies() -> None: +def install_subtensor_dependencies(verbose) -> None: """ Installs subtensor dependencies, including system-level dependencies and Rust. """ print_info("Installing subtensor system dependencies...", emoji="βš™οΈ ") + stdout = None if verbose else subprocess.DEVNULL + # Install required system dependencies missing_packages = [] if platform.system() == "Linux": @@ -215,9 +228,17 @@ def install_subtensor_dependencies() -> None: console.print( f"[yellow]Installing missing system packages: {', '.join(missing_packages)}" ) - subprocess.run(["sudo", "apt-get", "update"], check=True) subprocess.run( - ["sudo", "apt-get", "install", "-y"] + missing_packages, check=True + ["sudo", "apt-get", "update"], + check=True, + stdout=stdout, + stderr=stdout, + ) + subprocess.run( + ["sudo", "apt-get", "install", "-y"] + missing_packages, + check=True, + stdout=stdout, + stderr=stdout, ) else: print_info("All required system packages are already installed.") @@ -232,8 +253,18 @@ def install_subtensor_dependencies() -> None: console.print( f"[yellow]Installing missing macOS system packages: {', '.join(missing_packages)}" ) - subprocess.run(["brew", "update"], check=True) - subprocess.run(["brew", "install"] + missing_packages, check=True) + subprocess.run( + ["brew", "update"], + check=True, + stdout=stdout, + stderr=stdout, + ) + subprocess.run( + ["brew", "install"] + missing_packages, + check=True, + stdout=stdout, + stderr=stdout, + ) else: print_info( "All required macOS system packages are already installed.", emoji="πŸ””" @@ -262,10 +293,14 @@ def install_subtensor_dependencies() -> None: "rustup.sh", ], check=True, + stdout=stdout, + stderr=stdout, ) subprocess.run( ["sh", "rustup.sh", "-y", "--default-toolchain", RUST_INSTALLATION_VERSION], check=True, + stdout=stdout, + stderr=stdout, ) else: console.print( @@ -275,12 +310,17 @@ def install_subtensor_dependencies() -> None: # Add necessary Rust targets print_info("Configuring Rust toolchain...", emoji="πŸ› οΈ ") for target in RUST_TARGETS: - subprocess.run(target, check=True) + subprocess.run( + target, + check=True, + stdout=stdout, + stderr=stdout, + ) - print_info("Subtensor dependencies installed.", emoji="🧰 ") + print_info("Subtensor dependencies installed.\n", emoji="🧰 ") -def install_neuron_dependencies(venv_python: str, cwd: str) -> None: +def install_neuron_dependencies(venv_python: str, cwd: str, verbose: bool) -> None: """ Installs neuron dependencies into the virtual environment. @@ -288,14 +328,23 @@ def install_neuron_dependencies(venv_python: str, cwd: str) -> None: venv_python (str): Path to the Python executable in the virtual environment. cwd (str): Current working directory where the setup should run. """ - print_info("Installing neuron dependencies...", emoji="βš™οΈ ") + stdout = None if verbose else subprocess.DEVNULL + print_info("Installing neuron dependencies...", emoji="βš™οΈ ", style="bold green") subprocess.run( - [venv_python, "-m", "pip", "install", "--upgrade", "pip"], cwd=cwd, check=True + [venv_python, "-m", "pip", "install", "--upgrade", "pip"], + cwd=cwd, + check=True, + stdout=stdout, + stderr=stdout, ) subprocess.run( - [venv_python, "-m", "pip", "install", "-e", "."], cwd=cwd, check=True + [venv_python, "-m", "pip", "install", "-e", "."], + cwd=cwd, + check=True, + stdout=stdout, + stderr=stdout, ) - print_info("Neuron dependencies installed.", emoji="πŸ–₯️ ") + print_info("Neuron dependencies installed.", emoji="πŸ–₯️ ", style="bold green") def remove_ansi_escape_sequences(text: str) -> str: @@ -368,7 +417,7 @@ def exec_command( return result -def is_chain_running(config_file_path: str = CONFIG_FILE_PATH) -> bool: +def is_chain_running(config_file_path: str = BTQS_LOCK_CONFIG_FILE_PATH) -> bool: """ Checks if the local chain is running by verifying the PID in the config file. @@ -814,6 +863,7 @@ def start_miner( subnet_template_path: str, config_data: Dict[str, Any], venv_python: str, + verbose=False, ) -> bool: """ Starts a single miner and displays logs until user presses Ctrl+C. @@ -823,8 +873,6 @@ def start_miner( name=wallet_name, hotkey=wallet_info["hotkey"], ) - print_info(f"Starting miner: {wallet}...\nPress any key to continue.", emoji="🏁 ") - input() miner_command = get_miner_command(wallet_name, config_data) base_args = [ @@ -853,6 +901,11 @@ def start_miner( with open(log_file_path, "a") as log_file: try: + if verbose: + console.print( + "🏁 [bold yellow]Starting a miner...Press any key to start a miner [bold dark_orange]and wait for 3 to 5 seconds and press Ctrl + C to start the next neuron.\n" + ) + input() process = subprocess.Popen( cmd, cwd=subnet_template_path, @@ -866,7 +919,12 @@ def start_miner( # Update config_data config_data["Miners"][wallet_name] = wallet_info - attach_to_process_logs(log_file_path, f"Miner {wallet_name}", process.pid) + if verbose: + attach_to_process_logs( + log_file_path, f"Miner {wallet_name}", process.pid + ) + else: + console.print("[green]🎬 Started miner process!\n") return True except Exception as e: console.print(f"[red]Error starting miner {wallet_name}: {e}") @@ -878,6 +936,7 @@ def start_validator( subnet_template_path: str, config_data: Dict[str, Any], venv_python: str, + verbose=False, ) -> bool: """ Starts the validator process and displays logs until user presses Ctrl+C. @@ -887,8 +946,6 @@ def start_validator( name=owner_info["wallet_name"], hotkey=owner_info["hotkey"], ) - print_info("Starting validator...\nPress any key to continue.", emoji="🏁 ") - input() validator_command = get_validator_command(config_data) base_args = [ @@ -913,6 +970,12 @@ def start_validator( os.makedirs(logs_dir, exist_ok=True) log_file_path = os.path.join(logs_dir, "validator.log") + if verbose: + print_info( + "Starting a validator...\nAfter the validator starts press Ctrl + C to start the next neuron.\nPress any key to continue...", + emoji="🏁 ", + ) + input() with open(log_file_path, "a") as log_file: try: process = subprocess.Popen( @@ -928,8 +991,10 @@ def start_validator( # Update config_data config_data["Owner"] = owner_info - - attach_to_process_logs(log_file_path, "Validator", process.pid) + if verbose: + attach_to_process_logs(log_file_path, "Validator", process.pid) + else: + console.print("[green]πŸš“ Started validator process!\n") return True except Exception as e: console.print(f"[red]Error starting validator: {e}") From cd4663de6db629981abcf6279bf38707ddf4b6bd Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 18 Oct 2024 03:37:09 -0700 Subject: [PATCH 19/23] 4 am --- btqs/commands/chain.py | 8 ++++---- btqs/commands/neurons.py | 19 ++++++++++--------- btqs/config.py | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index f0310f2c..0d7225d4 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -226,21 +226,21 @@ def start(config_data, workspace_path, branch, fast_blocks=True, verbose=False): origin = repo.remotes.origin repo.git.checkout(branch) origin.pull() - print_info("Repository updated successfully.", emoji="πŸ“¦") + print_info("Repository updated successfully.", emoji="πŸ“¦ ") except GitCommandError as e: print_error(f"Error updating repository: {e}") return else: print_info( - "Using existing subtensor repository without updating.", emoji="πŸ“¦" + "Using existing subtensor repository without updating.", emoji="πŸ“¦ " ) else: try: - print_info("Cloning subtensor repository...", emoji="πŸ“¦") + print_info("Cloning subtensor repository...", emoji="πŸ“¦ ") repo = Repo.clone_from(SUBTENSOR_REPO_URL, subtensor_path) if branch: repo.git.checkout(branch) - print_success("Repository cloned successfully.", emoji="🏷") + print_success("Repository cloned successfully.", emoji="🏷 ") except GitCommandError as e: print_error(f"Error cloning repository: {e}") return diff --git a/btqs/commands/neurons.py b/btqs/commands/neurons.py index 2e6ec82a..0c74f04e 100644 --- a/btqs/commands/neurons.py +++ b/btqs/commands/neurons.py @@ -61,15 +61,16 @@ def setup_neurons(config_data): else: _create_miner_wallets(config_data) - print_info("Preparing for miner registrations. Please wait...\n", emoji="⏱️ ") - local_chain = AsyncSubstrateInterface(chain_endpoint=LOCALNET_ENDPOINT) - success = asyncio.run( - sudo_set_target_registrations_per_interval(local_chain, owner_wallet, 1, 1000) - ) - if success: - print_info("Proceeding with the miner registrations.\n", emoji="πŸ›£οΈ ") - else: - print_warning("All neurons might not be able to register at once.") + # In-case target regs are not correct + # print_info("Preparing for miner registrations. Please wait...\n", emoji="⏱️ ") + # local_chain = AsyncSubstrateInterface(chain_endpoint=LOCALNET_ENDPOINT) + # success = asyncio.run( + # sudo_set_target_registrations_per_interval(local_chain, owner_wallet, 1, 1000) + # ) + # if success: + # print_info("Proceeding with the miner registrations.\n", emoji="πŸ›£οΈ ") + # else: + # print_warning("All neurons might not be able to register at once.") _register_miners(config_data) diff --git a/btqs/config.py b/btqs/config.py index 85af78f8..aad2abd4 100644 --- a/btqs/config.py +++ b/btqs/config.py @@ -11,7 +11,7 @@ DEFAULT_VALIDATOR_COMMAND = "./neurons/validator.py" SUBTENSOR_REPO_URL = "https://github.com/opentensor/subtensor.git" -SUBTENSOR_BRANCH = "cam/junius/feat-localnet-improve" +SUBTENSOR_BRANCH = "junius/feat-localnet-improve" RUST_INSTALLATION_VERSION = "nightly-2024-03-05" RUST_CHECK_VERSION = "rustc 1.78.0-nightly" RUST_TARGETS = [ From 2c87141a162f6a91fdb5ce354d3282f96a818bbb Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 18 Oct 2024 09:28:55 -0700 Subject: [PATCH 20/23] Added option to skip rust deps --- btqs/btqs_cli.py | 10 ++++++++++ btqs/commands/chain.py | 19 +++++++++++++------ btqs/utils.py | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/btqs/btqs_cli.py b/btqs/btqs_cli.py index a2ba7940..90b47f2e 100644 --- a/btqs/btqs_cli.py +++ b/btqs/btqs_cli.py @@ -89,6 +89,7 @@ def __init__(self): self.fast_blocks = True self.workspace_path = None self.subtensor_branch = None + self.skip_rust = None self.steps = [ { "title": "Start Local Subtensor", @@ -100,6 +101,7 @@ def __init__(self): branch=self.subtensor_branch, fast_blocks=self.fast_blocks, verbose=self.verbose, + skip_rust=self.skip_rust, ), }, { @@ -142,6 +144,9 @@ def start_chain( verbose: bool = typer.Option( False, "--verbose", "-v", help="Enable verbose output" ), + skip_rust: bool = typer.Option( + False, "--skip-rust", help="Skip Rust installation" + ), ): """ Starts the local Subtensor chain. @@ -187,6 +192,7 @@ def start_chain( branch, fast_blocks=fast_blocks, verbose=verbose, + skip_rust=skip_rust, ) def stop_chain(self): @@ -237,6 +243,9 @@ def run_all( verbose: bool = typer.Option( False, "--verbose", "-v", help="Enable verbose output" ), + skip_rust: bool = typer.Option( + False, "--skip-rust", help="Skip Rust installation" + ), ): """ Runs all commands in sequence to set up and start the local chain, subnet, and neurons. @@ -245,6 +254,7 @@ def run_all( self.workspace_path = workspace_path self.fast_blocks = fast_blocks self.verbose = verbose + self.skip_rust = skip_rust console.clear() print_info("Welcome to the Bittensor Quick Start Tutorial", emoji="πŸš€") diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 0d7225d4..68db90b6 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -1,3 +1,4 @@ +import sys import os import subprocess import time @@ -209,7 +210,7 @@ async def sudo_set_tempo( return await response.is_success -def start(config_data, workspace_path, branch, fast_blocks=True, verbose=False): +def start(config_data, workspace_path, branch, fast_blocks=True, verbose=False, skip_rust=False): os.makedirs(workspace_path, exist_ok=True) subtensor_path = os.path.join(workspace_path, "subtensor") @@ -250,11 +251,17 @@ def start(config_data, workspace_path, branch, fast_blocks=True, verbose=False): else: print_info("Fast blocks are Off", emoji="🐌 ") - venv_subtensor_path = os.path.join(workspace_path, "venv_subtensor") - venv_python = create_virtualenv(venv_subtensor_path) - install_subtensor_dependencies(verbose) - - config_data["venv_subtensor"] = venv_python + if skip_rust: + venv_python = sys.executable + print_info("Skipping Rust installation", emoji="🦘 ") + config_data["venv_subtensor"] = "None" + venv_subtensor_path = "None" + else: + venv_subtensor_path = os.path.join(workspace_path, "venv_subtensor") + venv_python = create_virtualenv(venv_subtensor_path) + install_subtensor_dependencies(verbose) + print_info("Virtual environment created and dependencies installed.", emoji="🐍 ") + config_data["venv_subtensor"] = venv_python # Running localnet.sh using the virtual environment's Python localnet_path = os.path.join(subtensor_path, "scripts", "localnet.sh") diff --git a/btqs/utils.py b/btqs/utils.py index 71f5b8cc..cd4e7396 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -209,7 +209,7 @@ def create_virtualenv(venv_path: str) -> str: return venv_python -def install_subtensor_dependencies(verbose) -> None: +def install_subtensor_dependencies(verbose, skip_rust) -> None: """ Installs subtensor dependencies, including system-level dependencies and Rust. """ From 8f88701797c7b9a344213aaa57e0d2985abb4216 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 18 Oct 2024 09:42:34 -0700 Subject: [PATCH 21/23] Adds more retry patterns --- btqs/commands/subnet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index 16d73a6f..4b598563 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -149,12 +149,12 @@ def add_weights(config_data): hotkey=owner_data.get("hotkey"), ) print_info( - "Validator is now setting weights of subnet 1 on the root network.\n Please wait... (Timeout: 60 seconds)", + "Validator is now setting weights of subnet 1 on the root network.\n Please wait... (Timeout: ~ 120 seconds)", emoji="πŸ‹οΈ ", ) - max_retries = 30 + max_retries = 60 attempt = 0 - retry_patterns = ["ancient birth block", "Transaction has a bad signature"] + retry_patterns = ["ancient birth block", "Transaction has a bad signature", "SettingWeightsTooFast"] while attempt < max_retries: try: From 72d15a07aeb32d17d7ac13d46d81c820454e531a Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Fri, 18 Oct 2024 09:45:26 -0700 Subject: [PATCH 22/23] Fixes extra req --- btqs/commands/subnet.py | 6 +++++- btqs/utils.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/btqs/commands/subnet.py b/btqs/commands/subnet.py index 4b598563..9de3ca86 100644 --- a/btqs/commands/subnet.py +++ b/btqs/commands/subnet.py @@ -154,7 +154,11 @@ def add_weights(config_data): ) max_retries = 60 attempt = 0 - retry_patterns = ["ancient birth block", "Transaction has a bad signature", "SettingWeightsTooFast"] + retry_patterns = [ + "ancient birth block", + "Transaction has a bad signature", + "SettingWeightsTooFast", + ] while attempt < max_retries: try: diff --git a/btqs/utils.py b/btqs/utils.py index cd4e7396..71f5b8cc 100644 --- a/btqs/utils.py +++ b/btqs/utils.py @@ -209,7 +209,7 @@ def create_virtualenv(venv_path: str) -> str: return venv_python -def install_subtensor_dependencies(verbose, skip_rust) -> None: +def install_subtensor_dependencies(verbose) -> None: """ Installs subtensor dependencies, including system-level dependencies and Rust. """ From 85d3b65f2c909eef1c3964862064b61db0d2e364 Mon Sep 17 00:00:00 2001 From: ibraheem-opentensor Date: Tue, 22 Oct 2024 13:10:50 -0700 Subject: [PATCH 23/23] Increased timeout time --- btqs/commands/chain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btqs/commands/chain.py b/btqs/commands/chain.py index 68db90b6..304a881f 100644 --- a/btqs/commands/chain.py +++ b/btqs/commands/chain.py @@ -289,7 +289,7 @@ def start(config_data, workspace_path, branch, fast_blocks=True, verbose=False, alice_log = os.path.join(log_dir, "alice.log") # Waiting for chain compilation - timeout = 1200 # 17 minutes + timeout = 3000 start_time = time.time() while not os.path.exists(alice_log): if time.time() - start_time > timeout: