From 38abf7698f370a7d13027a60a444faa41a30a662 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sun, 18 May 2025 10:23:10 +0200 Subject: [PATCH 01/35] bootstrap charon launcher --- network_params.yaml | 23 +- src/package_io/constants.star | 1 + src/package_io/input_parser.star | 11 + src/package_io/sanity_check.star | 6 + src/participant_network.star | 49 ++ .../validator_keystores/keystore_files.star | 4 +- .../validator_keystore_generator.star | 23 + src/vc/charon.star | 431 ++++++++++++++++++ src/vc/charon_launcher.star | 288 ++++++++++++ src/vc/vc_launcher.star | 1 + 10 files changed, 826 insertions(+), 11 deletions(-) create mode 100644 src/vc/charon.star create mode 100644 src/vc/charon_launcher.star diff --git a/network_params.yaml b/network_params.yaml index e2b8539df..c6e1b5d87 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -1,5 +1,5 @@ participants: -# EL + # EL - el_type: geth el_image: ethereum/client-go:latest el_binary_path: "" @@ -15,7 +15,7 @@ participants: el_max_cpu: 0 el_min_mem: 0 el_max_mem: 0 -# CL + # CL cl_type: lighthouse cl_image: sigp/lighthouse:latest cl_binary_path: "" @@ -33,10 +33,12 @@ participants: cl_max_mem: 0 supernode: false use_separate_vc: true -# Validator - vc_type: lighthouse - vc_image: sigp/lighthouse:latest - vc_binary_path: "" + # Validator + # vc_type: lighthouse + # vc_image: sigp/lighthouse:latest + # vc_binary_path: "" + vc_type: charon + vc_image: obolnetwork/charon:latest vc_log_level: "" vc_extra_env_vars: {} vc_extra_labels: {} @@ -50,7 +52,9 @@ participants: vc_max_mem: 0 validator_count: null use_remote_signer: false -# Remote signer + charon_node_count: 4 + charon_validator_client: lighthouse + # Remote signer remote_signer_type: web3signer remote_signer_image: consensys/web3signer:latest remote_signer_extra_env_vars: {} @@ -61,7 +65,7 @@ participants: remote_signer_max_cpu: 0 remote_signer_min_mem: 0 remote_signer_max_mem: 0 -# participant specific + # participant specific node_selectors: {} tolerations: [] count: 1 @@ -82,8 +86,7 @@ network_params: seconds_per_slot: 12 slot_duration_ms: 12000 num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: - "giant issue aisle success illegal bike spike + preregistered_validator_keys_mnemonic: "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" preregistered_validator_count: 0 diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 71edfb5df..58213e34f 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -31,6 +31,7 @@ VC_TYPE = struct( vero="vero", grandine="grandine", consensoor="consensoor", + charon="charon", ) REMOTE_SIGNER_TYPE = struct(web3signer="web3signer") diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index bd9a99fb8..ff33c711a 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -48,6 +48,7 @@ DEFAULT_VC_IMAGES = { "grandine": "sifrai/grandine:stable", "vero": "ghcr.io/serenita-org/vero:latest", "consensoor": "ethpandaops/consensoor:main", + "charon": "obolnetwork/charon:latest", } DEFAULT_VC_IMAGES_MINIMAL = { @@ -59,6 +60,7 @@ DEFAULT_VC_IMAGES_MINIMAL = { "grandine": "ethpandaops/grandine:develop-minimal", "vero": "ghcr.io/serenita-org/vero:latest", "consensoor": "ethpandaops/consensoor:main", + "charon": "obolnetwork/charon:latest", } DEFAULT_REMOTE_SIGNER_IMAGES = { @@ -799,6 +801,9 @@ def input_parser(plan, input_args): vc_beacon_node_indices=participant["vc_beacon_node_indices"], checkpoint_sync_enabled=participant["checkpoint_sync_enabled"], skip_start=participant["skip_start"], + # Charon-specific parameters + charon_node_count=participant["charon_node_count"], + charon_validator_client=participant["charon_validator_client"], ) for participant in result["participants"] ], @@ -2020,6 +2025,9 @@ def default_participant(): "vc_min_cpu": 0, "vc_max_cpu": 0, "vc_min_mem": 0, + # Charon-specific parameters + "charon_node_count": 3, + "charon_validator_client": "lighthouse", "vc_max_mem": 0, "vc_force_restart": False, "use_remote_signer": None, @@ -2563,6 +2571,9 @@ def enrich_mev_extra_params(parsed_arguments_dict, mev_prefix, mev_port, mev_typ if participant["vc_type"] == "vero": participant["vc_extra_params"].append("--use-external-builder") + if participant["vc_type"] == "charon": + participant["vc_extra_params"].append("--builder-api=true") + num_participants = len(parsed_arguments_dict["participants"]) index_str = shared_utils.zfill_custom( num_participants + 1, len(str(num_participants + 1)) diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 547fbaf76..b69c44c89 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -55,6 +55,9 @@ PARTICIPANT_CATEGORIES = { "remote_signer_type", "remote_signer_image", "remote_signer_extra_env_vars", + # Charon-specific parameters + "charon_node_count", + "charon_validator_client", "remote_signer_extra_labels", "remote_signer_extra_params", "remote_signer_tolerations", @@ -156,6 +159,9 @@ PARTICIPANT_MATRIX_PARAMS = { "vc_max_mem", "vc_force_restart", "validator_count", + # Charon-specific parameters + "charon_node_count", + "charon_validator_client", ], "remote_signer": [ "remote_signer_type", diff --git a/src/participant_network.star b/src/participant_network.star index e47e41829..34dda2ec5 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -27,6 +27,7 @@ vc = import_module("./vc/vc_launcher.star") vc_shared = import_module("./vc/shared.star") vc_context_l = import_module("./vc/vc_context.star") node_metrics = import_module("./node_metrics_info.star") +charon_launcher = import_module("./vc/charon_launcher.star") remote_signer = import_module("./remote_signer/remote_signer_launcher.star") beacon_snooper = import_module("./snooper/snooper_beacon_launcher.star") @@ -584,6 +585,54 @@ def launch_participant_network( if vc_service_config == None: continue + # Use the charon_launcher for Charon validator clients + if vc_type == constants.VC_TYPE.charon: + vc_context = charon_launcher.launch( + plan=plan, + launcher=charon_launcher.new_charon_launcher(el_cl_genesis_data=el_cl_data, jwt_file=jwt_file), + keymanager_file=keymanager_file, + service_name="vc-{0}".format(full_name), + image=participant.vc_image, + global_log_level=args_with_right_defaults.global_log_level, + cl_context=cl_context, + el_context=el_context, + full_name=full_name, + node_keystore_files=vc_keystores, + participant=participant, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + network_params=network_params, + port_publisher=args_with_right_defaults.port_publisher, + vc_index=current_vc_index, + genesis_timestamp=final_genesis_timestamp, + ) + else: + vc_context = vc.launch( + plan=plan, + launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data), + keymanager_file=keymanager_file, + service_name="vc-{0}".format(full_name), + vc_type=vc_type, + image=participant.vc_image, + global_log_level=args_with_right_defaults.global_log_level, + cl_context=cl_context, + el_context=el_context, + remote_signer_context=remote_signer_context, + full_name=full_name, + snooper_enabled=participant.snooper_enabled, + snooper_beacon_context=snooper_beacon_context, + node_keystore_files=vc_keystores, + participant=participant, + prysm_password_relative_filepath=prysm_password_relative_filepath, + prysm_password_artifact_uuid=prysm_password_artifact_uuid, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + network_params=network_params, + port_publisher=args_with_right_defaults.port_publisher, + vc_index=current_vc_index, + ) + all_vc_contexts.append(vc_context) + vc_service_configs[service_name] = vc_service_config vc_service_info[service_name] = { "client_name": vc_type, diff --git a/src/prelaunch_data_generator/validator_keystores/keystore_files.star b/src/prelaunch_data_generator/validator_keystores/keystore_files.star index d10f359b4..c7fba4dca 100644 --- a/src/prelaunch_data_generator/validator_keystores/keystore_files.star +++ b/src/prelaunch_data_generator/validator_keystores/keystore_files.star @@ -8,6 +8,7 @@ def new_keystore_files( prysm_relative_dirpath, teku_keys_relative_dirpath, teku_secrets_relative_dirpath, + raw_keys_secrets_relative_dirpath, ): return struct( files_artifact_uuid=files_artifact_uuid, @@ -16,7 +17,8 @@ def new_keystore_files( raw_keys_relative_dirpath=raw_keys_relative_dirpath, raw_secrets_relative_dirpath=raw_secrets_relative_dirpath, nimbus_keys_relative_dirpath=nimbus_keys_relative_dirpath, - prysm_relative_dirpath=prysm_relative_dirpath, + prysm_relative_dirpath=prysm_relative_dirpath, teku_keys_relative_dirpath=teku_keys_relative_dirpath, teku_secrets_relative_dirpath=teku_secrets_relative_dirpath, + raw_keys_secrets_relative_dirpath=raw_keys_secrets_relative_dirpath, ) diff --git a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star index 8bc8d6d08..fb95b2630 100644 --- a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star +++ b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star @@ -17,6 +17,7 @@ SUCCESSFUL_EXEC_CMD_EXIT_CODE = 0 RAW_KEYS_DIRNAME = "keys" RAW_SECRETS_DIRNAME = "secrets" +RAW_KEYS_SECRETS_DIRNAME = "raw-keys-secrets" NIMBUS_KEYS_DIRNAME = "nimbus-keys" PRYSM_DIRNAME = "prysm" @@ -139,6 +140,25 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para running_total_validator_count += participant.validator_count + # add another folder which contains all the raw keys and secret + # create the folder + all_output_dirpaths.append(output_dirpath + RAW_KEYS_SECRETS_DIRNAME) + # copy + all_sub_command_strs.append( + "cp -r " + + output_dirpath + + "keys/ " + + output_dirpath + + "raw-keys-secrets" + ) + all_sub_command_strs.append( + "cp -r " + + output_dirpath + + "secrets/ " + + output_dirpath + + "raw-keys-secrets" + ) + command_str = " && ".join(all_sub_command_strs) command_result = plan.exec( @@ -169,6 +189,8 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para keystore_start_index, keystore_stop_index - 1, ) + + # copt the keys and secrets into a new directory and store the artifact artifact_name = plan.store_service_files( service_name, output_dirpath, @@ -186,6 +208,7 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), + shared_utils.path_join(base_dirname_in_artifact, RAW_KEYS_SECRETS_DIRNAME), ) keystore_files.append(to_add) diff --git a/src/vc/charon.star b/src/vc/charon.star new file mode 100644 index 000000000..913ecbf62 --- /dev/null +++ b/src/vc/charon.star @@ -0,0 +1,431 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +input_parser = import_module("../package_io/input_parser.star") +constants = import_module("../package_io/constants.star") +cl_context = import_module("../cl/cl_context.star") +vc_shared = import_module("./shared.star") + +# Charon specific ports +CHARON_VALIDATOR_API_PORT = 3600 +CHARON_P2P_TCP_PORT = 3610 +CHARON_MONITORING_PORT = 3620 +CHARON_METRICS_PORT = 8080 + +# Default Charon image +DEFAULT_CHARON_IMAGE = "obolnetwork/charon:latest" + +# Verbosity levels mapping +VERBOSITY_LEVELS = { + constants.GLOBAL_LOG_LEVEL.error: "error", + constants.GLOBAL_LOG_LEVEL.warn: "warn", + constants.GLOBAL_LOG_LEVEL.info: "info", + constants.GLOBAL_LOG_LEVEL.debug: "debug", + constants.GLOBAL_LOG_LEVEL.trace: "trace", +} + +def launch( + plan, + participant, + participant_index, + cl_context, + el_cl_genesis_data, + node_keystore_files, + global_node_selectors, + docker_cache_params, +): + """ + Launches a Charon distributed validator setup + """ + image = shared_utils.get_client_image( + participant.vc_type, + participant.vc_image, + DEFAULT_CHARON_IMAGE, + docker_cache_params, + ) + + # Get the number of Charon nodes to create (default to 3) + charon_node_count = 3 + if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: + charon_node_count = participant.charon_node_count + + # Get the beacon node endpoints for each Charon node + beacon_endpoints = [] + for i in range(charon_node_count): + # Use the same beacon node for all Charon nodes if we don't have enough participants + beacon_index = i % len(cl_context.all_beacon_http_urls) + beacon_endpoints.append(cl_context.all_beacon_http_urls[beacon_index]) + + # Get the genesis timestamp from the beacon node + genesis_timestamp = plan.exec( + service_name=cl_context.service_name, + recipe=ExecRecipe( + command=["curl", "-s", cl_context.beacon_http_url + "/eth/v1/beacon/genesis"], + ), + ) + genesis_time = plan.extract_from_json_result(genesis_timestamp, ".data.genesis_time") + + # Create a temporary directory for Charon cluster files + charon_cluster_dir = plan.store_service_files( + service_name=cl_context.service_name, + files={}, + ) + + # Create Charon cluster + cluster_creation = plan.exec( + service_name=cl_context.service_name, + recipe=ExecRecipe( + command=[ + "docker", "run", "--rm", + "-v", charon_cluster_dir + ":/opt/charon", + image, + "create", "cluster", + "--name=test", + "--nodes=" + str(charon_node_count), + "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", + "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", + "--split-existing-keys", + "--split-keys-dir=/opt/charon/validator_keys", + "--testnet-chain-id=3151908", + "--testnet-fork-version=0x10000038", + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=kurtosis-testnet", + ], + ), + ) + + # Launch Charon nodes + charon_services = [] + for i in range(charon_node_count): + node_name = participant.name + "-charon-" + str(i) + + cmd = [ + "run", + "--testnet-chain-id=3151908", + "--testnet-fork-version=0x10000038", + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=testnet", + "--testnet-capella-hard-fork=0x40000038", + ] + + if len(participant.vc_extra_params) > 0: + cmd.extend([param for param in participant.vc_extra_params]) + + env_vars = { + "CHARON_LOG_LEVEL": "debug", + "CHARON_LOG_FORMAT": "console", + "CHARON_P2P_RELAYS": "https://0.relay.obol.tech", + "CHARON_BUILDER_API": "true", + "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), + "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), + "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), + "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/charon-enr-private-key", + "CHARON_LOCK_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/cluster-lock.json", + "CHARON_JAEGER_SERVICE": "node" + str(i), + "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), + "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoints[i], + } + + # Add any extra environment variables + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + ports = { + "validator-api": shared_utils.new_port_spec( + CHARON_VALIDATOR_API_PORT, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + "p2p-tcp": shared_utils.new_port_spec( + CHARON_P2P_TCP_PORT, + shared_utils.TCP_PROTOCOL, + shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, + ), + "monitoring": shared_utils.new_port_spec( + CHARON_MONITORING_PORT, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + } + + files = { + "/opt/charon/.charon": charon_cluster_dir, + } + + charon_service = plan.add_service( + name=node_name, + config=ServiceConfig( + image=image, + ports=ports, + cmd=cmd, + env_vars=env_vars, + files=files, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.charon, + client_type=constants.CLIENT_TYPES.validator, + image=image[-constants.MAX_LABEL_LENGTH:], + connected_client=cl_context.client_name, + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + node_selectors=global_node_selectors, + ), + ) + charon_services.append(charon_service) + + # Now launch the validator clients that will connect to Charon nodes + vc_services = [] + for i in range(charon_node_count): + # Determine which validator client to use with Charon + vc_type = "lighthouse" # Default + if hasattr(participant, "charon_validator_client"): + vc_type = participant.charon_validator_client + + # Import the appropriate validator client module + vc_module = import_module("../vc/" + vc_type + ".star") + + # Create a modified participant for the validator client + vc_participant = struct( + name=participant.name + "-vc-" + str(i), + vc_type=vc_type, + vc_image=participant.vc_image if hasattr(participant, "vc_image") else None, + vc_extra_params=participant.vc_extra_params if hasattr(participant, "vc_extra_params") else [], + vc_extra_env_vars=participant.vc_extra_env_vars if hasattr(participant, "vc_extra_env_vars") else {}, + vc_extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ) + + # Create a modified CL context that points to the Charon node + charon_cl_context = struct( + service_name=charon_services[i].name, + ip_addr=charon_services[i].ip_address, + beacon_http_url="http://" + charon_services[i].ip_address + ":" + str(CHARON_VALIDATOR_API_PORT), + client_name=cl_context.client_name, + all_beacon_http_urls=cl_context.all_beacon_http_urls, + ) + + # Launch the validator client + vc_service = vc_module.launch( + plan=plan, + participant=vc_participant, + participant_index=i, + cl_context=charon_cl_context, + el_cl_genesis_data=el_cl_genesis_data, + node_keystore_files=node_keystore_files, + global_node_selectors=global_node_selectors, + docker_cache_params=docker_cache_params, + ) + vc_services.append(vc_service) + + # Return the first Charon service as the main service + return charon_services[0] + +def get_config( + participant, + el_cl_genesis_data, + image, + global_log_level, + beacon_http_url, + cl_context, + el_context, + full_name, + node_keystore_files, + tolerations, + node_selectors, + keymanager_enabled, + network_params, + port_publisher, + vc_index, +): + """ + Get the configuration for a Charon distributed validator client + """ + log_level = input_parser.get_client_log_level_or_default( + participant.vc_log_level, global_log_level, VERBOSITY_LEVELS + ) + + # We need to get the genesis timestamp from the beacon node + # This will be done when the service is started, so we'll use a script to get it + # and pass it to the Charon command + + # Get the number of Charon nodes to create (default to 4) + charon_node_count = 4 + if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: + charon_node_count = participant.charon_node_count + + # Get the validator keys directory path + validator_keys_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.raw_keys_relative_dirpath, + ) + + # Get the validator secrets directory path + validator_secrets_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.raw_secrets_relative_dirpath, + ) + + # Determine the validator client to use with Charon + validator_client = "lighthouse" # Default + if hasattr(participant, "charon_validator_client"): + validator_client = participant.charon_validator_client + + # Create a startup script that will get the genesis timestamp from the beacon node and pass it to Charon + startup_script = """#!/bin/sh +# Get the genesis timestamp from the beacon node +GENESIS_TIME=$(curl -s %s/eth/v1/beacon/genesis | grep -o '"genesis_time":[0-9]*' | cut -d':' -f2) + +# Run Charon with the genesis timestamp +exec charon run \\ + --beacon-node-endpoints=%s \\ + --validator-api-address=0.0.0.0:%d \\ + --p2p-tcp-address=0.0.0.0:%d \\ + --monitoring-address=0.0.0.0:%d \\ + --log-level=%s \\ + --log-format=console \\ + --builder-api=true \\ + --feature-set=alpha \\ + --testnet-genesis-timestamp=$GENESIS_TIME \\ +""" % ( + beacon_http_url, + beacon_http_url, + CHARON_VALIDATOR_API_PORT, + CHARON_P2P_TCP_PORT, + CHARON_MONITORING_PORT, + log_level, + ) + + # Add network-specific parameters to the startup script + if network_params.network == constants.NETWORK_NAME.kurtosis: + startup_script += """ --testnet-chain-id=3151908 \\ + --testnet-fork-version=0x10000038 \\ + --testnet-name=kurtosis-testnet \\ +""" + elif network_params.network in constants.PUBLIC_NETWORKS: + startup_script += """ --network=%s \\ +""" % network_params.network + else: + # For other networks, use the kurtosis defaults + startup_script += """ --testnet-chain-id=3151908 \\ + --testnet-fork-version=0x10000038 \\ + --testnet-name=kurtosis-testnet \\ +""" + + # Add any extra parameters to the startup script + if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + for param in participant.vc_extra_params: + startup_script += " %s \\\n" % param + + # Remove the trailing backslash and newline + startup_script = startup_script.rstrip("\\\n") + + # Basic command for Charon - just run the startup script + cmd = [ + "/bin/sh", + "/opt/charon/startup.sh", + ] + + + + # Environment variables + env_vars = { + "CHARON_DISTRIBUTED_VALIDATOR_ENABLED": "true", + "CHARON_VALIDATOR_CLIENT": validator_client, + "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), + "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), + "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), + "CHARON_LOG_LEVEL": log_level, + "CHARON_LOG_FORMAT": "console", + "CHARON_VALIDATOR_KEYS_DIR": validator_keys_dirpath, + "CHARON_VALIDATOR_SECRETS_DIR": validator_secrets_dirpath, + "CHARON_BEACON_NODE_ENDPOINTS": beacon_http_url, + "CHARON_JAEGER_SERVICE": full_name, + "CHARON_CLUSTER_ID": full_name, + "CHARON_NODE_COUNT": str(charon_node_count), + "CHARON_FEE_RECIPIENT_ADDRESS": constants.VALIDATING_REWARDS_ACCOUNT, + } + + # Add any extra environment variables + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Store the startup script + # startup_script_artifact = plan.store_service_files( + # service_name=cl_context.service_name, + # files={ + # "startup.sh": startup_script, + # }, + # ) + + # Files to mount + # files = { + # constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + # constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + # "/opt/charon": startup_script_artifact, + # } + + # Ports configuration + ports = {} + ports.update(vc_shared.VALIDATOR_CLIENT_USED_PORTS) + + # Add Charon-specific ports + ports.update({ + "validator-api": shared_utils.new_port_spec( + CHARON_VALIDATOR_API_PORT, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + "p2p-tcp": shared_utils.new_port_spec( + CHARON_P2P_TCP_PORT, + shared_utils.TCP_PROTOCOL, + shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, + ), + "monitoring": shared_utils.new_port_spec( + CHARON_MONITORING_PORT, + shared_utils.TCP_PROTOCOL, + shared_utils.HTTP_APPLICATION_PROTOCOL, + ), + }) + + # Public ports + public_ports = {} + + # Add public port assignments if port publisher is enabled + if port_publisher != None and port_publisher.enabled: + public_validator_api_port_assignment = port_publisher.get_port_assignment( + "validator-api", vc_index + ) + public_p2p_tcp_port_assignment = port_publisher.get_port_assignment( + "p2p-tcp", vc_index + ) + public_monitoring_port_assignment = port_publisher.get_port_assignment( + "monitoring", vc_index + ) + + public_ports.update( + shared_utils.get_port_specs(public_validator_api_port_assignment) + ) + public_ports.update( + shared_utils.get_port_specs(public_p2p_tcp_port_assignment) + ) + public_ports.update( + shared_utils.get_port_specs(public_monitoring_port_assignment) + ) + + # Return the configuration + return { + "image": image, + "ports": ports, + "public_ports": public_ports, + "cmd": cmd, + "files": files, + "env_vars": env_vars, + "labels": shared_utils.label_maker( + client=constants.VC_TYPE.charon, + client_type=constants.CLIENT_TYPES.validator, + image=image[-constants.MAX_LABEL_LENGTH:], + connected_client=cl_context.client_name, + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + "tolerations": tolerations, + "node_selectors": node_selectors, + } diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star new file mode 100644 index 000000000..95f1b1a53 --- /dev/null +++ b/src/vc/charon_launcher.star @@ -0,0 +1,288 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +input_parser = import_module("../package_io/input_parser.star") +constants = import_module("../package_io/constants.star") +cl_context = import_module("../cl/cl_context.star") +vc_shared = import_module("./shared.star") +vc_context = import_module("./vc_context.star") +node_metrics = import_module("../node_metrics_info.star") + +# Charon specific ports +CHARON_VALIDATOR_API_PORT = 3600 +CHARON_P2P_TCP_PORT = 3610 +CHARON_MONITORING_PORT = 3620 +CHARON_METRICS_PORT = 8080 + +# Default Charon image +DEFAULT_CHARON_IMAGE = "obolnetwork/charon:latest" + +# Verbosity levels mapping +VERBOSITY_LEVELS = { + constants.GLOBAL_LOG_LEVEL.error: "error", + constants.GLOBAL_LOG_LEVEL.warn: "warn", + constants.GLOBAL_LOG_LEVEL.info: "info", + constants.GLOBAL_LOG_LEVEL.debug: "debug", +} + +def launch( + plan, + launcher, + keymanager_file, + service_name, + image, + global_log_level, + cl_context, + el_context, + full_name, + node_keystore_files, + participant, + global_tolerations, + node_selectors, + network_params, + port_publisher, + vc_index, + genesis_timestamp, +): + """ + Launch a Charon distributed validator client + """ + VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" + + if node_keystore_files == None: + return None + + tolerations = input_parser.get_client_tolerations( + participant.vc_tolerations, participant.tolerations, global_tolerations + ) + + log_level = input_parser.get_client_log_level_or_default( + participant.vc_log_level, global_log_level, VERBOSITY_LEVELS + ) + + # Get the number of Charon nodes to create (default to 4) + charon_node_count = 4 + if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: + charon_node_count = participant.charon_node_count + + # Get the beacon node endpoints for each Charon node + beacon_endpoints = [] + for i in range(charon_node_count): + # Just use the same beacon node for all Charon nodes + beacon_endpoints.append(cl_context.beacon_http_url) + + # Use the genesis timestamp passed from the participant_network + genesis_time = genesis_timestamp + + # Create a temporary directory for Charon cluster files + # charon_cluster_dir = plan.store_service_files( + # service_name=service_name + "-charon-cluster-files", + # files={}, + # ) + + validator_keys_dirpath = "" + if node_keystore_files: + validator_keys_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.raw_keys_secrets_relative_dirpath, + ) + # join the validator keys directory path with the validator keys dirpath on the service container + # validator_keys_dirpath = shared_utils.path_ + + # Create a temporary service to run the Charon cluster creation + temp_service = plan.add_service( + name=service_name + "-temp", + config=ServiceConfig( + image=image, + cmd=[ + "create", "cluster", + "--name=test", + "--nodes=" + str(charon_node_count), + "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", + "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", + "--split-existing-keys", + "--split-keys-dir=" + validator_keys_dirpath, + "--testnet-chain-id=3151908", + "--testnet-fork-version=0x10000038", + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=kurtosis-testnet", + ], + files={ + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + }, + ), + ) + + # Create a directory for Charon cluster files + # plan.exec( + # service_name=temp_service.name, + # recipe=ExecRecipe( + # command=["mkdir", "-p", "/opt/charon/validator_keys"], + # ), + # ) + + # Copy validator keys to the Charon cluster directory + # plan.exec( + # service_name=temp_service.name, + # recipe=ExecRecipe( + # command=[ + # "cp", + # "-r", + # constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER + "/*", + # "/opt/charon/validator_keys/", + # ], + # ), + # ) + + # Create Charon cluster + # plan.exec( + # service_name=temp_service.name, + # recipe=ExecRecipe( + # command=[ + # "charon", "create", "cluster", + # "--name=test", + # "--nodes=" + str(charon_node_count), + # "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", + # "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", + # "--split-existing-keys", + # "--split-keys-dir=" + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + # "--testnet-chain-id=3151908", + # "--testnet-fork-version=0x10000038", + # "--testnet-genesis-timestamp=" + str(genesis_time), + # "--testnet-name=kurtosis-testnet", + # ], + # ), + # ) + + # Store the Charon cluster files + # charon_cluster_files = plan.store_service_files( + # service_name=temp_service.name, + # files={ + # ".charon": "/opt/charon/.charon", + # }, + # ) + + # We'll create the validator keys directory and copy the keys in the Charon service itself + # For now, we'll just create an empty directory structure + + # We'll create a temporary service to run the Charon cluster creation + # For now, we'll skip this step and just create the Charon services directly + + # Launch Charon nodes + charon_services = [] + for i in range(charon_node_count): + node_name = service_name + "-charon-" + str(i) + + cmd = [ + "run", + "--testnet-chain-id=3151908", + "--testnet-fork-version=0x10000038", + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=testnet", + "--testnet-capella-hard-fork=0x40000038", + "--split-keys-dir=" + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + ] + + if len(participant.vc_extra_params) > 0: + cmd.extend([param for param in participant.vc_extra_params]) + + env_vars = { + "CHARON_LOG_LEVEL": "debug", + "CHARON_LOG_FORMAT": "console", + "CHARON_P2P_RELAYS": "https://0.relay.obol.tech", + "CHARON_BUILDER_API": "true", + "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), + "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), + "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), + "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/charon-enr-private-key", + "CHARON_LOCK_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/cluster-lock.json", + "CHARON_JAEGER_SERVICE": "node" + str(i), + "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), + "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoints[i], + } + + # Add any extra environment variables + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Files to mount + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + } + + # Ports configuration + ports = { + "validator-api": PortSpec( + number=CHARON_VALIDATOR_API_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + "p2p-tcp": PortSpec( + number=CHARON_P2P_TCP_PORT, + transport_protocol="TCP", + ), + "monitoring": PortSpec( + number=CHARON_MONITORING_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + constants.METRICS_PORT_ID: PortSpec( + number=CHARON_METRICS_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Add the service + charon_service = plan.add_service( + name=node_name, + config=ServiceConfig( + image=image, + ports=ports, + cmd=cmd, + env_vars=env_vars, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.charon, + client_type=constants.CLIENT_TYPES.validator, + image=image[-constants.MAX_LABEL_LENGTH:], + connected_client=cl_context.client_name, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ), + ) + charon_services.append(charon_service) + + # Now launch the validator clients that will connect to Charon nodes + vc_services = [] + for i in range(charon_node_count): + # Determine which validator client to use with Charon + vc_type = "lighthouse" # Default + if hasattr(participant, "charon_validator_client"): + vc_type = participant.charon_validator_client + + # For now, we'll skip launching the validator clients + # In a real implementation, we would need to launch validator clients that connect to the Charon nodes + + # Return the first Charon service as the main service + validator_metrics_port = charon_services[0].ports[constants.METRICS_PORT_ID] + validator_metrics_url = "{0}:{1}".format( + charon_services[0].ip_address, validator_metrics_port.number + ) + validator_node_metrics_info = node_metrics.new_node_metrics_info( + charon_services[0].name, vc_shared.METRICS_PATH, validator_metrics_url + ) + + return vc_context.new_vc_context( + client_name=constants.VC_TYPE.charon, + service_name=charon_services[0].name, + metrics_info=validator_node_metrics_info, + ) + +def new_charon_launcher(el_cl_genesis_data, jwt_file): + return struct( + el_cl_genesis_data=el_cl_genesis_data, + jwt_file=jwt_file, + ) diff --git a/src/vc/vc_launcher.star b/src/vc/vc_launcher.star index 72c4e05fd..b073dcfd9 100644 --- a/src/vc/vc_launcher.star +++ b/src/vc/vc_launcher.star @@ -217,6 +217,7 @@ def get_vc_config( otel_otlp_grpc_url=otel_otlp_grpc_url, vc_binary_artifact=vc_binary_artifact, ) + # Charon is now handled by charon_launcher.star elif vc_type == constants.VC_TYPE.grandine: fail("Grandine VC is not yet supported") elif vc_type == constants.VC_TYPE.consensoor: From 96e44b5a49b19fea34542396f494cc866fc80232 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sun, 18 May 2025 14:29:44 +0200 Subject: [PATCH 02/35] adds charon key formatter service --- src/vc/charon_launcher.star | 147 ++++++++++++++++++++++++++++++++---- 1 file changed, 132 insertions(+), 15 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 95f1b1a53..0c72ae188 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -78,20 +78,110 @@ def launch( # files={}, # ) + # Get the raw validator keys directory path validator_keys_dirpath = "" if node_keystore_files: - validator_keys_dirpath = shared_utils.path_join( + validator_keys_dirpath = shared_utils.path_join( constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_keys_secrets_relative_dirpath, - ) - # join the validator keys directory path with the validator keys dirpath on the service container - # validator_keys_dirpath = shared_utils.path_ + node_keystore_files.raw_keys_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.raw_secrets_relative_dirpath, + ) + + # Create a temporary service to format the validator keys for Charon + # Use busybox as a lightweight image for key formatting + key_formatter_service = plan.add_service( + name=service_name + "-key-formatter-" + str(vc_index), + config=ServiceConfig( + image="busybox:latest", + cmd=["tail", "-f", "/dev/null"], # Keep the service running + files={ + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + }, + ), + ) + + # Create a directory for Charon-formatted keys + plan.exec( + service_name=key_formatter_service.name, + recipe=ExecRecipe( + command=["mkdir", "-p", "/opt/charon/charon-keys"], + ), + ) + + # Create a script to format the validator keys for Charon + format_keys_script = """#!/bin/sh +# Find all directories in the validator keys directory +keystore_directories="%s/*" + +index=0 +echo "Processing keystores from ${keystore_directories}" + +# Iterate over each directory +for keystore_dir in $keystore_directories; do + # Check if it's a directory + if [ -d "$keystore_dir" ]; then + # Copy 'voting-keystore.json' to 'charon-keys' with an indexed name + cp "$keystore_dir/voting-keystore.json" "/opt/charon/charon-keys/keystore-${index}.json" + + # Extract the directory name (pubkey) from the current keystore directory + dir_name=$(basename "$keystore_dir") + + # Check if a file with the same name exists in the secrets directory and copy it + if [ -f "%s/$dir_name" ]; then + cp "%s/$dir_name" "/opt/charon/charon-keys/keystore-${index}.txt" + else + echo "No matching file found in secrets directory for '$dir_name'." + fi + + # Increment the index for the next iteration (busybox compatible) + index=$(($index + 1)) + fi +done +""" % (validator_keys_dirpath, validator_secrets_dirpath, validator_secrets_dirpath) + + # Save the script to the service + plan.exec( + service_name=key_formatter_service.name, + recipe=ExecRecipe( + command=[ + "sh", "-c", "cat > /opt/charon/format_keys.sh << 'EOL'\n" + format_keys_script + "\nEOL" + ], + ), + ) + + # Make the script executable + plan.exec( + service_name=key_formatter_service.name, + recipe=ExecRecipe( + command=["chmod", "+x", "/opt/charon/format_keys.sh"], + ), + ) + + # Run the script to format the keys + plan.exec( + service_name=key_formatter_service.name, + recipe=ExecRecipe( + command=["/opt/charon/format_keys.sh"], + ), + ) + + # Store the formatted keys + charon_keys_artifact = plan.store_service_files( + service_name=key_formatter_service.name, src="/opt/charon/charon-keys", name="charon-keys" + str(vc_index), + ) + + # Set the path to the formatted keys for the Charon cluster creation + charon_keys_dir = "/opt/charon/charon-keys" # Create a temporary service to run the Charon cluster creation + # Use the Charon image for cluster creation with the direct command temp_service = plan.add_service( - name=service_name + "-temp", + name=service_name + "-temp-" + str(vc_index), config=ServiceConfig( - image=image, + image=image, # We need to use the Charon image for cluster creation cmd=[ "create", "cluster", "--name=test", @@ -99,19 +189,25 @@ def launch( "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", "--split-existing-keys", - "--split-keys-dir=" + validator_keys_dirpath, + "--split-keys-dir=/opt/charon/charon-keys", "--testnet-chain-id=3151908", "--testnet-fork-version=0x10000038", "--testnet-genesis-timestamp=" + str(genesis_time), "--testnet-name=kurtosis-testnet", ], files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + "/opt/charon/charon-keys": charon_keys_artifact, }, ), ) + # Store the Charon cluster files + charon_cluster_files = plan.store_service_files( + service_name=temp_service.name, + src="/opt/charon/.charon", + name="charon-cluster-files" + str(vc_index), + ) + # Create a directory for Charon cluster files # plan.exec( # service_name=temp_service.name, @@ -153,12 +249,33 @@ def launch( # ), # ) + # add a split keys service instead of executing it in a temporary service + split_keys_service = plan.add_service( + name=service_name + "-split-keys" + str(vc_index), + config=ServiceConfig( + image=image, + cmd=[ + "charon", "create", "cluster", + "--name=test", + "--nodes=" + str(charon_node_count), + "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", + "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", + "--split-existing-keys", + "--split-keys-dir=" + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + "--testnet-chain-id=3151908", + "--testnet-fork-version=0x10000038", + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=kurtosis-testnet", + ], + files={ + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: charon_cluster_files, + }, + ), + ) + # Store the Charon cluster files # charon_cluster_files = plan.store_service_files( - # service_name=temp_service.name, - # files={ - # ".charon": "/opt/charon/.charon", - # }, + # temp_service.name, charon_keys_dir, name="charon-cluster-files"+str(vc_index), # ) # We'll create the validator keys directory and copy the keys in the Charon service itself @@ -207,7 +324,7 @@ def launch( # Files to mount files = { constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + "/opt/charon/.charon": charon_cluster_files, } # Ports configuration From 6c883c9b6ade840e95e031f15b5461c869812db9 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sun, 18 May 2025 16:49:00 +0200 Subject: [PATCH 03/35] removes redundant code --- src/vc/charon_launcher.star | 94 +++++-------------------------------- 1 file changed, 12 insertions(+), 82 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 0c72ae188..47db2aaa6 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -119,12 +119,17 @@ keystore_directories="%s/*" index=0 echo "Processing keystores from ${keystore_directories}" +# Create directory with proper permissions +mkdir -p /opt/charon/charon-keys +chmod 755 /opt/charon/charon-keys + # Iterate over each directory for keystore_dir in $keystore_directories; do # Check if it's a directory if [ -d "$keystore_dir" ]; then # Copy 'voting-keystore.json' to 'charon-keys' with an indexed name cp "$keystore_dir/voting-keystore.json" "/opt/charon/charon-keys/keystore-${index}.json" + chmod 644 "/opt/charon/charon-keys/keystore-${index}.json" # Extract the directory name (pubkey) from the current keystore directory dir_name=$(basename "$keystore_dir") @@ -132,6 +137,7 @@ for keystore_dir in $keystore_directories; do # Check if a file with the same name exists in the secrets directory and copy it if [ -f "%s/$dir_name" ]; then cp "%s/$dir_name" "/opt/charon/charon-keys/keystore-${index}.txt" + chmod 644 "/opt/charon/charon-keys/keystore-${index}.txt" else echo "No matching file found in secrets directory for '$dir_name'." fi @@ -170,7 +176,7 @@ done # Store the formatted keys charon_keys_artifact = plan.store_service_files( - service_name=key_formatter_service.name, src="/opt/charon/charon-keys", name="charon-keys" + str(vc_index), + service_name=key_formatter_service.name, src="/opt/charon/charon-keys", name="charon-keys-" + str(vc_index), ) # Set the path to the formatted keys for the Charon cluster creation @@ -179,9 +185,9 @@ done # Create a temporary service to run the Charon cluster creation # Use the Charon image for cluster creation with the direct command temp_service = plan.add_service( - name=service_name + "-temp-" + str(vc_index), + name=service_name + "-charon-split-keys-" + str(vc_index), config=ServiceConfig( - image=image, # We need to use the Charon image for cluster creation + image=image, cmd=[ "create", "cluster", "--name=test", @@ -194,6 +200,7 @@ done "--testnet-fork-version=0x10000038", "--testnet-genesis-timestamp=" + str(genesis_time), "--testnet-name=kurtosis-testnet", + "tail", "-f", "/dev/null", # Keep the service running ], files={ "/opt/charon/charon-keys": charon_keys_artifact, @@ -204,86 +211,10 @@ done # Store the Charon cluster files charon_cluster_files = plan.store_service_files( service_name=temp_service.name, - src="/opt/charon/.charon", - name="charon-cluster-files" + str(vc_index), + src="/opt/charon/", + name="charon-cluster-files-" + str(vc_index), ) - # Create a directory for Charon cluster files - # plan.exec( - # service_name=temp_service.name, - # recipe=ExecRecipe( - # command=["mkdir", "-p", "/opt/charon/validator_keys"], - # ), - # ) - - # Copy validator keys to the Charon cluster directory - # plan.exec( - # service_name=temp_service.name, - # recipe=ExecRecipe( - # command=[ - # "cp", - # "-r", - # constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER + "/*", - # "/opt/charon/validator_keys/", - # ], - # ), - # ) - - # Create Charon cluster - # plan.exec( - # service_name=temp_service.name, - # recipe=ExecRecipe( - # command=[ - # "charon", "create", "cluster", - # "--name=test", - # "--nodes=" + str(charon_node_count), - # "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", - # "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", - # "--split-existing-keys", - # "--split-keys-dir=" + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - # "--testnet-chain-id=3151908", - # "--testnet-fork-version=0x10000038", - # "--testnet-genesis-timestamp=" + str(genesis_time), - # "--testnet-name=kurtosis-testnet", - # ], - # ), - # ) - - # add a split keys service instead of executing it in a temporary service - split_keys_service = plan.add_service( - name=service_name + "-split-keys" + str(vc_index), - config=ServiceConfig( - image=image, - cmd=[ - "charon", "create", "cluster", - "--name=test", - "--nodes=" + str(charon_node_count), - "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", - "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", - "--split-existing-keys", - "--split-keys-dir=" + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - "--testnet-chain-id=3151908", - "--testnet-fork-version=0x10000038", - "--testnet-genesis-timestamp=" + str(genesis_time), - "--testnet-name=kurtosis-testnet", - ], - files={ - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: charon_cluster_files, - }, - ), - ) - - # Store the Charon cluster files - # charon_cluster_files = plan.store_service_files( - # temp_service.name, charon_keys_dir, name="charon-cluster-files"+str(vc_index), - # ) - - # We'll create the validator keys directory and copy the keys in the Charon service itself - # For now, we'll just create an empty directory structure - - # We'll create a temporary service to run the Charon cluster creation - # For now, we'll skip this step and just create the Charon services directly - # Launch Charon nodes charon_services = [] for i in range(charon_node_count): @@ -296,7 +227,6 @@ done "--testnet-genesis-timestamp=" + str(genesis_time), "--testnet-name=testnet", "--testnet-capella-hard-fork=0x40000038", - "--split-keys-dir=" + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, ] if len(participant.vc_extra_params) > 0: From bf90e5f1cca06ba41d0cce78f8eac3a13276d1f0 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sun, 18 May 2025 20:32:36 +0200 Subject: [PATCH 04/35] Persists charon cluster artefacts --- src/vc/charon_launcher.star | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 47db2aaa6..ce77ce0b7 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -180,12 +180,22 @@ done ) # Set the path to the formatted keys for the Charon cluster creation - charon_keys_dir = "/opt/charon/charon-keys" + # charon_keys_dir = "/opt/charon/charon-keys" + + charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" + persistent_key = "data-{0}".format(charon_service_name) + + files = {} + files[CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key=persistent_key, + ) + files["/opt/charon/charon-keys"] = charon_keys_artifact # Create a temporary service to run the Charon cluster creation # Use the Charon image for cluster creation with the direct command temp_service = plan.add_service( - name=service_name + "-charon-split-keys-" + str(vc_index), + name=charon_service_name, config=ServiceConfig( image=image, cmd=[ @@ -200,18 +210,26 @@ done "--testnet-fork-version=0x10000038", "--testnet-genesis-timestamp=" + str(genesis_time), "--testnet-name=kurtosis-testnet", - "tail", "-f", "/dev/null", # Keep the service running + "--cluster-dir=" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, ], - files={ - "/opt/charon/charon-keys": charon_keys_artifact, - }, + files=files, + ), + ) + + # Restart the temporary service but with busy box image and keep running + temp_service = plan.add_service( + name=charon_service_name+"-keep-running", + config=ServiceConfig( + image="busybox:latest", + cmd=["tail", "-f", "/dev/null"], # Keep the service running + files=files, ), ) # Store the Charon cluster files charon_cluster_files = plan.store_service_files( service_name=temp_service.name, - src="/opt/charon/", + src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, name="charon-cluster-files-" + str(vc_index), ) From 658f00934227828fd46d288a13203107ad77f07a Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Thu, 22 May 2025 08:49:19 +0200 Subject: [PATCH 05/35] adds charon service, the feature is still broken --- src/vc/charon_launcher.star | 99 ++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 34 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index ce77ce0b7..8c764ea25 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -13,7 +13,7 @@ CHARON_MONITORING_PORT = 3620 CHARON_METRICS_PORT = 8080 # Default Charon image -DEFAULT_CHARON_IMAGE = "obolnetwork/charon:latest" +DEFAULT_CHARON_IMAGE = "obolnetwork/charon:local" # Verbosity levels mapping VERBOSITY_LEVELS = { @@ -183,7 +183,7 @@ done # charon_keys_dir = "/opt/charon/charon-keys" charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) - CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/.charon/cluster" persistent_key = "data-{0}".format(charon_service_name) files = {} @@ -197,7 +197,8 @@ done temp_service = plan.add_service( name=charon_service_name, config=ServiceConfig( - image=image, + # image=image, + image="obolnetwork/charon:latest", cmd=[ "create", "cluster", "--name=test", @@ -213,6 +214,7 @@ done "--cluster-dir=" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, ], files=files, + user = User(uid=0, gid=0), ), ) @@ -227,25 +229,43 @@ done ) # Store the Charon cluster files + # First store the entire cluster directory to get all shared files + # For e charon_cluster_files = plan.store_service_files( service_name=temp_service.name, src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, - name="charon-cluster-files-" + str(vc_index), + name="charon-cluster-files-" + str(vc_index) ) + # Then store each node's files separately for individual access + charon_node_files = [] + charon_lock = [] + for i in range(charon_node_count): + charon_node_files.append(plan.store_service_files( + service_name=temp_service.name, + src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "/node" + str(i), + name="charon-node-files-" + str(i) + "-" + str(vc_index) + )) + # charon_lock.append(plan.store_service_files( + # service_name=temp_service.name, + # src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "node" + str(i) + "/cluster-lock.json", + # name="charon-lock-" + str(i) + "-" + str(vc_index) + # )) + # Launch Charon nodes charon_services = [] for i in range(charon_node_count): node_name = service_name + "-charon-" + str(i) - cmd = [ - "run", - "--testnet-chain-id=3151908", - "--testnet-fork-version=0x10000038", - "--testnet-genesis-timestamp=" + str(genesis_time), - "--testnet-name=testnet", - "--testnet-capella-hard-fork=0x40000038", - ] + cmd=["tail", "-f", "/dev/null"] + + # cmd = [ + # "run", + # "--testnet-chain-id=3151908", + # "--testnet-fork-version=0x10000038", + # "--testnet-genesis-timestamp=" + str(genesis_time), + # "--testnet-name=testnet", + # ] if len(participant.vc_extra_params) > 0: cmd.extend([param for param in participant.vc_extra_params]) @@ -258,11 +278,15 @@ done "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), - "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/charon-enr-private-key", - "CHARON_LOCK_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/cluster-lock.json", + "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/node" + str(i) + "/charon-enr-private-key", + "CHARON_LOCK_FILE": "/opt/charon/.charon/node" + str(i) + "/cluster-lock.json", "CHARON_JAEGER_SERVICE": "node" + str(i), "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoints[i], + "CHARON_TESTNET_CHAIN_ID": "3151908", + "CHARON_TESTNET_FORK_VERSION": "0x10000038", + "CHARON_TESTNET_GENESIS_TIMESTAMP": str(genesis_time), + "CHARON_TESTNET_NAME": "kurtosis-testnet", } # Add any extra environment variables @@ -270,10 +294,10 @@ done env_vars.update(participant.vc_extra_env_vars) # Files to mount - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - "/opt/charon/.charon": charon_cluster_files, - } + # files = { + # "/opt/charon/.charon/cluster": charon_node_files[i], + # # "/opt/charon/.charon/cluster/cluster-lock.json": charon_lock[i], + # } # Ports configuration ports = { @@ -291,18 +315,19 @@ done transport_protocol="TCP", application_protocol="http", ), - constants.METRICS_PORT_ID: PortSpec( - number=CHARON_METRICS_PORT, - transport_protocol="TCP", - application_protocol="http", - ), + # constants.METRICS_PORT_ID: PortSpec( + # number=CHARON_METRICS_PORT, + # transport_protocol="TCP", + # application_protocol="http", + # ), } # Add the service charon_service = plan.add_service( name=node_name, config=ServiceConfig( - image=image, + # image=image, + image="obolnetwork/charon:local", ports=ports, cmd=cmd, env_vars=env_vars, @@ -316,30 +341,36 @@ done ), tolerations=tolerations, node_selectors=node_selectors, + files={ + "/opt/charon/.charon/": Directory( + persistent_key=persistent_key + ), + }, + user = User(uid=0, gid=0), ), ) charon_services.append(charon_service) # Now launch the validator clients that will connect to Charon nodes - vc_services = [] - for i in range(charon_node_count): - # Determine which validator client to use with Charon - vc_type = "lighthouse" # Default - if hasattr(participant, "charon_validator_client"): - vc_type = participant.charon_validator_client + # vc_services = [] + # for i in range(charon_node_count): + # # Determine which validator client to use with Charon + # vc_type = "lighthouse" # Default + # if hasattr(participant, "charon_validator_client"): + # vc_type = participant.charon_validator_client - # For now, we'll skip launching the validator clients - # In a real implementation, we would need to launch validator clients that connect to the Charon nodes + # # For now, we'll skip launching the validator clients + # # In a real implementation, we would need to launch validator clients that connect to the Charon nodes # Return the first Charon service as the main service - validator_metrics_port = charon_services[0].ports[constants.METRICS_PORT_ID] + validator_metrics_port = charon_services[0].ports["monitoring"] validator_metrics_url = "{0}:{1}".format( charon_services[0].ip_address, validator_metrics_port.number ) validator_node_metrics_info = node_metrics.new_node_metrics_info( charon_services[0].name, vc_shared.METRICS_PATH, validator_metrics_url ) - + return vc_context.new_vc_context( client_name=constants.VC_TYPE.charon, service_name=charon_services[0].name, From 2df8c98e6602d75a0ac2e44c1320ea623044a2d1 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Mon, 26 May 2025 06:27:34 +0200 Subject: [PATCH 06/35] adds an experimental charon:local to debug run --- src/vc/charon_launcher.star | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 8c764ea25..dc9bea665 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -13,7 +13,7 @@ CHARON_MONITORING_PORT = 3620 CHARON_METRICS_PORT = 8080 # Default Charon image -DEFAULT_CHARON_IMAGE = "obolnetwork/charon:local" +DEFAULT_CHARON_IMAGE = "obolnetwork/charon:latest" # Verbosity levels mapping VERBOSITY_LEVELS = { @@ -183,7 +183,7 @@ done # charon_keys_dir = "/opt/charon/charon-keys" charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) - CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/.charon/cluster" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" persistent_key = "data-{0}".format(charon_service_name) files = {} @@ -197,8 +197,7 @@ done temp_service = plan.add_service( name=charon_service_name, config=ServiceConfig( - # image=image, - image="obolnetwork/charon:latest", + image=image, cmd=[ "create", "cluster", "--name=test", @@ -225,6 +224,21 @@ done image="busybox:latest", cmd=["tail", "-f", "/dev/null"], # Keep the service running files=files, + user = User(uid=0, gid=0), + ), + ) + + plan.exec( + service_name=temp_service.name, + recipe=ExecRecipe( + command=["ls", "-laR", CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER], + ), + ) + + plan.exec( + service_name=temp_service.name, + recipe=ExecRecipe( + command=["find", CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, "-name", "cluster-lock.json", "-exec", "chmod", "644", "{}", "+"], ), ) @@ -267,8 +281,8 @@ done # "--testnet-name=testnet", # ] - if len(participant.vc_extra_params) > 0: - cmd.extend([param for param in participant.vc_extra_params]) + # if len(participant.vc_extra_params) > 0: + # cmd.extend([param for param in participant.vc_extra_params]) env_vars = { "CHARON_LOG_LEVEL": "debug", From 4bb5837fe9a82eceee289581a1c60d17594b4142 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Thu, 29 May 2025 18:22:20 +0200 Subject: [PATCH 07/35] adds lighthouse vc integration with Charon --- src/vc/charon_launcher.star | 204 ++++++++++++++++++++++++++++++------ 1 file changed, 173 insertions(+), 31 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index dc9bea665..c00537c07 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -69,14 +69,15 @@ def launch( # Just use the same beacon node for all Charon nodes beacon_endpoints.append(cl_context.beacon_http_url) - # Use the genesis timestamp passed from the participant_network - genesis_time = genesis_timestamp + # Fetch the actual genesis timestamp from the beacon node + genesis_response = plan.run_sh( + name="get-genesis-timestamp", + description="Get the genesis timestamp from the beacon node", + run="curl -s " + cl_context.beacon_http_url + "/eth/v1/beacon/genesis | jq -r '.data.genesis_time' | tr -d '\\n'", + ) - # Create a temporary directory for Charon cluster files - # charon_cluster_dir = plan.store_service_files( - # service_name=service_name + "-charon-cluster-files", - # files={}, - # ) + # Extract the genesis timestamp from the response + genesis_time = genesis_response.output # Get the raw validator keys directory path validator_keys_dirpath = "" @@ -228,17 +229,11 @@ done ), ) + # Wait a moment for files to be fully written plan.exec( service_name=temp_service.name, recipe=ExecRecipe( - command=["ls", "-laR", CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER], - ), - ) - - plan.exec( - service_name=temp_service.name, - recipe=ExecRecipe( - command=["find", CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, "-name", "cluster-lock.json", "-exec", "chmod", "644", "{}", "+"], + command=["sleep", "2"], ), ) @@ -260,18 +255,13 @@ done src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "/node" + str(i), name="charon-node-files-" + str(i) + "-" + str(vc_index) )) - # charon_lock.append(plan.store_service_files( - # service_name=temp_service.name, - # src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "node" + str(i) + "/cluster-lock.json", - # name="charon-lock-" + str(i) + "-" + str(vc_index) - # )) # Launch Charon nodes charon_services = [] for i in range(charon_node_count): node_name = service_name + "-charon-" + str(i) - cmd=["tail", "-f", "/dev/null"] + # cmd=["tail", "-f", "/dev/null"] # cmd = [ # "run", @@ -336,12 +326,21 @@ done # ), } + # Charon run command + cmd = [ + "run", + "--testnet-chain-id=3151908", + "--testnet-fork-version=0x10000038", + "--testnet-genesis-timestamp=" + str(genesis_time), + "--testnet-name=kurtosis-testnet", + ] + # Add the service charon_service = plan.add_service( name=node_name, config=ServiceConfig( # image=image, - image="obolnetwork/charon:local", + image="obolnetwork/charon:latest", ports=ports, cmd=cmd, env_vars=env_vars, @@ -366,15 +365,48 @@ done charon_services.append(charon_service) # Now launch the validator clients that will connect to Charon nodes - # vc_services = [] - # for i in range(charon_node_count): - # # Determine which validator client to use with Charon - # vc_type = "lighthouse" # Default - # if hasattr(participant, "charon_validator_client"): - # vc_type = participant.charon_validator_client + vc_services = [] + for i in range(charon_node_count): + # Determine which validator client to use with Charon + vc_type = "lighthouse" # Default + if hasattr(participant, "charon_validator_client"): + vc_type = participant.charon_validator_client + + # Create VC service name + vc_service_name = service_name + "-vc-" + str(i) + "-" + vc_type + + # Get the Charon node's validator API URL + charon_validator_api_url = "http://{0}:{1}".format( + charon_services[i].ip_address, + CHARON_VALIDATOR_API_PORT + ) - # # For now, we'll skip launching the validator clients - # # In a real implementation, we would need to launch validator clients that connect to the Charon nodes + # Create validator keys directory for this specific node + validator_keys_for_node = plan.store_service_files( + service_name=temp_service.name, + src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "/node" + str(i) + "/validator_keys", + name="validator-keys-node-" + str(i) + "-" + str(vc_index) + ) + + # Launch the validator client based on type + if vc_type == "lighthouse": + vc_service = launch_lighthouse_vc( + plan=plan, + vc_service_name=vc_service_name, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i + ) + vc_services.append(vc_service) + else: + # For now, only lighthouse is supported + fail("Only lighthouse validator client is currently supported with Charon") # Return the first Charon service as the main service validator_metrics_port = charon_services[0].ports["monitoring"] @@ -384,13 +416,123 @@ done validator_node_metrics_info = node_metrics.new_node_metrics_info( charon_services[0].name, vc_shared.METRICS_PATH, validator_metrics_url ) - + return vc_context.new_vc_context( client_name=constants.VC_TYPE.charon, service_name=charon_services[0].name, metrics_info=validator_node_metrics_info, ) +def launch_lighthouse_vc( + plan, + vc_service_name, + charon_validator_api_url, + validator_keys_artifact, + launcher, + participant, + tolerations, + node_selectors, + full_name, + vc_index, + node_index +): + """ + Launch a Lighthouse validator client that connects to a Charon node + Uses the two-stage approach: import keys, then run validator + """ + + # Create the startup script that implements the two-stage approach + startup_script = """#!/bin/bash +set -e + +# Install required packages +apt-get update && apt-get install -y curl jq wget + +# Wait for Charon node to be available +# while ! curl "${LIGHTHOUSE_BEACON_NODE_ADDRESS}/eth/v1/node/health" 2>/dev/null; do +# echo "Waiting for ${LIGHTHOUSE_BEACON_NODE_ADDRESS} to become available..." +# sleep 5 +# done + +echo "Charon node is available, proceeding with key import..." + +# Stage 1: Import validator keys +for f in /opt/charon/keys/keystore-*.json; do + if [ -f "$f" ]; then + echo "Importing key ${f}" + lighthouse account validator import \\ + --reuse-password \\ + --keystore "${f}" \\ + --password-file "${f//json/txt}" \\ + --testnet-dir "/opt/lighthouse/network-configs" + fi +done + +echo "Starting lighthouse validator client for node""" + str(node_index) + """" +# Stage 2: Run the validator client +exec lighthouse validator \\ + --beacon-nodes ${LIGHTHOUSE_BEACON_NODE_ADDRESS} \\ + --suggested-fee-recipient """ + constants.VALIDATING_REWARDS_ACCOUNT + """ \\ + --metrics \\ + --metrics-address "0.0.0.0" \\ + --metrics-allow-origin "*" \\ + --metrics-port """ + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --use-long-timeouts \\ + --testnet-dir "/opt/lighthouse/network-configs" \\ + --builder-proposals \\ + --distributed \\ + --debug-level "debug" +""" + + # Environment variables + env_vars = { + "LIGHTHOUSE_BEACON_NODE_ADDRESS": charon_validator_api_url, + "NODE": "node" + str(node_index), + "RUST_BACKTRACE": "full" + } + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Files to mount + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, + "/opt/charon/keys": validator_keys_artifact, + "/opt/lighthouse/network-configs": launcher.el_cl_genesis_data.files_artifact_uuid, + } + + # Ports configuration + ports = { + constants.METRICS_PORT_ID: PortSpec( + number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Create the service with the startup script + vc_service = plan.add_service( + name=vc_service_name, + config=ServiceConfig( + image="sigp/lighthouse:latest", + ports=ports, + cmd=["bash", "-c", startup_script], + env_vars=env_vars, + files=files, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.lighthouse, + client_type=constants.CLIENT_TYPES.validator, + image="sigp/lighthouse:latest"[-constants.MAX_LABEL_LENGTH:], + connected_client="charon-node-" + str(node_index), + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ), + ) + + return vc_service + def new_charon_launcher(el_cl_genesis_data, jwt_file): return struct( el_cl_genesis_data=el_cl_genesis_data, From e4cb57a1dee8ad4fa0b38550832c6baabd450ebc Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sat, 7 Jun 2025 12:38:28 +0200 Subject: [PATCH 08/35] adds lodestar and teku vc --- src/vc/charon_launcher.star | 305 +++++++++++++++++++++++++++++++++++- 1 file changed, 302 insertions(+), 3 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index c00537c07..7f8122568 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -233,7 +233,7 @@ done plan.exec( service_name=temp_service.name, recipe=ExecRecipe( - command=["sleep", "2"], + command=["sleep", "5"], ), ) @@ -404,9 +404,39 @@ done node_index=i ) vc_services.append(vc_service) + elif vc_type == "lodestar": + vc_service = launch_lodestar_vc( + plan=plan, + vc_service_name=vc_service_name, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i + ) + vc_services.append(vc_service) + elif vc_type == "teku": + vc_service = launch_teku_vc( + plan=plan, + vc_service_name=vc_service_name, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i + ) + vc_services.append(vc_service) else: - # For now, only lighthouse is supported - fail("Only lighthouse validator client is currently supported with Charon") + # For now, only lighthouse, lodestar, and teku are supported + fail("Only lighthouse, lodestar, and teku validator clients are currently supported with Charon") # Return the first Charon service as the main service validator_metrics_port = charon_services[0].ports["monitoring"] @@ -533,6 +563,275 @@ exec lighthouse validator \\ return vc_service +def launch_lodestar_vc( + plan, + vc_service_name, + charon_validator_api_url, + validator_keys_artifact, + launcher, + participant, + tolerations, + node_selectors, + full_name, + vc_index, + node_index +): + """ + Launch a Lodestar validator client that connects to a Charon node + Uses Charon-specific key management with standard Lodestar parameters + """ + + # Create the run.sh script content (similar to kurtosis-charon/lodestar/run.sh) + run_script_content = """#!/bin/sh + +BUILDER_SELECTION="executiononly" + +# If the builder API is enabled, override the builder selection to signal Lodestar to always prefer proposing blinded blocks, but fall back on EL blocks if unavailable. +if [ "$BUILDER_API_ENABLED" = "true" ]; then + BUILDER_SELECTION="builderalways" +fi + +DATA_DIR="/opt/data" +KEYSTORES_DIR="${DATA_DIR}/keystores" +SECRETS_DIR="${DATA_DIR}/secrets" + +mkdir -p "${KEYSTORES_DIR}" "${SECRETS_DIR}" + +IMPORTED_COUNT=0 +EXISTING_COUNT=0 + +for f in /home/charon/validator_keys/keystore-*.json; do + echo "Importing key ${f}" + + # Extract pubkey from keystore file + PUBKEY="0x$(grep '"pubkey"' "$f" | awk -F'"' '{print $4}')" + + PUBKEY_DIR="${KEYSTORES_DIR}/${PUBKEY}" + + # Skip import if keystore already exists + if [ -d "${PUBKEY_DIR}" ]; then + EXISTING_COUNT=$((EXISTING_COUNT + 1)) + continue + fi + + mkdir -p "${PUBKEY_DIR}" + chown 1000:1000 "${PUBKEY_DIR}" + + # Copy the keystore file to persisted keys backend + install -m 600 "$f" "${PUBKEY_DIR}/voting-keystore.json" + chown 1000:1000 "${PUBKEY_DIR}/voting-keystore.json" + + # Copy the corresponding password file + PASSWORD_FILE="${f%.json}.txt" + install -m 600 "${PASSWORD_FILE}" "${SECRETS_DIR}/${PUBKEY}" + + IMPORTED_COUNT=$((IMPORTED_COUNT + 1)) +done + +echo "Processed all keys imported=${IMPORTED_COUNT}, existing=${EXISTING_COUNT}, total=$(ls /home/charon/validator_keys/keystore-*.json | wc -l)" + +exec node /usr/app/packages/cli/bin/lodestar validator \\ + --dataDir="$DATA_DIR" \\ + --keystoresDir="$KEYSTORES_DIR" \\ + --secretsDir="$SECRETS_DIR" \\ + --metrics=true \\ + --metrics.address="0.0.0.0" \\ + --metrics.port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --beaconNodes="$BEACON_NODE_ADDRESS" \\ + --builder="$BUILDER_API_ENABLED" \\ + --builder.selection="$BUILDER_SELECTION" \\ + --distributed \\ + --paramsFile="/opt/lodestar/config.yaml" +""" + + # Add extra params if specified + if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) + run_script_content += extra_params + + # Create the script file artifact using render_templates + script_artifact = plan.render_templates( + config={ + "run.sh": struct( + template=run_script_content, + data={}, + ), + }, + name="lodestar-run-script-" + str(node_index) + "-" + str(vc_index), + ) + + # Debug: Print that the script artifact has been created + plan.print("Created Lodestar run script artifact: lodestar-run-script-" + str(node_index) + "-" + str(vc_index)) + plan.print("You can download this script using: kurtosis files download lodestar-run-script-" + str(node_index) + "-" + str(vc_index)) + + # Environment variables + env_vars = { + "BEACON_NODE_ADDRESS": charon_validator_api_url, + "BUILDER_API_ENABLED": "true", + "NODE": "node" + str(node_index), + } + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Files to mount - Charon keys + standard genesis data + run script + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, + "/home/charon/validator_keys": validator_keys_artifact, + "/opt/lodestar": launcher.el_cl_genesis_data.files_artifact_uuid, + "/opt/charon": script_artifact, + } + + # Ports configuration + ports = { + constants.METRICS_PORT_ID: PortSpec( + number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Create the service - execute the script file + vc_service = plan.add_service( + name=vc_service_name, + config=ServiceConfig( + image="chainsafe/lodestar:latest", + ports=ports, + cmd=["chmod +x /opt/charon/run.sh && /opt/charon/run.sh"], + entrypoint=["sh", "-c"], + env_vars=env_vars, + files=files, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.lodestar, + client_type=constants.CLIENT_TYPES.validator, + image="chainsafe/lodestar:latest"[-constants.MAX_LABEL_LENGTH:], + connected_client="charon-node-" + str(node_index), + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + tolerations=tolerations, + node_selectors=node_selectors, + ), + ) + + return vc_service + +def launch_teku_vc( + plan, + vc_service_name, + charon_validator_api_url, + validator_keys_artifact, + launcher, + participant, + tolerations, + node_selectors, + full_name, + vc_index, + node_index +): + """ + Launch a Teku validator client that connects to a Charon node + Uses config file approach similar to compose.teku.yaml + """ + + # Create the teku-config.yaml content based on kurtosis-charon/teku/teku-config.yaml + teku_config_content = """metrics-enabled: true +metrics-host-allowlist: "*" +metrics-interface: "0.0.0.0" +metrics-port: \"""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """\" +validators-keystore-locking-enabled: false +network: "/opt/teku/network-configs/config.yaml" +validator-keys: "/opt/charon/validator_keys:/opt/charon/validator_keys" +validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ACCOUNT + """\" +""" + + # Create the config file artifact using render_templates + config_artifact = plan.render_templates( + config={ + "teku-config.yaml": struct( + template=teku_config_content, + data={}, + ), + }, + name="teku-config-" + str(node_index) + "-" + str(vc_index), + ) + + # Debug: Print that the config artifact has been created + plan.print("Created Teku config artifact: teku-config-" + str(node_index) + "-" + str(vc_index)) + plan.print("You can download this config using: kurtosis files download teku-config-" + str(node_index) + "-" + str(vc_index)) + + # Teku validator command based on standard teku.star but with Charon-specific flags + cmd = [ + "validator-client", + "--network=/opt/teku/network-configs/config.yaml", + "--beacon-node-api-endpoint=" + charon_validator_api_url, + "--config-file=/opt/charon/teku/teku-config.yaml", + "--validators-external-signer-slashing-protection-enabled=true", + "--validators-proposer-blinded-blocks-enabled=true", + "--validators-builder-registration-default-enabled=true", + "--Xobol-dvt-integration-enabled=true", + "--logging=DEBUG", + "--metrics-enabled=true", + "--metrics-host-allowlist=*", + "--metrics-interface=0.0.0.0", + "--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), + ] + + # print the cmd + plan.print("cmd: " + str(cmd)) + + # Add extra params if specified + if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + cmd.extend([param for param in participant.vc_extra_params]) + + # Environment variables + env_vars = {} + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Files to mount - Charon keys + standard genesis data + teku config + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, + "/opt/charon/validator_keys": validator_keys_artifact, + "/opt/charon/teku": config_artifact, + "/opt/teku/network-configs": launcher.el_cl_genesis_data.files_artifact_uuid, + } + + # Ports configuration + ports = { + constants.METRICS_PORT_ID: PortSpec( + number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Create the service + vc_service = plan.add_service( + name=vc_service_name, + config=ServiceConfig( + image="consensys/teku:latest", + ports=ports, + cmd = cmd, + # cmd=["tail", "-f", "/dev/null"], + # entrypoint= ["sh", "-c"], + env_vars=env_vars, + files=files, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.teku, + client_type=constants.CLIENT_TYPES.validator, + image="consensys/teku:latest"[-constants.MAX_LABEL_LENGTH:], + connected_client="charon-node-" + str(node_index), + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + tolerations=tolerations, + node_selectors=node_selectors, + user = User(uid=0, gid=0), + ), + ) + + return vc_service + def new_charon_launcher(el_cl_genesis_data, jwt_file): return struct( el_cl_genesis_data=el_cl_genesis_data, From bff3886cd27fbf2736ec9d9adb401cee24dd4737 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sat, 7 Jun 2025 14:53:50 +0200 Subject: [PATCH 09/35] adds nimbus vc --- src/vc/charon_launcher.star | 268 +++++++++++++++++++++++++++++++++++- 1 file changed, 266 insertions(+), 2 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 7f8122568..af2b572ec 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -434,9 +434,24 @@ done node_index=i ) vc_services.append(vc_service) + elif vc_type == "nimbus": + vc_service = launch_nimbus_vc( + plan=plan, + vc_service_name=vc_service_name, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i + ) + vc_services.append(vc_service) else: - # For now, only lighthouse, lodestar, and teku are supported - fail("Only lighthouse, lodestar, and teku validator clients are currently supported with Charon") + # For now, only lighthouse, lodestar, teku, and nimbus are supported + fail("Only lighthouse, lodestar, teku, and nimbus validator clients are currently supported with Charon") # Return the first Charon service as the main service validator_metrics_port = charon_services[0].ports["monitoring"] @@ -832,6 +847,255 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ return vc_service +def launch_nimbus_vc( + plan, + vc_service_name, + charon_validator_api_url, + validator_keys_artifact, + launcher, + participant, + tolerations, + node_selectors, + full_name, + vc_index, + node_index +): + """ + Launch a Nimbus validator client that connects to a Charon node + Uses a two-service approach: + 1. Key import service using nimbus-eth2 (beacon node) to import keys + 2. Validator client service using nimbus-validator-client with imported keys + """ + + # Step 1: Create key import service using beacon node image + key_import_service_name = vc_service_name + "-key-import" + + # Create the key import script + key_import_script = """#!/usr/bin/env bash + +# Cleanup nimbus directories if they already exist. +rm -rf /home/user/data/${NODE} + +# Refer: https://nimbus.guide/keys.html +# Running a nimbus VC involves two steps which need to run in order: +# 1. Importing the validator keys +# 2. And then actually running the VC +tmpkeys="/home/validator_keys/tmpkeys" +mkdir -p ${tmpkeys} + +for f in /home/validator_keys/keystore-*.json; do + echo "Importing key ${f}" + + # Read password from keystore-*.txt into $password variable. + password=$(<"${f//json/txt}") + echo "Password length: ${#password}" + + # Copy keystore file to tmpkeys/ directory. + cp "${f}" "${tmpkeys}" + echo "Copied ${f} to ${tmpkeys}" + + # List files in tmpkeys before import + echo "Files in tmpkeys before import:" + ls -la "${tmpkeys}" + + # Import keystore with the password. + echo "Running nimbus import command..." + echo "$password" | \\ + /home/user/nimbus_beacon_node deposits import \\ + --data-dir=/home/user/data/${NODE} \\ + /home/validator_keys/tmpkeys + + IMPORT_RESULT=$? + echo "Import command exit code: $IMPORT_RESULT" + + # Check what was created + echo "Contents of data directory after import:" + ls -la /home/user/data/${NODE}/ || echo "Data directory does not exist" + if [ -d "/home/user/data/${NODE}/validators" ]; then + echo "Validators directory contents:" + ls -la /home/user/data/${NODE}/validators/ + fi + + # Delete tmpkeys/keystore-*.json file that was copied before. + filename="$(basename ${f})" + rm "${tmpkeys}/${filename}" + echo "Deleted ${tmpkeys}/${filename}" +done + +# Delete the tmpkeys/ directory since it's no longer needed. +rm -r ${tmpkeys} + +echo "Imported all keys successfully" +echo "Key import process completed" + +# Create a completion marker file to signal that import is done +echo "IMPORT_COMPLETE" > /home/user/data/import_complete.txt +echo "Created completion marker file" + +# Keep the container running so we can extract the data +tail -f /dev/null +""" + + # Create the key import script artifact + key_import_script_artifact = plan.render_templates( + config={ + "import_keys.sh": struct( + template=key_import_script, + data={}, + ), + }, + name="nimbus-key-import-script-" + str(node_index) + "-" + str(vc_index), + ) + + # Environment variables for key import + import_env_vars = { + "NODE": "node" + str(node_index), + } + + # Files to mount for key import + import_files = { + "/home/validator_keys": validator_keys_artifact, + "/home/user/scripts": key_import_script_artifact, + } + + # Create the key import service + plan.print("Creating Nimbus key import service: " + key_import_service_name) + key_import_service = plan.add_service( + name=key_import_service_name, + config=ServiceConfig( + image="statusim/nimbus-eth2:multiarch-latest", + cmd=["chmod +x /home/user/scripts/import_keys.sh && /home/user/scripts/import_keys.sh"], + entrypoint=["bash", "-c"], + env_vars=import_env_vars, + files=import_files, + user=User(uid=0, gid=0), + ), + ) + + # Step 2: Wait for key import to complete and then extract the keys as an artifact + plan.print("Waiting for key import to complete...") + + # Wait for the completion marker file to be created + plan.exec( + service_name=key_import_service_name, + recipe=ExecRecipe( + command=["bash", "-c", "while [ ! -f /home/user/data/import_complete.txt ]; do echo 'Waiting for import to complete...'; sleep 2; done; echo 'Import completed! Found completion marker.'"] + ), + description="Wait for key import completion", + ) + + # Store the imported keys from the key import service + # Note: The beacon node imports to /home/user/data/${NODE}, so we store that specific directory + imported_keys_artifact = plan.store_service_files( + service_name=key_import_service_name, + src="/home/user/data/node" + str(node_index), + name="nimbus-imported-keys-" + str(node_index) + "-" + str(vc_index), + description="Nimbus imported validator keys for node " + str(node_index), + ) + + # Step 3: Create the actual validator client service + # Create the VC run script + vc_run_script = """#!/usr/bin/env bash + +# Find the nimbus_validator_client binary +if [ -f "/home/user/nimbus_validator_client" ]; then + NIMBUS_VC_PATH="/home/user/nimbus_validator_client" +elif [ -f "/usr/bin/nimbus_validator_client" ]; then + NIMBUS_VC_PATH="/usr/bin/nimbus_validator_client" +elif [ -f "/usr/local/bin/nimbus_validator_client" ]; then + NIMBUS_VC_PATH="/usr/local/bin/nimbus_validator_client" +else + echo "Error: Could not find nimbus_validator_client binary" + echo "Available files in /home/user:" + ls -la /home/user/ + exit 1 +fi + +echo "Using Nimbus VC binary at: $NIMBUS_VC_PATH" +echo "Using imported keys from: /home/user/imported_data" + +# List what's available in the imported data +echo "Contents of imported_data:" +ls -la /home/user/imported_data/ + +# Run nimbus validator client with imported keys +exec "$NIMBUS_VC_PATH" \\ + --data-dir="/home/user/imported_data" \\ + --beacon-node="$BEACON_NODE_ADDRESS" \\ + --doppelganger-detection=false \\ + --metrics \\ + --metrics-address=0.0.0.0 \\ + --metrics-port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --payload-builder=true \\ + --distributed +""" + + # Add extra params if specified + if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) + vc_run_script = vc_run_script.replace("--distributed", "--distributed" + extra_params) + + # Create the VC script artifact + vc_script_artifact = plan.render_templates( + config={ + "run_vc.sh": struct( + template=vc_run_script, + data={}, + ), + }, + name="nimbus-vc-script-" + str(node_index) + "-" + str(vc_index), + ) + + # Environment variables for VC + vc_env_vars = { + "BEACON_NODE_ADDRESS": charon_validator_api_url, + "NODE": "node" + str(node_index), + } + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + vc_env_vars.update(participant.vc_extra_env_vars) + + # Files to mount for VC - imported keys + VC script + vc_files = { + "/home/user/imported_data": imported_keys_artifact, + "/home/user/scripts": vc_script_artifact, + } + + # Ports configuration + ports = { + constants.METRICS_PORT_ID: PortSpec( + number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Create the actual validator client service + plan.print("Creating Nimbus validator client service: " + vc_service_name) + vc_service = plan.add_service( + name=vc_service_name, + config=ServiceConfig( + image="statusim/nimbus-validator-client:multiarch-latest", + ports=ports, + cmd=["chmod +x /home/user/scripts/run_vc.sh && /home/user/scripts/run_vc.sh"], + entrypoint=["bash", "-c"], + env_vars=vc_env_vars, + files=vc_files, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.nimbus, + client_type=constants.CLIENT_TYPES.validator, + image="statusim/nimbus-validator-client:multiarch-latest"[-constants.MAX_LABEL_LENGTH:], + connected_client="charon-node-" + str(node_index), + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + tolerations=tolerations, + node_selectors=node_selectors, + user=User(uid=0, gid=0), + ), + ) + + return vc_service + def new_charon_launcher(el_cl_genesis_data, jwt_file): return struct( el_cl_genesis_data=el_cl_genesis_data, From ff6acecdaa4fcaa53dea837e84bde2115961c17e Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Sat, 7 Jun 2025 15:26:19 +0200 Subject: [PATCH 10/35] adds prysm vc --- src/vc/charon_launcher.star | 166 +++++++++++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 2 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index af2b572ec..737423457 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -449,9 +449,24 @@ done node_index=i ) vc_services.append(vc_service) + elif vc_type == "prysm": + vc_service = launch_prysm_vc( + plan=plan, + vc_service_name=vc_service_name, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i + ) + vc_services.append(vc_service) else: - # For now, only lighthouse, lodestar, teku, and nimbus are supported - fail("Only lighthouse, lodestar, teku, and nimbus validator clients are currently supported with Charon") + # For now, only lighthouse, lodestar, teku, nimbus, and prysm are supported + fail("Only lighthouse, lodestar, teku, nimbus, and prysm validator clients are currently supported with Charon") # Return the first Charon service as the main service validator_metrics_port = charon_services[0].ports["monitoring"] @@ -1096,6 +1111,153 @@ exec "$NIMBUS_VC_PATH" \\ return vc_service +def launch_prysm_vc( + plan, + vc_service_name, + charon_validator_api_url, + validator_keys_artifact, + launcher, + participant, + tolerations, + node_selectors, + full_name, + vc_index, + node_index +): + """ + Launch a Prysm validator client that connects to a Charon node + Uses script approach similar to kurtosis-charon/prysm/run.sh + """ + + # Create the run.sh script content based on kurtosis-charon/prysm/run.sh + run_script_content = """#!/usr/bin/env bash + +WALLET_DIR="/prysm-wallet" + +# Cleanup wallet directories if already exists. +rm -rf $WALLET_DIR +mkdir $WALLET_DIR + +# Refer: https://docs.prylabs.network/docs/install/install-with-script#step-5-run-a-validator-using-prysm +# Running a prysm VC involves two steps which need to run in order: +# 1. Import validator keys in a prysm wallet account. +# 2. Run the validator client. +WALLET_PASSWORD="prysm-validator-secret" +echo $WALLET_PASSWORD > /wallet-password.txt +/app/cmd/validator/validator wallet create --accept-terms-of-use --wallet-password-file=wallet-password.txt --keymanager-kind=direct --wallet-dir="$WALLET_DIR" + +tmpkeys="/home/validator_keys/tmpkeys" +mkdir -p ${tmpkeys} + +for f in /home/charon/validator_keys/keystore-*.json; do + echo "Importing key ${f}" + + # Copy keystore file to tmpkeys/ directory. + cp "${f}" "${tmpkeys}" + + # Import keystore with password. + /app/cmd/validator/validator accounts import \\ + --accept-terms-of-use=true \\ + --wallet-dir="$WALLET_DIR" \\ + --keys-dir="${tmpkeys}" \\ + --account-password-file="${f//json/txt}" \\ + --wallet-password-file=wallet-password.txt + + # Delete tmpkeys/keystore-*.json file that was copied before. + filename="$(basename ${f})" + rm "${tmpkeys}/${filename}" +done + +# Delete the tmpkeys/ directory since it's no longer needed. +rm -r ${tmpkeys} + +echo "Imported all keys" + +# Now run prysm VC +exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ + --accept-terms-of-use=true \\ + --datadir="/data/vc" \\ + --wallet-password-file="/wallet-password.txt" \\ + --enable-beacon-rest-api \\ + --beacon-rest-api-provider="$BEACON_NODE_ADDRESS" \\ + --beacon-rpc-provider="$BEACON_NODE_ADDRESS" \\ + --beacon-rpc-gateway-provider="$BEACON_NODE_ADDRESS" \\ + --chain-config-file="/opt/prysm/config.yaml" \\ + --monitoring-host=0.0.0.0 \\ + --monitoring-port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --distributed +""" + + # Add extra params if specified + if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) + run_script_content = run_script_content.replace("--distributed", "--distributed" + extra_params) + + # Create the script file artifact using render_templates + script_artifact = plan.render_templates( + config={ + "run.sh": struct( + template=run_script_content, + data={}, + ), + }, + name="prysm-run-script-" + str(node_index) + "-" + str(vc_index), + ) + + # Debug: Print that the script artifact has been created + plan.print("Created Prysm run script artifact: prysm-run-script-" + str(node_index) + "-" + str(vc_index)) + plan.print("You can download this script using: kurtosis files download prysm-run-script-" + str(node_index) + "-" + str(vc_index)) + + # Environment variables + env_vars = { + "BEACON_NODE_ADDRESS": charon_validator_api_url, + } + if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + env_vars.update(participant.vc_extra_env_vars) + + # Files to mount - Charon keys + standard genesis data + run script + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, + "/home/charon/validator_keys": validator_keys_artifact, + "/opt/prysm": launcher.el_cl_genesis_data.files_artifact_uuid, + "/opt/charon": script_artifact, + } + + # Ports configuration + ports = { + constants.METRICS_PORT_ID: PortSpec( + number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, + transport_protocol="TCP", + application_protocol="http", + ), + } + + # Create the service - execute the script file + vc_service = plan.add_service( + name=vc_service_name, + config=ServiceConfig( + image="gcr.io/prysmaticlabs/prysm/validator:latest", + ports=ports, + cmd=["chmod +x /opt/charon/run.sh && /opt/charon/run.sh"], + entrypoint=["bash", "-c"], + env_vars=env_vars, + files=files, + labels=shared_utils.label_maker( + client=constants.VC_TYPE.prysm, + client_type=constants.CLIENT_TYPES.validator, + image="gcr.io/prysmaticlabs/prysm/validator:latest"[-constants.MAX_LABEL_LENGTH:], + connected_client="charon-node-" + str(node_index), + extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, + supernode=participant.supernode if hasattr(participant, "supernode") else False, + ), + tolerations=tolerations, + node_selectors=node_selectors, + user=User(uid=0, gid=0), + ), + ) + + return vc_service + def new_charon_launcher(el_cl_genesis_data, jwt_file): return struct( el_cl_genesis_data=el_cl_genesis_data, From 92f5c12ab7a2e3d7cf7e6eb5294d1eb4cc72f15e Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Tue, 10 Jun 2025 07:50:35 +0200 Subject: [PATCH 11/35] replaces hard coded values with constants --- src/package_io/constants.star | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/package_io/constants.star b/src/package_io/constants.star index 58213e34f..a1718dd32 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -73,6 +73,8 @@ LITTLE_BIGTABLE_PORT_ID = "littlebigtable" VALIDATOR_GRPC_PORT_ID = "grpc" VALIDATING_REWARDS_ACCOUNT = "0x8943545177806ED17B9F23F0a21ee5948eCaa776" +CHARON_VALIDATING_REWARDS_ACCOUNT = "0x8943545177806ED17B9F23F0a21ee5948eCaa776" +CHARON_WITHDRAWAL_ADDRESS = "0xBc7c960C1097ef1Af0FD32407701465f3c03e407" MAX_ENR_ENTRIES = 20 MAX_ENODE_ENTRIES = 20 MIN_PEERS = 0 From 24e022f47bc608187bdd9d96fc122f84dd1df186 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Tue, 10 Jun 2025 07:50:54 +0200 Subject: [PATCH 12/35] replaces hard coded values with constants --- src/vc/charon_launcher.star | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 737423457..e8401ab0f 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -203,12 +203,12 @@ done "create", "cluster", "--name=test", "--nodes=" + str(charon_node_count), - "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", - "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", + "--fee-recipient-addresses=" + constants.CHARON_VALIDATING_REWARDS_ACCOUNT, + "--withdrawal-addresses=" + constants.CHARON_WITHDRAWAL_ADDRESS, "--split-existing-keys", "--split-keys-dir=/opt/charon/charon-keys", - "--testnet-chain-id=3151908", - "--testnet-fork-version=0x10000038", + "--testnet-chain-id=" + network_params.network_id, + "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, "--testnet-genesis-timestamp=" + str(genesis_time), "--testnet-name=kurtosis-testnet", "--cluster-dir=" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, @@ -239,7 +239,6 @@ done # Store the Charon cluster files # First store the entire cluster directory to get all shared files - # For e charon_cluster_files = plan.store_service_files( service_name=temp_service.name, src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, @@ -265,8 +264,8 @@ done # cmd = [ # "run", - # "--testnet-chain-id=3151908", - # "--testnet-fork-version=0x10000038", + # "--testnet-chain-id=" + network_params.network_id, + # "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, # "--testnet-genesis-timestamp=" + str(genesis_time), # "--testnet-name=testnet", # ] @@ -287,8 +286,8 @@ done "CHARON_JAEGER_SERVICE": "node" + str(i), "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoints[i], - "CHARON_TESTNET_CHAIN_ID": "3151908", - "CHARON_TESTNET_FORK_VERSION": "0x10000038", + "CHARON_TESTNET_CHAIN_ID": network_params.network_id, + "CHARON_TESTNET_FORK_VERSION": constants.GENESIS_FORK_VERSION, "CHARON_TESTNET_GENESIS_TIMESTAMP": str(genesis_time), "CHARON_TESTNET_NAME": "kurtosis-testnet", } @@ -329,8 +328,8 @@ done # Charon run command cmd = [ "run", - "--testnet-chain-id=3151908", - "--testnet-fork-version=0x10000038", + "--testnet-chain-id=" + network_params.network_id, + "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, "--testnet-genesis-timestamp=" + str(genesis_time), "--testnet-name=kurtosis-testnet", ] @@ -339,8 +338,7 @@ done charon_service = plan.add_service( name=node_name, config=ServiceConfig( - # image=image, - image="obolnetwork/charon:latest", + image=image, ports=ports, cmd=cmd, env_vars=env_vars, @@ -611,7 +609,7 @@ def launch_lodestar_vc( Uses Charon-specific key management with standard Lodestar parameters """ - # Create the run.sh script content (similar to kurtosis-charon/lodestar/run.sh) + # Create the run.sh script content run_script_content = """#!/bin/sh BUILDER_SELECTION="executiononly" @@ -763,7 +761,7 @@ def launch_teku_vc( Uses config file approach similar to compose.teku.yaml """ - # Create the teku-config.yaml content based on kurtosis-charon/teku/teku-config.yaml + # Create the teku-config.yaml content teku_config_content = """metrics-enabled: true metrics-host-allowlist: "*" metrics-interface: "0.0.0.0" @@ -1129,7 +1127,7 @@ def launch_prysm_vc( Uses script approach similar to kurtosis-charon/prysm/run.sh """ - # Create the run.sh script content based on kurtosis-charon/prysm/run.sh + # Create the run.sh script content run_script_content = """#!/usr/bin/env bash WALLET_DIR="/prysm-wallet" From 36586e430877e76ce7c397f0f9503e5667bad1e6 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Wed, 11 Jun 2025 17:28:53 +0200 Subject: [PATCH 13/35] uses charon_oparams field to configure charon VCs --- src/package_io/input_parser.star | 7 ++- src/package_io/sanity_check.star | 4 +- src/vc/charon_launcher.star | 75 +++++++++++++++++++++----------- 3 files changed, 57 insertions(+), 29 deletions(-) diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index ff33c711a..39a5e75b0 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -803,7 +803,7 @@ def input_parser(plan, input_args): skip_start=participant["skip_start"], # Charon-specific parameters charon_node_count=participant["charon_node_count"], - charon_validator_client=participant["charon_validator_client"], + charon_params=participant["charon_params"], ) for participant in result["participants"] ], @@ -2027,7 +2027,10 @@ def default_participant(): "vc_min_mem": 0, # Charon-specific parameters "charon_node_count": 3, - "charon_validator_client": "lighthouse", + "charon_params": { + "charon_vc": "lighthouse", + "charon_vc_image": DEFAULT_CL_IMAGES[constants.CL_TYPE.lighthouse], + }, "vc_max_mem": 0, "vc_force_restart": False, "use_remote_signer": None, diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index b69c44c89..abf90bfd2 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -57,7 +57,7 @@ PARTICIPANT_CATEGORIES = { "remote_signer_extra_env_vars", # Charon-specific parameters "charon_node_count", - "charon_validator_client", + "charon_params", "remote_signer_extra_labels", "remote_signer_extra_params", "remote_signer_tolerations", @@ -161,7 +161,7 @@ PARTICIPANT_MATRIX_PARAMS = { "validator_count", # Charon-specific parameters "charon_node_count", - "charon_validator_client", + "charon_params", ], "remote_signer": [ "remote_signer_type", diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index e8401ab0f..4ebe0e5d5 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -13,7 +13,7 @@ CHARON_MONITORING_PORT = 3620 CHARON_METRICS_PORT = 8080 # Default Charon image -DEFAULT_CHARON_IMAGE = "obolnetwork/charon:latest" +DEFAULT_CHARON_IMAGE = input_parser.DEFAULT_VC_IMAGES[constants.VC_TYPE.charon] # Verbosity levels mapping VERBOSITY_LEVELS = { @@ -63,6 +63,24 @@ def launch( if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: charon_node_count = participant.charon_node_count + # Get Charon validator client parameters + vc_type = constants.CL_TYPE.lighthouse # Default + vc_image = input_parser.DEFAULT_CL_IMAGES[vc_type] + + # Extract charon_params (it's a dictionary, not a struct) + if hasattr(participant, "charon_params") and participant.charon_params != None: + charon_params = participant.charon_params + plan.print("DEBUG: charon_params is a dictionary: " + str(charon_params)) + + # Access dictionary keys + if "charon_vc" in charon_params and charon_params["charon_vc"] != None: + vc_type = charon_params["charon_vc"] + plan.print("DEBUG: Set vc_type to: " + str(vc_type)) + + if "charon_vc_image" in charon_params and charon_params["charon_vc_image"] != None: + vc_image = charon_params["charon_vc_image"] + plan.print("DEBUG: Set vc_image to: " + str(vc_image)) + # Get the beacon node endpoints for each Charon node beacon_endpoints = [] for i in range(charon_node_count): @@ -365,10 +383,7 @@ done # Now launch the validator clients that will connect to Charon nodes vc_services = [] for i in range(charon_node_count): - # Determine which validator client to use with Charon - vc_type = "lighthouse" # Default - if hasattr(participant, "charon_validator_client"): - vc_type = participant.charon_validator_client + # Use the vc_type and vc_image determined earlier # Create VC service name vc_service_name = service_name + "-vc-" + str(i) + "-" + vc_type @@ -399,7 +414,8 @@ done node_selectors=node_selectors, full_name=full_name + "-node" + str(i), vc_index=vc_index, - node_index=i + node_index=i, + vc_image=vc_image ) vc_services.append(vc_service) elif vc_type == "lodestar": @@ -414,7 +430,8 @@ done node_selectors=node_selectors, full_name=full_name + "-node" + str(i), vc_index=vc_index, - node_index=i + node_index=i, + vc_image=vc_image ) vc_services.append(vc_service) elif vc_type == "teku": @@ -429,7 +446,8 @@ done node_selectors=node_selectors, full_name=full_name + "-node" + str(i), vc_index=vc_index, - node_index=i + node_index=i, + vc_image=vc_image ) vc_services.append(vc_service) elif vc_type == "nimbus": @@ -444,7 +462,8 @@ done node_selectors=node_selectors, full_name=full_name + "-node" + str(i), vc_index=vc_index, - node_index=i + node_index=i, + vc_image=vc_image ) vc_services.append(vc_service) elif vc_type == "prysm": @@ -459,7 +478,8 @@ done node_selectors=node_selectors, full_name=full_name + "-node" + str(i), vc_index=vc_index, - node_index=i + node_index=i, + vc_image=vc_image ) vc_services.append(vc_service) else: @@ -492,7 +512,8 @@ def launch_lighthouse_vc( node_selectors, full_name, vc_index, - node_index + node_index, + vc_image ): """ Launch a Lighthouse validator client that connects to a Charon node @@ -571,7 +592,7 @@ exec lighthouse validator \\ vc_service = plan.add_service( name=vc_service_name, config=ServiceConfig( - image="sigp/lighthouse:latest", + image=vc_image, ports=ports, cmd=["bash", "-c", startup_script], env_vars=env_vars, @@ -579,7 +600,7 @@ exec lighthouse validator \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.lighthouse, client_type=constants.CLIENT_TYPES.validator, - image="sigp/lighthouse:latest"[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, supernode=participant.supernode if hasattr(participant, "supernode") else False, @@ -602,7 +623,8 @@ def launch_lodestar_vc( node_selectors, full_name, vc_index, - node_index + node_index, + vc_image ): """ Launch a Lodestar validator client that connects to a Charon node @@ -722,7 +744,7 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ vc_service = plan.add_service( name=vc_service_name, config=ServiceConfig( - image="chainsafe/lodestar:latest", + image=vc_image, ports=ports, cmd=["chmod +x /opt/charon/run.sh && /opt/charon/run.sh"], entrypoint=["sh", "-c"], @@ -731,7 +753,7 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.lodestar, client_type=constants.CLIENT_TYPES.validator, - image="chainsafe/lodestar:latest"[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, supernode=participant.supernode if hasattr(participant, "supernode") else False, @@ -754,7 +776,8 @@ def launch_teku_vc( node_selectors, full_name, vc_index, - node_index + node_index, + vc_image ): """ Launch a Teku validator client that connects to a Charon node @@ -837,7 +860,7 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ vc_service = plan.add_service( name=vc_service_name, config=ServiceConfig( - image="consensys/teku:latest", + image=vc_image, ports=ports, cmd = cmd, # cmd=["tail", "-f", "/dev/null"], @@ -847,7 +870,7 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ labels=shared_utils.label_maker( client=constants.VC_TYPE.teku, client_type=constants.CLIENT_TYPES.validator, - image="consensys/teku:latest"[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, supernode=participant.supernode if hasattr(participant, "supernode") else False, @@ -871,7 +894,8 @@ def launch_nimbus_vc( node_selectors, full_name, vc_index, - node_index + node_index, + vc_image ): """ Launch a Nimbus validator client that connects to a Charon node @@ -1087,7 +1111,7 @@ exec "$NIMBUS_VC_PATH" \\ vc_service = plan.add_service( name=vc_service_name, config=ServiceConfig( - image="statusim/nimbus-validator-client:multiarch-latest", + image=vc_image, ports=ports, cmd=["chmod +x /home/user/scripts/run_vc.sh && /home/user/scripts/run_vc.sh"], entrypoint=["bash", "-c"], @@ -1096,7 +1120,7 @@ exec "$NIMBUS_VC_PATH" \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.nimbus, client_type=constants.CLIENT_TYPES.validator, - image="statusim/nimbus-validator-client:multiarch-latest"[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, supernode=participant.supernode if hasattr(participant, "supernode") else False, @@ -1120,7 +1144,8 @@ def launch_prysm_vc( node_selectors, full_name, vc_index, - node_index + node_index, + vc_image ): """ Launch a Prysm validator client that connects to a Charon node @@ -1234,7 +1259,7 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ vc_service = plan.add_service( name=vc_service_name, config=ServiceConfig( - image="gcr.io/prysmaticlabs/prysm/validator:latest", + image=vc_image, ports=ports, cmd=["chmod +x /opt/charon/run.sh && /opt/charon/run.sh"], entrypoint=["bash", "-c"], @@ -1243,7 +1268,7 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.prysm, client_type=constants.CLIENT_TYPES.validator, - image="gcr.io/prysmaticlabs/prysm/validator:latest"[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, supernode=participant.supernode if hasattr(participant, "supernode") else False, From 41505725f7ca21ebc71ab389271482b1cfb08c06 Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Wed, 11 Jun 2025 18:01:55 +0200 Subject: [PATCH 14/35] removes unused files --- network_params.yaml | 9 +- src/vc/charon.star | 431 -------------------------------------------- 2 files changed, 2 insertions(+), 438 deletions(-) delete mode 100644 src/vc/charon.star diff --git a/network_params.yaml b/network_params.yaml index c6e1b5d87..660497e2b 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -34,11 +34,8 @@ participants: supernode: false use_separate_vc: true # Validator - # vc_type: lighthouse - # vc_image: sigp/lighthouse:latest - # vc_binary_path: "" - vc_type: charon - vc_image: obolnetwork/charon:latest + vc_type: lighthouse + vc_image: sigp/lighthouse:latest vc_log_level: "" vc_extra_env_vars: {} vc_extra_labels: {} @@ -52,8 +49,6 @@ participants: vc_max_mem: 0 validator_count: null use_remote_signer: false - charon_node_count: 4 - charon_validator_client: lighthouse # Remote signer remote_signer_type: web3signer remote_signer_image: consensys/web3signer:latest diff --git a/src/vc/charon.star b/src/vc/charon.star deleted file mode 100644 index 913ecbf62..000000000 --- a/src/vc/charon.star +++ /dev/null @@ -1,431 +0,0 @@ -shared_utils = import_module("../shared_utils/shared_utils.star") -input_parser = import_module("../package_io/input_parser.star") -constants = import_module("../package_io/constants.star") -cl_context = import_module("../cl/cl_context.star") -vc_shared = import_module("./shared.star") - -# Charon specific ports -CHARON_VALIDATOR_API_PORT = 3600 -CHARON_P2P_TCP_PORT = 3610 -CHARON_MONITORING_PORT = 3620 -CHARON_METRICS_PORT = 8080 - -# Default Charon image -DEFAULT_CHARON_IMAGE = "obolnetwork/charon:latest" - -# Verbosity levels mapping -VERBOSITY_LEVELS = { - constants.GLOBAL_LOG_LEVEL.error: "error", - constants.GLOBAL_LOG_LEVEL.warn: "warn", - constants.GLOBAL_LOG_LEVEL.info: "info", - constants.GLOBAL_LOG_LEVEL.debug: "debug", - constants.GLOBAL_LOG_LEVEL.trace: "trace", -} - -def launch( - plan, - participant, - participant_index, - cl_context, - el_cl_genesis_data, - node_keystore_files, - global_node_selectors, - docker_cache_params, -): - """ - Launches a Charon distributed validator setup - """ - image = shared_utils.get_client_image( - participant.vc_type, - participant.vc_image, - DEFAULT_CHARON_IMAGE, - docker_cache_params, - ) - - # Get the number of Charon nodes to create (default to 3) - charon_node_count = 3 - if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: - charon_node_count = participant.charon_node_count - - # Get the beacon node endpoints for each Charon node - beacon_endpoints = [] - for i in range(charon_node_count): - # Use the same beacon node for all Charon nodes if we don't have enough participants - beacon_index = i % len(cl_context.all_beacon_http_urls) - beacon_endpoints.append(cl_context.all_beacon_http_urls[beacon_index]) - - # Get the genesis timestamp from the beacon node - genesis_timestamp = plan.exec( - service_name=cl_context.service_name, - recipe=ExecRecipe( - command=["curl", "-s", cl_context.beacon_http_url + "/eth/v1/beacon/genesis"], - ), - ) - genesis_time = plan.extract_from_json_result(genesis_timestamp, ".data.genesis_time") - - # Create a temporary directory for Charon cluster files - charon_cluster_dir = plan.store_service_files( - service_name=cl_context.service_name, - files={}, - ) - - # Create Charon cluster - cluster_creation = plan.exec( - service_name=cl_context.service_name, - recipe=ExecRecipe( - command=[ - "docker", "run", "--rm", - "-v", charon_cluster_dir + ":/opt/charon", - image, - "create", "cluster", - "--name=test", - "--nodes=" + str(charon_node_count), - "--fee-recipient-addresses=0x8943545177806ED17B9F23F0a21ee5948eCaa776", - "--withdrawal-addresses=0xBc7c960C1097ef1Af0FD32407701465f3c03e407", - "--split-existing-keys", - "--split-keys-dir=/opt/charon/validator_keys", - "--testnet-chain-id=3151908", - "--testnet-fork-version=0x10000038", - "--testnet-genesis-timestamp=" + str(genesis_time), - "--testnet-name=kurtosis-testnet", - ], - ), - ) - - # Launch Charon nodes - charon_services = [] - for i in range(charon_node_count): - node_name = participant.name + "-charon-" + str(i) - - cmd = [ - "run", - "--testnet-chain-id=3151908", - "--testnet-fork-version=0x10000038", - "--testnet-genesis-timestamp=" + str(genesis_time), - "--testnet-name=testnet", - "--testnet-capella-hard-fork=0x40000038", - ] - - if len(participant.vc_extra_params) > 0: - cmd.extend([param for param in participant.vc_extra_params]) - - env_vars = { - "CHARON_LOG_LEVEL": "debug", - "CHARON_LOG_FORMAT": "console", - "CHARON_P2P_RELAYS": "https://0.relay.obol.tech", - "CHARON_BUILDER_API": "true", - "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), - "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), - "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), - "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/charon-enr-private-key", - "CHARON_LOCK_FILE": "/opt/charon/.charon/cluster/node" + str(i) + "/cluster-lock.json", - "CHARON_JAEGER_SERVICE": "node" + str(i), - "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), - "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoints[i], - } - - # Add any extra environment variables - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: - env_vars.update(participant.vc_extra_env_vars) - - ports = { - "validator-api": shared_utils.new_port_spec( - CHARON_VALIDATOR_API_PORT, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - "p2p-tcp": shared_utils.new_port_spec( - CHARON_P2P_TCP_PORT, - shared_utils.TCP_PROTOCOL, - shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, - ), - "monitoring": shared_utils.new_port_spec( - CHARON_MONITORING_PORT, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - } - - files = { - "/opt/charon/.charon": charon_cluster_dir, - } - - charon_service = plan.add_service( - name=node_name, - config=ServiceConfig( - image=image, - ports=ports, - cmd=cmd, - env_vars=env_vars, - files=files, - labels=shared_utils.label_maker( - client=constants.VC_TYPE.charon, - client_type=constants.CLIENT_TYPES.validator, - image=image[-constants.MAX_LABEL_LENGTH:], - connected_client=cl_context.client_name, - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, - ), - node_selectors=global_node_selectors, - ), - ) - charon_services.append(charon_service) - - # Now launch the validator clients that will connect to Charon nodes - vc_services = [] - for i in range(charon_node_count): - # Determine which validator client to use with Charon - vc_type = "lighthouse" # Default - if hasattr(participant, "charon_validator_client"): - vc_type = participant.charon_validator_client - - # Import the appropriate validator client module - vc_module = import_module("../vc/" + vc_type + ".star") - - # Create a modified participant for the validator client - vc_participant = struct( - name=participant.name + "-vc-" + str(i), - vc_type=vc_type, - vc_image=participant.vc_image if hasattr(participant, "vc_image") else None, - vc_extra_params=participant.vc_extra_params if hasattr(participant, "vc_extra_params") else [], - vc_extra_env_vars=participant.vc_extra_env_vars if hasattr(participant, "vc_extra_env_vars") else {}, - vc_extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, - ) - - # Create a modified CL context that points to the Charon node - charon_cl_context = struct( - service_name=charon_services[i].name, - ip_addr=charon_services[i].ip_address, - beacon_http_url="http://" + charon_services[i].ip_address + ":" + str(CHARON_VALIDATOR_API_PORT), - client_name=cl_context.client_name, - all_beacon_http_urls=cl_context.all_beacon_http_urls, - ) - - # Launch the validator client - vc_service = vc_module.launch( - plan=plan, - participant=vc_participant, - participant_index=i, - cl_context=charon_cl_context, - el_cl_genesis_data=el_cl_genesis_data, - node_keystore_files=node_keystore_files, - global_node_selectors=global_node_selectors, - docker_cache_params=docker_cache_params, - ) - vc_services.append(vc_service) - - # Return the first Charon service as the main service - return charon_services[0] - -def get_config( - participant, - el_cl_genesis_data, - image, - global_log_level, - beacon_http_url, - cl_context, - el_context, - full_name, - node_keystore_files, - tolerations, - node_selectors, - keymanager_enabled, - network_params, - port_publisher, - vc_index, -): - """ - Get the configuration for a Charon distributed validator client - """ - log_level = input_parser.get_client_log_level_or_default( - participant.vc_log_level, global_log_level, VERBOSITY_LEVELS - ) - - # We need to get the genesis timestamp from the beacon node - # This will be done when the service is started, so we'll use a script to get it - # and pass it to the Charon command - - # Get the number of Charon nodes to create (default to 4) - charon_node_count = 4 - if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: - charon_node_count = participant.charon_node_count - - # Get the validator keys directory path - validator_keys_dirpath = shared_utils.path_join( - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_keys_relative_dirpath, - ) - - # Get the validator secrets directory path - validator_secrets_dirpath = shared_utils.path_join( - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_secrets_relative_dirpath, - ) - - # Determine the validator client to use with Charon - validator_client = "lighthouse" # Default - if hasattr(participant, "charon_validator_client"): - validator_client = participant.charon_validator_client - - # Create a startup script that will get the genesis timestamp from the beacon node and pass it to Charon - startup_script = """#!/bin/sh -# Get the genesis timestamp from the beacon node -GENESIS_TIME=$(curl -s %s/eth/v1/beacon/genesis | grep -o '"genesis_time":[0-9]*' | cut -d':' -f2) - -# Run Charon with the genesis timestamp -exec charon run \\ - --beacon-node-endpoints=%s \\ - --validator-api-address=0.0.0.0:%d \\ - --p2p-tcp-address=0.0.0.0:%d \\ - --monitoring-address=0.0.0.0:%d \\ - --log-level=%s \\ - --log-format=console \\ - --builder-api=true \\ - --feature-set=alpha \\ - --testnet-genesis-timestamp=$GENESIS_TIME \\ -""" % ( - beacon_http_url, - beacon_http_url, - CHARON_VALIDATOR_API_PORT, - CHARON_P2P_TCP_PORT, - CHARON_MONITORING_PORT, - log_level, - ) - - # Add network-specific parameters to the startup script - if network_params.network == constants.NETWORK_NAME.kurtosis: - startup_script += """ --testnet-chain-id=3151908 \\ - --testnet-fork-version=0x10000038 \\ - --testnet-name=kurtosis-testnet \\ -""" - elif network_params.network in constants.PUBLIC_NETWORKS: - startup_script += """ --network=%s \\ -""" % network_params.network - else: - # For other networks, use the kurtosis defaults - startup_script += """ --testnet-chain-id=3151908 \\ - --testnet-fork-version=0x10000038 \\ - --testnet-name=kurtosis-testnet \\ -""" - - # Add any extra parameters to the startup script - if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: - for param in participant.vc_extra_params: - startup_script += " %s \\\n" % param - - # Remove the trailing backslash and newline - startup_script = startup_script.rstrip("\\\n") - - # Basic command for Charon - just run the startup script - cmd = [ - "/bin/sh", - "/opt/charon/startup.sh", - ] - - - - # Environment variables - env_vars = { - "CHARON_DISTRIBUTED_VALIDATOR_ENABLED": "true", - "CHARON_VALIDATOR_CLIENT": validator_client, - "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), - "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), - "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), - "CHARON_LOG_LEVEL": log_level, - "CHARON_LOG_FORMAT": "console", - "CHARON_VALIDATOR_KEYS_DIR": validator_keys_dirpath, - "CHARON_VALIDATOR_SECRETS_DIR": validator_secrets_dirpath, - "CHARON_BEACON_NODE_ENDPOINTS": beacon_http_url, - "CHARON_JAEGER_SERVICE": full_name, - "CHARON_CLUSTER_ID": full_name, - "CHARON_NODE_COUNT": str(charon_node_count), - "CHARON_FEE_RECIPIENT_ADDRESS": constants.VALIDATING_REWARDS_ACCOUNT, - } - - # Add any extra environment variables - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: - env_vars.update(participant.vc_extra_env_vars) - - # Store the startup script - # startup_script_artifact = plan.store_service_files( - # service_name=cl_context.service_name, - # files={ - # "startup.sh": startup_script, - # }, - # ) - - # Files to mount - # files = { - # constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - # constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - # "/opt/charon": startup_script_artifact, - # } - - # Ports configuration - ports = {} - ports.update(vc_shared.VALIDATOR_CLIENT_USED_PORTS) - - # Add Charon-specific ports - ports.update({ - "validator-api": shared_utils.new_port_spec( - CHARON_VALIDATOR_API_PORT, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - "p2p-tcp": shared_utils.new_port_spec( - CHARON_P2P_TCP_PORT, - shared_utils.TCP_PROTOCOL, - shared_utils.NOT_PROVIDED_APPLICATION_PROTOCOL, - ), - "monitoring": shared_utils.new_port_spec( - CHARON_MONITORING_PORT, - shared_utils.TCP_PROTOCOL, - shared_utils.HTTP_APPLICATION_PROTOCOL, - ), - }) - - # Public ports - public_ports = {} - - # Add public port assignments if port publisher is enabled - if port_publisher != None and port_publisher.enabled: - public_validator_api_port_assignment = port_publisher.get_port_assignment( - "validator-api", vc_index - ) - public_p2p_tcp_port_assignment = port_publisher.get_port_assignment( - "p2p-tcp", vc_index - ) - public_monitoring_port_assignment = port_publisher.get_port_assignment( - "monitoring", vc_index - ) - - public_ports.update( - shared_utils.get_port_specs(public_validator_api_port_assignment) - ) - public_ports.update( - shared_utils.get_port_specs(public_p2p_tcp_port_assignment) - ) - public_ports.update( - shared_utils.get_port_specs(public_monitoring_port_assignment) - ) - - # Return the configuration - return { - "image": image, - "ports": ports, - "public_ports": public_ports, - "cmd": cmd, - "files": files, - "env_vars": env_vars, - "labels": shared_utils.label_maker( - client=constants.VC_TYPE.charon, - client_type=constants.CLIENT_TYPES.validator, - image=image[-constants.MAX_LABEL_LENGTH:], - connected_client=cl_context.client_name, - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, - ), - "tolerations": tolerations, - "node_selectors": node_selectors, - } From fa0376b42671e3d7ceefa95877958c815762ab1e Mon Sep 17 00:00:00 2001 From: sugh01 <19183308+sugh01@users.noreply.github.com> Date: Fri, 8 Aug 2025 10:58:40 +0200 Subject: [PATCH 15/35] adds readme --- README.md | 160 +++++++++++++++++++++++++-- network_params_charon_example.yaml | 169 +++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+), 8 deletions(-) create mode 100644 network_params_charon_example.yaml diff --git a/README.md b/README.md index 3e8504220..94a0ef117 100644 --- a/README.md +++ b/README.md @@ -571,6 +571,33 @@ participants: # network parameter num_validator_keys_per_node validator_count: null + # Charon Distributed Validator Configuration + # The number of Charon nodes to create for distributed validation + # Each Charon node will run as a middleware between the beacon node and validator client + # Minimum 4 nodes recommended for fault tolerance + # Only used when vc_type is set to "charon" + # Defaults to 4 + charon_node_count: 4 + + # Charon-specific parameters for distributed validator setup + # Only used when vc_type is set to "charon" + charon_params: + # The type of validator client to run with Charon + # Valid values are: lighthouse, lodestar, teku, nimbus, prysm + # Each Charon node will run this validator client type + # Defaults to "lighthouse" + charon_vc: lighthouse + + # The Docker image for the validator client used with Charon + # This allows you to specify custom validator client images + # Defaults by client: + # - lighthouse: sigp/lighthouse:latest + # - lodestar: chainsafe/lodestar:latest + # - teku: consensys/teku:latest + # - nimbus: statusim/nimbus-validator-client:multiarch-latest + # - prysm: gcr.io/prysmaticlabs/prysm/validator:latest + charon_vc_image: sigp/lighthouse:latest + # Whether to use a remote signer instead of the vc directly handling keys # Note Lighthouse VC does not support this flag # Defaults to false @@ -2117,16 +2144,133 @@ participants: "/configs": "validator_config.json" # File available at: /configs/validator_config.json ``` +### Notes + +- All file paths must be relative to the package root directory +- Files outside the package directory cannot be mounted directly +- The entire directory structure is preserved when mounting directories + +## Charon Distributed Validator Technology (DVT) + +[Charon](https://github.com/ObolNetwork/charon) is a distributed validator middleware that enables fault-tolerant Ethereum validation by running validator duties across multiple nodes. This package supports deploying Charon clusters with any of the supported validator clients. + +### What is Distributed Validation? + +Distributed validation splits validator duties across multiple nodes (typically 4-7), providing: + +- **Fault Tolerance**: Continue validating even if some nodes go offline +- **Reduced Slashing Risk**: Consensus mechanisms prevent double-signing +- **Improved Uptime**: No single point of failure +- **Decentralization**: Distribute validator operations across multiple operators + +### Charon Configuration + +To use Charon distributed validators, set `vc_type: charon` in your participant configuration: + +```yaml +participants: + - el_type: geth + el_image: ethereum/client-go:latest + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + use_separate_vc: true + # Charon Configuration + vc_type: charon + vc_image: obolnetwork/charon:latest + charon_node_count: 4 + charon_params: + charon_vc: teku + charon_vc_image: consensys/teku:latest +``` + +### Supported Validator Clients with Charon + +All major Ethereum validator clients are supported with Charon: + +| Validator Client | Status | Implementation | Notes | +| ---------------- | ------ | ------------------------ | ----------------------------- | +| **Lighthouse** | ✅ | Two-stage health checks | Default choice, most tested | +| **Lodestar** | ✅ | Script-based execution | Custom key management | +| **Teku** | ✅ | Config file approach | External signer mode | +| **Nimbus** | ✅ | Two-service architecture | Key import + validator client | +| **Prysm** | ✅ | Wallet-based import | Prysm wallet integration | + +### Example Configurations + +#### Basic Charon Setup (4 nodes with Lighthouse) + +```yaml +participants: + - vc_type: charon + vc_image: obolnetwork/charon:latest + charon_node_count: 4 + charon_params: + charon_vc: lighthouse + charon_vc_image: sigp/lighthouse:latest-unstable +``` + +#### Mixed Network (Charon + Standard Validators) + +```yaml +participants: + # Distributed validator with Charon + - vc_type: charon + charon_node_count: 4 + charon_params: + charon_vc: prysm + charon_vc_image: gcr.io/prysmaticlabs/prysm/validator:latest + # Standard validator + - vc_type: lighthouse + vc_image: sigp/lighthouse:latest-unstable +``` + +### Running Charon Networks + +```bash +# Create your network_params.yaml with Charon configuration +kurtosis run --enclave charon-testnet github.com/ethpandaops/ethereum-package --args-file network_params.yaml +``` +### Running Charon Networks from source code +```bash +# Create your network_params.yaml with Charon configuration +kurtosis run . --args-file network_params_charon_example.yaml +``` + + +### Charon Architecture + +When you deploy a Charon participant, the package creates: + +1. **Charon Cluster**: Multiple Charon nodes that form a distributed validator cluster +2. **Validator Clients**: Each Charon node connects to a validator client of your chosen type +3. **Key Distribution**: Validator keys are split and distributed across the Charon nodes +4. **Consensus Layer**: All nodes connect to the same beacon node for chain data + +``` +Beacon Node + ↓ +┌─────────────────────────────────────┐ +│ Charon Cluster │ +│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ +│ │Charon-1 │ │Charon-2 │ │Charon-3 │ │ +│ │ ↓ │ │ ↓ │ │ ↓ │ │ +│ │ Teku VC │ │ Teku VC │ │ Teku VC │ │ +│ └─────────┘ └─────────┘ └─────────┘ │ +└─────────────────────────────────────┘ +``` + ## Beacon Node <> Validator Client compatibility -| | Lighthouse VC | Prysm VC | Teku VC | Lodestar VC | Nimbus VC -|---------------|---------------|----------|---------|-------------|-----------| -| Lighthouse BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Prysm BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Teku BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Lodestar BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Nimbus BN | ✅ | ✅ | ✅ | ✅ | ✅ -| Grandine BN | ✅ | ✅ | ✅ | ✅ | ✅ +| | Lighthouse VC | Prysm VC | Teku VC | Lodestar VC | Nimbus VC | Charon DVT | +|---------------|---------------|----------|---------|-------------|-----------|------------| +| Lighthouse BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Prysm BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Teku BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Lodestar BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Nimbus BN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Grandine BN | ✅ | ✅ | ✅ | ✅ | ✅ | | + +**Note**: Charon DVT (Distributed Validator Technology) is compatible with all beacon node clients and can run any of the supported validator clients (Lighthouse, Lodestar, Teku, Nimbus, Prysm) in a distributed configuration. ## Custom labels for Docker and Kubernetes diff --git a/network_params_charon_example.yaml b/network_params_charon_example.yaml new file mode 100644 index 000000000..00bc4d5c8 --- /dev/null +++ b/network_params_charon_example.yaml @@ -0,0 +1,169 @@ +participants: + # EL + - el_type: geth + el_image: ethereum/client-go:latest + # CL + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + supernode: false + use_separate_vc: true + # Validator + vc_type: charon + vc_image: obolnetwork/charon:latest + charon_node_count: 4 + charon_params: + charon_vc: teku + charon_vc_image: consensys/teku:latest + validator_count: null + use_remote_signer: false + - el_type: geth + el_image: ethereum/client-go:latest + # CL + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + supernode: false + use_separate_vc: true + # Validator + vc_type: lighthouse + vc_image: sigp/lighthouse:latest-unstable + use_remote_signer: false + - el_type: geth + el_image: ethereum/client-go:latest + # CL + cl_type: lighthouse + cl_image: sigp/lighthouse:latest-unstable + supernode: false + use_separate_vc: true + # Validator + vc_type: lighthouse + vc_image: sigp/lighthouse:latest-unstable + use_remote_signer: false +network_params: + network: kurtosis + network_id: "3151908" + deposit_contract_address: "0x00000000219ab540356cBB839Cbe05303d7705Fa" + seconds_per_slot: 12 + num_validator_keys_per_node: 64 + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike + question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy + very lucky have athlete" + preregistered_validator_count: 0 + genesis_delay: 20 + genesis_gaslimit: 30000000 + max_per_epoch_activation_churn_limit: 8 + churn_limit_quotient: 65536 + ejection_balance: 16000000000 + eth1_follow_distance: 2048 + min_validator_withdrawability_delay: 256 + shard_committee_period: 256 + deneb_fork_epoch: 0 + electra_fork_epoch: 0 + fulu_fork_epoch: 18446744073709551615 + network_sync_base_url: https://snapshots.ethpandaops.io/ + force_snapshot_sync: false + data_column_sidecar_subnet_count: 128 + samples_per_slot: 8 + custody_requirement: 4 + max_blobs_per_block_electra: 9 + target_blobs_per_block_electra: 6 + base_fee_update_fraction_electra: 5007716 + additional_preloaded_contracts: {} + devnet_repo: ethpandaops + prefunded_accounts: {} +additional_services: [] +dora_params: + image: "" +tx_fuzz_params: + tx_fuzz_extra_args: [] +spamoor_params: + min_cpu: 100 + max_cpu: 1000 + min_mem: 20 + max_mem: 300 + extra_args: [] +prometheus_params: + storage_tsdb_retention_time: "1d" + storage_tsdb_retention_size: "512MB" + min_cpu: 10 + max_cpu: 1000 + min_mem: 128 + max_mem: 2048 +grafana_params: + additional_dashboards: [] + min_cpu: 10 + max_cpu: 1000 + min_mem: 128 + max_mem: 2048 +assertoor_params: + image: "" + run_stability_check: false + run_block_proposal_check: false + run_transaction_test: false + run_blob_transaction_test: false + run_opcodes_transaction_test: false + run_lifecycle_test: false + tests: [] +wait_for_finalization: false +global_log_level: info +snooper_enabled: false +ethereum_metrics_exporter_enabled: false +parallel_keystore_generation: false +disable_peer_scoring: false +persistent: false +mev_type: null +mev_params: + mev_relay_image: ethpandaops/mev-boost-relay:main + mev_builder_image: ethpandaops/flashbots-builder:main + mev_builder_cl_image: sigp/lighthouse:latest + mev_boost_image: ethpandaops/mev-boost:develop + mev_boost_args: ["mev-boost", "--relay-check"] + mev_relay_api_extra_args: [] + mev_relay_housekeeper_extra_args: [] + mev_relay_website_extra_args: [] + mev_builder_extra_args: [] + mev_builder_prometheus_config: + scrape_interval: 15s + labels: {} + custom_flood_params: + interval_between_transactions: 1 +xatu_sentry_enabled: false +xatu_sentry_params: + xatu_sentry_image: ethpandaops/xatu-sentry + xatu_server_addr: localhost:8000 + xatu_server_tls: false + xatu_server_headers: {} + beacon_subscriptions: + - attestation + - block + - chain_reorg + - finalized_checkpoint + - head + - voluntary_exit + - contribution_and_proof + - blob_sidecar +apache_port: 40000 +global_tolerations: [] +global_node_selectors: {} +keymanager_enabled: false +checkpoint_sync_enabled: false +checkpoint_sync_url: "" +ethereum_genesis_generator_params: + image: ethpandaops/ethereum-genesis-generator:4.0.4 +port_publisher: + nat_exit_ip: KURTOSIS_IP_ADDR_PLACEHOLDER + el: + enabled: false + public_port_start: 32000 + cl: + enabled: false + public_port_start: 33000 + vc: + enabled: false + public_port_start: 34000 + remote_signer: + enabled: false + public_port_start: 35000 + additional_services: + enabled: false + public_port_start: 36000 From 0b0ff272548bf8be11e8f5c084d0fbe262665a40 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:19:08 +0300 Subject: [PATCH 16/35] Change from ethpandaops to ObolNetwork --- kurtosis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kurtosis.yml b/kurtosis.yml index 6b37a1c12..01f4dbe0e 100644 --- a/kurtosis.yml +++ b/kurtosis.yml @@ -1 +1 @@ -name: "github.com/ethpandaops/ethereum-package" +name: "github.com/ObolNetwork/ethereum-package" From ed3ca901c0678b89078b4b7e1730a1952a1857d7 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:08:58 +0300 Subject: [PATCH 17/35] Various fixes after rebase --- src/package_io/constants.star | 1 - src/participant_network.star | 87 +++-- .../validator_keystores/keystore_files.star | 2 +- .../validator_keystore_generator.star | 49 +-- src/vc/charon_launcher.star | 324 +++++------------- src/vc/vc_launcher.star | 4 +- 6 files changed, 156 insertions(+), 311 deletions(-) diff --git a/src/package_io/constants.star b/src/package_io/constants.star index a1718dd32..b29110a70 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -73,7 +73,6 @@ LITTLE_BIGTABLE_PORT_ID = "littlebigtable" VALIDATOR_GRPC_PORT_ID = "grpc" VALIDATING_REWARDS_ACCOUNT = "0x8943545177806ED17B9F23F0a21ee5948eCaa776" -CHARON_VALIDATING_REWARDS_ACCOUNT = "0x8943545177806ED17B9F23F0a21ee5948eCaa776" CHARON_WITHDRAWAL_ADDRESS = "0xBc7c960C1097ef1Af0FD32407701465f3c03e407" MAX_ENR_ENTRIES = 20 MAX_ENODE_ENTRIES = 20 diff --git a/src/participant_network.star b/src/participant_network.star index 34dda2ec5..67c923892 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -346,6 +346,10 @@ def launch_participant_network( vc_service_configs = {} vc_service_info = {} + # Charon launches its own cluster of services immediately (rather than + # deferring to the parallel launch below), so its contexts are collected + # here keyed by participant index and merged in afterwards. + charon_vc_contexts = {} for index, participant in enumerate(args_with_right_defaults.participants): el_type = participant.el_type cl_type = participant.cl_type @@ -552,6 +556,33 @@ def launch_participant_network( remote_signer_context.metrics_info["config"] = participant.prometheus_config service_name = "vc-{0}".format(full_name) + + # Charon launches a full distributed-validator cluster (its own Charon + # nodes plus their validator clients) immediately and returns a ready + # vc_context, so it bypasses the deferred config / parallel-launch path + # used by the other validator clients. + if vc_type == constants.VC_TYPE.charon: + charon_vc_contexts[index] = charon_launcher.launch( + plan=plan, + launcher=charon_launcher.new_charon_launcher( + el_cl_genesis_data=el_cl_data + ), + service_name=service_name, + image=participant.vc_image, + global_log_level=args_with_right_defaults.global_log_level, + cl_context=cl_context, + full_name=full_name, + node_keystore_files=vc_keystores, + participant=participant, + global_tolerations=global_tolerations, + node_selectors=node_selectors, + network_params=network_params, + port_publisher=args_with_right_defaults.port_publisher, + vc_index=current_vc_index, + ) + current_vc_index += 1 + continue + vc_binary_artifact = binary_artifacts.get(index, {}).get("vc", None) vc_service_config = vc.get_vc_config( plan=plan, @@ -585,54 +616,6 @@ def launch_participant_network( if vc_service_config == None: continue - # Use the charon_launcher for Charon validator clients - if vc_type == constants.VC_TYPE.charon: - vc_context = charon_launcher.launch( - plan=plan, - launcher=charon_launcher.new_charon_launcher(el_cl_genesis_data=el_cl_data, jwt_file=jwt_file), - keymanager_file=keymanager_file, - service_name="vc-{0}".format(full_name), - image=participant.vc_image, - global_log_level=args_with_right_defaults.global_log_level, - cl_context=cl_context, - el_context=el_context, - full_name=full_name, - node_keystore_files=vc_keystores, - participant=participant, - global_tolerations=global_tolerations, - node_selectors=node_selectors, - network_params=network_params, - port_publisher=args_with_right_defaults.port_publisher, - vc_index=current_vc_index, - genesis_timestamp=final_genesis_timestamp, - ) - else: - vc_context = vc.launch( - plan=plan, - launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data), - keymanager_file=keymanager_file, - service_name="vc-{0}".format(full_name), - vc_type=vc_type, - image=participant.vc_image, - global_log_level=args_with_right_defaults.global_log_level, - cl_context=cl_context, - el_context=el_context, - remote_signer_context=remote_signer_context, - full_name=full_name, - snooper_enabled=participant.snooper_enabled, - snooper_beacon_context=snooper_beacon_context, - node_keystore_files=vc_keystores, - participant=participant, - prysm_password_relative_filepath=prysm_password_relative_filepath, - prysm_password_artifact_uuid=prysm_password_artifact_uuid, - global_tolerations=global_tolerations, - node_selectors=node_selectors, - network_params=network_params, - port_publisher=args_with_right_defaults.port_publisher, - vc_index=current_vc_index, - ) - all_vc_contexts.append(vc_context) - vc_service_configs[service_name] = vc_service_config vc_service_info[service_name] = { "client_name": vc_type, @@ -664,6 +647,14 @@ def launch_participant_network( vc_contexts_temp[participant_index] = vc_context + # Charon contexts were launched outside the parallel path above; fold them in. + for participant_index, vc_context in charon_vc_contexts.items(): + if vc_context and vc_context.metrics_info: + vc_context.metrics_info["config"] = args_with_right_defaults.participants[ + participant_index + ].prometheus_config + vc_contexts_temp[participant_index] = vc_context + # Convert to ordered list all_vc_contexts = [] for i in range(len(args_with_right_defaults.participants)): diff --git a/src/prelaunch_data_generator/validator_keystores/keystore_files.star b/src/prelaunch_data_generator/validator_keystores/keystore_files.star index c7fba4dca..3a4622d28 100644 --- a/src/prelaunch_data_generator/validator_keystores/keystore_files.star +++ b/src/prelaunch_data_generator/validator_keystores/keystore_files.star @@ -17,7 +17,7 @@ def new_keystore_files( raw_keys_relative_dirpath=raw_keys_relative_dirpath, raw_secrets_relative_dirpath=raw_secrets_relative_dirpath, nimbus_keys_relative_dirpath=nimbus_keys_relative_dirpath, - prysm_relative_dirpath=prysm_relative_dirpath, + prysm_relative_dirpath=prysm_relative_dirpath, teku_keys_relative_dirpath=teku_keys_relative_dirpath, teku_secrets_relative_dirpath=teku_secrets_relative_dirpath, raw_keys_secrets_relative_dirpath=raw_keys_secrets_relative_dirpath, diff --git a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star index fb95b2630..7968586eb 100644 --- a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star +++ b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star @@ -140,24 +140,27 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para running_total_validator_count += participant.validator_count - # add another folder which contains all the raw keys and secret - # create the folder - all_output_dirpaths.append(output_dirpath + RAW_KEYS_SECRETS_DIRNAME) - # copy - all_sub_command_strs.append( - "cp -r " - + output_dirpath - + "keys/ " - + output_dirpath - + "raw-keys-secrets" - ) - all_sub_command_strs.append( - "cp -r " - + output_dirpath - + "secrets/ " - + output_dirpath - + "raw-keys-secrets" - ) + # Collect the raw keys and secrets together in a single folder, which + # Charon consumes when splitting keys across its cluster nodes. Only + # Charon participants need this, so vanilla VCs skip the extra copy. + if participant.vc_type == constants.VC_TYPE.charon: + all_output_dirpaths.append(output_dirpath + RAW_KEYS_SECRETS_DIRNAME) + all_sub_command_strs.append( + "cp -r " + + output_dirpath + + RAW_KEYS_DIRNAME + + "/ " + + output_dirpath + + RAW_KEYS_SECRETS_DIRNAME + ) + all_sub_command_strs.append( + "cp -r " + + output_dirpath + + RAW_SECRETS_DIRNAME + + "/ " + + output_dirpath + + RAW_KEYS_SECRETS_DIRNAME + ) command_str = " && ".join(all_sub_command_strs) @@ -190,7 +193,6 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para keystore_stop_index - 1, ) - # copt the keys and secrets into a new directory and store the artifact artifact_name = plan.store_service_files( service_name, output_dirpath, @@ -199,6 +201,13 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para ) base_dirname_in_artifact = shared_utils.path_base(output_dirpath) + # The raw-keys-secrets folder is only generated for Charon participants + # (see above), so only reference it for them; vanilla VCs leave it empty. + raw_keys_secrets_relative_dirpath = "" + if participant.vc_type == constants.VC_TYPE.charon: + raw_keys_secrets_relative_dirpath = shared_utils.path_join( + base_dirname_in_artifact, RAW_KEYS_SECRETS_DIRNAME + ) to_add = keystore_files_module.new_keystore_files( artifact_name, shared_utils.path_join(base_dirname_in_artifact), @@ -208,7 +217,7 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), - shared_utils.path_join(base_dirname_in_artifact, RAW_KEYS_SECRETS_DIRNAME), + raw_keys_secrets_relative_dirpath, ) keystore_files.append(to_add) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 4ebe0e5d5..6eaed572e 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -1,7 +1,6 @@ shared_utils = import_module("../shared_utils/shared_utils.star") input_parser = import_module("../package_io/input_parser.star") constants = import_module("../package_io/constants.star") -cl_context = import_module("../cl/cl_context.star") vc_shared = import_module("./shared.star") vc_context = import_module("./vc_context.star") node_metrics = import_module("../node_metrics_info.star") @@ -10,10 +9,9 @@ node_metrics = import_module("../node_metrics_info.star") CHARON_VALIDATOR_API_PORT = 3600 CHARON_P2P_TCP_PORT = 3610 CHARON_MONITORING_PORT = 3620 -CHARON_METRICS_PORT = 8080 -# Default Charon image -DEFAULT_CHARON_IMAGE = input_parser.DEFAULT_VC_IMAGES[constants.VC_TYPE.charon] +# Fallback node count if the participant doesn't request a valid one. +DEFAULT_CHARON_NODE_COUNT = 4 # Verbosity levels mapping VERBOSITY_LEVELS = { @@ -26,12 +24,10 @@ VERBOSITY_LEVELS = { def launch( plan, launcher, - keymanager_file, service_name, image, global_log_level, cl_context, - el_context, full_name, node_keystore_files, participant, @@ -40,52 +36,37 @@ def launch( network_params, port_publisher, vc_index, - genesis_timestamp, ): """ Launch a Charon distributed validator client """ - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" - if node_keystore_files == None: return None - tolerations = input_parser.get_client_tolerations( - participant.vc_tolerations, participant.tolerations, global_tolerations + tolerations = shared_utils.get_tolerations( + specific_container_tolerations=participant.vc_tolerations, + participant_tolerations=participant.tolerations, + global_tolerations=global_tolerations, ) log_level = input_parser.get_client_log_level_or_default( participant.vc_log_level, global_log_level, VERBOSITY_LEVELS ) - # Get the number of Charon nodes to create (default to 4) - charon_node_count = 4 - if hasattr(participant, "charon_node_count") and participant.charon_node_count > 0: - charon_node_count = participant.charon_node_count - - # Get Charon validator client parameters - vc_type = constants.CL_TYPE.lighthouse # Default - vc_image = input_parser.DEFAULT_CL_IMAGES[vc_type] - - # Extract charon_params (it's a dictionary, not a struct) - if hasattr(participant, "charon_params") and participant.charon_params != None: - charon_params = participant.charon_params - plan.print("DEBUG: charon_params is a dictionary: " + str(charon_params)) + # Number of Charon nodes to create. + charon_node_count = participant.charon_node_count + if charon_node_count <= 0: + charon_node_count = DEFAULT_CHARON_NODE_COUNT - # Access dictionary keys - if "charon_vc" in charon_params and charon_params["charon_vc"] != None: - vc_type = charon_params["charon_vc"] - plan.print("DEBUG: Set vc_type to: " + str(vc_type)) + # Validator client type/image to run behind each Charon node. + vc_type = constants.CL_TYPE.lighthouse + vc_image = input_parser.DEFAULT_CL_IMAGES[constants.CL_TYPE.lighthouse] + if participant.charon_params != None: + vc_type = participant.charon_params.get("charon_vc", vc_type) + vc_image = participant.charon_params.get("charon_vc_image", vc_image) - if "charon_vc_image" in charon_params and charon_params["charon_vc_image"] != None: - vc_image = charon_params["charon_vc_image"] - plan.print("DEBUG: Set vc_image to: " + str(vc_image)) - - # Get the beacon node endpoints for each Charon node - beacon_endpoints = [] - for i in range(charon_node_count): - # Just use the same beacon node for all Charon nodes - beacon_endpoints.append(cl_context.beacon_http_url) + # All Charon nodes connect to the same beacon node. + beacon_endpoint = cl_context.beacon_http_url # Fetch the actual genesis timestamp from the beacon node genesis_response = plan.run_sh( @@ -198,9 +179,6 @@ done service_name=key_formatter_service.name, src="/opt/charon/charon-keys", name="charon-keys-" + str(vc_index), ) - # Set the path to the formatted keys for the Charon cluster creation - # charon_keys_dir = "/opt/charon/charon-keys" - charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" persistent_key = "data-{0}".format(charon_service_name) @@ -211,9 +189,8 @@ done ) files["/opt/charon/charon-keys"] = charon_keys_artifact - # Create a temporary service to run the Charon cluster creation - # Use the Charon image for cluster creation with the direct command - temp_service = plan.add_service( + # Run the Charon cluster creation (splits the existing keys across nodes). + plan.add_service( name=charon_service_name, config=ServiceConfig( image=image, @@ -221,7 +198,7 @@ done "create", "cluster", "--name=test", "--nodes=" + str(charon_node_count), - "--fee-recipient-addresses=" + constants.CHARON_VALIDATING_REWARDS_ACCOUNT, + "--fee-recipient-addresses=" + constants.VALIDATING_REWARDS_ACCOUNT, "--withdrawal-addresses=" + constants.CHARON_WITHDRAWAL_ADDRESS, "--split-existing-keys", "--split-keys-dir=/opt/charon/charon-keys", @@ -236,8 +213,9 @@ done ), ) - # Restart the temporary service but with busy box image and keep running - temp_service = plan.add_service( + # Keep a busybox service running on the cluster volume so we can read the + # generated per-node files back out as artifacts. + cluster_files_service = plan.add_service( name=charon_service_name+"-keep-running", config=ServiceConfig( image="busybox:latest", @@ -249,50 +227,19 @@ done # Wait a moment for files to be fully written plan.exec( - service_name=temp_service.name, + service_name=cluster_files_service.name, recipe=ExecRecipe( command=["sleep", "5"], ), ) - # Store the Charon cluster files - # First store the entire cluster directory to get all shared files - charon_cluster_files = plan.store_service_files( - service_name=temp_service.name, - src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, - name="charon-cluster-files-" + str(vc_index) - ) - - # Then store each node's files separately for individual access - charon_node_files = [] - charon_lock = [] - for i in range(charon_node_count): - charon_node_files.append(plan.store_service_files( - service_name=temp_service.name, - src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "/node" + str(i), - name="charon-node-files-" + str(i) + "-" + str(vc_index) - )) - # Launch Charon nodes charon_services = [] for i in range(charon_node_count): node_name = service_name + "-charon-" + str(i) - # cmd=["tail", "-f", "/dev/null"] - - # cmd = [ - # "run", - # "--testnet-chain-id=" + network_params.network_id, - # "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, - # "--testnet-genesis-timestamp=" + str(genesis_time), - # "--testnet-name=testnet", - # ] - - # if len(participant.vc_extra_params) > 0: - # cmd.extend([param for param in participant.vc_extra_params]) - env_vars = { - "CHARON_LOG_LEVEL": "debug", + "CHARON_LOG_LEVEL": log_level, "CHARON_LOG_FORMAT": "console", "CHARON_P2P_RELAYS": "https://0.relay.obol.tech", "CHARON_BUILDER_API": "true", @@ -303,7 +250,7 @@ done "CHARON_LOCK_FILE": "/opt/charon/.charon/node" + str(i) + "/cluster-lock.json", "CHARON_JAEGER_SERVICE": "node" + str(i), "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), - "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoints[i], + "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoint, "CHARON_TESTNET_CHAIN_ID": network_params.network_id, "CHARON_TESTNET_FORK_VERSION": constants.GENESIS_FORK_VERSION, "CHARON_TESTNET_GENESIS_TIMESTAMP": str(genesis_time), @@ -311,15 +258,9 @@ done } # Add any extra environment variables - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + if participant.vc_extra_env_vars: env_vars.update(participant.vc_extra_env_vars) - # Files to mount - # files = { - # "/opt/charon/.charon/cluster": charon_node_files[i], - # # "/opt/charon/.charon/cluster/cluster-lock.json": charon_lock[i], - # } - # Ports configuration ports = { "validator-api": PortSpec( @@ -336,11 +277,6 @@ done transport_protocol="TCP", application_protocol="http", ), - # constants.METRICS_PORT_ID: PortSpec( - # number=CHARON_METRICS_PORT, - # transport_protocol="TCP", - # application_protocol="http", - # ), } # Charon run command @@ -380,111 +316,49 @@ done ) charon_services.append(charon_service) - # Now launch the validator clients that will connect to Charon nodes - vc_services = [] - for i in range(charon_node_count): - # Use the vc_type and vc_image determined earlier - - # Create VC service name - vc_service_name = service_name + "-vc-" + str(i) + "-" + vc_type + # Map each supported VC type to its launcher function. + vc_launchers = { + constants.VC_TYPE.lighthouse: launch_lighthouse_vc, + constants.VC_TYPE.lodestar: launch_lodestar_vc, + constants.VC_TYPE.teku: launch_teku_vc, + constants.VC_TYPE.nimbus: launch_nimbus_vc, + constants.VC_TYPE.prysm: launch_prysm_vc, + } + if vc_type not in vc_launchers: + fail( + "Unsupported Charon validator client '{0}'. Supported clients: {1}".format( + vc_type, ", ".join(vc_launchers.keys()) + ) + ) + launch_vc = vc_launchers[vc_type] - # Get the Charon node's validator API URL + # Launch one validator client per Charon node, connected to that node's validator API. + for i in range(charon_node_count): charon_validator_api_url = "http://{0}:{1}".format( - charon_services[i].ip_address, - CHARON_VALIDATOR_API_PORT + charon_services[i].ip_address, CHARON_VALIDATOR_API_PORT ) - # Create validator keys directory for this specific node + # Each node's validator keys come from the cluster-creation output. validator_keys_for_node = plan.store_service_files( - service_name=temp_service.name, + service_name=cluster_files_service.name, src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "/node" + str(i) + "/validator_keys", - name="validator-keys-node-" + str(i) + "-" + str(vc_index) + name="validator-keys-node-" + str(i) + "-" + str(vc_index), ) - # Launch the validator client based on type - if vc_type == "lighthouse": - vc_service = launch_lighthouse_vc( - plan=plan, - vc_service_name=vc_service_name, - charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, - launcher=launcher, - participant=participant, - tolerations=tolerations, - node_selectors=node_selectors, - full_name=full_name + "-node" + str(i), - vc_index=vc_index, - node_index=i, - vc_image=vc_image - ) - vc_services.append(vc_service) - elif vc_type == "lodestar": - vc_service = launch_lodestar_vc( - plan=plan, - vc_service_name=vc_service_name, - charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, - launcher=launcher, - participant=participant, - tolerations=tolerations, - node_selectors=node_selectors, - full_name=full_name + "-node" + str(i), - vc_index=vc_index, - node_index=i, - vc_image=vc_image - ) - vc_services.append(vc_service) - elif vc_type == "teku": - vc_service = launch_teku_vc( - plan=plan, - vc_service_name=vc_service_name, - charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, - launcher=launcher, - participant=participant, - tolerations=tolerations, - node_selectors=node_selectors, - full_name=full_name + "-node" + str(i), - vc_index=vc_index, - node_index=i, - vc_image=vc_image - ) - vc_services.append(vc_service) - elif vc_type == "nimbus": - vc_service = launch_nimbus_vc( - plan=plan, - vc_service_name=vc_service_name, - charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, - launcher=launcher, - participant=participant, - tolerations=tolerations, - node_selectors=node_selectors, - full_name=full_name + "-node" + str(i), - vc_index=vc_index, - node_index=i, - vc_image=vc_image - ) - vc_services.append(vc_service) - elif vc_type == "prysm": - vc_service = launch_prysm_vc( - plan=plan, - vc_service_name=vc_service_name, - charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, - launcher=launcher, - participant=participant, - tolerations=tolerations, - node_selectors=node_selectors, - full_name=full_name + "-node" + str(i), - vc_index=vc_index, - node_index=i, - vc_image=vc_image - ) - vc_services.append(vc_service) - else: - # For now, only lighthouse, lodestar, teku, nimbus, and prysm are supported - fail("Only lighthouse, lodestar, teku, nimbus, and prysm validator clients are currently supported with Charon") + launch_vc( + plan=plan, + vc_service_name=service_name + "-vc-" + str(i) + "-" + vc_type, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i, + vc_image=vc_image, + ) # Return the first Charon service as the main service validator_metrics_port = charon_services[0].ports["monitoring"] @@ -527,14 +401,6 @@ set -e # Install required packages apt-get update && apt-get install -y curl jq wget -# Wait for Charon node to be available -# while ! curl "${LIGHTHOUSE_BEACON_NODE_ADDRESS}/eth/v1/node/health" 2>/dev/null; do -# echo "Waiting for ${LIGHTHOUSE_BEACON_NODE_ADDRESS} to become available..." -# sleep 5 -# done - -echo "Charon node is available, proceeding with key import..." - # Stage 1: Import validator keys for f in /opt/charon/keys/keystore-*.json; do if [ -f "$f" ]; then @@ -569,7 +435,7 @@ exec lighthouse validator \\ "NODE": "node" + str(node_index), "RUST_BACKTRACE": "full" } - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + if participant.vc_extra_env_vars: env_vars.update(participant.vc_extra_env_vars) # Files to mount @@ -602,8 +468,8 @@ exec lighthouse validator \\ client_type=constants.CLIENT_TYPES.validator, image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, ), tolerations=tolerations, node_selectors=node_selectors, @@ -695,7 +561,7 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ """ # Add extra params if specified - if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + if participant.vc_extra_params: extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) run_script_content += extra_params @@ -710,17 +576,13 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ name="lodestar-run-script-" + str(node_index) + "-" + str(vc_index), ) - # Debug: Print that the script artifact has been created - plan.print("Created Lodestar run script artifact: lodestar-run-script-" + str(node_index) + "-" + str(vc_index)) - plan.print("You can download this script using: kurtosis files download lodestar-run-script-" + str(node_index) + "-" + str(vc_index)) - # Environment variables env_vars = { "BEACON_NODE_ADDRESS": charon_validator_api_url, "BUILDER_API_ENABLED": "true", "NODE": "node" + str(node_index), } - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + if participant.vc_extra_env_vars: env_vars.update(participant.vc_extra_env_vars) # Files to mount - Charon keys + standard genesis data + run script @@ -755,8 +617,8 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ client_type=constants.CLIENT_TYPES.validator, image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, ), tolerations=tolerations, node_selectors=node_selectors, @@ -806,10 +668,6 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ name="teku-config-" + str(node_index) + "-" + str(vc_index), ) - # Debug: Print that the config artifact has been created - plan.print("Created Teku config artifact: teku-config-" + str(node_index) + "-" + str(vc_index)) - plan.print("You can download this config using: kurtosis files download teku-config-" + str(node_index) + "-" + str(vc_index)) - # Teku validator command based on standard teku.star but with Charon-specific flags cmd = [ "validator-client", @@ -827,16 +685,13 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ "--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), ] - # print the cmd - plan.print("cmd: " + str(cmd)) - # Add extra params if specified - if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + if participant.vc_extra_params: cmd.extend([param for param in participant.vc_extra_params]) # Environment variables env_vars = {} - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + if participant.vc_extra_env_vars: env_vars.update(participant.vc_extra_env_vars) # Files to mount - Charon keys + standard genesis data + teku config @@ -862,9 +717,7 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ config=ServiceConfig( image=vc_image, ports=ports, - cmd = cmd, - # cmd=["tail", "-f", "/dev/null"], - # entrypoint= ["sh", "-c"], + cmd=cmd, env_vars=env_vars, files=files, labels=shared_utils.label_maker( @@ -872,8 +725,8 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ client_type=constants.CLIENT_TYPES.validator, image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, ), tolerations=tolerations, node_selectors=node_selectors, @@ -996,7 +849,6 @@ tail -f /dev/null } # Create the key import service - plan.print("Creating Nimbus key import service: " + key_import_service_name) key_import_service = plan.add_service( name=key_import_service_name, config=ServiceConfig( @@ -1010,8 +862,6 @@ tail -f /dev/null ) # Step 2: Wait for key import to complete and then extract the keys as an artifact - plan.print("Waiting for key import to complete...") - # Wait for the completion marker file to be created plan.exec( service_name=key_import_service_name, @@ -1068,7 +918,7 @@ exec "$NIMBUS_VC_PATH" \\ """ # Add extra params if specified - if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + if participant.vc_extra_params: extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) vc_run_script = vc_run_script.replace("--distributed", "--distributed" + extra_params) @@ -1088,7 +938,7 @@ exec "$NIMBUS_VC_PATH" \\ "BEACON_NODE_ADDRESS": charon_validator_api_url, "NODE": "node" + str(node_index), } - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + if participant.vc_extra_env_vars: vc_env_vars.update(participant.vc_extra_env_vars) # Files to mount for VC - imported keys + VC script @@ -1107,7 +957,6 @@ exec "$NIMBUS_VC_PATH" \\ } # Create the actual validator client service - plan.print("Creating Nimbus validator client service: " + vc_service_name) vc_service = plan.add_service( name=vc_service_name, config=ServiceConfig( @@ -1122,8 +971,8 @@ exec "$NIMBUS_VC_PATH" \\ client_type=constants.CLIENT_TYPES.validator, image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, ), tolerations=tolerations, node_selectors=node_selectors, @@ -1212,7 +1061,7 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ """ # Add extra params if specified - if hasattr(participant, "vc_extra_params") and len(participant.vc_extra_params) > 0: + if participant.vc_extra_params: extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) run_script_content = run_script_content.replace("--distributed", "--distributed" + extra_params) @@ -1227,15 +1076,11 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ name="prysm-run-script-" + str(node_index) + "-" + str(vc_index), ) - # Debug: Print that the script artifact has been created - plan.print("Created Prysm run script artifact: prysm-run-script-" + str(node_index) + "-" + str(vc_index)) - plan.print("You can download this script using: kurtosis files download prysm-run-script-" + str(node_index) + "-" + str(vc_index)) - # Environment variables env_vars = { "BEACON_NODE_ADDRESS": charon_validator_api_url, } - if hasattr(participant, "vc_extra_env_vars") and participant.vc_extra_env_vars: + if participant.vc_extra_env_vars: env_vars.update(participant.vc_extra_env_vars) # Files to mount - Charon keys + standard genesis data + run script @@ -1270,8 +1115,8 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ client_type=constants.CLIENT_TYPES.validator, image=vc_image[-constants.MAX_LABEL_LENGTH:], connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels if hasattr(participant, "vc_extra_labels") else {}, - supernode=participant.supernode if hasattr(participant, "supernode") else False, + extra_labels=participant.vc_extra_labels, + supernode=participant.supernode, ), tolerations=tolerations, node_selectors=node_selectors, @@ -1281,8 +1126,7 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ return vc_service -def new_charon_launcher(el_cl_genesis_data, jwt_file): +def new_charon_launcher(el_cl_genesis_data): return struct( el_cl_genesis_data=el_cl_genesis_data, - jwt_file=jwt_file, ) diff --git a/src/vc/vc_launcher.star b/src/vc/vc_launcher.star index b073dcfd9..28b0792e8 100644 --- a/src/vc/vc_launcher.star +++ b/src/vc/vc_launcher.star @@ -217,7 +217,9 @@ def get_vc_config( otel_otlp_grpc_url=otel_otlp_grpc_url, vc_binary_artifact=vc_binary_artifact, ) - # Charon is now handled by charon_launcher.star + elif vc_type == constants.VC_TYPE.charon: + # Charon is launched separately by charon_launcher.star. + return None elif vc_type == constants.VC_TYPE.grandine: fail("Grandine VC is not yet supported") elif vc_type == constants.VC_TYPE.consensoor: From 7488a41ab8bffd59a393a506758de65df4b8ca14 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:13:51 +0300 Subject: [PATCH 18/35] Default to 4 charon nodes --- README.md | 2 ++ network_params.yaml | 1 + src/package_io/input_parser.star | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94a0ef117..d336f4e31 100644 --- a/README.md +++ b/README.md @@ -2230,7 +2230,9 @@ participants: # Create your network_params.yaml with Charon configuration kurtosis run --enclave charon-testnet github.com/ethpandaops/ethereum-package --args-file network_params.yaml ``` + ### Running Charon Networks from source code + ```bash # Create your network_params.yaml with Charon configuration kurtosis run . --args-file network_params_charon_example.yaml diff --git a/network_params.yaml b/network_params.yaml index 660497e2b..8dc36d93e 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -36,6 +36,7 @@ participants: # Validator vc_type: lighthouse vc_image: sigp/lighthouse:latest + vc_binary_path: "" vc_log_level: "" vc_extra_env_vars: {} vc_extra_labels: {} diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 39a5e75b0..4a60b7431 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -2026,7 +2026,7 @@ def default_participant(): "vc_max_cpu": 0, "vc_min_mem": 0, # Charon-specific parameters - "charon_node_count": 3, + "charon_node_count": 4, "charon_params": { "charon_vc": "lighthouse", "charon_vc_image": DEFAULT_CL_IMAGES[constants.CL_TYPE.lighthouse], From 709f92588f5baa881870b8f66170f2fb8b243d4e Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:41:22 +0300 Subject: [PATCH 19/35] Gate the DV path --- .../validator_keystore_generator.star | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star index 7968586eb..cf73882e1 100644 --- a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star +++ b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star @@ -305,6 +305,28 @@ def generate_validator_keystores_in_parallel( ) generate_keystores_cmd += teku_permissions_cmd generate_keystores_cmd += raw_secret_permissions_cmd + + # Collect the raw keys and secrets together in a single folder, which + # Charon consumes when splitting keys across its cluster nodes. Only + # Charon participants need this, so vanilla VCs skip the extra copy. + if participant.vc_type == constants.VC_TYPE.charon: + generate_keystores_cmd += ( + " && cp -r " + + output_dirpath + + RAW_KEYS_DIRNAME + + "/ " + + output_dirpath + + RAW_KEYS_SECRETS_DIRNAME + ) + generate_keystores_cmd += ( + " && cp -r " + + output_dirpath + + RAW_SECRETS_DIRNAME + + "/ " + + output_dirpath + + RAW_KEYS_SECRETS_DIRNAME + ) + all_generation_commands.append(generate_keystores_cmd) all_output_dirpaths.append(output_dirpath) @@ -368,6 +390,13 @@ def generate_validator_keystores_in_parallel( # This is necessary because the way Kurtosis currently implements artifact-storing is base_dirname_in_artifact = shared_utils.path_base(output_dirpath) + # The raw-keys-secrets folder is only generated for Charon participants + # (see above), so only reference it for them; vanilla VCs leave it empty. + raw_keys_secrets_relative_dirpath = "" + if participant.vc_type == constants.VC_TYPE.charon: + raw_keys_secrets_relative_dirpath = shared_utils.path_join( + base_dirname_in_artifact, RAW_KEYS_SECRETS_DIRNAME + ) to_add = keystore_files_module.new_keystore_files( artifact_name, shared_utils.path_join(base_dirname_in_artifact), @@ -377,6 +406,7 @@ def generate_validator_keystores_in_parallel( shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), + raw_keys_secrets_relative_dirpath, ) keystore_files.append(to_add) From 5239d4ed8a338bec5b60eda6882275a7343f3525 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Thu, 18 Jun 2026 11:41:05 +0300 Subject: [PATCH 20/35] Spin up own relay; fix metrics; cleanup unused calls --- main.star | 5 ++ network_params_charon_example.yaml | 1 - src/package_io/input_parser.star | 5 +- src/participant_network.star | 10 ++- src/vc/charon_launcher.star | 122 ++++++++++++++++++++++------- 5 files changed, 109 insertions(+), 34 deletions(-) diff --git a/main.star b/main.star index 201ee3caa..a3995cc7f 100644 --- a/main.star +++ b/main.star @@ -530,6 +530,7 @@ def run(plan, args={}): network_id, osaka_time, shadowfork_block_height, + charon_metrics_jobs, ) = participant_network.launch_participant_network( plan, args_with_right_defaults, @@ -549,6 +550,10 @@ def run(plan, args={}): detected_backend, ) + # Charon clusters expose extra per-node/per-VC metrics endpoints that aren't + # captured by the single vc_context per participant; register them directly. + prometheus_additional_metrics_jobs.extend(charon_metrics_jobs) + for p in all_participants: if p.el_context != None: plan.print( diff --git a/network_params_charon_example.yaml b/network_params_charon_example.yaml index 00bc4d5c8..7b64b14b3 100644 --- a/network_params_charon_example.yaml +++ b/network_params_charon_example.yaml @@ -62,7 +62,6 @@ network_params: fulu_fork_epoch: 18446744073709551615 network_sync_base_url: https://snapshots.ethpandaops.io/ force_snapshot_sync: false - data_column_sidecar_subnet_count: 128 samples_per_slot: 8 custody_requirement: 4 max_blobs_per_block_electra: 9 diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 4a60b7431..569502ff2 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -2574,8 +2574,9 @@ def enrich_mev_extra_params(parsed_arguments_dict, mev_prefix, mev_port, mev_typ if participant["vc_type"] == "vero": participant["vc_extra_params"].append("--use-external-builder") - if participant["vc_type"] == "charon": - participant["vc_extra_params"].append("--builder-api=true") + # Note: Charon enables the builder API via the CHARON_BUILDER_API env var + # set in charon_launcher.star, not through vc_extra_params (those flow into + # the inner validator client, where "--builder-api" is not a valid flag). num_participants = len(parsed_arguments_dict["participants"]) index_str = shared_utils.zfill_custom( diff --git a/src/participant_network.star b/src/participant_network.star index 67c923892..d5d7922f6 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -348,8 +348,11 @@ def launch_participant_network( vc_service_info = {} # Charon launches its own cluster of services immediately (rather than # deferring to the parallel launch below), so its contexts are collected - # here keyed by participant index and merged in afterwards. + # here keyed by participant index and merged in afterwards. It also produces + # extra Prometheus scrape jobs (one per Charon node and validator client) + # that are returned to the caller to register with Prometheus. charon_vc_contexts = {} + charon_metrics_jobs = [] for index, participant in enumerate(args_with_right_defaults.participants): el_type = participant.el_type cl_type = participant.cl_type @@ -562,7 +565,7 @@ def launch_participant_network( # vc_context, so it bypasses the deferred config / parallel-launch path # used by the other validator clients. if vc_type == constants.VC_TYPE.charon: - charon_vc_contexts[index] = charon_launcher.launch( + charon_vc_contexts[index], charon_jobs = charon_launcher.launch( plan=plan, launcher=charon_launcher.new_charon_launcher( el_cl_genesis_data=el_cl_data @@ -579,7 +582,9 @@ def launch_participant_network( network_params=network_params, port_publisher=args_with_right_defaults.port_publisher, vc_index=current_vc_index, + genesis_timestamp=final_genesis_timestamp, ) + charon_metrics_jobs.extend(charon_jobs) current_vc_index += 1 continue @@ -737,4 +742,5 @@ def launch_participant_network( network_id, el_cl_data.osaka_time, el_cl_data.shadowfork_block_height, + charon_metrics_jobs, ) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 6eaed572e..e41e863a0 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -4,11 +4,13 @@ constants = import_module("../package_io/constants.star") vc_shared = import_module("./shared.star") vc_context = import_module("./vc_context.star") node_metrics = import_module("../node_metrics_info.star") +prometheus = import_module("../prometheus/prometheus_launcher.star") # Charon specific ports CHARON_VALIDATOR_API_PORT = 3600 CHARON_P2P_TCP_PORT = 3610 CHARON_MONITORING_PORT = 3620 +CHARON_RELAY_HTTP_PORT = 3640 # Fallback node count if the participant doesn't request a valid one. DEFAULT_CHARON_NODE_COUNT = 4 @@ -36,12 +38,13 @@ def launch( network_params, port_publisher, vc_index, + genesis_timestamp, ): """ Launch a Charon distributed validator client """ if node_keystore_files == None: - return None + return None, [] tolerations = shared_utils.get_tolerations( specific_container_tolerations=participant.vc_tolerations, @@ -68,27 +71,20 @@ def launch( # All Charon nodes connect to the same beacon node. beacon_endpoint = cl_context.beacon_http_url - # Fetch the actual genesis timestamp from the beacon node - genesis_response = plan.run_sh( - name="get-genesis-timestamp", - description="Get the genesis timestamp from the beacon node", - run="curl -s " + cl_context.beacon_http_url + "/eth/v1/beacon/genesis | jq -r '.data.genesis_time' | tr -d '\\n'", - ) - - # Extract the genesis timestamp from the response - genesis_time = genesis_response.output + # The genesis timestamp is already known from genesis generation, so use it + # directly rather than querying the (possibly not-yet-ready) beacon node. + genesis_time = genesis_timestamp - # Get the raw validator keys directory path - validator_keys_dirpath = "" - if node_keystore_files: - validator_keys_dirpath = shared_utils.path_join( - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_keys_relative_dirpath, - ) - validator_secrets_dirpath = shared_utils.path_join( - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_secrets_relative_dirpath, - ) + # Raw validator key/secret directory paths. node_keystore_files is non-None + # here (we returned early above otherwise). + validator_keys_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.raw_keys_relative_dirpath, + ) + validator_secrets_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.raw_secrets_relative_dirpath, + ) # Create a temporary service to format the validator keys for Charon # Use busybox as a lightweight image for key formatting @@ -179,6 +175,9 @@ done service_name=key_formatter_service.name, src="/opt/charon/charon-keys", name="charon-keys-" + str(vc_index), ) + # The formatter has served its purpose; tear it down so it doesn't linger. + plan.remove_service(name=key_formatter_service.name) + charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" persistent_key = "data-{0}".format(charon_service_name) @@ -233,6 +232,37 @@ done ), ) + # Spin up a local Charon relay so the nodes can discover each other within + # the enclave instead of depending on the public Obol relay network. + relay_service = plan.add_service( + name=service_name + "-charon-relay-" + str(vc_index), + config=ServiceConfig( + image=image, + cmd=[ + "relay", + "--data-dir=/opt/charon", + "--http-address=0.0.0.0:" + str(CHARON_RELAY_HTTP_PORT), + "--p2p-tcp-address=0.0.0.0:" + str(CHARON_P2P_TCP_PORT), + "--monitoring-address=0.0.0.0:" + str(CHARON_MONITORING_PORT), + ], + ports={ + "relay-http": PortSpec( + number=CHARON_RELAY_HTTP_PORT, + transport_protocol="TCP", + application_protocol="http", + ), + "p2p-tcp": PortSpec( + number=CHARON_P2P_TCP_PORT, + transport_protocol="TCP", + ), + }, + user=User(uid=0, gid=0), + ), + ) + charon_relay_url = "http://{0}:{1}".format( + relay_service.ip_address, CHARON_RELAY_HTTP_PORT + ) + # Launch Charon nodes charon_services = [] for i in range(charon_node_count): @@ -241,7 +271,7 @@ done env_vars = { "CHARON_LOG_LEVEL": log_level, "CHARON_LOG_FORMAT": "console", - "CHARON_P2P_RELAYS": "https://0.relay.obol.tech", + "CHARON_P2P_RELAYS": charon_relay_url, "CHARON_BUILDER_API": "true", "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), @@ -333,6 +363,7 @@ done launch_vc = vc_launchers[vc_type] # Launch one validator client per Charon node, connected to that node's validator API. + vc_services = [] for i in range(charon_node_count): charon_validator_api_url = "http://{0}:{1}".format( charon_services[i].ip_address, CHARON_VALIDATOR_API_PORT @@ -345,7 +376,7 @@ done name="validator-keys-node-" + str(i) + "-" + str(vc_index), ) - launch_vc( + vc_services.append(launch_vc( plan=plan, vc_service_name=service_name + "-vc-" + str(i) + "-" + vc_type, charon_validator_api_url=charon_validator_api_url, @@ -358,9 +389,45 @@ done vc_index=vc_index, node_index=i, vc_image=vc_image, - ) + )) + + # The cluster files have all been extracted as artifacts; drop the busybox + # helper that was only kept alive (tail -f) to read them. + plan.remove_service(name=cluster_files_service.name) - # Return the first Charon service as the main service + # Node 0 is surfaced as the participant's primary vc_context (below). Register + # every other Charon node and all validator clients as additional Prometheus + # scrape jobs so the whole cluster is monitored, not just node 0. + metrics_jobs = [] + for i in range(charon_node_count): + if i != 0: + charon_service = charon_services[i] + metrics_jobs.append(prometheus.new_metrics_job( + job_name=charon_service.name, + endpoint="{0}:{1}".format( + charon_service.ip_address, CHARON_MONITORING_PORT + ), + metrics_path=vc_shared.METRICS_PATH, + labels={ + "service": charon_service.name, + "client_type": constants.CLIENT_TYPES.validator, + "client_name": constants.VC_TYPE.charon, + }, + )) + vc_service = vc_services[i] + vc_metrics_port = vc_service.ports[constants.METRICS_PORT_ID] + metrics_jobs.append(prometheus.new_metrics_job( + job_name=vc_service.name, + endpoint="{0}:{1}".format(vc_service.ip_address, vc_metrics_port.number), + metrics_path=vc_shared.METRICS_PATH, + labels={ + "service": vc_service.name, + "client_type": constants.CLIENT_TYPES.validator, + "client_name": vc_type, + }, + )) + + # Surface Charon node 0 as the participant's primary vc_context. validator_metrics_port = charon_services[0].ports["monitoring"] validator_metrics_url = "{0}:{1}".format( charon_services[0].ip_address, validator_metrics_port.number @@ -373,7 +440,7 @@ done client_name=constants.VC_TYPE.charon, service_name=charon_services[0].name, metrics_info=validator_node_metrics_info, - ) + ), metrics_jobs def launch_lighthouse_vc( plan, @@ -398,9 +465,6 @@ def launch_lighthouse_vc( startup_script = """#!/bin/bash set -e -# Install required packages -apt-get update && apt-get install -y curl jq wget - # Stage 1: Import validator keys for f in /opt/charon/keys/keystore-*.json; do if [ -f "$f" ]; then From 318b6b589975c1e479cddfe70037b8b47cdfad50 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:10:39 +0300 Subject: [PATCH 21/35] Various charon updates; add vouch VC --- src/package_io/constants.star | 1 + src/vc/charon_launcher.star | 179 ++++++++++++++++++++++++++++++++-- 2 files changed, 174 insertions(+), 6 deletions(-) diff --git a/src/package_io/constants.star b/src/package_io/constants.star index b29110a70..ba55cab62 100644 --- a/src/package_io/constants.star +++ b/src/package_io/constants.star @@ -32,6 +32,7 @@ VC_TYPE = struct( grandine="grandine", consensoor="consensoor", charon="charon", + vouch="vouch", ) REMOTE_SIGNER_TYPE = struct(web3signer="web3signer") diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index e41e863a0..dc6852f5d 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -195,7 +195,9 @@ done image=image, cmd=[ "create", "cluster", - "--name=test", + # cluster_name label shown in Charon dashboards; mirror the + # docker-compose convention "kurtosis--". + "--name=kurtosis-" + cl_context.client_name + "-" + vc_type, "--nodes=" + str(charon_node_count), "--fee-recipient-addresses=" + constants.VALIDATING_REWARDS_ACCOUNT, "--withdrawal-addresses=" + constants.CHARON_WITHDRAWAL_ADDRESS, @@ -204,7 +206,7 @@ done "--testnet-chain-id=" + network_params.network_id, "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, "--testnet-genesis-timestamp=" + str(genesis_time), - "--testnet-name=kurtosis-testnet", + "--testnet-name=kurtosis", "--cluster-dir=" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, ], files=files, @@ -244,6 +246,10 @@ done "--http-address=0.0.0.0:" + str(CHARON_RELAY_HTTP_PORT), "--p2p-tcp-address=0.0.0.0:" + str(CHARON_P2P_TCP_PORT), "--monitoring-address=0.0.0.0:" + str(CHARON_MONITORING_PORT), + # The relay lives on a private Kurtosis network; without this it + # advertises no addresses in its ENR and nodes fail to resolve it + # ("timeout resolving bootnode ENR"). + "--p2p-advertise-private-addresses=true", ], ports={ "relay-http": PortSpec( @@ -284,7 +290,7 @@ done "CHARON_TESTNET_CHAIN_ID": network_params.network_id, "CHARON_TESTNET_FORK_VERSION": constants.GENESIS_FORK_VERSION, "CHARON_TESTNET_GENESIS_TIMESTAMP": str(genesis_time), - "CHARON_TESTNET_NAME": "kurtosis-testnet", + "CHARON_TESTNET_NAME": "kurtosis", } # Add any extra environment variables @@ -315,7 +321,7 @@ done "--testnet-chain-id=" + network_params.network_id, "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, "--testnet-genesis-timestamp=" + str(genesis_time), - "--testnet-name=kurtosis-testnet", + "--testnet-name=kurtosis", ] # Add the service @@ -353,6 +359,7 @@ done constants.VC_TYPE.teku: launch_teku_vc, constants.VC_TYPE.nimbus: launch_nimbus_vc, constants.VC_TYPE.prysm: launch_prysm_vc, + constants.VC_TYPE.vouch: launch_vouch_vc, } if vc_type not in vc_launchers: fail( @@ -515,6 +522,9 @@ exec lighthouse validator \\ number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, transport_protocol="TCP", application_protocol="http", + # Importing 256 keystores sequentially can exceed the default port + # readiness timeout; allow plenty of time so the VC isn't rolled back. + wait="15m", ), } @@ -663,6 +673,9 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, transport_protocol="TCP", application_protocol="http", + # Importing 256 keystores sequentially can exceed the default port + # readiness timeout; allow plenty of time so the VC isn't rolled back. + wait="15m", ), } @@ -739,7 +752,6 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ "--beacon-node-api-endpoint=" + charon_validator_api_url, "--config-file=/opt/charon/teku/teku-config.yaml", "--validators-external-signer-slashing-protection-enabled=true", - "--validators-proposer-blinded-blocks-enabled=true", "--validators-builder-registration-default-enabled=true", "--Xobol-dvt-integration-enabled=true", "--logging=DEBUG", @@ -772,6 +784,9 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, transport_protocol="TCP", application_protocol="http", + # Importing 256 keystores sequentially can exceed the default port + # readiness timeout; allow plenty of time so the VC isn't rolled back. + wait="15m", ), } @@ -1017,6 +1032,9 @@ exec "$NIMBUS_VC_PATH" \\ number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, transport_protocol="TCP", application_protocol="http", + # Importing 256 keystores sequentially can exceed the default port + # readiness timeout; allow plenty of time so the VC isn't rolled back. + wait="15m", ), } @@ -1117,7 +1135,6 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ --enable-beacon-rest-api \\ --beacon-rest-api-provider="$BEACON_NODE_ADDRESS" \\ --beacon-rpc-provider="$BEACON_NODE_ADDRESS" \\ - --beacon-rpc-gateway-provider="$BEACON_NODE_ADDRESS" \\ --chain-config-file="/opt/prysm/config.yaml" \\ --monitoring-host=0.0.0.0 \\ --monitoring-port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ @@ -1161,6 +1178,9 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, transport_protocol="TCP", application_protocol="http", + # Importing 256 keystores sequentially can exceed the default port + # readiness timeout; allow plenty of time so the VC isn't rolled back. + wait="15m", ), } @@ -1190,6 +1210,153 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ return vc_service +def launch_vouch_vc( + plan, + vc_service_name, + charon_validator_api_url, + validator_keys_artifact, + launcher, + participant, + tolerations, + node_selectors, + full_name, + vc_index, + node_index, + vc_image +): + """ + Launch a Vouch validator client that connects to a Charon node. + + Imports the Charon-split keystores into an ethdo wallet, + writes ~/.vouch.yml pointing at the Charon validator API, and runs vouch. + ethdo/yq are fetched at runtime (the attestant/vouch image ships neither), + with arch detection so it works on amd64 and arm64 hosts alike. + Prometheus metrics are exposed on the standard VC metrics port so Kurtosis' + readiness check passes and the cluster is scraped like every other VC. + """ + + ETHDO_VERSION = "1.37.3" + + startup_script = """#!/usr/bin/env bash +set -e + +# Only wget+ca-certificates are needed (for the ethdo download). Installing curl +# pulls a large dependency chain that pushed startup past Kurtosis' port-readiness +# window, so keep this minimal. +apt-get update +apt-get install -y --no-install-recommends wget ca-certificates + +# Match the host architecture so the downloaded ethdo binary actually runs. +ARCH="$(uname -m)" +case "${ARCH}" in + x86_64) DL_ARCH="amd64" ;; + aarch64|arm64) DL_ARCH="arm64" ;; + *) echo "Unsupported arch ${ARCH}"; exit 1 ;; +esac + +mkdir -p /opt/vouch +cd /opt/vouch + +# Install ethdo (used to import the Charon keystores into a wallet vouch can read). +wget -q "https://github.com/wealdtech/ethdo/releases/download/v""" + ETHDO_VERSION + """/ethdo-""" + ETHDO_VERSION + """-linux-${DL_ARCH}.tar.gz" -O ethdo.tar.gz +tar -xf ethdo.tar.gz +rm ethdo.tar.gz + +# Passphrase protecting every account in the local wallet. +account_passphrase="1234" + +./ethdo wallet create --wallet=vals --passphrase="" + +accounts_list=() +for keystore_file in /home/charon/validator_keys/keystore-*.json; do + basename="$(basename "${keystore_file%.json}")" + password_file="/home/charon/validator_keys/${basename}.txt" + index="${basename##*-}" + account_name="vals/val${index}" + passphrase_content=$(cat "$password_file") + + echo "Importing account ${account_name} from ${keystore_file}" + ./ethdo account import \\ + --account="$account_name" \\ + --keystore="$keystore_file" \\ + --keystore-passphrase="$passphrase_content" \\ + --passphrase="$account_passphrase" --allow-weak-passphrases + + accounts_list+=("$account_name") +done + +yq_accounts=$(printf " - %s\\n" "${accounts_list[@]}") +echo -n "$account_passphrase" > /opt/vouch/account_passphrase.txt + +cat > ~/.vouch.yml < Date: Mon, 22 Jun 2026 11:11:03 +0300 Subject: [PATCH 22/35] Add remote write option to Prometheus --- src/package_io/input_parser.star | 14 +++ src/package_io/sanity_check.star | 3 + src/prometheus/prometheus_launcher.star | 118 +++++++++++++++++++++--- 3 files changed, 121 insertions(+), 14 deletions(-) diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 569502ff2..edac225fb 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -1030,6 +1030,11 @@ def input_parser(plan, input_args): min_mem=result["prometheus_params"]["min_mem"], max_mem=result["prometheus_params"]["max_mem"], image=result["prometheus_params"]["image"], + remote_write_url=result["prometheus_params"]["remote_write_url"], + remote_write_token=result["prometheus_params"]["remote_write_token"], + remote_write_job_regex=result["prometheus_params"][ + "remote_write_job_regex" + ], ), grafana_params=struct( additional_dashboards=result["grafana_params"]["additional_dashboards"], @@ -2253,6 +2258,15 @@ def get_default_prometheus_params(): "min_mem": 128, "max_mem": 2048, "image": "prom/prometheus:v3.2.1", + # remote_write: ship metrics to an external endpoint (e.g. Obol central + # monitoring). Disabled unless remote_write_token is non-empty. The token + # is intended to be supplied at runtime via --args-file / Kurtosis args. + "remote_write_url": "https://vm.monitoring.gcp.obol.tech/write", + "remote_write_token": "", + # Native scrape jobs are named after the Kurtosis service (e.g. + # vc-1-geth-lighthouse-charon-charon-0), so match jobs *containing* charon + # rather than starting with it; keeps Charon nodes + their VCs, drops EL/CL. + "remote_write_job_regex": ".*charon.*", } diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index abf90bfd2..5e0c67c3f 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -339,6 +339,9 @@ SUBCATEGORY_PARAMS = { "storage_tsdb_retention_time", "storage_tsdb_retention_size", "image", + "remote_write_url", + "remote_write_token", + "remote_write_job_regex", ], "grafana_params": [ "additional_dashboards", diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index 54572566a..fece8eea3 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -14,6 +14,47 @@ METRICS_INFO_ADDITIONAL_CONFIG_KEY = "config" PROMETHEUS_DEFAULT_SCRAPE_INTERVAL = "15s" +PROMETHEUS_CONFIG_DIR = "/config" +PROMETHEUS_CONFIG_FILENAME = "prometheus-config.yml" + +# Rendered in-package (rather than via the external prometheus-package) so we can +# append an optional remote_write block when a token is supplied. The scrape_config +# section mirrors the upstream prometheus-package template. +PROMETHEUS_CONFIG_TEMPLATE = """global: + scrape_interval: 15s +scrape_configs: + {{- range $job := .MetricsJobs }} + - job_name: "{{ $job.Name }}" + metrics_path: "{{ $job.MetricsPath }}" + {{- if $job.ScrapeInterval }} + scrape_interval: {{ $job.ScrapeInterval }} + {{- end }} + static_configs: + - targets: ['{{ $job.Endpoint }}'] + labels:{{ range $labelName, $labelValue := $job.Labels }} + {{ $labelName }}: "{{ $labelValue }}" + {{- end }} + {{- end }} +{{- if .RemoteWriteToken }} +remote_write: + - url: {{ .RemoteWriteUrl }} + authorization: + credentials: "{{ .RemoteWriteToken }}" + write_relabel_configs: + - source_labels: [job] + regex: '{{ .RemoteWriteJobRegex }}' + action: keep + # Charon dashboards query job="charon". Native scrape jobs are named after + # the service, so rewrite the job label to "charon" for Charon-node series + # (identified by client_name=charon; VCs keep their own job label). + - source_labels: [client_name] + regex: 'charon' + target_label: job + replacement: 'charon' + action: replace +{{- end }} +""" + def launch_prometheus( plan, @@ -48,22 +89,71 @@ def launch_prometheus( 0, ) - prometheus_url = prometheus.run( - plan, - metrics_jobs, - "prometheus", - min_cpu=prometheus_params.min_cpu, - max_cpu=prometheus_params.max_cpu, - min_memory=prometheus_params.min_mem, - max_memory=prometheus_params.max_mem, - node_selectors=global_node_selectors, - storage_tsdb_retention_time=prometheus_params.storage_tsdb_retention_time, - storage_tsdb_retention_size=prometheus_params.storage_tsdb_retention_size, - image=prometheus_params.image, - public_ports=public_ports, + # Render the Prometheus config in-package so a remote_write block can be added + # when prometheus_params.remote_write_token is supplied (e.g. via Kurtosis args + # for shipping metrics to Obol central monitoring). remote_write stays off when + # the token is empty. + config_artifact = plan.render_templates( + config={ + PROMETHEUS_CONFIG_FILENAME: struct( + template=PROMETHEUS_CONFIG_TEMPLATE, + data={ + "MetricsJobs": metrics_jobs, + "RemoteWriteUrl": prometheus_params.remote_write_url, + "RemoteWriteToken": prometheus_params.remote_write_token, + "RemoteWriteJobRegex": prometheus_params.remote_write_job_regex, + }, + ), + }, + name="prometheus-config", + ) + + image = prometheus_params.image + if image == "": + image = "prom/prometheus:latest" + + prometheus_service = plan.add_service( + name="prometheus", + config=ServiceConfig( + image=image, + ports={ + constants.HTTP_PORT_ID: PortSpec( + number=9090, + transport_protocol="TCP", + application_protocol="http", + ), + }, + files={ + PROMETHEUS_CONFIG_DIR: config_artifact, + }, + cmd=[ + "--config.file=" + + PROMETHEUS_CONFIG_DIR + + "/" + + PROMETHEUS_CONFIG_FILENAME, + "--storage.tsdb.path=/prometheus", + "--storage.tsdb.retention.time=" + + str(prometheus_params.storage_tsdb_retention_time), + "--storage.tsdb.retention.size=" + + str(prometheus_params.storage_tsdb_retention_size), + "--storage.tsdb.wal-compression", + "--web.console.libraries=/etc/prometheus/console_libraries", + "--web.console.templates=/etc/prometheus/consoles", + "--web.enable-lifecycle", + ], + min_cpu=prometheus_params.min_cpu, + max_cpu=prometheus_params.max_cpu, + min_memory=prometheus_params.min_mem, + max_memory=prometheus_params.max_mem, + node_selectors=global_node_selectors, + public_ports=public_ports, + ), ) - return prometheus_url + return "http://{0}:{1}".format( + prometheus_service.ip_address, + prometheus_service.ports[constants.HTTP_PORT_ID].number, + ) def get_metrics_jobs( From 902261cb70a028dd954289cd8a15d22f93dd428a Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:06:07 +0300 Subject: [PATCH 23/35] Don't launch Vouch outside of DVs --- src/vc/charon_launcher.star | 2 +- src/vc/vc_launcher.star | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index dc6852f5d..ebff66719 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -1229,7 +1229,7 @@ def launch_vouch_vc( Imports the Charon-split keystores into an ethdo wallet, writes ~/.vouch.yml pointing at the Charon validator API, and runs vouch. - ethdo/yq are fetched at runtime (the attestant/vouch image ships neither), + ethdo is fetched at runtime (the attestant/vouch image doesn't ship it), with arch detection so it works on amd64 and arm64 hosts alike. Prometheus metrics are exposed on the standard VC metrics port so Kurtosis' readiness check passes and the cluster is scraped like every other VC. diff --git a/src/vc/vc_launcher.star b/src/vc/vc_launcher.star index 28b0792e8..08c1fb3f0 100644 --- a/src/vc/vc_launcher.star +++ b/src/vc/vc_launcher.star @@ -220,6 +220,11 @@ def get_vc_config( elif vc_type == constants.VC_TYPE.charon: # Charon is launched separately by charon_launcher.star. return None + elif vc_type == constants.VC_TYPE.vouch: + fail( + "vouch VC is only supported as a Charon distributed-validator client; " + + "set vc_type=charon with charon_params.charon_vc=vouch" + ) elif vc_type == constants.VC_TYPE.grandine: fail("Grandine VC is not yet supported") elif vc_type == constants.VC_TYPE.consensoor: From 6c441d4a0a071439dc3dc5650a41f13e2ae4eba1 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 22 Jun 2026 12:42:22 +0300 Subject: [PATCH 24/35] Test remote write with changes in the original prometheus kurtosis package --- src/prometheus/prometheus_launcher.star | 147 +++++++----------------- 1 file changed, 43 insertions(+), 104 deletions(-) diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index fece8eea3..710aa2a47 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -1,5 +1,5 @@ shared_utils = import_module("../shared_utils/shared_utils.star") -prometheus = import_module("github.com/kurtosis-tech/prometheus-package/main.star") +prometheus = import_module("github.com/KaloyanTanev/prometheus-package/main.star@kalo/add-remote-write-kt-package-name") constants = import_module("../package_io/constants.star") EXECUTION_CLIENT_TYPE = "execution" @@ -14,47 +14,6 @@ METRICS_INFO_ADDITIONAL_CONFIG_KEY = "config" PROMETHEUS_DEFAULT_SCRAPE_INTERVAL = "15s" -PROMETHEUS_CONFIG_DIR = "/config" -PROMETHEUS_CONFIG_FILENAME = "prometheus-config.yml" - -# Rendered in-package (rather than via the external prometheus-package) so we can -# append an optional remote_write block when a token is supplied. The scrape_config -# section mirrors the upstream prometheus-package template. -PROMETHEUS_CONFIG_TEMPLATE = """global: - scrape_interval: 15s -scrape_configs: - {{- range $job := .MetricsJobs }} - - job_name: "{{ $job.Name }}" - metrics_path: "{{ $job.MetricsPath }}" - {{- if $job.ScrapeInterval }} - scrape_interval: {{ $job.ScrapeInterval }} - {{- end }} - static_configs: - - targets: ['{{ $job.Endpoint }}'] - labels:{{ range $labelName, $labelValue := $job.Labels }} - {{ $labelName }}: "{{ $labelValue }}" - {{- end }} - {{- end }} -{{- if .RemoteWriteToken }} -remote_write: - - url: {{ .RemoteWriteUrl }} - authorization: - credentials: "{{ .RemoteWriteToken }}" - write_relabel_configs: - - source_labels: [job] - regex: '{{ .RemoteWriteJobRegex }}' - action: keep - # Charon dashboards query job="charon". Native scrape jobs are named after - # the service, so rewrite the job label to "charon" for Charon-node series - # (identified by client_name=charon; VCs keep their own job label). - - source_labels: [client_name] - regex: 'charon' - target_label: job - replacement: 'charon' - action: replace -{{- end }} -""" - def launch_prometheus( plan, @@ -89,70 +48,50 @@ def launch_prometheus( 0, ) - # Render the Prometheus config in-package so a remote_write block can be added - # when prometheus_params.remote_write_token is supplied (e.g. via Kurtosis args - # for shipping metrics to Obol central monitoring). remote_write stays off when - # the token is empty. - config_artifact = plan.render_templates( - config={ - PROMETHEUS_CONFIG_FILENAME: struct( - template=PROMETHEUS_CONFIG_TEMPLATE, - data={ - "MetricsJobs": metrics_jobs, - "RemoteWriteUrl": prometheus_params.remote_write_url, - "RemoteWriteToken": prometheus_params.remote_write_token, - "RemoteWriteJobRegex": prometheus_params.remote_write_job_regex, - }, - ), - }, - name="prometheus-config", - ) - - image = prometheus_params.image - if image == "": - image = "prom/prometheus:latest" - - prometheus_service = plan.add_service( - name="prometheus", - config=ServiceConfig( - image=image, - ports={ - constants.HTTP_PORT_ID: PortSpec( - number=9090, - transport_protocol="TCP", - application_protocol="http", - ), - }, - files={ - PROMETHEUS_CONFIG_DIR: config_artifact, + # remote_write is enabled only when a token is supplied (e.g. via Kurtosis args + # for shipping Charon metrics to Obol central monitoring); empty token => no + # remote_write block, identical to upstream behaviour. + remote_write_configs = [] + if prometheus_params.remote_write_token != "": + remote_write_configs = [ + { + "Url": prometheus_params.remote_write_url, + "BearerToken": prometheus_params.remote_write_token, + "WriteRelabelConfigs": [ + # Only ship jobs matching the configured regex (Charon nodes + VCs). + { + "SourceLabels": ["job"], + "Regex": prometheus_params.remote_write_job_regex, + "Action": "keep", + }, + # Charon dashboards query job="charon"; native scrape jobs are + # named after the service, so rewrite the job label to "charon" + # for Charon-node series (VCs keep their own job label). + { + "SourceLabels": ["client_name"], + "Regex": "charon", + "TargetLabel": "job", + "Replacement": "charon", + "Action": "replace", + }, + ], }, - cmd=[ - "--config.file=" - + PROMETHEUS_CONFIG_DIR - + "/" - + PROMETHEUS_CONFIG_FILENAME, - "--storage.tsdb.path=/prometheus", - "--storage.tsdb.retention.time=" - + str(prometheus_params.storage_tsdb_retention_time), - "--storage.tsdb.retention.size=" - + str(prometheus_params.storage_tsdb_retention_size), - "--storage.tsdb.wal-compression", - "--web.console.libraries=/etc/prometheus/console_libraries", - "--web.console.templates=/etc/prometheus/consoles", - "--web.enable-lifecycle", - ], - min_cpu=prometheus_params.min_cpu, - max_cpu=prometheus_params.max_cpu, - min_memory=prometheus_params.min_mem, - max_memory=prometheus_params.max_mem, - node_selectors=global_node_selectors, - public_ports=public_ports, - ), - ) + ] - return "http://{0}:{1}".format( - prometheus_service.ip_address, - prometheus_service.ports[constants.HTTP_PORT_ID].number, + return prometheus.run( + plan, + metrics_jobs, + "prometheus", + min_cpu=prometheus_params.min_cpu, + max_cpu=prometheus_params.max_cpu, + min_memory=prometheus_params.min_mem, + max_memory=prometheus_params.max_mem, + node_selectors=global_node_selectors, + storage_tsdb_retention_time=prometheus_params.storage_tsdb_retention_time, + storage_tsdb_retention_size=prometheus_params.storage_tsdb_retention_size, + image=prometheus_params.image, + public_ports=public_ports, + remote_write_configs=remote_write_configs, ) From d1d03183dd6b59294730873b3c551e292be85e04 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:49:27 +0300 Subject: [PATCH 25/35] Fix prometheus metrics --- network_params_charon_example.yaml | 15 +++++++++++++ src/package_io/input_parser.star | 20 +++++++++--------- src/package_io/sanity_check.star | 2 +- src/prometheus/prometheus_launcher.star | 28 ++++++------------------- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/network_params_charon_example.yaml b/network_params_charon_example.yaml index 7b64b14b3..4bfa83df0 100644 --- a/network_params_charon_example.yaml +++ b/network_params_charon_example.yaml @@ -88,6 +88,21 @@ prometheus_params: max_cpu: 1000 min_mem: 128 max_mem: 2048 + # remote_write stays off until you set remote_write_url (and, if the endpoint + # needs auth, remote_write_token), e.g.: + # remote_write_url: "https://vm.monitoring.gcp.obol.tech/write" + # remote_write_token: "" + # The relabel rules below only ship Charon node/VC jobs and rewrite their job + # label to "charon" so Obol dashboards (which query job="charon") line up. + remote_write_relabel_configs: + - SourceLabels: ["job"] + Regex: ".*charon.*" + Action: keep + - SourceLabels: ["client_name"] + Regex: "charon" + TargetLabel: job + Replacement: charon + Action: replace grafana_params: additional_dashboards: [] min_cpu: 10 diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index edac225fb..88da1e735 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -1032,8 +1032,8 @@ def input_parser(plan, input_args): image=result["prometheus_params"]["image"], remote_write_url=result["prometheus_params"]["remote_write_url"], remote_write_token=result["prometheus_params"]["remote_write_token"], - remote_write_job_regex=result["prometheus_params"][ - "remote_write_job_regex" + remote_write_relabel_configs=result["prometheus_params"][ + "remote_write_relabel_configs" ], ), grafana_params=struct( @@ -2258,15 +2258,15 @@ def get_default_prometheus_params(): "min_mem": 128, "max_mem": 2048, "image": "prom/prometheus:v3.2.1", - # remote_write: ship metrics to an external endpoint (e.g. Obol central - # monitoring). Disabled unless remote_write_token is non-empty. The token - # is intended to be supplied at runtime via --args-file / Kurtosis args. - "remote_write_url": "https://vm.monitoring.gcp.obol.tech/write", + # remote_write: ship scraped metrics to an external endpoint. Disabled + # unless remote_write_url is set. remote_write_token is sent as a bearer + # credential (optional). remote_write_relabel_configs is an optional list + # of Prometheus write_relabel_configs (each a dict with SourceLabels and + # optionally Regex/Action/TargetLabel/Replacement) to filter/rewrite what + # is shipped. All are intended to be supplied at runtime via Kurtosis args. + "remote_write_url": "", "remote_write_token": "", - # Native scrape jobs are named after the Kurtosis service (e.g. - # vc-1-geth-lighthouse-charon-charon-0), so match jobs *containing* charon - # rather than starting with it; keeps Charon nodes + their VCs, drops EL/CL. - "remote_write_job_regex": ".*charon.*", + "remote_write_relabel_configs": [], } diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 5e0c67c3f..b659c512d 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -341,7 +341,7 @@ SUBCATEGORY_PARAMS = { "image", "remote_write_url", "remote_write_token", - "remote_write_job_regex", + "remote_write_relabel_configs", ], "grafana_params": [ "additional_dashboards", diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index 710aa2a47..b2769d712 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -48,33 +48,17 @@ def launch_prometheus( 0, ) - # remote_write is enabled only when a token is supplied (e.g. via Kurtosis args - # for shipping Charon metrics to Obol central monitoring); empty token => no - # remote_write block, identical to upstream behaviour. + # remote_write is enabled only when a URL is configured; empty => no + # remote_write block, identical to upstream behaviour. The relabel configs are + # passed through verbatim, so any filtering/rewriting (e.g. for a Charon + # cluster) is supplied via args rather than hardcoded here. remote_write_configs = [] - if prometheus_params.remote_write_token != "": + if prometheus_params.remote_write_url != "": remote_write_configs = [ { "Url": prometheus_params.remote_write_url, "BearerToken": prometheus_params.remote_write_token, - "WriteRelabelConfigs": [ - # Only ship jobs matching the configured regex (Charon nodes + VCs). - { - "SourceLabels": ["job"], - "Regex": prometheus_params.remote_write_job_regex, - "Action": "keep", - }, - # Charon dashboards query job="charon"; native scrape jobs are - # named after the service, so rewrite the job label to "charon" - # for Charon-node series (VCs keep their own job label). - { - "SourceLabels": ["client_name"], - "Regex": "charon", - "TargetLabel": "job", - "Replacement": "charon", - "Action": "replace", - }, - ], + "WriteRelabelConfigs": prometheus_params.remote_write_relabel_configs, }, ] From b09c7885656fd348327638f7899834813c931335 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:30:11 +0300 Subject: [PATCH 26/35] Remove remote write; use upstream prometheus package Charon DV monitoring (local scraping of all Charon nodes/VCs) is kept; only the remote_write feature and the personal prometheus-package fork import are removed, so this branch depends solely on the upstream kurtosis-tech/prometheus-package. Co-Authored-By: Claude Opus 4.8 (1M context) --- network_params_charon_example.yaml | 15 --------------- src/package_io/input_parser.star | 14 -------------- src/package_io/sanity_check.star | 3 --- src/prometheus/prometheus_launcher.star | 21 ++++----------------- 4 files changed, 4 insertions(+), 49 deletions(-) diff --git a/network_params_charon_example.yaml b/network_params_charon_example.yaml index 4bfa83df0..7b64b14b3 100644 --- a/network_params_charon_example.yaml +++ b/network_params_charon_example.yaml @@ -88,21 +88,6 @@ prometheus_params: max_cpu: 1000 min_mem: 128 max_mem: 2048 - # remote_write stays off until you set remote_write_url (and, if the endpoint - # needs auth, remote_write_token), e.g.: - # remote_write_url: "https://vm.monitoring.gcp.obol.tech/write" - # remote_write_token: "" - # The relabel rules below only ship Charon node/VC jobs and rewrite their job - # label to "charon" so Obol dashboards (which query job="charon") line up. - remote_write_relabel_configs: - - SourceLabels: ["job"] - Regex: ".*charon.*" - Action: keep - - SourceLabels: ["client_name"] - Regex: "charon" - TargetLabel: job - Replacement: charon - Action: replace grafana_params: additional_dashboards: [] min_cpu: 10 diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 88da1e735..569502ff2 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -1030,11 +1030,6 @@ def input_parser(plan, input_args): min_mem=result["prometheus_params"]["min_mem"], max_mem=result["prometheus_params"]["max_mem"], image=result["prometheus_params"]["image"], - remote_write_url=result["prometheus_params"]["remote_write_url"], - remote_write_token=result["prometheus_params"]["remote_write_token"], - remote_write_relabel_configs=result["prometheus_params"][ - "remote_write_relabel_configs" - ], ), grafana_params=struct( additional_dashboards=result["grafana_params"]["additional_dashboards"], @@ -2258,15 +2253,6 @@ def get_default_prometheus_params(): "min_mem": 128, "max_mem": 2048, "image": "prom/prometheus:v3.2.1", - # remote_write: ship scraped metrics to an external endpoint. Disabled - # unless remote_write_url is set. remote_write_token is sent as a bearer - # credential (optional). remote_write_relabel_configs is an optional list - # of Prometheus write_relabel_configs (each a dict with SourceLabels and - # optionally Regex/Action/TargetLabel/Replacement) to filter/rewrite what - # is shipped. All are intended to be supplied at runtime via Kurtosis args. - "remote_write_url": "", - "remote_write_token": "", - "remote_write_relabel_configs": [], } diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index b659c512d..abf90bfd2 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -339,9 +339,6 @@ SUBCATEGORY_PARAMS = { "storage_tsdb_retention_time", "storage_tsdb_retention_size", "image", - "remote_write_url", - "remote_write_token", - "remote_write_relabel_configs", ], "grafana_params": [ "additional_dashboards", diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index b2769d712..54572566a 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -1,5 +1,5 @@ shared_utils = import_module("../shared_utils/shared_utils.star") -prometheus = import_module("github.com/KaloyanTanev/prometheus-package/main.star@kalo/add-remote-write-kt-package-name") +prometheus = import_module("github.com/kurtosis-tech/prometheus-package/main.star") constants = import_module("../package_io/constants.star") EXECUTION_CLIENT_TYPE = "execution" @@ -48,21 +48,7 @@ def launch_prometheus( 0, ) - # remote_write is enabled only when a URL is configured; empty => no - # remote_write block, identical to upstream behaviour. The relabel configs are - # passed through verbatim, so any filtering/rewriting (e.g. for a Charon - # cluster) is supplied via args rather than hardcoded here. - remote_write_configs = [] - if prometheus_params.remote_write_url != "": - remote_write_configs = [ - { - "Url": prometheus_params.remote_write_url, - "BearerToken": prometheus_params.remote_write_token, - "WriteRelabelConfigs": prometheus_params.remote_write_relabel_configs, - }, - ] - - return prometheus.run( + prometheus_url = prometheus.run( plan, metrics_jobs, "prometheus", @@ -75,9 +61,10 @@ def launch_prometheus( storage_tsdb_retention_size=prometheus_params.storage_tsdb_retention_size, image=prometheus_params.image, public_ports=public_ports, - remote_write_configs=remote_write_configs, ) + return prometheus_url + def get_metrics_jobs( el_contexts, From df92becef7ba8aaac8fc5d8c4c945097b43c778a Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:50:35 +0300 Subject: [PATCH 27/35] Revert unrelated formatting --- network_params.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/network_params.yaml b/network_params.yaml index 8dc36d93e..e2b8539df 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -1,5 +1,5 @@ participants: - # EL +# EL - el_type: geth el_image: ethereum/client-go:latest el_binary_path: "" @@ -15,7 +15,7 @@ participants: el_max_cpu: 0 el_min_mem: 0 el_max_mem: 0 - # CL +# CL cl_type: lighthouse cl_image: sigp/lighthouse:latest cl_binary_path: "" @@ -33,7 +33,7 @@ participants: cl_max_mem: 0 supernode: false use_separate_vc: true - # Validator +# Validator vc_type: lighthouse vc_image: sigp/lighthouse:latest vc_binary_path: "" @@ -50,7 +50,7 @@ participants: vc_max_mem: 0 validator_count: null use_remote_signer: false - # Remote signer +# Remote signer remote_signer_type: web3signer remote_signer_image: consensys/web3signer:latest remote_signer_extra_env_vars: {} @@ -61,7 +61,7 @@ participants: remote_signer_max_cpu: 0 remote_signer_min_mem: 0 remote_signer_max_mem: 0 - # participant specific +# participant specific node_selectors: {} tolerations: [] count: 1 @@ -82,7 +82,8 @@ network_params: seconds_per_slot: 12 slot_duration_ms: 12000 num_validator_keys_per_node: 128 - preregistered_validator_keys_mnemonic: "giant issue aisle success illegal bike spike + preregistered_validator_keys_mnemonic: + "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete" preregistered_validator_count: 0 From 8fb6f53b9ede262ab76f7ec229306362c3d479a3 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:23:03 +0300 Subject: [PATCH 28/35] Fix linting --- src/vc/charon_launcher.star | 265 ++++++++++++++++++++++++------------ 1 file changed, 175 insertions(+), 90 deletions(-) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index ebff66719..922b9da65 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -23,6 +23,7 @@ VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.debug: "debug", } + def launch( plan, launcher, @@ -142,14 +143,22 @@ for keystore_dir in $keystore_directories; do index=$(($index + 1)) fi done -""" % (validator_keys_dirpath, validator_secrets_dirpath, validator_secrets_dirpath) +""" % ( + validator_keys_dirpath, + validator_secrets_dirpath, + validator_secrets_dirpath, + ) # Save the script to the service plan.exec( service_name=key_formatter_service.name, recipe=ExecRecipe( command=[ - "sh", "-c", "cat > /opt/charon/format_keys.sh << 'EOL'\n" + format_keys_script + "\nEOL" + "sh", + "-c", + "cat > /opt/charon/format_keys.sh << 'EOL'\n" + + format_keys_script + + "\nEOL", ], ), ) @@ -172,7 +181,9 @@ done # Store the formatted keys charon_keys_artifact = plan.store_service_files( - service_name=key_formatter_service.name, src="/opt/charon/charon-keys", name="charon-keys-" + str(vc_index), + service_name=key_formatter_service.name, + src="/opt/charon/charon-keys", + name="charon-keys-" + str(vc_index), ) # The formatter has served its purpose; tear it down so it doesn't linger. @@ -184,7 +195,7 @@ done files = {} files[CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( - persistent_key=persistent_key, + persistent_key=persistent_key, ) files["/opt/charon/charon-keys"] = charon_keys_artifact @@ -194,7 +205,8 @@ done config=ServiceConfig( image=image, cmd=[ - "create", "cluster", + "create", + "cluster", # cluster_name label shown in Charon dashboards; mirror the # docker-compose convention "kurtosis--". "--name=kurtosis-" + cl_context.client_name + "-" + vc_type, @@ -210,19 +222,19 @@ done "--cluster-dir=" + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER, ], files=files, - user = User(uid=0, gid=0), + user=User(uid=0, gid=0), ), ) # Keep a busybox service running on the cluster volume so we can read the # generated per-node files back out as artifacts. cluster_files_service = plan.add_service( - name=charon_service_name+"-keep-running", + name=charon_service_name + "-keep-running", config=ServiceConfig( image="busybox:latest", cmd=["tail", "-f", "/dev/null"], # Keep the service running files=files, - user = User(uid=0, gid=0), + user=User(uid=0, gid=0), ), ) @@ -282,8 +294,12 @@ done "CHARON_VALIDATOR_API_ADDRESS": "0.0.0.0:" + str(CHARON_VALIDATOR_API_PORT), "CHARON_P2P_TCP_ADDRESS": "0.0.0.0:" + str(CHARON_P2P_TCP_PORT), "CHARON_MONITORING_ADDRESS": "0.0.0.0:" + str(CHARON_MONITORING_PORT), - "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/node" + str(i) + "/charon-enr-private-key", - "CHARON_LOCK_FILE": "/opt/charon/.charon/node" + str(i) + "/cluster-lock.json", + "CHARON_PRIVATE_KEY_FILE": "/opt/charon/.charon/node" + + str(i) + + "/charon-enr-private-key", + "CHARON_LOCK_FILE": "/opt/charon/.charon/node" + + str(i) + + "/cluster-lock.json", "CHARON_JAEGER_SERVICE": "node" + str(i), "CHARON_P2P_EXTERNAL_HOSTNAME": "node" + str(i), "CHARON_BEACON_NODE_ENDPOINTS": beacon_endpoint, @@ -335,7 +351,7 @@ done labels=shared_utils.label_maker( client=constants.VC_TYPE.charon, client_type=constants.CLIENT_TYPES.validator, - image=image[-constants.MAX_LABEL_LENGTH:], + image=image[-constants.MAX_LABEL_LENGTH :], connected_client=cl_context.client_name, extra_labels=participant.vc_extra_labels, supernode=participant.supernode, @@ -343,11 +359,9 @@ done tolerations=tolerations, node_selectors=node_selectors, files={ - "/opt/charon/.charon/": Directory( - persistent_key=persistent_key - ), + "/opt/charon/.charon/": Directory(persistent_key=persistent_key), }, - user = User(uid=0, gid=0), + user=User(uid=0, gid=0), ), ) charon_services.append(charon_service) @@ -379,24 +393,29 @@ done # Each node's validator keys come from the cluster-creation output. validator_keys_for_node = plan.store_service_files( service_name=cluster_files_service.name, - src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + "/node" + str(i) + "/validator_keys", + src=CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER + + "/node" + + str(i) + + "/validator_keys", name="validator-keys-node-" + str(i) + "-" + str(vc_index), ) - vc_services.append(launch_vc( - plan=plan, - vc_service_name=service_name + "-vc-" + str(i) + "-" + vc_type, - charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, - launcher=launcher, - participant=participant, - tolerations=tolerations, - node_selectors=node_selectors, - full_name=full_name + "-node" + str(i), - vc_index=vc_index, - node_index=i, - vc_image=vc_image, - )) + vc_services.append( + launch_vc( + plan=plan, + vc_service_name=service_name + "-vc-" + str(i) + "-" + vc_type, + charon_validator_api_url=charon_validator_api_url, + validator_keys_artifact=validator_keys_for_node, + launcher=launcher, + participant=participant, + tolerations=tolerations, + node_selectors=node_selectors, + full_name=full_name + "-node" + str(i), + vc_index=vc_index, + node_index=i, + vc_image=vc_image, + ) + ) # The cluster files have all been extracted as artifacts; drop the busybox # helper that was only kept alive (tail -f) to read them. @@ -409,30 +428,36 @@ done for i in range(charon_node_count): if i != 0: charon_service = charon_services[i] - metrics_jobs.append(prometheus.new_metrics_job( - job_name=charon_service.name, + metrics_jobs.append( + prometheus.new_metrics_job( + job_name=charon_service.name, + endpoint="{0}:{1}".format( + charon_service.ip_address, CHARON_MONITORING_PORT + ), + metrics_path=vc_shared.METRICS_PATH, + labels={ + "service": charon_service.name, + "client_type": constants.CLIENT_TYPES.validator, + "client_name": constants.VC_TYPE.charon, + }, + ) + ) + vc_service = vc_services[i] + vc_metrics_port = vc_service.ports[constants.METRICS_PORT_ID] + metrics_jobs.append( + prometheus.new_metrics_job( + job_name=vc_service.name, endpoint="{0}:{1}".format( - charon_service.ip_address, CHARON_MONITORING_PORT + vc_service.ip_address, vc_metrics_port.number ), metrics_path=vc_shared.METRICS_PATH, labels={ - "service": charon_service.name, + "service": vc_service.name, "client_type": constants.CLIENT_TYPES.validator, - "client_name": constants.VC_TYPE.charon, + "client_name": vc_type, }, - )) - vc_service = vc_services[i] - vc_metrics_port = vc_service.ports[constants.METRICS_PORT_ID] - metrics_jobs.append(prometheus.new_metrics_job( - job_name=vc_service.name, - endpoint="{0}:{1}".format(vc_service.ip_address, vc_metrics_port.number), - metrics_path=vc_shared.METRICS_PATH, - labels={ - "service": vc_service.name, - "client_type": constants.CLIENT_TYPES.validator, - "client_name": vc_type, - }, - )) + ) + ) # Surface Charon node 0 as the participant's primary vc_context. validator_metrics_port = charon_services[0].ports["monitoring"] @@ -443,11 +468,15 @@ done charon_services[0].name, vc_shared.METRICS_PATH, validator_metrics_url ) - return vc_context.new_vc_context( - client_name=constants.VC_TYPE.charon, - service_name=charon_services[0].name, - metrics_info=validator_node_metrics_info, - ), metrics_jobs + return ( + vc_context.new_vc_context( + client_name=constants.VC_TYPE.charon, + service_name=charon_services[0].name, + metrics_info=validator_node_metrics_info, + ), + metrics_jobs, + ) + def launch_lighthouse_vc( plan, @@ -461,7 +490,7 @@ def launch_lighthouse_vc( full_name, vc_index, node_index, - vc_image + vc_image, ): """ Launch a Lighthouse validator client that connects to a Charon node @@ -469,7 +498,8 @@ def launch_lighthouse_vc( """ # Create the startup script that implements the two-stage approach - startup_script = """#!/bin/bash + startup_script = ( + """#!/bin/bash set -e # Stage 1: Import validator keys @@ -484,27 +514,34 @@ for f in /opt/charon/keys/keystore-*.json; do fi done -echo "Starting lighthouse validator client for node""" + str(node_index) + """" +echo "Starting lighthouse validator client for node""" + + str(node_index) + + """" # Stage 2: Run the validator client exec lighthouse validator \\ --beacon-nodes ${LIGHTHOUSE_BEACON_NODE_ADDRESS} \\ - --suggested-fee-recipient """ + constants.VALIDATING_REWARDS_ACCOUNT + """ \\ + --suggested-fee-recipient """ + + constants.VALIDATING_REWARDS_ACCOUNT + + """ \\ --metrics \\ --metrics-address "0.0.0.0" \\ --metrics-allow-origin "*" \\ - --metrics-port """ + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --metrics-port """ + + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + + """ \\ --use-long-timeouts \\ --testnet-dir "/opt/lighthouse/network-configs" \\ --builder-proposals \\ --distributed \\ --debug-level "debug" """ + ) # Environment variables env_vars = { "LIGHTHOUSE_BEACON_NODE_ADDRESS": charon_validator_api_url, "NODE": "node" + str(node_index), - "RUST_BACKTRACE": "full" + "RUST_BACKTRACE": "full", } if participant.vc_extra_env_vars: env_vars.update(participant.vc_extra_env_vars) @@ -540,7 +577,7 @@ exec lighthouse validator \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.lighthouse, client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH :], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels, supernode=participant.supernode, @@ -552,6 +589,7 @@ exec lighthouse validator \\ return vc_service + def launch_lodestar_vc( plan, vc_service_name, @@ -564,7 +602,7 @@ def launch_lodestar_vc( full_name, vc_index, node_index, - vc_image + vc_image, ): """ Launch a Lodestar validator client that connects to a Charon node @@ -572,7 +610,8 @@ def launch_lodestar_vc( """ # Create the run.sh script content - run_script_content = """#!/bin/sh + run_script_content = ( + """#!/bin/sh BUILDER_SELECTION="executiononly" @@ -626,13 +665,16 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ --secretsDir="$SECRETS_DIR" \\ --metrics=true \\ --metrics.address="0.0.0.0" \\ - --metrics.port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --metrics.port=""" + + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + + """ \\ --beaconNodes="$BEACON_NODE_ADDRESS" \\ --builder="$BUILDER_API_ENABLED" \\ --builder.selection="$BUILDER_SELECTION" \\ --distributed \\ --paramsFile="/opt/lodestar/config.yaml" """ + ) # Add extra params if specified if participant.vc_extra_params: @@ -692,7 +734,7 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.lodestar, client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH :], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels, supernode=participant.supernode, @@ -704,6 +746,7 @@ exec node /usr/app/packages/cli/bin/lodestar validator \\ return vc_service + def launch_teku_vc( plan, vc_service_name, @@ -716,7 +759,7 @@ def launch_teku_vc( full_name, vc_index, node_index, - vc_image + vc_image, ): """ Launch a Teku validator client that connects to a Charon node @@ -724,15 +767,21 @@ def launch_teku_vc( """ # Create the teku-config.yaml content - teku_config_content = """metrics-enabled: true + teku_config_content = ( + """metrics-enabled: true metrics-host-allowlist: "*" metrics-interface: "0.0.0.0" -metrics-port: \"""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """\" +metrics-port: \"""" + + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + + """\" validators-keystore-locking-enabled: false network: "/opt/teku/network-configs/config.yaml" validator-keys: "/opt/charon/validator_keys:/opt/charon/validator_keys" -validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ACCOUNT + """\" +validators-proposer-default-fee-recipient: \"""" + + constants.VALIDATING_REWARDS_ACCOUNT + + """\" """ + ) # Create the config file artifact using render_templates config_artifact = plan.render_templates( @@ -802,19 +851,20 @@ validators-proposer-default-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ labels=shared_utils.label_maker( client=constants.VC_TYPE.teku, client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH :], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels, supernode=participant.supernode, ), tolerations=tolerations, node_selectors=node_selectors, - user = User(uid=0, gid=0), + user=User(uid=0, gid=0), ), ) return vc_service + def launch_nimbus_vc( plan, vc_service_name, @@ -827,7 +877,7 @@ def launch_nimbus_vc( full_name, vc_index, node_index, - vc_image + vc_image, ): """ Launch a Nimbus validator client that connects to a Charon node @@ -932,7 +982,9 @@ tail -f /dev/null name=key_import_service_name, config=ServiceConfig( image="statusim/nimbus-eth2:multiarch-latest", - cmd=["chmod +x /home/user/scripts/import_keys.sh && /home/user/scripts/import_keys.sh"], + cmd=[ + "chmod +x /home/user/scripts/import_keys.sh && /home/user/scripts/import_keys.sh" + ], entrypoint=["bash", "-c"], env_vars=import_env_vars, files=import_files, @@ -945,7 +997,11 @@ tail -f /dev/null plan.exec( service_name=key_import_service_name, recipe=ExecRecipe( - command=["bash", "-c", "while [ ! -f /home/user/data/import_complete.txt ]; do echo 'Waiting for import to complete...'; sleep 2; done; echo 'Import completed! Found completion marker.'"] + command=[ + "bash", + "-c", + "while [ ! -f /home/user/data/import_complete.txt ]; do echo 'Waiting for import to complete...'; sleep 2; done; echo 'Import completed! Found completion marker.'", + ] ), description="Wait for key import completion", ) @@ -961,7 +1017,8 @@ tail -f /dev/null # Step 3: Create the actual validator client service # Create the VC run script - vc_run_script = """#!/usr/bin/env bash + vc_run_script = ( + """#!/usr/bin/env bash # Find the nimbus_validator_client binary if [ -f "/home/user/nimbus_validator_client" ]; then @@ -991,15 +1048,20 @@ exec "$NIMBUS_VC_PATH" \\ --doppelganger-detection=false \\ --metrics \\ --metrics-address=0.0.0.0 \\ - --metrics-port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --metrics-port=""" + + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + + """ \\ --payload-builder=true \\ --distributed """ + ) # Add extra params if specified if participant.vc_extra_params: extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) - vc_run_script = vc_run_script.replace("--distributed", "--distributed" + extra_params) + vc_run_script = vc_run_script.replace( + "--distributed", "--distributed" + extra_params + ) # Create the VC script artifact vc_script_artifact = plan.render_templates( @@ -1044,14 +1106,16 @@ exec "$NIMBUS_VC_PATH" \\ config=ServiceConfig( image=vc_image, ports=ports, - cmd=["chmod +x /home/user/scripts/run_vc.sh && /home/user/scripts/run_vc.sh"], + cmd=[ + "chmod +x /home/user/scripts/run_vc.sh && /home/user/scripts/run_vc.sh" + ], entrypoint=["bash", "-c"], env_vars=vc_env_vars, files=vc_files, labels=shared_utils.label_maker( client=constants.VC_TYPE.nimbus, client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH :], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels, supernode=participant.supernode, @@ -1064,6 +1128,7 @@ exec "$NIMBUS_VC_PATH" \\ return vc_service + def launch_prysm_vc( plan, vc_service_name, @@ -1076,7 +1141,7 @@ def launch_prysm_vc( full_name, vc_index, node_index, - vc_image + vc_image, ): """ Launch a Prysm validator client that connects to a Charon node @@ -1084,7 +1149,8 @@ def launch_prysm_vc( """ # Create the run.sh script content - run_script_content = """#!/usr/bin/env bash + run_script_content = ( + """#!/usr/bin/env bash WALLET_DIR="/prysm-wallet" @@ -1137,14 +1203,19 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ --beacon-rpc-provider="$BEACON_NODE_ADDRESS" \\ --chain-config-file="/opt/prysm/config.yaml" \\ --monitoring-host=0.0.0.0 \\ - --monitoring-port=""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """ \\ + --monitoring-port=""" + + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + + """ \\ --distributed """ + ) # Add extra params if specified if participant.vc_extra_params: extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) - run_script_content = run_script_content.replace("--distributed", "--distributed" + extra_params) + run_script_content = run_script_content.replace( + "--distributed", "--distributed" + extra_params + ) # Create the script file artifact using render_templates script_artifact = plan.render_templates( @@ -1197,7 +1268,7 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ labels=shared_utils.label_maker( client=constants.VC_TYPE.prysm, client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH :], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels, supernode=participant.supernode, @@ -1210,6 +1281,7 @@ exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ return vc_service + def launch_vouch_vc( plan, vc_service_name, @@ -1222,7 +1294,7 @@ def launch_vouch_vc( full_name, vc_index, node_index, - vc_image + vc_image, ): """ Launch a Vouch validator client that connects to a Charon node. @@ -1237,7 +1309,8 @@ def launch_vouch_vc( ETHDO_VERSION = "1.37.3" - startup_script = """#!/usr/bin/env bash + startup_script = ( + """#!/usr/bin/env bash set -e # Only wget+ca-certificates are needed (for the ethdo download). Installing curl @@ -1258,7 +1331,11 @@ mkdir -p /opt/vouch cd /opt/vouch # Install ethdo (used to import the Charon keystores into a wallet vouch can read). -wget -q "https://github.com/wealdtech/ethdo/releases/download/v""" + ETHDO_VERSION + """/ethdo-""" + ETHDO_VERSION + """-linux-${DL_ARCH}.tar.gz" -O ethdo.tar.gz +wget -q "https://github.com/wealdtech/ethdo/releases/download/v""" + + ETHDO_VERSION + + """/ethdo-""" + + ETHDO_VERSION + + """-linux-${DL_ARCH}.tar.gz" -O ethdo.tar.gz tar -xf ethdo.tar.gz rm ethdo.tar.gz @@ -1298,16 +1375,23 @@ $yq_accounts passphrases: - file:///opt/vouch/account_passphrase.txt blockrelay: - fallback-fee-recipient: \"""" + constants.VALIDATING_REWARDS_ACCOUNT + """\" + fallback-fee-recipient: \"""" + + constants.VALIDATING_REWARDS_ACCOUNT + + """\" metrics: prometheus: - listen-address: "0.0.0.0:""" + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + """" + listen-address: "0.0.0.0:""" + + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) + + """" EOF -echo "Starting vouch for charon node""" + str(node_index) + """" +echo "Starting vouch for charon node""" + + str(node_index) + + """" # vouch binary lives at /app/vouch in the attestant/vouch image (not on PATH). exec /app/vouch """ + ) env_vars = { "BEACON_NODE_ADDRESS": charon_validator_api_url, @@ -1344,7 +1428,7 @@ exec /app/vouch labels=shared_utils.label_maker( client=constants.VC_TYPE.vouch, client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH:], + image=vc_image[-constants.MAX_LABEL_LENGTH :], connected_client="charon-node-" + str(node_index), extra_labels=participant.vc_extra_labels, supernode=participant.supernode, @@ -1357,6 +1441,7 @@ exec /app/vouch return vc_service + def new_charon_launcher(el_cl_genesis_data): return struct( el_cl_genesis_data=el_cl_genesis_data, From 10d44e36c07834057753ba96be508766102a7a3a Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:33:04 +0300 Subject: [PATCH 29/35] Add remote write via KaloyanTanev/prometheus-package fork Adds the optional Prometheus remote_write feature on top of the cleaned charon branch: prometheus_params gains remote_write_url/token/relabel_configs (generic, inert unless a URL is set) and the prometheus import is pinned to the fork branch that implements remote_write. The Charon example documents the Charon-specific relabel rules. Co-Authored-By: Claude Opus 4.8 (1M context) --- network_params_charon_example.yaml | 15 +++++++++++++++ src/package_io/input_parser.star | 14 ++++++++++++++ src/package_io/sanity_check.star | 3 +++ src/prometheus/prometheus_launcher.star | 21 +++++++++++++++++---- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/network_params_charon_example.yaml b/network_params_charon_example.yaml index 7b64b14b3..4bfa83df0 100644 --- a/network_params_charon_example.yaml +++ b/network_params_charon_example.yaml @@ -88,6 +88,21 @@ prometheus_params: max_cpu: 1000 min_mem: 128 max_mem: 2048 + # remote_write stays off until you set remote_write_url (and, if the endpoint + # needs auth, remote_write_token), e.g.: + # remote_write_url: "https://vm.monitoring.gcp.obol.tech/write" + # remote_write_token: "" + # The relabel rules below only ship Charon node/VC jobs and rewrite their job + # label to "charon" so Obol dashboards (which query job="charon") line up. + remote_write_relabel_configs: + - SourceLabels: ["job"] + Regex: ".*charon.*" + Action: keep + - SourceLabels: ["client_name"] + Regex: "charon" + TargetLabel: job + Replacement: charon + Action: replace grafana_params: additional_dashboards: [] min_cpu: 10 diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 569502ff2..88da1e735 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -1030,6 +1030,11 @@ def input_parser(plan, input_args): min_mem=result["prometheus_params"]["min_mem"], max_mem=result["prometheus_params"]["max_mem"], image=result["prometheus_params"]["image"], + remote_write_url=result["prometheus_params"]["remote_write_url"], + remote_write_token=result["prometheus_params"]["remote_write_token"], + remote_write_relabel_configs=result["prometheus_params"][ + "remote_write_relabel_configs" + ], ), grafana_params=struct( additional_dashboards=result["grafana_params"]["additional_dashboards"], @@ -2253,6 +2258,15 @@ def get_default_prometheus_params(): "min_mem": 128, "max_mem": 2048, "image": "prom/prometheus:v3.2.1", + # remote_write: ship scraped metrics to an external endpoint. Disabled + # unless remote_write_url is set. remote_write_token is sent as a bearer + # credential (optional). remote_write_relabel_configs is an optional list + # of Prometheus write_relabel_configs (each a dict with SourceLabels and + # optionally Regex/Action/TargetLabel/Replacement) to filter/rewrite what + # is shipped. All are intended to be supplied at runtime via Kurtosis args. + "remote_write_url": "", + "remote_write_token": "", + "remote_write_relabel_configs": [], } diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index abf90bfd2..b659c512d 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -339,6 +339,9 @@ SUBCATEGORY_PARAMS = { "storage_tsdb_retention_time", "storage_tsdb_retention_size", "image", + "remote_write_url", + "remote_write_token", + "remote_write_relabel_configs", ], "grafana_params": [ "additional_dashboards", diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index 54572566a..b2769d712 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -1,5 +1,5 @@ shared_utils = import_module("../shared_utils/shared_utils.star") -prometheus = import_module("github.com/kurtosis-tech/prometheus-package/main.star") +prometheus = import_module("github.com/KaloyanTanev/prometheus-package/main.star@kalo/add-remote-write-kt-package-name") constants = import_module("../package_io/constants.star") EXECUTION_CLIENT_TYPE = "execution" @@ -48,7 +48,21 @@ def launch_prometheus( 0, ) - prometheus_url = prometheus.run( + # remote_write is enabled only when a URL is configured; empty => no + # remote_write block, identical to upstream behaviour. The relabel configs are + # passed through verbatim, so any filtering/rewriting (e.g. for a Charon + # cluster) is supplied via args rather than hardcoded here. + remote_write_configs = [] + if prometheus_params.remote_write_url != "": + remote_write_configs = [ + { + "Url": prometheus_params.remote_write_url, + "BearerToken": prometheus_params.remote_write_token, + "WriteRelabelConfigs": prometheus_params.remote_write_relabel_configs, + }, + ] + + return prometheus.run( plan, metrics_jobs, "prometheus", @@ -61,10 +75,9 @@ def launch_prometheus( storage_tsdb_retention_size=prometheus_params.storage_tsdb_retention_size, image=prometheus_params.image, public_ports=public_ports, + remote_write_configs=remote_write_configs, ) - return prometheus_url - def get_metrics_jobs( el_contexts, From 40d5c0849b90a7d996f25440545bd330c758fae4 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:34:24 +0300 Subject: [PATCH 30/35] Fix linting --- src/prometheus/prometheus_launcher.star | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/prometheus/prometheus_launcher.star b/src/prometheus/prometheus_launcher.star index b2769d712..a44693a62 100644 --- a/src/prometheus/prometheus_launcher.star +++ b/src/prometheus/prometheus_launcher.star @@ -1,5 +1,7 @@ shared_utils = import_module("../shared_utils/shared_utils.star") -prometheus = import_module("github.com/KaloyanTanev/prometheus-package/main.star@kalo/add-remote-write-kt-package-name") +prometheus = import_module( + "github.com/KaloyanTanev/prometheus-package/main.star@kalo/add-remote-write-kt-package-name" +) constants = import_module("../package_io/constants.star") EXECUTION_CLIENT_TYPE = "execution" From 087e3eed854a177210f994f540d418b8c8764da2 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:22:28 +0300 Subject: [PATCH 31/35] Reuse pre-existing VC modules; add Vouch VC module --- src/participant_network.star | 1 + src/vc/charon_launcher.star | 1284 +++++++++++----------------------- src/vc/lighthouse.star | 7 + src/vc/lodestar.star | 6 + src/vc/nimbus.star | 5 + src/vc/prysm.star | 18 +- src/vc/teku.star | 5 + src/vc/vouch.star | 138 ++++ 8 files changed, 588 insertions(+), 876 deletions(-) create mode 100644 src/vc/vouch.star diff --git a/src/participant_network.star b/src/participant_network.star index d5d7922f6..13de0d931 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -570,6 +570,7 @@ def launch_participant_network( launcher=charon_launcher.new_charon_launcher( el_cl_genesis_data=el_cl_data ), + keymanager_file=keymanager_file, service_name=service_name, image=participant.vc_image, global_log_level=args_with_right_defaults.global_log_level, diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 922b9da65..81c35ddb3 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -5,6 +5,15 @@ vc_shared = import_module("./shared.star") vc_context = import_module("./vc_context.star") node_metrics = import_module("../node_metrics_info.star") prometheus = import_module("../prometheus/prometheus_launcher.star") +lighthouse = import_module("./lighthouse.star") +lodestar = import_module("./lodestar.star") +teku = import_module("./teku.star") +nimbus = import_module("./nimbus.star") +prysm = import_module("./prysm.star") +vouch = import_module("./vouch.star") +keystore_files_module = import_module( + "../prelaunch_data_generator/validator_keystores/keystore_files.star" +) # Charon specific ports CHARON_VALIDATOR_API_PORT = 3600 @@ -15,6 +24,10 @@ CHARON_RELAY_HTTP_PORT = 3640 # Fallback node count if the participant doesn't request a valid one. DEFAULT_CHARON_NODE_COUNT = 4 +# Official ethdo image, used to build the Vouch wallet from the split keystores +# (so the Vouch container itself needs no ethdo download). +ETHDO_IMAGE = "wealdtech/ethdo:latest" + # Verbosity levels mapping VERBOSITY_LEVELS = { constants.GLOBAL_LOG_LEVEL.error: "error", @@ -27,6 +40,7 @@ VERBOSITY_LEVELS = { def launch( plan, launcher, + keymanager_file, service_name, image, global_log_level, @@ -366,14 +380,13 @@ done ) charon_services.append(charon_service) - # Map each supported VC type to its launcher function. vc_launchers = { - constants.VC_TYPE.lighthouse: launch_lighthouse_vc, - constants.VC_TYPE.lodestar: launch_lodestar_vc, - constants.VC_TYPE.teku: launch_teku_vc, - constants.VC_TYPE.nimbus: launch_nimbus_vc, - constants.VC_TYPE.prysm: launch_prysm_vc, - constants.VC_TYPE.vouch: launch_vouch_vc, + constants.VC_TYPE.lighthouse: launch_lighthouse, + constants.VC_TYPE.lodestar: launch_lodestar, + constants.VC_TYPE.teku: launch_teku, + constants.VC_TYPE.nimbus: launch_nimbus, + constants.VC_TYPE.prysm: launch_prysm, + constants.VC_TYPE.vouch: launch_vouch, } if vc_type not in vc_launchers: fail( @@ -381,7 +394,6 @@ done vc_type, ", ".join(vc_launchers.keys()) ) ) - launch_vc = vc_launchers[vc_type] # Launch one validator client per Charon node, connected to that node's validator API. vc_services = [] @@ -389,6 +401,7 @@ done charon_validator_api_url = "http://{0}:{1}".format( charon_services[i].ip_address, CHARON_VALIDATOR_API_PORT ) + vc_service_name = service_name + "-vc-" + str(i) + "-" + vc_type # Each node's validator keys come from the cluster-creation output. validator_keys_for_node = plan.store_service_files( @@ -401,15 +414,20 @@ done ) vc_services.append( - launch_vc( + vc_launchers[vc_type]( plan=plan, - vc_service_name=service_name + "-vc-" + str(i) + "-" + vc_type, + vc_service_name=vc_service_name, charon_validator_api_url=charon_validator_api_url, - validator_keys_artifact=validator_keys_for_node, + split_keys_artifact=validator_keys_for_node, launcher=launcher, + keymanager_file=keymanager_file, participant=participant, + global_log_level=global_log_level, + cl_context=cl_context, tolerations=tolerations, node_selectors=node_selectors, + network_params=network_params, + port_publisher=port_publisher, full_name=full_name + "-node" + str(i), vc_index=vc_index, node_index=i, @@ -478,968 +496,488 @@ done ) -def launch_lighthouse_vc( - plan, - vc_service_name, - charon_validator_api_url, - validator_keys_artifact, - launcher, - participant, - tolerations, - node_selectors, - full_name, - vc_index, - node_index, - vc_image, +def _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index ): """ - Launch a Lighthouse validator client that connects to a Charon node - Uses the two-stage approach: import keys, then run validator + Convert a Charon node's split keystores (a flat dir of keystore-N.json + + keystore-N.txt) into the on-disk layouts the stock vc/.star launchers + consume, and wrap them in a node_keystore_files struct. + + Produces, in the returned artifact: + keys//voting-keystore.json + secrets/ (lighthouse, lodestar) + nimbus-keys//keystore.json (nimbus; reuses secrets/) + teku-keys/.json + teku-secrets/.txt (teku) + matching the eth2-val-tools layouts, so no per-client import step is needed. + (prysm and vouch instead need a wallet, built separately in + _charon_split_keys_to_prysm_wallet / _charon_split_keys_to_vouch_wallet.) """ + converter_name = "charon-keys-convert-" + str(node_index) + "-" + str(vc_index) + converter = plan.add_service( + name=converter_name, + config=ServiceConfig( + image="busybox:latest", + cmd=["tail", "-f", "/dev/null"], + files={"/split-keys": split_keys_artifact}, + ), + ) - # Create the startup script that implements the two-stage approach - startup_script = ( - """#!/bin/bash + # Reorganise each keystore-N.json/.txt pair into the on-disk layouts the stock + # launchers expect: raw (lighthouse/lodestar), nimbus-keys (nimbus), and + # teku-keys/teku-secrets (teku). Prysm needs a wallet, handled separately. + convert_script = """#!/bin/sh set -e - -# Stage 1: Import validator keys -for f in /opt/charon/keys/keystore-*.json; do - if [ -f "$f" ]; then - echo "Importing key ${f}" - lighthouse account validator import \\ - --reuse-password \\ - --keystore "${f}" \\ - --password-file "${f//json/txt}" \\ - --testnet-dir "/opt/lighthouse/network-configs" - fi +mkdir -p /out/keys /out/secrets /out/nimbus-keys /out/teku-keys /out/teku-secrets +for f in /split-keys/keystore-*.json; do + [ -f "$f" ] || continue + pubkey="0x$(grep '"pubkey"' "$f" | head -1 | awk -F'"' '{print $4}')" + pw="${f%.json}.txt" + + # raw layout (lighthouse, lodestar): /voting-keystore.json + secrets/ + mkdir -p "/out/keys/${pubkey}" + cp "$f" "/out/keys/${pubkey}/voting-keystore.json" + cp "$pw" "/out/secrets/${pubkey}" + + # nimbus layout: /keystore.json (secrets reuse the raw secrets dir) + mkdir -p "/out/nimbus-keys/${pubkey}" + cp "$f" "/out/nimbus-keys/${pubkey}/keystore.json" + + # teku layout: flat .json + .txt + cp "$f" "/out/teku-keys/${pubkey}.json" + cp "$pw" "/out/teku-secrets/${pubkey}.txt" done - -echo "Starting lighthouse validator client for node""" - + str(node_index) - + """" -# Stage 2: Run the validator client -exec lighthouse validator \\ - --beacon-nodes ${LIGHTHOUSE_BEACON_NODE_ADDRESS} \\ - --suggested-fee-recipient """ - + constants.VALIDATING_REWARDS_ACCOUNT - + """ \\ - --metrics \\ - --metrics-address "0.0.0.0" \\ - --metrics-allow-origin "*" \\ - --metrics-port """ - + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) - + """ \\ - --use-long-timeouts \\ - --testnet-dir "/opt/lighthouse/network-configs" \\ - --builder-proposals \\ - --distributed \\ - --debug-level "debug" """ + plan.exec( + service_name=converter.name, + recipe=ExecRecipe(command=["sh", "-c", convert_script]), ) - # Environment variables - env_vars = { - "LIGHTHOUSE_BEACON_NODE_ADDRESS": charon_validator_api_url, - "NODE": "node" + str(node_index), - "RUST_BACKTRACE": "full", - } - if participant.vc_extra_env_vars: - env_vars.update(participant.vc_extra_env_vars) - - # Files to mount - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - "/opt/charon/keys": validator_keys_artifact, - "/opt/lighthouse/network-configs": launcher.el_cl_genesis_data.files_artifact_uuid, - } - - # Ports configuration - ports = { - constants.METRICS_PORT_ID: PortSpec( - number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, - transport_protocol="TCP", - application_protocol="http", - # Importing 256 keystores sequentially can exceed the default port - # readiness timeout; allow plenty of time so the VC isn't rolled back. - wait="15m", - ), - } - - # Create the service with the startup script - vc_service = plan.add_service( - name=vc_service_name, - config=ServiceConfig( - image=vc_image, - ports=ports, - cmd=["bash", "-c", startup_script], - env_vars=env_vars, - files=files, - labels=shared_utils.label_maker( - client=constants.VC_TYPE.lighthouse, - client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH :], - connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels, - supernode=participant.supernode, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ), + keystore_artifact = plan.store_service_files( + service_name=converter.name, + src="/out", + name="charon-raw-keys-" + str(node_index) + "-" + str(vc_index), ) + plan.remove_service(name=converter.name) - return vc_service + return keystore_files_module.new_keystore_files( + files_artifact_uuid=keystore_artifact, + raw_root_dirpath="", + raw_keys_relative_dirpath="keys", + raw_secrets_relative_dirpath="secrets", + nimbus_keys_relative_dirpath="nimbus-keys", + prysm_relative_dirpath="", + teku_keys_relative_dirpath="teku-keys", + teku_secrets_relative_dirpath="teku-secrets", + raw_keys_secrets_relative_dirpath="", + ) -def launch_lodestar_vc( +def launch_lighthouse( plan, vc_service_name, charon_validator_api_url, - validator_keys_artifact, + split_keys_artifact, launcher, + keymanager_file, participant, + global_log_level, + cl_context, tolerations, node_selectors, + network_params, + port_publisher, full_name, vc_index, node_index, vc_image, ): - """ - Launch a Lodestar validator client that connects to a Charon node - Uses Charon-specific key management with standard Lodestar parameters - """ - - # Create the run.sh script content - run_script_content = ( - """#!/bin/sh - -BUILDER_SELECTION="executiononly" - -# If the builder API is enabled, override the builder selection to signal Lodestar to always prefer proposing blinded blocks, but fall back on EL blocks if unavailable. -if [ "$BUILDER_API_ENABLED" = "true" ]; then - BUILDER_SELECTION="builderalways" -fi - -DATA_DIR="/opt/data" -KEYSTORES_DIR="${DATA_DIR}/keystores" -SECRETS_DIR="${DATA_DIR}/secrets" - -mkdir -p "${KEYSTORES_DIR}" "${SECRETS_DIR}" - -IMPORTED_COUNT=0 -EXISTING_COUNT=0 - -for f in /home/charon/validator_keys/keystore-*.json; do - echo "Importing key ${f}" - - # Extract pubkey from keystore file - PUBKEY="0x$(grep '"pubkey"' "$f" | awk -F'"' '{print $4}')" - - PUBKEY_DIR="${KEYSTORES_DIR}/${PUBKEY}" - - # Skip import if keystore already exists - if [ -d "${PUBKEY_DIR}" ]; then - EXISTING_COUNT=$((EXISTING_COUNT + 1)) - continue - fi - - mkdir -p "${PUBKEY_DIR}" - chown 1000:1000 "${PUBKEY_DIR}" - - # Copy the keystore file to persisted keys backend - install -m 600 "$f" "${PUBKEY_DIR}/voting-keystore.json" - chown 1000:1000 "${PUBKEY_DIR}/voting-keystore.json" - - # Copy the corresponding password file - PASSWORD_FILE="${f%.json}.txt" - install -m 600 "${PASSWORD_FILE}" "${SECRETS_DIR}/${PUBKEY}" - - IMPORTED_COUNT=$((IMPORTED_COUNT + 1)) -done - -echo "Processed all keys imported=${IMPORTED_COUNT}, existing=${EXISTING_COUNT}, total=$(ls /home/charon/validator_keys/keystore-*.json | wc -l)" - -exec node /usr/app/packages/cli/bin/lodestar validator \\ - --dataDir="$DATA_DIR" \\ - --keystoresDir="$KEYSTORES_DIR" \\ - --secretsDir="$SECRETS_DIR" \\ - --metrics=true \\ - --metrics.address="0.0.0.0" \\ - --metrics.port=""" - + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) - + """ \\ - --beaconNodes="$BEACON_NODE_ADDRESS" \\ - --builder="$BUILDER_API_ENABLED" \\ - --builder.selection="$BUILDER_SELECTION" \\ - --distributed \\ - --paramsFile="/opt/lodestar/config.yaml" -""" - ) - - # Add extra params if specified - if participant.vc_extra_params: - extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) - run_script_content += extra_params - - # Create the script file artifact using render_templates - script_artifact = plan.render_templates( - config={ - "run.sh": struct( - template=run_script_content, - data={}, - ), - }, - name="lodestar-run-script-" + str(node_index) + "-" + str(vc_index), - ) - - # Environment variables - env_vars = { - "BEACON_NODE_ADDRESS": charon_validator_api_url, - "BUILDER_API_ENABLED": "true", - "NODE": "node" + str(node_index), - } - if participant.vc_extra_env_vars: - env_vars.update(participant.vc_extra_env_vars) - - # Files to mount - Charon keys + standard genesis data + run script - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - "/home/charon/validator_keys": validator_keys_artifact, - "/opt/lodestar": launcher.el_cl_genesis_data.files_artifact_uuid, - "/opt/charon": script_artifact, - } - - # Ports configuration - ports = { - constants.METRICS_PORT_ID: PortSpec( - number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, - transport_protocol="TCP", - application_protocol="http", - # Importing 256 keystores sequentially can exceed the default port - # readiness timeout; allow plenty of time so the VC isn't rolled back. - wait="15m", - ), - } - - # Create the service - execute the script file - vc_service = plan.add_service( - name=vc_service_name, - config=ServiceConfig( - image=vc_image, - ports=ports, - cmd=["chmod +x /opt/charon/run.sh && /opt/charon/run.sh"], - entrypoint=["sh", "-c"], - env_vars=env_vars, - files=files, - labels=shared_utils.label_maker( - client=constants.VC_TYPE.lodestar, - client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH :], - connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels, - supernode=participant.supernode, - ), - tolerations=tolerations, - node_selectors=node_selectors, - ), - ) - - return vc_service - - -def launch_teku_vc( + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = lighthouse.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=vc_image, + service_name=vc_service_name, + global_log_level=global_log_level, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, # unused by lighthouse.get_config + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_lodestar( plan, vc_service_name, charon_validator_api_url, - validator_keys_artifact, + split_keys_artifact, launcher, + keymanager_file, participant, + global_log_level, + cl_context, tolerations, node_selectors, + network_params, + port_publisher, full_name, vc_index, node_index, vc_image, ): - """ - Launch a Teku validator client that connects to a Charon node - Uses config file approach similar to compose.teku.yaml - """ - - # Create the teku-config.yaml content - teku_config_content = ( - """metrics-enabled: true -metrics-host-allowlist: "*" -metrics-interface: "0.0.0.0" -metrics-port: \"""" - + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) - + """\" -validators-keystore-locking-enabled: false -network: "/opt/teku/network-configs/config.yaml" -validator-keys: "/opt/charon/validator_keys:/opt/charon/validator_keys" -validators-proposer-default-fee-recipient: \"""" - + constants.VALIDATING_REWARDS_ACCOUNT - + """\" -""" - ) - - # Create the config file artifact using render_templates - config_artifact = plan.render_templates( - config={ - "teku-config.yaml": struct( - template=teku_config_content, - data={}, - ), - }, - name="teku-config-" + str(node_index) + "-" + str(vc_index), - ) - - # Teku validator command based on standard teku.star but with Charon-specific flags - cmd = [ - "validator-client", - "--network=/opt/teku/network-configs/config.yaml", - "--beacon-node-api-endpoint=" + charon_validator_api_url, - "--config-file=/opt/charon/teku/teku-config.yaml", - "--validators-external-signer-slashing-protection-enabled=true", - "--validators-builder-registration-default-enabled=true", - "--Xobol-dvt-integration-enabled=true", - "--logging=DEBUG", - "--metrics-enabled=true", - "--metrics-host-allowlist=*", - "--metrics-interface=0.0.0.0", - "--metrics-port={0}".format(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM), - ] - - # Add extra params if specified - if participant.vc_extra_params: - cmd.extend([param for param in participant.vc_extra_params]) - - # Environment variables - env_vars = {} - if participant.vc_extra_env_vars: - env_vars.update(participant.vc_extra_env_vars) - - # Files to mount - Charon keys + standard genesis data + teku config - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - "/opt/charon/validator_keys": validator_keys_artifact, - "/opt/charon/teku": config_artifact, - "/opt/teku/network-configs": launcher.el_cl_genesis_data.files_artifact_uuid, - } - - # Ports configuration - ports = { - constants.METRICS_PORT_ID: PortSpec( - number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, - transport_protocol="TCP", - application_protocol="http", - # Importing 256 keystores sequentially can exceed the default port - # readiness timeout; allow plenty of time so the VC isn't rolled back. - wait="15m", - ), - } - - # Create the service - vc_service = plan.add_service( - name=vc_service_name, - config=ServiceConfig( - image=vc_image, - ports=ports, - cmd=cmd, - env_vars=env_vars, - files=files, - labels=shared_utils.label_maker( - client=constants.VC_TYPE.teku, - client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH :], - connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels, - supernode=participant.supernode, - ), - tolerations=tolerations, - node_selectors=node_selectors, - user=User(uid=0, gid=0), - ), - ) - - return vc_service - - -def launch_nimbus_vc( + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = lodestar.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + keymanager_file=keymanager_file, + image=vc_image, + global_log_level=global_log_level, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_teku( plan, vc_service_name, charon_validator_api_url, - validator_keys_artifact, + split_keys_artifact, launcher, + keymanager_file, participant, + global_log_level, + cl_context, tolerations, node_selectors, + network_params, + port_publisher, full_name, vc_index, node_index, vc_image, +): + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = teku.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + keymanager_file=keymanager_file, + image=vc_image, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_nimbus( + plan, + vc_service_name, + charon_validator_api_url, + split_keys_artifact, + launcher, + keymanager_file, + participant, + global_log_level, + cl_context, + tolerations, + node_selectors, + network_params, + port_publisher, + full_name, + vc_index, + node_index, + vc_image, +): + node_keystore_files = _charon_split_keys_to_keystore_files( + plan, split_keys_artifact, vc_index, node_index + ) + + config = nimbus.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + image=vc_image, + keymanager_file=keymanager_file, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def _charon_split_keys_to_prysm_wallet( + plan, split_keys_artifact, vc_index, node_index, prysm_image ): """ - Launch a Nimbus validator client that connects to a Charon node - Uses a two-service approach: - 1. Key import service using nimbus-eth2 (beacon node) to import keys - 2. Validator client service using nimbus-validator-client with imported keys - """ - - # Step 1: Create key import service using beacon node image - key_import_service_name = vc_service_name + "-key-import" - - # Create the key import script - key_import_script = """#!/usr/bin/env bash - -# Cleanup nimbus directories if they already exist. -rm -rf /home/user/data/${NODE} - -# Refer: https://nimbus.guide/keys.html -# Running a nimbus VC involves two steps which need to run in order: -# 1. Importing the validator keys -# 2. And then actually running the VC -tmpkeys="/home/validator_keys/tmpkeys" -mkdir -p ${tmpkeys} - -for f in /home/validator_keys/keystore-*.json; do - echo "Importing key ${f}" - - # Read password from keystore-*.txt into $password variable. - password=$(<"${f//json/txt}") - echo "Password length: ${#password}" - - # Copy keystore file to tmpkeys/ directory. - cp "${f}" "${tmpkeys}" - echo "Copied ${f} to ${tmpkeys}" - - # List files in tmpkeys before import - echo "Files in tmpkeys before import:" - ls -la "${tmpkeys}" - - # Import keystore with the password. - echo "Running nimbus import command..." - echo "$password" | \\ - /home/user/nimbus_beacon_node deposits import \\ - --data-dir=/home/user/data/${NODE} \\ - /home/validator_keys/tmpkeys - - IMPORT_RESULT=$? - echo "Import command exit code: $IMPORT_RESULT" - - # Check what was created - echo "Contents of data directory after import:" - ls -la /home/user/data/${NODE}/ || echo "Data directory does not exist" - if [ -d "/home/user/data/${NODE}/validators" ]; then - echo "Validators directory contents:" - ls -la /home/user/data/${NODE}/validators/ - fi - - # Delete tmpkeys/keystore-*.json file that was copied before. - filename="$(basename ${f})" - rm "${tmpkeys}/${filename}" - echo "Deleted ${tmpkeys}/${filename}" -done - -# Delete the tmpkeys/ directory since it's no longer needed. -rm -r ${tmpkeys} - -echo "Imported all keys successfully" -echo "Key import process completed" - -# Create a completion marker file to signal that import is done -echo "IMPORT_COMPLETE" > /home/user/data/import_complete.txt -echo "Created completion marker file" - -# Keep the container running so we can extract the data -tail -f /dev/null -""" + Build a Prysm wallet from a Charon node's split keystores so the stock + vc/prysm.star launcher (which expects a wallet, not raw keystores) can run it. - # Create the key import script artifact - key_import_script_artifact = plan.render_templates( - config={ - "import_keys.sh": struct( - template=key_import_script, - data={}, - ), - }, - name="nimbus-key-import-script-" + str(node_index) + "-" + str(vc_index), - ) - - # Environment variables for key import - import_env_vars = { - "NODE": "node" + str(node_index), - } - - # Files to mount for key import - import_files = { - "/home/validator_keys": validator_keys_artifact, - "/home/user/scripts": key_import_script_artifact, - } - - # Create the key import service - key_import_service = plan.add_service( - name=key_import_service_name, + Returns an artifact containing: + prysm/ the imported direct-keymanager wallet + wallet-password.txt the wallet password + plus the node_keystore_files struct that points at them. + """ + builder_name = "charon-prysm-wallet-" + str(node_index) + "-" + str(vc_index) + builder = plan.add_service( + name=builder_name, config=ServiceConfig( - image="statusim/nimbus-eth2:multiarch-latest", - cmd=[ - "chmod +x /home/user/scripts/import_keys.sh && /home/user/scripts/import_keys.sh" - ], + image=prysm_image, entrypoint=["bash", "-c"], - env_vars=import_env_vars, - files=import_files, + cmd=["tail -f /dev/null"], + files={"/split-keys": split_keys_artifact}, user=User(uid=0, gid=0), ), ) - # Step 2: Wait for key import to complete and then extract the keys as an artifact - # Wait for the completion marker file to be created + build_script = """#!/usr/bin/env bash +set -e +mkdir -p /out +echo "prysm-validator-secret" > /out/wallet-password.txt +/app/cmd/validator/validator wallet create \\ + --accept-terms-of-use \\ + --keymanager-kind=direct \\ + --wallet-dir=/out/prysm \\ + --wallet-password-file=/out/wallet-password.txt +tmpkeys=/tmp/keys +mkdir -p "$tmpkeys" +for f in /split-keys/keystore-*.json; do + [ -f "$f" ] || continue + cp "$f" "$tmpkeys/" + /app/cmd/validator/validator accounts import \\ + --accept-terms-of-use=true \\ + --wallet-dir=/out/prysm \\ + --keys-dir="$tmpkeys" \\ + --account-password-file="${f%.json}.txt" \\ + --wallet-password-file=/out/wallet-password.txt + rm "$tmpkeys/$(basename "$f")" +done +""" plan.exec( - service_name=key_import_service_name, - recipe=ExecRecipe( - command=[ - "bash", - "-c", - "while [ ! -f /home/user/data/import_complete.txt ]; do echo 'Waiting for import to complete...'; sleep 2; done; echo 'Import completed! Found completion marker.'", - ] - ), - description="Wait for key import completion", + service_name=builder.name, + recipe=ExecRecipe(command=["bash", "-c", build_script]), ) - # Store the imported keys from the key import service - # Note: The beacon node imports to /home/user/data/${NODE}, so we store that specific directory - imported_keys_artifact = plan.store_service_files( - service_name=key_import_service_name, - src="/home/user/data/node" + str(node_index), - name="nimbus-imported-keys-" + str(node_index) + "-" + str(vc_index), - description="Nimbus imported validator keys for node " + str(node_index), + wallet_artifact = plan.store_service_files( + service_name=builder.name, + src="/out", + name="charon-prysm-wallet-files-" + str(node_index) + "-" + str(vc_index), ) + plan.remove_service(name=builder.name) - # Step 3: Create the actual validator client service - # Create the VC run script - vc_run_script = ( - """#!/usr/bin/env bash - -# Find the nimbus_validator_client binary -if [ -f "/home/user/nimbus_validator_client" ]; then - NIMBUS_VC_PATH="/home/user/nimbus_validator_client" -elif [ -f "/usr/bin/nimbus_validator_client" ]; then - NIMBUS_VC_PATH="/usr/bin/nimbus_validator_client" -elif [ -f "/usr/local/bin/nimbus_validator_client" ]; then - NIMBUS_VC_PATH="/usr/local/bin/nimbus_validator_client" -else - echo "Error: Could not find nimbus_validator_client binary" - echo "Available files in /home/user:" - ls -la /home/user/ - exit 1 -fi - -echo "Using Nimbus VC binary at: $NIMBUS_VC_PATH" -echo "Using imported keys from: /home/user/imported_data" - -# List what's available in the imported data -echo "Contents of imported_data:" -ls -la /home/user/imported_data/ - -# Run nimbus validator client with imported keys -exec "$NIMBUS_VC_PATH" \\ - --data-dir="/home/user/imported_data" \\ - --beacon-node="$BEACON_NODE_ADDRESS" \\ - --doppelganger-detection=false \\ - --metrics \\ - --metrics-address=0.0.0.0 \\ - --metrics-port=""" - + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) - + """ \\ - --payload-builder=true \\ - --distributed -""" + node_keystore_files = keystore_files_module.new_keystore_files( + files_artifact_uuid=wallet_artifact, + raw_root_dirpath="", + raw_keys_relative_dirpath="", + raw_secrets_relative_dirpath="", + nimbus_keys_relative_dirpath="", + prysm_relative_dirpath="prysm", + teku_keys_relative_dirpath="", + teku_secrets_relative_dirpath="", + raw_keys_secrets_relative_dirpath="", ) + return wallet_artifact, node_keystore_files - # Add extra params if specified - if participant.vc_extra_params: - extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) - vc_run_script = vc_run_script.replace( - "--distributed", "--distributed" + extra_params - ) - # Create the VC script artifact - vc_script_artifact = plan.render_templates( - config={ - "run_vc.sh": struct( - template=vc_run_script, - data={}, - ), - }, - name="nimbus-vc-script-" + str(node_index) + "-" + str(vc_index), - ) - - # Environment variables for VC - vc_env_vars = { - "BEACON_NODE_ADDRESS": charon_validator_api_url, - "NODE": "node" + str(node_index), - } - if participant.vc_extra_env_vars: - vc_env_vars.update(participant.vc_extra_env_vars) - - # Files to mount for VC - imported keys + VC script - vc_files = { - "/home/user/imported_data": imported_keys_artifact, - "/home/user/scripts": vc_script_artifact, - } - - # Ports configuration - ports = { - constants.METRICS_PORT_ID: PortSpec( - number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, - transport_protocol="TCP", - application_protocol="http", - # Importing 256 keystores sequentially can exceed the default port - # readiness timeout; allow plenty of time so the VC isn't rolled back. - wait="15m", - ), - } - - # Create the actual validator client service - vc_service = plan.add_service( - name=vc_service_name, +def _charon_split_keys_to_vouch_wallet(plan, split_keys_artifact, vc_index, node_index): + """ + Build an ethdo wallet from a Charon node's split keystores by running the + official ethdo image, so the Vouch container itself needs no ethdo (and no + apt/download) — it just mounts the result. + + Returns an artifact containing: + wallets/ the ethdo wallet store + accounts.txt one account path per line (vals/valN) + account-passphrase.txt the account passphrase + """ + builder_name = "charon-vouch-wallet-" + str(node_index) + "-" + str(vc_index) + builder = plan.add_service( + name=builder_name, config=ServiceConfig( - image=vc_image, - ports=ports, - cmd=[ - "chmod +x /home/user/scripts/run_vc.sh && /home/user/scripts/run_vc.sh" - ], - entrypoint=["bash", "-c"], - env_vars=vc_env_vars, - files=vc_files, - labels=shared_utils.label_maker( - client=constants.VC_TYPE.nimbus, - client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH :], - connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels, - supernode=participant.supernode, - ), - tolerations=tolerations, - node_selectors=node_selectors, + image=ETHDO_IMAGE, + entrypoint=["sh", "-c"], + cmd=["tail -f /dev/null"], + files={"/split-keys": split_keys_artifact}, user=User(uid=0, gid=0), ), ) - return vc_service + build_script = """set -e +mkdir -p /out/wallets +echo "1234" > /out/account-passphrase.txt +: > /out/accounts.txt +/app/ethdo --base-dir=/out/wallets wallet create --wallet=vals --passphrase="" +index=0 +for f in /split-keys/keystore-*.json; do + [ -f "$f" ] || continue + /app/ethdo --base-dir=/out/wallets account import \\ + --account="vals/val${index}" \\ + --keystore="$f" \\ + --keystore-passphrase="$(cat "${f%.json}.txt")" \\ + --passphrase="1234" --allow-weak-passphrases + echo "vals/val${index}" >> /out/accounts.txt + index=$((index + 1)) +done +""" + plan.exec( + service_name=builder.name, + recipe=ExecRecipe(command=["sh", "-c", build_script]), + ) + + wallet_artifact = plan.store_service_files( + service_name=builder.name, + src="/out", + name="charon-vouch-wallet-files-" + str(node_index) + "-" + str(vc_index), + ) + plan.remove_service(name=builder.name) + return wallet_artifact -def launch_prysm_vc( +def launch_prysm( plan, vc_service_name, charon_validator_api_url, - validator_keys_artifact, + split_keys_artifact, launcher, + keymanager_file, participant, + global_log_level, + cl_context, tolerations, node_selectors, + network_params, + port_publisher, full_name, vc_index, node_index, vc_image, ): - """ - Launch a Prysm validator client that connects to a Charon node - Uses script approach similar to kurtosis-charon/prysm/run.sh - """ - - # Create the run.sh script content - run_script_content = ( - """#!/usr/bin/env bash - -WALLET_DIR="/prysm-wallet" - -# Cleanup wallet directories if already exists. -rm -rf $WALLET_DIR -mkdir $WALLET_DIR - -# Refer: https://docs.prylabs.network/docs/install/install-with-script#step-5-run-a-validator-using-prysm -# Running a prysm VC involves two steps which need to run in order: -# 1. Import validator keys in a prysm wallet account. -# 2. Run the validator client. -WALLET_PASSWORD="prysm-validator-secret" -echo $WALLET_PASSWORD > /wallet-password.txt -/app/cmd/validator/validator wallet create --accept-terms-of-use --wallet-password-file=wallet-password.txt --keymanager-kind=direct --wallet-dir="$WALLET_DIR" - -tmpkeys="/home/validator_keys/tmpkeys" -mkdir -p ${tmpkeys} - -for f in /home/charon/validator_keys/keystore-*.json; do - echo "Importing key ${f}" - - # Copy keystore file to tmpkeys/ directory. - cp "${f}" "${tmpkeys}" - - # Import keystore with password. - /app/cmd/validator/validator accounts import \\ - --accept-terms-of-use=true \\ - --wallet-dir="$WALLET_DIR" \\ - --keys-dir="${tmpkeys}" \\ - --account-password-file="${f//json/txt}" \\ - --wallet-password-file=wallet-password.txt - - # Delete tmpkeys/keystore-*.json file that was copied before. - filename="$(basename ${f})" - rm "${tmpkeys}/${filename}" -done - -# Delete the tmpkeys/ directory since it's no longer needed. -rm -r ${tmpkeys} - -echo "Imported all keys" - -# Now run prysm VC -exec /app/cmd/validator/validator --wallet-dir="$WALLET_DIR" \\ - --accept-terms-of-use=true \\ - --datadir="/data/vc" \\ - --wallet-password-file="/wallet-password.txt" \\ - --enable-beacon-rest-api \\ - --beacon-rest-api-provider="$BEACON_NODE_ADDRESS" \\ - --beacon-rpc-provider="$BEACON_NODE_ADDRESS" \\ - --chain-config-file="/opt/prysm/config.yaml" \\ - --monitoring-host=0.0.0.0 \\ - --monitoring-port=""" - + str(vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM) - + """ \\ - --distributed -""" - ) - - # Add extra params if specified - if participant.vc_extra_params: - extra_params = " \\\n " + " \\\n ".join(participant.vc_extra_params) - run_script_content = run_script_content.replace( - "--distributed", "--distributed" + extra_params - ) - - # Create the script file artifact using render_templates - script_artifact = plan.render_templates( - config={ - "run.sh": struct( - template=run_script_content, - data={}, - ), - }, - name="prysm-run-script-" + str(node_index) + "-" + str(vc_index), - ) - - # Environment variables - env_vars = { - "BEACON_NODE_ADDRESS": charon_validator_api_url, - } - if participant.vc_extra_env_vars: - env_vars.update(participant.vc_extra_env_vars) - - # Files to mount - Charon keys + standard genesis data + run script - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: launcher.el_cl_genesis_data.files_artifact_uuid, - "/home/charon/validator_keys": validator_keys_artifact, - "/opt/prysm": launcher.el_cl_genesis_data.files_artifact_uuid, - "/opt/charon": script_artifact, - } - - # Ports configuration - ports = { - constants.METRICS_PORT_ID: PortSpec( - number=vc_shared.VALIDATOR_CLIENT_METRICS_PORT_NUM, - transport_protocol="TCP", - application_protocol="http", - # Importing 256 keystores sequentially can exceed the default port - # readiness timeout; allow plenty of time so the VC isn't rolled back. - wait="15m", - ), - } - - # Create the service - execute the script file - vc_service = plan.add_service( - name=vc_service_name, - config=ServiceConfig( - image=vc_image, - ports=ports, - cmd=["chmod +x /opt/charon/run.sh && /opt/charon/run.sh"], - entrypoint=["bash", "-c"], - env_vars=env_vars, - files=files, - labels=shared_utils.label_maker( - client=constants.VC_TYPE.prysm, - client_type=constants.CLIENT_TYPES.validator, - image=vc_image[-constants.MAX_LABEL_LENGTH :], - connected_client="charon-node-" + str(node_index), - extra_labels=participant.vc_extra_labels, - supernode=participant.supernode, - ), - tolerations=tolerations, - node_selectors=node_selectors, - user=User(uid=0, gid=0), - ), - ) - - return vc_service - - -def launch_vouch_vc( + wallet_artifact, node_keystore_files = _charon_split_keys_to_prysm_wallet( + plan, split_keys_artifact, vc_index, node_index, vc_image + ) + + config = prysm.get_config( + plan=plan, + participant=participant, + el_cl_genesis_data=launcher.el_cl_genesis_data, + keymanager_file=keymanager_file, + image=vc_image, + beacon_http_urls=[charon_validator_api_url], + cl_context=cl_context, + el_context=None, + remote_signer_context=None, + full_name=full_name, + node_keystore_files=node_keystore_files, + prysm_password_relative_filepath="wallet-password.txt", + prysm_password_artifact_uuid=wallet_artifact, + tolerations=tolerations, + node_selectors=node_selectors, + keymanager_enabled=False, + network_params=network_params, + port_publisher=port_publisher, + vc_index=vc_index, + extra_files_artifacts=[], + distributed=True, + ) + + return plan.add_service(name=vc_service_name, config=config) + + +def launch_vouch( plan, vc_service_name, charon_validator_api_url, - validator_keys_artifact, + split_keys_artifact, launcher, + keymanager_file, participant, + global_log_level, + cl_context, tolerations, node_selectors, + network_params, + port_publisher, full_name, vc_index, node_index, vc_image, ): - """ - Launch a Vouch validator client that connects to a Charon node. - - Imports the Charon-split keystores into an ethdo wallet, - writes ~/.vouch.yml pointing at the Charon validator API, and runs vouch. - ethdo is fetched at runtime (the attestant/vouch image doesn't ship it), - with arch detection so it works on amd64 and arm64 hosts alike. - Prometheus metrics are exposed on the standard VC metrics port so Kurtosis' - readiness check passes and the cluster is scraped like every other VC. - """ - - ETHDO_VERSION = "1.37.3" - - startup_script = ( - """#!/usr/bin/env bash -set -e - -# Only wget+ca-certificates are needed (for the ethdo download). Installing curl -# pulls a large dependency chain that pushed startup past Kurtosis' port-readiness -# window, so keep this minimal. -apt-get update -apt-get install -y --no-install-recommends wget ca-certificates - -# Match the host architecture so the downloaded ethdo binary actually runs. -ARCH="$(uname -m)" -case "${ARCH}" in - x86_64) DL_ARCH="amd64" ;; - aarch64|arm64) DL_ARCH="arm64" ;; - *) echo "Unsupported arch ${ARCH}"; exit 1 ;; -esac - -mkdir -p /opt/vouch -cd /opt/vouch - -# Install ethdo (used to import the Charon keystores into a wallet vouch can read). -wget -q "https://github.com/wealdtech/ethdo/releases/download/v""" - + ETHDO_VERSION - + """/ethdo-""" - + ETHDO_VERSION - + """-linux-${DL_ARCH}.tar.gz" -O ethdo.tar.gz -tar -xf ethdo.tar.gz -rm ethdo.tar.gz - -# Passphrase protecting every account in the local wallet. -account_passphrase="1234" - -./ethdo wallet create --wallet=vals --passphrase="" - -accounts_list=() -for keystore_file in /home/charon/validator_keys/keystore-*.json; do - basename="$(basename "${keystore_file%.json}")" - password_file="/home/charon/validator_keys/${basename}.txt" - index="${basename##*-}" - account_name="vals/val${index}" - passphrase_content=$(cat "$password_file") - - echo "Importing account ${account_name} from ${keystore_file}" - ./ethdo account import \\ - --account="$account_name" \\ - --keystore="$keystore_file" \\ - --keystore-passphrase="$passphrase_content" \\ - --passphrase="$account_passphrase" --allow-weak-passphrases - - accounts_list+=("$account_name") -done - -yq_accounts=$(printf " - %s\\n" "${accounts_list[@]}") -echo -n "$account_passphrase" > /opt/vouch/account_passphrase.txt - -cat > ~/.vouch.yml < 0: cmd.append("--defaultGasLimit={0}".format(network_params.gas_limit)) + if distributed: + cmd.append("--distributed") + cmd.append("--builder") + cmd.append("--builder.selection=builderalways") + if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.vc_extra_params]) diff --git a/src/vc/nimbus.star b/src/vc/nimbus.star index f124d5278..00f7f5f07 100644 --- a/src/vc/nimbus.star +++ b/src/vc/nimbus.star @@ -24,6 +24,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -77,6 +78,10 @@ def get_config( if network_params.gas_limit > 0: cmd.append("--suggested-gas-limit={0}".format(network_params.gas_limit)) + if distributed: + cmd.append("--distributed") + cmd.append("--payload-builder=true") + if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.vc_extra_params]) diff --git a/src/vc/prysm.star b/src/vc/prysm.star index cdd010df1..4f56ccba3 100644 --- a/src/vc/prysm.star +++ b/src/vc/prysm.star @@ -29,6 +29,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): validator_keys_dirpath = shared_utils.path_join( constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, @@ -54,8 +55,11 @@ def get_config( ] # Only add RPC provider if we're not using a blobber (blobber doesn't proxy RPC) - # Blobber uses port 5000, so check if that's in the URL - if ":5000" not in beacon_http_urls[0]: + # Blobber uses port 5000, so check if that's in the URL. + # In DV mode the only beacon endpoint is the Charon REST + # validator API, so skip the gRPC provider (which would point at the real + # beacon and bypass Charon). + if not distributed and ":5000" not in beacon_http_urls[0]: cmd.append("--beacon-rpc-provider=" + cl_context.beacon_grpc_url) if remote_signer_context == None: @@ -78,6 +82,9 @@ def get_config( if network_params.gas_limit > 0: cmd.append("--suggested-gas-limit={0}".format(network_params.gas_limit)) + if distributed: + cmd.append("--distributed") + keymanager_api_cmd = [ "--rpc", "--http-port={0}".format(vc_shared.VALIDATOR_HTTP_PORT_NUM), @@ -88,10 +95,15 @@ def get_config( # Check if we're using a blobber by checking for port 5000 is_using_blobber = ":5000" in beacon_http_urls[0] - if cl_context.client_name != constants.CL_TYPE.prysm or is_using_blobber: + if ( + cl_context.client_name != constants.CL_TYPE.prysm + or is_using_blobber + or distributed + ): # Use Beacon API if: # 1. Prysm VC wants to connect to a non-Prysm BN, OR # 2. Blobber is enabled (since blobber only proxies REST, not RPC) + # 3. Distributed (Charon) mode — Charon only exposes a REST validator API, OR cmd.append("--enable-beacon-rest-api") if len(participant.vc_extra_params) > 0: diff --git a/src/vc/teku.star b/src/vc/teku.star index ec2dab662..badd76a41 100644 --- a/src/vc/teku.star +++ b/src/vc/teku.star @@ -24,6 +24,7 @@ def get_config( extra_files_artifacts, otel_otlp_grpc_url=None, vc_binary_artifact=None, + distributed=False, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -88,6 +89,10 @@ def get_config( "--Xvalidator-api-unsafe-hosts-enabled=true", ] + if distributed: + cmd.append("--Xobol-dvt-integration-enabled=true") + cmd.append("--validators-builder-registration-default-enabled=true") + if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in participant.vc_extra_params]) diff --git a/src/vc/vouch.star b/src/vc/vouch.star new file mode 100644 index 000000000..6728fe0df --- /dev/null +++ b/src/vc/vouch.star @@ -0,0 +1,138 @@ +shared_utils = import_module("../shared_utils/shared_utils.star") +constants = import_module("../package_io/constants.star") +input_parser = import_module("../package_io/input_parser.star") +vc_shared = import_module("./shared.star") + +# Where the prebuilt ethdo wallet artifact is mounted in the Vouch container. +VOUCH_WALLET_DIRPATH = "/vouch-wallet" + +VERBOSITY_LEVELS = { + constants.GLOBAL_LOG_LEVEL.error: "error", + constants.GLOBAL_LOG_LEVEL.warn: "warn", + constants.GLOBAL_LOG_LEVEL.info: "info", + constants.GLOBAL_LOG_LEVEL.debug: "debug", +} + + +def get_config( + plan, + participant, + image, + global_log_level, + beacon_http_urls, + cl_context, + vouch_wallet_artifact, + tolerations, + node_selectors, +): + """ + Vouch validator client config. + + The ethdo wallet is built up-front by the caller and passed in as + vouch_wallet_artifact, so this container needs no ethdo/apt/download: it just + writes ~/.vouch.yml pointing at the mounted wallet and the beacon API, then + runs vouch. + + The wallet artifact contains: + wallets/ the ethdo wallet store + accounts.txt one account path per line + account-passphrase.txt the account passphrase + """ + log_level = input_parser.get_client_log_level_or_default( + participant.vc_log_level, global_log_level, VERBOSITY_LEVELS + ) + + startup_script = ( + """#!/usr/bin/env bash +set -e + +# Turn the account list shipped with the wallet into the YAML the wallet +# account manager expects. +accounts_yaml=$(sed 's/^/ - /' """ + + VOUCH_WALLET_DIRPATH + + """/accounts.txt) + +cat > ~/.vouch.yml < 0: + config_args["min_cpu"] = participant.vc_min_cpu + if participant.vc_max_cpu > 0: + config_args["max_cpu"] = participant.vc_max_cpu + if participant.vc_min_mem > 0: + config_args["min_memory"] = participant.vc_min_mem + if participant.vc_max_mem > 0: + config_args["max_memory"] = participant.vc_max_mem + + return ServiceConfig(**config_args) From 1b18387bd7dba5af87c74ca7d8dab53db5d0159e Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:10:00 +0300 Subject: [PATCH 32/35] Move Charon key formatting to the generator --- .../validator_keystores/keystore_files.star | 6 +- .../validator_keystore_generator.star | 96 ++++++------- src/vc/charon_launcher.star | 133 ++---------------- 3 files changed, 62 insertions(+), 173 deletions(-) diff --git a/src/prelaunch_data_generator/validator_keystores/keystore_files.star b/src/prelaunch_data_generator/validator_keystores/keystore_files.star index 3a4622d28..967e12bf0 100644 --- a/src/prelaunch_data_generator/validator_keystores/keystore_files.star +++ b/src/prelaunch_data_generator/validator_keystores/keystore_files.star @@ -8,7 +8,7 @@ def new_keystore_files( prysm_relative_dirpath, teku_keys_relative_dirpath, teku_secrets_relative_dirpath, - raw_keys_secrets_relative_dirpath, + charon_keys_relative_dirpath, ): return struct( files_artifact_uuid=files_artifact_uuid, @@ -20,5 +20,7 @@ def new_keystore_files( prysm_relative_dirpath=prysm_relative_dirpath, teku_keys_relative_dirpath=teku_keys_relative_dirpath, teku_secrets_relative_dirpath=teku_secrets_relative_dirpath, - raw_keys_secrets_relative_dirpath=raw_keys_secrets_relative_dirpath, + # Flat keystore-N.json + keystore-N.txt pairs that Charon's + # `create cluster --split-keys-dir` consumes. Empty for non-Charon VCs. + charon_keys_relative_dirpath=charon_keys_relative_dirpath, ) diff --git a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star index cf73882e1..8838cb43b 100644 --- a/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star +++ b/src/prelaunch_data_generator/validator_keystores/validator_keystore_generator.star @@ -17,7 +17,7 @@ SUCCESSFUL_EXEC_CMD_EXIT_CODE = 0 RAW_KEYS_DIRNAME = "keys" RAW_SECRETS_DIRNAME = "secrets" -RAW_KEYS_SECRETS_DIRNAME = "raw-keys-secrets" +CHARON_KEYS_DIRNAME = "charon-keys" NIMBUS_KEYS_DIRNAME = "nimbus-keys" PRYSM_DIRNAME = "prysm" @@ -54,6 +54,35 @@ def keystore_artifact_basename( ) +# Reshape the raw eth2-val-tools layout (keys//voting-keystore.json + +# secrets/) into the flat keystore-N.json + keystore-N.txt pairs that +# Charon's `create cluster --split-keys-dir` consumes, written to charon-keys/. +# Returned as a single shell command so it can be appended to the existing +# keystore-generation command (same pattern as the per-client chmod steps). +def charon_keystore_format_cmd(output_dirpath): + charon_dir = output_dirpath + CHARON_KEYS_DIRNAME + keys_dir = output_dirpath + RAW_KEYS_DIRNAME + secrets_dir = output_dirpath + RAW_SECRETS_DIRNAME + return ( + "mkdir -p " + + charon_dir + + " && i=0" + + " && for d in " + + keys_dir + + '/*/; do [ -d "$d" ] || continue;' + + ' cp "${d}voting-keystore.json" "' + + charon_dir + + '/keystore-${i}.json";' + + ' pubkey=$(basename "$d");' + + ' cp "' + + secrets_dir + + '/${pubkey}" "' + + charon_dir + + '/keystore-${i}.txt";' + + " i=$((i+1)); done" + ) + + # Launches a prelaunch data generator IMAGE, for use in various of the genesis generation def launch_prelaunch_data_generator( plan, @@ -140,27 +169,10 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para running_total_validator_count += participant.validator_count - # Collect the raw keys and secrets together in a single folder, which - # Charon consumes when splitting keys across its cluster nodes. Only - # Charon participants need this, so vanilla VCs skip the extra copy. + # Charon consumes a single dir of flat keystore-N.json + keystore-N.txt + # pairs; reshape the raw layout into charon-keys/. if participant.vc_type == constants.VC_TYPE.charon: - all_output_dirpaths.append(output_dirpath + RAW_KEYS_SECRETS_DIRNAME) - all_sub_command_strs.append( - "cp -r " - + output_dirpath - + RAW_KEYS_DIRNAME - + "/ " - + output_dirpath - + RAW_KEYS_SECRETS_DIRNAME - ) - all_sub_command_strs.append( - "cp -r " - + output_dirpath - + RAW_SECRETS_DIRNAME - + "/ " - + output_dirpath - + RAW_KEYS_SECRETS_DIRNAME - ) + all_sub_command_strs.append(charon_keystore_format_cmd(output_dirpath)) command_str = " && ".join(all_sub_command_strs) @@ -201,12 +213,10 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para ) base_dirname_in_artifact = shared_utils.path_base(output_dirpath) - # The raw-keys-secrets folder is only generated for Charon participants - # (see above), so only reference it for them; vanilla VCs leave it empty. - raw_keys_secrets_relative_dirpath = "" + charon_keys_relative_dirpath = "" if participant.vc_type == constants.VC_TYPE.charon: - raw_keys_secrets_relative_dirpath = shared_utils.path_join( - base_dirname_in_artifact, RAW_KEYS_SECRETS_DIRNAME + charon_keys_relative_dirpath = shared_utils.path_join( + base_dirname_in_artifact, CHARON_KEYS_DIRNAME ) to_add = keystore_files_module.new_keystore_files( artifact_name, @@ -217,7 +227,7 @@ def generate_validator_keystores(plan, mnemonic, participants, docker_cache_para shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), - raw_keys_secrets_relative_dirpath, + charon_keys_relative_dirpath, ) keystore_files.append(to_add) @@ -306,25 +316,11 @@ def generate_validator_keystores_in_parallel( generate_keystores_cmd += teku_permissions_cmd generate_keystores_cmd += raw_secret_permissions_cmd - # Collect the raw keys and secrets together in a single folder, which - # Charon consumes when splitting keys across its cluster nodes. Only - # Charon participants need this, so vanilla VCs skip the extra copy. + # Charon consumes a single dir of flat keystore-N.json + keystore-N.txt + # pairs; reshape the raw layout into charon-keys/. if participant.vc_type == constants.VC_TYPE.charon: - generate_keystores_cmd += ( - " && cp -r " - + output_dirpath - + RAW_KEYS_DIRNAME - + "/ " - + output_dirpath - + RAW_KEYS_SECRETS_DIRNAME - ) - generate_keystores_cmd += ( - " && cp -r " - + output_dirpath - + RAW_SECRETS_DIRNAME - + "/ " - + output_dirpath - + RAW_KEYS_SECRETS_DIRNAME + generate_keystores_cmd += " && " + charon_keystore_format_cmd( + output_dirpath ) all_generation_commands.append(generate_keystores_cmd) @@ -390,12 +386,10 @@ def generate_validator_keystores_in_parallel( # This is necessary because the way Kurtosis currently implements artifact-storing is base_dirname_in_artifact = shared_utils.path_base(output_dirpath) - # The raw-keys-secrets folder is only generated for Charon participants - # (see above), so only reference it for them; vanilla VCs leave it empty. - raw_keys_secrets_relative_dirpath = "" + charon_keys_relative_dirpath = "" if participant.vc_type == constants.VC_TYPE.charon: - raw_keys_secrets_relative_dirpath = shared_utils.path_join( - base_dirname_in_artifact, RAW_KEYS_SECRETS_DIRNAME + charon_keys_relative_dirpath = shared_utils.path_join( + base_dirname_in_artifact, CHARON_KEYS_DIRNAME ) to_add = keystore_files_module.new_keystore_files( artifact_name, @@ -406,7 +400,7 @@ def generate_validator_keystores_in_parallel( shared_utils.path_join(base_dirname_in_artifact, PRYSM_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_KEYS_DIRNAME), shared_utils.path_join(base_dirname_in_artifact, TEKU_SECRETS_DIRNAME), - raw_keys_secrets_relative_dirpath, + charon_keys_relative_dirpath, ) keystore_files.append(to_add) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 81c35ddb3..96d9fd4a8 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -90,128 +90,21 @@ def launch( # directly rather than querying the (possibly not-yet-ready) beacon node. genesis_time = genesis_timestamp - # Raw validator key/secret directory paths. node_keystore_files is non-None - # here (we returned early above otherwise). - validator_keys_dirpath = shared_utils.path_join( - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_keys_relative_dirpath, - ) - validator_secrets_dirpath = shared_utils.path_join( - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, - node_keystore_files.raw_secrets_relative_dirpath, - ) - - # Create a temporary service to format the validator keys for Charon - # Use busybox as a lightweight image for key formatting - key_formatter_service = plan.add_service( - name=service_name + "-key-formatter-" + str(vc_index), - config=ServiceConfig( - image="busybox:latest", - cmd=["tail", "-f", "/dev/null"], # Keep the service running - files={ - constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - }, - ), - ) - - # Create a directory for Charon-formatted keys - plan.exec( - service_name=key_formatter_service.name, - recipe=ExecRecipe( - command=["mkdir", "-p", "/opt/charon/charon-keys"], - ), - ) - - # Create a script to format the validator keys for Charon - format_keys_script = """#!/bin/sh -# Find all directories in the validator keys directory -keystore_directories="%s/*" - -index=0 -echo "Processing keystores from ${keystore_directories}" - -# Create directory with proper permissions -mkdir -p /opt/charon/charon-keys -chmod 755 /opt/charon/charon-keys - -# Iterate over each directory -for keystore_dir in $keystore_directories; do - # Check if it's a directory - if [ -d "$keystore_dir" ]; then - # Copy 'voting-keystore.json' to 'charon-keys' with an indexed name - cp "$keystore_dir/voting-keystore.json" "/opt/charon/charon-keys/keystore-${index}.json" - chmod 644 "/opt/charon/charon-keys/keystore-${index}.json" - - # Extract the directory name (pubkey) from the current keystore directory - dir_name=$(basename "$keystore_dir") - - # Check if a file with the same name exists in the secrets directory and copy it - if [ -f "%s/$dir_name" ]; then - cp "%s/$dir_name" "/opt/charon/charon-keys/keystore-${index}.txt" - chmod 644 "/opt/charon/charon-keys/keystore-${index}.txt" - else - echo "No matching file found in secrets directory for '$dir_name'." - fi - - # Increment the index for the next iteration (busybox compatible) - index=$(($index + 1)) - fi -done -""" % ( - validator_keys_dirpath, - validator_secrets_dirpath, - validator_secrets_dirpath, - ) - - # Save the script to the service - plan.exec( - service_name=key_formatter_service.name, - recipe=ExecRecipe( - command=[ - "sh", - "-c", - "cat > /opt/charon/format_keys.sh << 'EOL'\n" - + format_keys_script - + "\nEOL", - ], - ), - ) - - # Make the script executable - plan.exec( - service_name=key_formatter_service.name, - recipe=ExecRecipe( - command=["chmod", "+x", "/opt/charon/format_keys.sh"], - ), - ) - - # Run the script to format the keys - plan.exec( - service_name=key_formatter_service.name, - recipe=ExecRecipe( - command=["/opt/charon/format_keys.sh"], - ), - ) - - # Store the formatted keys - charon_keys_artifact = plan.store_service_files( - service_name=key_formatter_service.name, - src="/opt/charon/charon-keys", - name="charon-keys-" + str(vc_index), - ) - - # The formatter has served its purpose; tear it down so it doesn't linger. - plan.remove_service(name=key_formatter_service.name) - charon_service_name = service_name + "-charon-split-keys-" + str(vc_index) CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/charon/" persistent_key = "data-{0}".format(charon_service_name) - files = {} - files[CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( - persistent_key=persistent_key, + charon_keys_dirpath = shared_utils.path_join( + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER, + node_keystore_files.charon_keys_relative_dirpath, ) - files["/opt/charon/charon-keys"] = charon_keys_artifact + + files = { + CHARON_DATA_DIRPATH_ON_CLIENT_CONTAINER: Directory( + persistent_key=persistent_key, + ), + constants.VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + } # Run the Charon cluster creation (splits the existing keys across nodes). plan.add_service( @@ -228,7 +121,7 @@ done "--fee-recipient-addresses=" + constants.VALIDATING_REWARDS_ACCOUNT, "--withdrawal-addresses=" + constants.CHARON_WITHDRAWAL_ADDRESS, "--split-existing-keys", - "--split-keys-dir=/opt/charon/charon-keys", + "--split-keys-dir=" + charon_keys_dirpath, "--testnet-chain-id=" + network_params.network_id, "--testnet-fork-version=" + constants.GENESIS_FORK_VERSION, "--testnet-genesis-timestamp=" + str(genesis_time), @@ -568,7 +461,7 @@ done prysm_relative_dirpath="", teku_keys_relative_dirpath="teku-keys", teku_secrets_relative_dirpath="teku-secrets", - raw_keys_secrets_relative_dirpath="", + charon_keys_relative_dirpath="", ) @@ -833,7 +726,7 @@ done prysm_relative_dirpath="prysm", teku_keys_relative_dirpath="", teku_secrets_relative_dirpath="", - raw_keys_secrets_relative_dirpath="", + charon_keys_relative_dirpath="", ) return wallet_artifact, node_keystore_files From c07501e99092a876d1ad226f56bc1f2cd21c4307 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Mon, 29 Jun 2026 12:27:34 +0300 Subject: [PATCH 33/35] Make split keystore files readable/writable for non-root VC images Charon writes split-key password files as 0400/root. Non-root VC images (teku, lodestar) could not read the password files nor create teku .lock files. chmod the converter output so they are accessible. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/vc/charon_launcher.star | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/vc/charon_launcher.star b/src/vc/charon_launcher.star index 96d9fd4a8..0e65cfff3 100644 --- a/src/vc/charon_launcher.star +++ b/src/vc/charon_launcher.star @@ -439,6 +439,9 @@ for f in /split-keys/keystore-*.json; do cp "$f" "/out/teku-keys/${pubkey}.json" cp "$pw" "/out/teku-secrets/${pubkey}.txt" done +# charon writes password files as 0400/root; make them readable by non-root VC +# images (teku, lodestar), and make dirs writable so teku can create its .lock files +chmod -R a+rwX /out """ plan.exec( service_name=converter.name, From 7e22a12f4a47f39109f1c109c5fbf7af1e1a7f68 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:35:26 +0200 Subject: [PATCH 34/35] Increase max threads Teku VC can keep to Charon to 50 --- src/vc/teku.star | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vc/teku.star b/src/vc/teku.star index badd76a41..18751ca77 100644 --- a/src/vc/teku.star +++ b/src/vc/teku.star @@ -92,6 +92,7 @@ def get_config( if distributed: cmd.append("--Xobol-dvt-integration-enabled=true") cmd.append("--validators-builder-registration-default-enabled=true") + cmd.append("--Xvalidator-client-beacon-api-executor-threads=50") if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark From cab5508be753664854e0068318c38b190158ba3f Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:14:29 +0200 Subject: [PATCH 35/35] Disable doppelganger detection for distributed (Charon) Nimbus VCs A Charon distributed validator shares the same validator pubkeys across all cluster nodes (threshold signing). Nimbus VC enables doppelganger detection by default: at startup it watches ~2 epochs for its validators attesting elsewhere as an anti-slashing check. Because the cluster as a whole attests those pubkeys, each Nimbus VC sees its own validators live on the network and self-terminates at the epoch-1 boundary (FATAL, exit 127) -- taking down every VC in the DV and leaving Charon reporting 'VC not connected'. Add --doppelganger-detection=false in the distributed branch only; standalone Nimbus VCs keep it enabled. --- src/vc/nimbus.star | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/vc/nimbus.star b/src/vc/nimbus.star index 00f7f5f07..336d43b94 100644 --- a/src/vc/nimbus.star +++ b/src/vc/nimbus.star @@ -81,6 +81,13 @@ def get_config( if distributed: cmd.append("--distributed") cmd.append("--payload-builder=true") + # Disable doppelganger detection for distributed (Charon) validators. A + # Charon cluster legitimately shares the same validator pubkeys across + # all nodes (threshold signing), so each Nimbus VC sees "its" validators + # already attesting on the network and would self-terminate at the epoch + # boundary (FATAL, exit 127), taking down the whole DV. Standalone + # (non-distributed) Nimbus VCs keep doppelganger detection enabled. + cmd.append("--doppelganger-detection=false") if len(participant.vc_extra_params) > 0: # this is a repeated, we convert it into Starlark