fix(bootnodoor): hand discv5-only EL clients the EL ENR instead of the enode

README.md

@@ -1233,6 +1233,9 @@ bootnodoor_params:
   max_cpu: 1000
   min_mem: 128
   max_mem: 512
+  # Use separate randomly-generated EL (discv4) and CL (discv5) identity keys
+  # instead of one shared key
+  separate_keys: false
   # A list of optional extra args the bootnodoor container should spin up with
   extra_args: []
 

main.star

@@ -55,6 +55,7 @@ buildoor = import_module("./src/mev/buildoor/buildoor_launcher.star")
 mev_custom_flood = import_module(
     "./src/mev/flashbots/mev_custom_flood/mev_custom_flood_launcher.star"
 )
+bootnodoor = import_module("./src/bootnodoor/bootnodoor_launcher.star")
 broadcaster = import_module("./src/broadcaster/broadcaster.star")
 mempool_bridge = import_module("./src/mempool_bridge/mempool_bridge_launcher.star")
 assertoor = import_module("./src/assertoor/assertoor_launcher.star")
@@ -522,6 +523,8 @@ def run(plan, args={}):
             num_participants, network_params
         )
     )
+    # launch_participant_network removes bootnodoor from additional_services, so check first
+    bootnodoor_enabled = "bootnodoor" in args_with_right_defaults.additional_services
     (
         all_participants,
         final_genesis_timestamp,
@@ -549,6 +552,9 @@ def run(plan, args={}):
         detected_backend,
     )
 
+    if bootnodoor_enabled:
+        prometheus_additional_metrics_jobs.append(bootnodoor.get_metrics_job())
+
     for p in all_participants:
         if p.el_context != None:
             plan.print(

src/bootnodoor/bootnodoor_launcher.star

@@ -5,18 +5,6 @@ SERVICE_NAME = "bootnodoor"
 HTTP_PORT_NUMBER = 8080
 DISCOVERY_PORT_NUMBER = 9000
 
-USED_PORTS = {
-    constants.HTTP_PORT_ID: shared_utils.new_port_spec(
-        HTTP_PORT_NUMBER,
-        shared_utils.TCP_PROTOCOL,
-        shared_utils.HTTP_APPLICATION_PROTOCOL,
-    ),
-    constants.UDP_DISCOVERY_PORT_ID: shared_utils.new_port_spec(
-        DISCOVERY_PORT_NUMBER,
-        shared_utils.UDP_PROTOCOL,
-    ),
-}
-
 
 def launch_bootnodoor(
     plan,
@@ -26,11 +14,27 @@ def launch_bootnodoor(
     global_node_selectors,
     global_tolerations,
     docker_cache_params,
+    port_publisher,
+    additional_service_index,
+    backend,
 ):
     tolerations = shared_utils.get_tolerations(global_tolerations=global_tolerations)
 
-    # Generate a random 32-byte (64 hex char) private key
-    private_key = generate_random_private_key(plan)
+    # Generate random 32-byte (64 hex char) private keys
+    private_key = None
+    el_private_key = None
+    cl_private_key = None
+    if bootnodoor_params.separate_keys:
+        el_private_key = generate_random_private_key(
+            plan, "generate-bootnodoor-el-private-key"
+        )
+        cl_private_key = generate_random_private_key(
+            plan, "generate-bootnodoor-cl-private-key"
+        )
+    else:
+        private_key = generate_random_private_key(
+            plan, "generate-bootnodoor-private-key"
+        )
 
     # Get the genesis validators root from the genesis data
     genesis_validators_root = el_cl_genesis_data.genesis_validators_root
@@ -40,8 +44,10 @@ def launch_bootnodoor(
 
     # Fetch external bootnodes for non-kurtosis networks
     # For kurtosis/shadowfork: bootnodoor is the primary bootnode, no external bootnodes needed
-    # For ephemery/devnets: read bootstrap_nodes.txt from genesis data
-    external_bootnodes = None
+    # For ephemery/devnets: read bootstrap_nodes.txt (CL ENRs) plus el_enrs.txt and
+    # enodes.txt (EL discv5 + discv4) from genesis data
+    external_cl_bootnodes = None
+    external_el_bootnodes = None
     if network_params.network == constants.NETWORK_NAME.ephemery or (
         network_params.network not in constants.PUBLIC_NETWORKS
         and network_params.network != constants.NETWORK_NAME.kurtosis
@@ -52,28 +58,37 @@ def launch_bootnodoor(
                 network_params.network
             )
         )
-        external_bootnodes = shared_utils.get_devnet_enrs_list(
+        external_cl_bootnodes = shared_utils.get_devnet_enrs_list(
+            plan, el_cl_genesis_data.files_artifact_uuid
+        )
+        external_el_bootnodes = shared_utils.get_devnet_el_bootnodes(
             plan, el_cl_genesis_data.files_artifact_uuid
         )
 
     config = get_config(
         bootnodoor_params,
         el_cl_genesis_data,
         private_key,
+        el_private_key,
+        cl_private_key,
         genesis_validators_root,
         el_genesis_hash,
-        external_bootnodes,
+        external_cl_bootnodes,
+        external_el_bootnodes,
         global_node_selectors,
         tolerations,
         docker_cache_params,
+        port_publisher,
+        additional_service_index,
+        backend,
     )
 
     plan.add_service(SERVICE_NAME, config)
 
     # Request ENR from the running bootnodoor service
-    # The /enr endpoint returns a plain string (not JSON)
+    # /cl-enr strips the EL-only "eth" field, which some CL clients reject
     enr_recipe = GetHttpRequestRecipe(
-        endpoint="/enr",
+        endpoint="/cl-enr",
         port_id=constants.HTTP_PORT_ID,
     )
 
@@ -97,20 +112,64 @@ def launch_bootnodoor(
 
     bootnodoor_enode = enode_response["body"]
 
-    return bootnodoor_enr, bootnodoor_enode
+    # Request the EL ENR for discv5-only EL clients
+    # /el-enr strips the CL-only "eth2" field, which some EL clients reject; with
+    # separate_keys it is a different identity than the CL ENR altogether
+    el_enr_recipe = GetHttpRequestRecipe(
+        endpoint="/el-enr",
+        port_id=constants.HTTP_PORT_ID,
+    )
+
+    el_enr_response = plan.request(
+        recipe=el_enr_recipe,
+        service_name=SERVICE_NAME,
+    )
+
+    bootnodoor_el_enr = el_enr_response["body"]
+
+    return bootnodoor_enr, bootnodoor_enode, bootnodoor_el_enr
 
 
 def get_config(
     bootnodoor_params,
     el_cl_genesis_data,
     private_key,
+    el_private_key,
+    cl_private_key,
     genesis_validators_root,
     el_genesis_hash,
-    external_bootnodes,
+    external_cl_bootnodes,
+    external_el_bootnodes,
     node_selectors,
     tolerations,
     docker_cache_params,
+    port_publisher,
+    additional_service_index,
+    backend,
 ):
+    # Bind the published port number itself so the ENR (ip, port) is valid both
+    # in-enclave (private IP) and externally (nat_exit_ip)
+    discovery_port = DISCOVERY_PORT_NUMBER
+    public_ports = {}
+    if port_publisher.additional_services_enabled:
+        public_ports_for_component = shared_utils.get_public_ports_for_component(
+            "additional_services", port_publisher, additional_service_index
+        )
+        discovery_port = public_ports_for_component[1]
+        public_ports = shared_utils.get_port_specs(
+            {
+                constants.HTTP_PORT_ID: public_ports_for_component[0],
+                constants.UDP_DISCOVERY_PORT_ID: discovery_port,
+            }
+        )
+
+    used_ports = shared_utils.get_port_specs(
+        {
+            constants.HTTP_PORT_ID: HTTP_PORT_NUMBER,
+            constants.UDP_DISCOVERY_PORT_ID: discovery_port,
+        }
+    )
+
     cmd = [
         "--cl-config",
         "{0}/config.yaml".format(constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS),
@@ -120,34 +179,57 @@ def get_config(
         "{0}/genesis.json".format(constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS),
         "--el-genesis-hash",
         el_genesis_hash,
-        "--private-key",
-        private_key,
         "--bind-addr",
         "0.0.0.0",
+        "--bind-port",
+        str(discovery_port),
+        # Without a pinned IP, bootnodoor rewrites its ENR to the WAN address learned from PONGs
+        "--enr-ip",
+        "${K8S_POD_IP}"
+        if backend == "kubernetes"
+        else port_publisher.additional_services_nat_exit_ip,
         "--web-ui",
         "--nodedb",
         "/nodes.db",
     ]
 
+    if private_key != None:
+        cmd.append("--private-key")
+        cmd.append(private_key)
+    if el_private_key != None:
+        cmd.append("--el-private-key")
+        cmd.append(el_private_key)
+    if cl_private_key != None:
+        cmd.append("--cl-private-key")
+        cmd.append(cl_private_key)
+
     # Add external bootnodes for non-kurtosis networks (ephemery, devnets)
-    if external_bootnodes != None:
-        cmd.append("--bootnodes")
-        cmd.append(external_bootnodes)
+    if external_cl_bootnodes != None:
+        cmd.append("--cl-bootnodes")
+        cmd.append(external_cl_bootnodes)
+    if external_el_bootnodes != None:
+        cmd.append("--el-bootnodes")
+        cmd.append(external_el_bootnodes)
 
     # Add any extra args from the params
     if len(bootnodoor_params.extra_args) > 0:
         cmd.extend(bootnodoor_params.extra_args)
 
+    cmd_str = "exec /app/bootnodoor " + " ".join(cmd)
+
     return ServiceConfig(
         image=shared_utils.docker_cache_image_calc(
             docker_cache_params,
             bootnodoor_params.image,
         ),
-        ports=USED_PORTS,
+        ports=used_ports,
+        public_ports=public_ports,
         files={
             constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
         },
-        cmd=cmd,
+        entrypoint=["sh", "-c"],
+        cmd=[cmd_str],
+        private_ip_address_placeholder=constants.PRIVATE_IP_ADDRESS_PLACEHOLDER,
         min_cpu=bootnodoor_params.min_cpu,
         max_cpu=bootnodoor_params.max_cpu,
         min_memory=bootnodoor_params.min_mem,
@@ -177,12 +259,24 @@ def get_el_genesis_hash(plan, network_params, el_cl_genesis_data):
     return result.output
 
 
-def generate_random_private_key(plan):
+def generate_random_private_key(plan, name):
     # Generate a random 32-byte (64 hex char) private key using /dev/urandom
     result = plan.run_sh(
-        name="generate-bootnodoor-private-key",
+        name=name,
         description="Generating random private key for bootnodoor",
         run="head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \\n'",
         wait=None,
     )
     return result.output
+
+
+def get_metrics_job():
+    return {
+        "Name": SERVICE_NAME,
+        "Endpoint": "{0}:{1}".format(SERVICE_NAME, HTTP_PORT_NUMBER),
+        "MetricsPath": "/metrics",
+        "Labels": {
+            "service": SERVICE_NAME,
+        },
+        "ScrapeInterval": "15s",
+    }

src/el/besu/besu_launcher.star

@@ -45,7 +45,7 @@ def launch(
     participant_index,
     network_params,
     extra_files_artifacts,
-    bootnodoor_enode=None,
+    bootnodoor_el_enr=None,
     el_binary_artifact=None,
     otel_otlp_grpc_url=None,
 ):
@@ -66,7 +66,7 @@ def launch(
         participant_index,
         network_params,
         extra_files_artifacts,
-        bootnodoor_enode,
+        bootnodoor_el_enr,
         el_binary_artifact,
         otel_otlp_grpc_url,
     )
@@ -98,7 +98,7 @@ def get_config(
     participant_index,
     network_params,
     extra_files_artifacts,
-    bootnodoor_enode=None,
+    bootnodoor_el_enr=None,
     el_binary_artifact=None,
     otel_otlp_grpc_url=None,
 ):
@@ -200,9 +200,9 @@ def get_config(
     else:
         cmd.append("--network=" + network_params.network)
 
-    # Handle bootnode configuration with bootnodoor_enode override
-    if bootnodoor_enode != None:
-        cmd.append("--bootnodes=" + bootnodoor_enode)
+    # Handle bootnode configuration with bootnodoor_el_enr override
+    if bootnodoor_el_enr != None:
+        cmd.append("--bootnodes=" + bootnodoor_el_enr)
     elif (
         network_params.network == constants.NETWORK_NAME.kurtosis
         or constants.NETWORK_NAME.shadowfork in network_params.network

src/el/el_launcher.star

@@ -32,6 +32,7 @@ def launch(
     mev_params,
     extra_files_artifacts={},
     bootnodoor_enode=None,
+    bootnodoor_el_enr=None,
     binary_artifacts={},
     otel_otlp_grpc_url=None,
 ):
@@ -128,11 +129,12 @@ def launch(
     el_service_configs = {}
     el_participant_info = {}
 
-    # Generic bootnode ENODE override - can be set from bootnodoor or any other bootnode service
-    if bootnodoor_enode != None:
+    # Generic bootnode override - can be set from bootnodoor or any other bootnode service.
+    # Discv5-only clients get the EL ENR; discv4-only ethereumjs keeps the enode.
+    if bootnodoor_el_enr != None or bootnodoor_enode != None:
         plan.print(
-            "Using bootnode ENODE override for all EL clients: {0}".format(
-                bootnodoor_enode
+            "Using bootnode override for all EL clients: ENR {0} / ENODE {1}".format(
+                bootnodoor_el_enr, bootnodoor_enode
             )
         )
 
@@ -175,6 +177,14 @@ def launch(
         el_service_name = "el-{0}-{1}-{2}".format(index_str, el_type, cl_type)
         el_binary_artifact = binary_artifacts.get(index, {}).get("el", None)
 
+        # ethereumjs is discv4-only and needs the enode; every other client is
+        # discv5-only and needs the EL ENR
+        el_bootnode_override = (
+            bootnodoor_enode
+            if el_type == constants.EL_TYPE.ethereumjs
+            else bootnodoor_el_enr
+        )
+
         if index == 0:
             el_context = launch_method(
                 plan,
@@ -190,7 +200,7 @@ def launch(
                 index,
                 network_params,
                 extra_files_artifacts,
-                bootnodoor_enode,
+                el_bootnode_override,
                 el_binary_artifact,
                 otel_otlp_grpc_url,
             )
@@ -217,7 +227,7 @@ def launch(
                 index,
                 network_params,
                 extra_files_artifacts,
-                bootnodoor_enode,
+                el_bootnode_override,
                 el_binary_artifact,
                 otel_otlp_grpc_url,
             )