Stop stale saved/appended startup-configs from hijacking the management IP

cisco/asav/docker/launch.py

@@ -184,6 +184,16 @@ def _open(conn):
             with open(STARTUP_CONFIG_FILE, "r") as config:
                 startup_config = config.read()
                 self.logger.debug("Applying startup configuration")
+                # Strip any address the startup config sets on the mgmt
+                # interface before applying it, so a stale saved/hand-written
+                # mgmt address can never override the interface Management0/0
+                # stanza pushed above in this same session -- otherwise the
+                # firewall comes up "healthy" but unreachable at the address
+                # clab/DNS expect. Keyed on interface name, not address value
+                # (the stale value differs from the current one).
+                startup_config = vrnetlab.strip_mgmt_interface_config(
+                    startup_config, "Management0/0", "ios"
+                )
                 con.send_configs(startup_config.splitlines())
         else:
             self.logger.info("User provided startup configuration is not found.")

cisco/c8000v/docker/launch.py

@@ -87,8 +87,17 @@ def __init__(self, hostname, username, password, conn_mode, install_mode=False):
                     self.logger.info("MIME-wrapped config detected, using as-is")
                     cfg = startup_cfg
                 else:
-                    # Otherwise, append to bootstrap config
-                    cfg = self.gen_bootstrap_config() + startup_cfg
+                    # Otherwise, append to bootstrap config. Strip any address
+                    # the appended config sets on the mgmt interface so a stale
+                    # saved/hand-written mgmt address can never re-set it after
+                    # the bootstrap GigabitEthernet1 stanza (IOS-XE applies day0
+                    # config top to bottom, so a later "ip address" under the
+                    # same interface wins) -- otherwise the router comes up
+                    # "healthy" but unreachable at the address clab/DNS expect.
+                    # Keyed on interface name, not address value.
+                    cfg = self.gen_bootstrap_config() + vrnetlab.strip_mgmt_interface_config(
+                        startup_cfg, "GigabitEthernet1", "ios"
+                    )
             else:
                 self.logger.warning("User provided startup configuration is not found.")
                 cfg = self.gen_bootstrap_config()

cisco/cat9kv/docker/launch.py

@@ -143,7 +143,16 @@ def create_boot_image(self):
         if os.path.exists(STARTUP_CONFIG_FILE):
             self.logger.info("Startup configuration file found")
             with open(STARTUP_CONFIG_FILE, "r") as startup_config:
-                cat9kv_config += startup_config.read()
+                # Strip any address the appended startup-config sets on the mgmt
+                # interface so a stale saved/hand-written mgmt address can never
+                # re-set it after the GigabitEthernet0/0 stanza configured above
+                # (IOS-XE applies day0 config top to bottom, so a later "ip
+                # address"/"ipv6 address" under the same interface wins) --
+                # otherwise the switch comes up "healthy" but unreachable at the
+                # address clab/DNS expect. Keyed on interface name, not value.
+                cat9kv_config += vrnetlab.strip_mgmt_interface_config(
+                    startup_config.read(), "GigabitEthernet0/0", "ios"
+                )
         else:
             self.logger.warning(f"User provided startup configuration is not found.")
 

cisco/csr1000v/docker/launch.py

@@ -74,7 +74,18 @@ def __init__(
             if os.path.exists(STARTUP_CONFIG_FILE):
                 self.logger.info("Startup configuration file found")
                 with open(STARTUP_CONFIG_FILE, "r") as startup_config:
-                    cfg += startup_config.read()
+                    # Strip any address the appended startup-config sets on the
+                    # mgmt interface so a stale saved/hand-written mgmt address
+                    # can never re-set it after the GigabitEthernet1 stanza
+                    # configured above (IOS-XE applies day0 config top to bottom,
+                    # so a later "ip address" under the same interface wins) --
+                    # otherwise the router comes up "healthy" but unreachable at
+                    # the address clab/DNS expect. Keyed on interface name, not
+                    # address value (the stale value differs from the current
+                    # one, so only the name is invariant).
+                    cfg += vrnetlab.strip_mgmt_interface_config(
+                        startup_config.read(), "GigabitEthernet1", "ios"
+                    )
             else:
                 self.logger.warning(
                     f"User provided startup configuration is not found."

cisco/n9kv/docker/launch.py

@@ -184,7 +184,16 @@ def apply_config(self):
         if os.path.exists(STARTUP_CONFIG_FILE):
             self.logger.info("Startup configuration file found")
             with open(STARTUP_CONFIG_FILE, "r") as config:
-                n9kv_config += config.read()
+                # Strip any address the appended startup-config sets on the mgmt
+                # interface so a stale saved/hand-written mgmt address can never
+                # re-set it after the interface mgmt0 stanza configured above
+                # (config is applied top to bottom, so a later "ip address"/"ipv6
+                # address" under the same interface wins) -- otherwise the switch
+                # comes up "healthy" but unreachable at the address clab/DNS
+                # expect. Keyed on interface name, not address value.
+                n9kv_config += vrnetlab.strip_mgmt_interface_config(
+                    config.read(), "mgmt0", "ios"
+                )
         else:
             self.logger.warning("User provided startup configuration is not found.")
 

cisco/nxos/docker/launch.py

@@ -142,7 +142,16 @@ def apply_config(self):
         if os.path.exists(STARTUP_CONFIG_FILE):
             self.logger.info("Startup configuration file found")
             with open(STARTUP_CONFIG_FILE, "r") as config:
-                nxos_config += config.read()
+                # Strip any address the appended startup-config sets on the mgmt
+                # interface so a stale saved/hand-written mgmt address can never
+                # re-set it after the interface mgmt0 stanza configured above
+                # (config is applied top to bottom, so a later "ip address"/"ipv6
+                # address" under the same interface wins) -- otherwise the switch
+                # comes up "healthy" but unreachable at the address clab/DNS
+                # expect. Keyed on interface name, not address value.
+                nxos_config += vrnetlab.strip_mgmt_interface_config(
+                    config.read(), "mgmt0", "ios"
+                )
         else:
             self.logger.warning("User provided startup configuration is not found.")
 

cisco/xrd-vrouter/docker/Dockerfile

@@ -4,8 +4,15 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends cloud-image-utils curl ca-certificates && \
     rm -rf /var/lib/apt/lists/*
 
+# vrnetlab.py is the shared launcher library. The generic build step
+# (docker-build-common in makefile.include) stages common/vrnetlab.py into this
+# build context, but because this Dockerfile lists files explicitly rather than
+# `COPY *.py /`, vrnetlab.py must be named here too -- otherwise it is NOT copied
+# in and the image keeps the (older) /vrnetlab.py baked into vrnetlab-base, so
+# launch.py's calls into the current common lib (strip_mgmt_interface_config,
+# the qemu-guest-agent channel) fail with AttributeError.
 COPY make-golden.sh guest-init.sh xrd-guest-init.service netplan-mgmt.yaml \
-     cloud-init-disable-net.cfg xrd-unconfined launch.py /
+     cloud-init-disable-net.cfg xrd-unconfined launch.py vrnetlab.py /
 RUN chmod +x /make-golden.sh
 
 EXPOSE 22 161/udp 830 9339 57400 10000-10099

cisco/xrd-vrouter/docker/guest-init.sh

@@ -96,27 +96,48 @@ xrcli(){ timeout 25 docker exec xrd /pkg/bin/xr_cli "$1" 2>/dev/null; }
 # containerlab healthcheck, so we do not probe the address from the Linux netns
 # (XR owns it, so it does not live there).
 echo "waiting for XR control plane + management..."
+mgmt_up=0
 for i in $(seq 1 120); do
     if xrcli "show version" | grep -qi "cisco IOS XR"; then
         # '|| true' is essential: under `set -e`, a non-matching grep would
-        # otherwise abort guest-init before it ever emits the readiness marker.
+        # otherwise abort guest-init before it decides success/failure.
         mgmtline=$(xrcli "show ipv4 vrf all interface brief" | grep -i Mgmt || true)
         echo "[t=$((i*10))s] XR up; MgmtEth: ${mgmtline:-pending}"
         case "$mgmtline" in
-            *Up*Up*) echo "XR management is up"; break ;;
+            *Up*Up*) echo "XR management is up"; mgmt_up=1; break ;;
         esac
     else
         echo "[t=$((i*10))s] XR converging..."
     fi
     sleep 10
 done
 
-# Emit the readiness marker on the serial console for the vrnetlab launcher.
-# Repeat a few times so the launcher's serial poller reliably catches it.
+if [ "$mgmt_up" != 1 ]; then
+    # Management never came up. Emitting the readiness marker here is exactly what
+    # made the node report "healthy" while unreachable — the launcher sets
+    # running=True on the marker and writes "0 running" to /health regardless of
+    # mgmt. So we DON'T signal ready: the launcher keeps waiting (and eventually
+    # restarts the VM), and the node stays unhealthy. First, dump why — this goes
+    # to the console, so `docker logs`/serial shows the root cause on every attempt.
+    echo "!!! MgmtEth never reached Up/Up — NOT signalling ready. Diagnostics: !!!"
+    xrcli "show configuration failed startup"                 || true
+    xrcli "show running-config interface MgmtEth0/RP0/CPU0/0"  || true
+    xrcli "show interfaces MgmtEth0/RP0/CPU0/0"                || true
+    xrcli "show ipv4 interface MgmtEth0/RP0/CPU0/0 brief"      || true
+    xrcli "show logging last 80"                               || true
+    echo "=== guest-init FAILED (management down) ==="
+    # For interactive triage, reach the guest out-of-band via the qemu-guest-agent
+    # channel (the /run/qga<n>.sock unix socket inside the launcher container):
+    # guest-exec docker exec xrd /pkg/bin/xr_cli.
+    exit 1
+fi
+
+# Management is up — signal readiness to the vrnetlab launcher. Repeat a few
+# times so the launcher's serial poller reliably catches it.
 for n in 1 2 3 4 5; do
     for tty in /dev/ttyS0 /dev/console; do
         [ -w "$tty" ] && echo "${READY_MARKER}:${NODENAME}" > "$tty" 2>/dev/null || true
     done
     sleep 2
 done
-echo "=== guest-init done ==="
+echo "=== guest-init done (management up) ==="

cisco/xrd-vrouter/docker/launch.py

@@ -124,8 +124,16 @@ def _gen_xr_config(self):
         so no default route is needed and the data routing table stays clean.
         Data IPs come from the user's startup-config."""
         import ipaddress
-        mgmt_if = ipaddress.ip_interface(self.mgmt_address_ipv4)  # e.g. 10.0.0.15/24
+        mgmt_if = ipaddress.ip_interface(self.mgmt_address_ipv4)  # real clab IP, e.g. 172.20.20.2/24
         mgmt_v4 = f"{mgmt_if.ip} {mgmt_if.netmask}"
+        # Also configure IPv6 mgmt when the clab network provides it, so the
+        # launcher owns both families on MgmtEth (matches xrv9k/xrv). Without
+        # this, strip_mgmt_interface_config -- which removes every ip/ipv4/ipv6
+        # address the appended startup-config sets on MgmtEth -- would leave a
+        # dual-stack node with no launcher-managed IPv6 mgmt address.
+        mgmt_v6_line = ""
+        if self.mgmt_address_ipv6 and self.mgmt_address_ipv6 != "dhcp":
+            mgmt_v6_line = f"\n ipv6 address {self.mgmt_address_ipv6}"
         # Modelled on containerlab's official nodes/xrd/xrd.cfg. The bits XRd
         # needs for working SSH: `line default / transport input ssh` and
         # `secret` (not `secret 0`). MgmtEth gets the clab management address
@@ -144,7 +152,7 @@ def _gen_xr_config(self):
  ssh
 !
 interface MgmtEth0/RP0/CPU0/0
- ipv4 address {mgmt_v4}
+ ipv4 address {mgmt_v4}{mgmt_v6_line}
  no shutdown
 !
 ssh server v2
@@ -158,7 +166,18 @@ def _gen_xr_config(self):
 """
         if os.path.exists(STARTUP_CONFIG_FILE):
             with open(STARTUP_CONFIG_FILE) as f:
-                cfg += f.read()
+                startup = f.read()
+            # The launcher owns MgmtEth0/RP0/CPU0/0 (configured above from the
+            # container's *actual* IP). Strip any address the appended
+            # startup-config sets on that interface so a stale saved/hand-written
+            # mgmt address can never win -- otherwise XR comes up on the old IP:
+            # "healthy" (XR is up) but unreachable at the address clab/DNS
+            # expect. Keyed on the interface name, not the address value: the
+            # stale value differs from the one just configured, so only the
+            # interface identity is invariant.
+            cfg += vrnetlab.strip_mgmt_interface_config(
+                startup, "MgmtEth0/RP0/CPU0/0", "ios"
+            )
             if not cfg.rstrip().endswith("end"):
                 cfg += "\nend\n"
         else:

cisco/xrd-vrouter/docker/make-golden.sh

@@ -99,6 +99,12 @@ write_files:
       set -euo pipefail
       curl -fsSL https://get.docker.com | sh
       systemctl restart docker
+      # qemu-guest-agent: lets the host drive guest-exec over the virtio-serial
+      # channel org.qemu.guest_agent.0 the launcher now attaches -- out-of-band
+      # access into the guest even when management is down and no serial getty.
+      apt-get update
+      apt-get install -y --no-install-recommends qemu-guest-agent
+      systemctl enable qemu-guest-agent
       mount -t 9p -o trans=virtio,version=9p2000.L,ro xrdshare /mnt
       docker load -i /mnt/xrd.tar
       umount /mnt

cisco/xrv/docker/launch.py

@@ -181,7 +181,17 @@ def apply_config(self):
         if os.path.exists(STARTUP_CONFIG_FILE):
             self.logger.info("Startup configuration file found")
             with open(STARTUP_CONFIG_FILE, "r") as config:
-                xrv_config += config.read()
+                # Strip any address the appended startup-config sets on the mgmt
+                # interface so a stale saved/hand-written mgmt address can never
+                # win over the interface MgmtEth stanza configured above
+                # (last-applied wins under IOS-XR) -- otherwise XR comes up
+                # "healthy" but unreachable at the address clab/DNS expect.
+                # Keyed on interface name, not address value (the stale value
+                # differs from the current one, so only the name is invariant).
+                # Note: classic XRv's mgmt is MgmtEth0/0/CPU0/0, not .../RP0/... .
+                xrv_config += vrnetlab.strip_mgmt_interface_config(
+                    config.read(), "MgmtEth0/0/CPU0/0", "ios"
+                )
         else:
             self.logger.warning("User provided startup configuration is not found.")
 

cisco/xrv9k/docker/launch.py

@@ -293,7 +293,16 @@ def apply_config(self):
         if os.path.exists(STARTUP_CONFIG_FILE):
             self.logger.info("Startup configuration file found")
             with open(STARTUP_CONFIG_FILE, "r") as config:
-                xrv9k_config += config.read()
+                # Strip any address the appended startup-config sets on the mgmt
+                # interface so a stale saved/hand-written mgmt address can never
+                # win over the interface MgmtEth stanza configured above
+                # (last-applied wins under IOS-XR) -- otherwise XR comes up
+                # "healthy" but unreachable at the address clab/DNS expect.
+                # Keyed on interface name, not address value (the stale value
+                # differs from the current one, so only the name is invariant).
+                xrv9k_config += vrnetlab.strip_mgmt_interface_config(
+                    config.read(), "MgmtEth0/RP0/CPU0/0", "ios"
+                )
         else:
             self.logger.warning("User provided startup configuration is not found.")