dhcp: T9085: metric-aware stale delete; JSON presence; no-default-route skip

alexk37 on-box follow-up (2026-08-22):

1. Two-router option 3 installs at distance 210 and 211. A wrapper delete
   without metric always issues 210, so the 211 leftover stays in FRR and
   can become the kernel default after option 3 is cleared. Delete with
   the same per-router metric 01-vyos-cleanup uses.

2. Two DHCP WANs at the same distance install a multipath default.
   `ip route show default via GW dev IF` returns nothing for a nexthop
   group, so every renew logged a false reinstall. Match gateways in
   `ip -j` output.

3. no-default-route blanks new_routers while a force-sent option 3 still
   fills old_routers, so cleanup logged and opened vtysh every renew.
   06-vyos-nodefaultroute now sets VYOS_NO_DEFAULT_ROUTE; skip the delete
   unless a leftover kernel default is actually present.

Author: BradKollmyer

src/etc/dhcp/dhclient-enter-hooks.d/06-vyos-nodefaultroute

@@ -17,4 +17,7 @@ PYEND
 
 if [[ "$(get_no_default_route)" == 'True' ]]; then
     new_routers=""
+    # Exit-hooks (same dhclient-script shell) use this to skip stale-default
+    # cleanup when the lease still carries a force-sent option 3.
+    VYOS_NO_DEFAULT_ROUTE=1
 fi

src/etc/dhcp/dhclient-exit-hooks.d/02-vyos-dhcp-renew-default-route

@@ -10,16 +10,27 @@
 # Presence check MUST use /usr/sbin/ip, not the enter-hook ip() wrapper.
 # That wrapper sends any `ip … route` that is not `del` through vtysh as an
 # add, so `ip -4 route show` both misses the kernel table and writes a
-# staticd duplicate. delroute() already reads via /usr/sbin/ip; match that,
-# and append ${VRF_OPTION} so VRF leases see the same table the add uses.
+# staticd duplicate. Use `ip -j` so a multipath default (two DHCP WANs at
+# the same distance) is visible — `ip route show default via GW dev IF`
+# returns nothing for a nexthop group. Append ${VRF_OPTION} so VRF leases
+# see the same table the add uses.
 #
 # If option 3 changes across a same-IP RENEW, drop defaults via gateways
 # that disappeared from $new_routers (including option 3 going away entirely).
 # Otherwise FRR keeps the old nexthop and the kernel shows ECMP over old+new.
 # Compare $old_routers vs $new_routers (same pattern as
-# 98-vyos-static-routes-dhclient-hook). The no-default-route enter hook blanks
-# new_routers; removing a leftover option-3 default is the right outcome there
-# too, so cleanup runs before the empty-new_routers return.
+# 98-vyos-static-routes-dhclient-hook). Delete with the same per-router
+# metric 01-vyos-cleanup uses (${IF_METRIC}, then +1 per old_routers
+# entry): a wrapper delete without metric always issues distance 210, which
+# misses the 211+ FRR leftovers and can leave a ghost as the kernel default
+# after option 3 is cleared.
+#
+# 06-vyos-nodefaultroute blanks new_routers and sets VYOS_NO_DEFAULT_ROUTE.
+# Skip the stale-delete (and its vtysh session) when that flag is set and
+# the kernel has no default via that gateway — otherwise a server that
+# force-sends option 3 logs "removing stale default" on every renew.
+# If a leftover kernel default exists (no-default-route just set), still
+# delete it.
 #
 # Adds still use the enter-hook ip() wrapper so FRR gets tag/distance
 # IF_METRIC. Skip when classless static routes are present (stock ignores
@@ -42,16 +53,26 @@ if [ -n "$new_rfc3442_classless_static_routes" ]; then
     return 0 2>/dev/null || exit 0
 fi
 
-# Kernel/FRR-installed default present? Real ip(8) only — never the wrapper.
+# Kernel default via $1 on $interface? Real ip(8) JSON — never the wrapper.
+# Matches both a single-nexthop default and a multipath nexthop group.
 _default_via_present() {
-    /usr/sbin/ip -4 route show default via "$1" dev "${interface}" ${VRF_OPTION} 2>/dev/null | grep -q .
+    local gw="$1"
+    local json
+    json=$(/usr/sbin/ip -j -4 route show default ${VRF_OPTION} 2>/dev/null) || return 1
+    [ -n "$json" ] || return 1
+    echo "$json" | jq -e --arg gw "$gw" --arg dev "$interface" \
+        'any(.[];
+            (.gateway == $gw and .dev == $dev)
+            or any(.nexthops[]?; .gateway == $gw and .dev == $dev)
+        )' >/dev/null 2>&1
 }
 
-if_metric="${IF_METRIC:-210}"
-
 # Drop defaults via gateways the server no longer announces, including when
-# option 3 disappeared (empty new_routers) or no-default-route blanked it.
+# option 3 disappeared (empty new_routers). Metric numbering matches the
+# original install / 01-vyos-cleanup: IF_METRIC, then +1 per old_routers
+# entry (keepers included, or the 211+ leftover is missed).
 if [ -n "$old_routers" ] && [ "$old_routers" != "$new_routers" ]; then
+    if_metric="${IF_METRIC:-210}"
     for router in $old_routers; do
         skip=
         for keep in $new_routers; do
@@ -61,16 +82,25 @@ if [ -n "$old_routers" ] && [ "$old_routers" != "$new_routers" ]; then
             fi
         done
         if [ -n "$skip" ]; then
+            if_metric=$((if_metric + 1))
+            continue
+        fi
+        # no-default-route blanks new_routers every renew; only act if a
+        # leftover default is actually present.
+        if [ -n "$VYOS_NO_DEFAULT_ROUTE" ] && ! _default_via_present "${router}"; then
+            if_metric=$((if_metric + 1))
             continue
         fi
         logmsg info \
-            "RENEW/REBIND: removing stale default via ${router} dev ${interface}"
-        # Wrapper: FRR-aware del (staticd + kernel).
-        ip -4 route del default via "${router}" dev "${interface}" >/dev/null 2>&1
+            "RENEW/REBIND: removing stale default via ${router} dev ${interface} metric ${if_metric}"
+        # Wrapper: FRR-aware del (staticd + kernel) at the install distance.
+        ip -4 route del default via "${router}" dev "${interface}" \
+            metric "${if_metric}" >/dev/null 2>&1
         if _default_via_present "${router}"; then
             logmsg warn \
                 "RENEW/REBIND: stale default still present via ${router} dev ${interface}"
         fi
+        if_metric=$((if_metric + 1))
     done
 fi
 
@@ -79,6 +109,7 @@ if [ -z "$new_routers" ]; then
     return 0 2>/dev/null || exit 0
 fi
 
+if_metric="${IF_METRIC:-210}"
 for router in $new_routers; do
     if _default_via_present "${router}"; then
         if_metric=$((if_metric + 1))