WIP 14 vpn: add gateway support

Author: pymq

README.md

@@ -243,7 +243,8 @@ In addition to the per-application SOCKS5 proxy, awl can route **all** of your I
 - **Linux:** supported (this is the platform we have tested on).
 - **Android:** client mode is supported via the awl Android app (uses `VpnService` for routing). Serving as an exit node from Android is **not** supported.
 - **macOS / Windows / other:** **not supported yet** — awl will refuse to start with VPN gateway enabled. Windows-side code exists but is unfinished; see `vpn/sockmark/sockmark_windows.go` and `vpn/routes/nat_windows.go` for the open work items.
-- IPv6 traffic is not tunnelled through the gateway in either direction (it is forwarded as a regular awl peer packet instead).
+- **IPv6 is not tunnelled; it is fenced off (fail-closed) while the gateway is on.** The gateway only carries IPv4. To stop IPv6 from leaking your real address past the exit node, the Linux client installs an `unreachable ::/0` route while gateway mode is on, so IPv6 connect()s fail fast and apps fall back to IPv4 through the tunnel. This is applied unconditionally — even if the host has no IPv6 right now, and even when IPv6 is disabled via sysctl (`disable_ipv6=1` only blocks address assignment, not routes), so that IPv6 appearing later (a hot-plugged uplink, a fresh router advertisement) is already fenced. It is skipped only when the IPv6 stack is absent entirely (kernel `ipv6.disable=1`), where nothing can leak. On Android the `VpnService` captures `::/0` into the TUN where it is dropped, so IPv6 is fenced there too. Full IPv6 tunnelling through the gateway is not implemented yet.
+- **Host default-route changes mid-session are not tracked.** The original default route(s) are snapshotted when the gateway is enabled and copied into the policy table for libp2p's use; if the host default later changes (DHCP renew, Wi-Fi↔Ethernet roaming, or an IPv6 RA advertising a new router/prefix), libp2p's physical-NIC exit can go stale until the gateway is toggled off and on. This degrades p2p connectivity but does not leak — the fail-closed routes are static.
 - Both client and exit-node sides require **CAP_NET_ADMIN** on Linux. AWL needs that already to bring up the TUN, so there is no extra capability to grant.
 - The exit node sets up NAT through the host's `iptables` binary, which must be installed (on modern distros it resolves to `iptables-nft`). AWL only sees rules on whatever backend that binary uses; rules created against the other backend by unrelated software are invisible to it.
 

vpn/routes/routes_linux.go

@@ -9,6 +9,7 @@ import (
 	"syscall"
 
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -45,6 +46,15 @@ type RouteState struct {
 	// instead of restoring originals.
 	origDefaults  []netlink.Route
 	tunRouteAdded bool
+
+	// IPv6 fail-closed state. The gateway only tunnels IPv4; to stop IPv6 from
+	// leaking past the exit node on a dual-stack host we fence it with an
+	// `unreachable ::/0` route, and exempt marked libp2p sockets via a v6
+	// fwmark rule + a copy of the host's IPv6 default(s) into tableID. Mirrors
+	// the IPv4 fields above. origDefaultsV6 may be empty (host has no IPv6).
+	origDefaultsV6 []netlink.Route
+	v6RuleAdded    bool
+	v6UnreachAdded bool
 }
 
 // SetupGatewayRoutes configures the system to route all traffic through the
@@ -77,6 +87,16 @@ func SetupGatewayRoutes(tunIfName string, fwmark uint32) (*RouteState, error) {
 		logger.Warnf("recovered from leftover gateway route state (previous run was likely killed before teardown)")
 	}
 
+	// TODO(gateway-route-staleness): origDefaults (and origDefaultsV6 below) are
+	// snapshotted once here and copied into tableID; they are never refreshed.
+	// Applies to BOTH families. If the host default changes mid-session (IPv4:
+	// DHCP renew, Wi-Fi<->Ethernet roaming; IPv6: RA re-advertising a new
+	// router/prefix — far more frequent), the copy in tableID goes stale and
+	// marked libp2p sockets lose their physical-NIC exit until the gateway is
+	// re-toggled. This does NOT cause a leak — the catch-all TUN default / v6
+	// unreachable route is static — only degraded p2p connectivity. Fix:
+	// subscribe to netlink RTM_NEWROUTE/RTM_DELROUTE and re-copy the live
+	// default(s) into tableID for both families.
 	origDefaults, err := getDefaultRoutes()
 	if err != nil {
 		return nil, fmt.Errorf("get default routes: %w", err)
@@ -127,9 +147,88 @@ func SetupGatewayRoutes(tunIfName string, fwmark uint32) (*RouteState, error) {
 	}
 	state.tunRouteAdded = true
 
+	// IPv6 fail-closed fence (`unreachable ::/0` + libp2p exemption). Installed
+	// unconditionally — see setupIPv6Fence for why that is safe even when IPv6 is
+	// disabled via sysctl, and how a genuinely absent IPv6 stack is tolerated.
+	if err := setupIPv6Fence(state, fwmark); err != nil {
+		_ = TeardownGatewayRoutes(state)
+		return nil, err
+	}
+
 	return state, nil
 }
 
+// setupIPv6Fence installs the IPv6 fail-closed fencing onto state: a v6
+// fwmark->tableID rule, a copy of the host's current IPv6 default(s) into
+// tableID (the libp2p exemption path), and an `unreachable ::/0` route that wins
+// LPM over any host default so locally generated IPv6 connect()s fail fast with
+// EHOSTUNREACH and apps fall back to IPv4 through the tunnel (Happy Eyeballs,
+// RFC 8305). Without it a dual-stack host egresses IPv6 straight out its
+// physical interface, exposing the real address past the exit node.
+//
+// It is applied UNCONDITIONALLY, even when the host has no IPv6 default right
+// now. The unreachable route and the rule install fine even with IPv6
+// administratively disabled via sysctl (`disable_ipv6=1` only blocks address
+// assignment, not route/rule additions — verified on Linux 6.8). Installing it
+// regardless means IPv6 that appears later — a hot-plugged uplink, a runtime
+// sysctl flip, a fresh RA — is already fenced and loses LPM to our metric-5
+// unreachable, rather than leaking. An empty IPv6 default set is therefore not
+// an error (unlike the IPv4 default above).
+//
+// The one case where the IPv6 stack genuinely isn't there is a kernel-level
+// disable (`ipv6.disable=1` on the cmdline): the module is absent, AF_INET6 ops
+// fail with EAFNOSUPPORT, and there is nothing to leak. We detect that from the
+// netlink ops and skip the fence (leaving state.v6* unset) rather than failing
+// the otherwise-working IPv4 gateway setup.
+func setupIPv6Fence(state *RouteState, fwmark uint32) error {
+	origDefaultsV6, err := getDefaultRoutesV6()
+	if err != nil {
+		if ipv6Unavailable(err) {
+			logger.Infof("IPv6 stack unavailable (%v); skipping IPv6 leak fence", err)
+			return nil
+		}
+		return fmt.Errorf("get IPv6 default routes: %w", err)
+	}
+
+	if err := netlink.RuleAdd(buildFwmarkRuleV6(fwmark)); err != nil {
+		if ipv6Unavailable(err) {
+			logger.Infof("IPv6 stack unavailable (%v); skipping IPv6 leak fence", err)
+			return nil
+		}
+		return fmt.Errorf("add IPv6 ip rule: %w", err)
+	}
+	state.v6RuleAdded = true
+	state.origDefaultsV6 = origDefaultsV6
+
+	for i := range origDefaultsV6 {
+		tableRoute := origDefaultsV6[i]
+		tableRoute.Table = tableID
+		if err := netlink.RouteAdd(&tableRoute); err != nil {
+			return fmt.Errorf("add original IPv6 default to table %d: %w", tableID, err)
+		}
+	}
+
+	if err := netlink.RouteAdd(buildV6UnreachableRoute()); err != nil {
+		if errors.Is(err, syscall.EEXIST) {
+			return fmt.Errorf("add IPv6 unreachable default route: %w — a ::/0 route at metric %d "+
+				"already exists (likely another VPN or a manual route, not awl: cleanupStaleRoutes "+
+				"already removed any of ours); inspect with `ip -6 route show` and resolve the conflict",
+				err, tunRouteMetric)
+		}
+		return fmt.Errorf("add IPv6 unreachable default route: %w", err)
+	}
+	state.v6UnreachAdded = true
+
+	return nil
+}
+
+// ipv6Unavailable reports whether a netlink error means the IPv6 stack is not
+// present at all (kernel-level ipv6.disable=1), as opposed to a real failure to
+// be surfaced. When it is, there is nothing to fence.
+func ipv6Unavailable(err error) bool {
+	return errors.Is(err, syscall.EAFNOSUPPORT) || errors.Is(err, syscall.EPROTONOSUPPORT)
+}
+
 // cleanupStaleRoutes removes leftover state from a previous SetupGatewayRoutes
 // call: the fwmark→tableID ip rule and every route currently in tableID. All
 // errors are intentionally swallowed — this is a best-effort pre-clean before
@@ -159,10 +258,14 @@ func cleanupStaleRoutes(fwmark uint32) bool {
 	// 2. Every route currently in tableID. We own the table by convention
 	// (its value is "awl" in ASCII), so anything inside it is leftover.
 	// Filter on Table only — LinkIndex of the original routes may differ
-	// from run to run if the physical NIC was renumbered.
-	routesInTable, err := netlink.RouteListFiltered(netlink.FAMILY_V4,
-		&netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE)
-	if err == nil {
+	// from run to run if the physical NIC was renumbered. Both families share
+	// the table, so sweep it for v4 and v6.
+	for _, family := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} {
+		routesInTable, err := netlink.RouteListFiltered(family,
+			&netlink.Route{Table: tableID}, netlink.RT_FILTER_TABLE)
+		if err != nil {
+			continue
+		}
 		for i := range routesInTable {
 			r := routesInTable[i]
 			if delErr := netlink.RouteDel(&r); delErr == nil {
@@ -171,6 +274,19 @@ func cleanupStaleRoutes(fwmark uint32) bool {
 		}
 	}
 
+	// 3. Stale IPv6 fwmark rule and the `unreachable ::/0` fence. Unlike the
+	// IPv4 TUN default route (which the kernel auto-removes when the TUN fd dies
+	// with the process), the v6 unreachable route is not bound to any interface,
+	// so a SIGKILL'd run leaves it behind — fencing off IPv6 host-wide until it
+	// is removed. It IS owner-tagged (its low metric + ::/0 + RTN_UNREACHABLE
+	// shape is ours), so we clean it here rather than surfacing an EEXIST.
+	if err := netlink.RuleDel(buildFwmarkRuleV6(fwmark)); err == nil {
+		cleaned = true
+	}
+	if err := netlink.RouteDel(buildV6UnreachableRoute()); err == nil {
+		cleaned = true
+	}
+
 	return cleaned
 }
 
@@ -202,6 +318,27 @@ func TeardownGatewayRoutes(state *RouteState) error {
 		errs = append(errs, fmt.Errorf("del ip rule: %w", err))
 	}
 
+	// IPv6 fail-closed teardown, reverse order of setup: unreachable fence,
+	// copied defaults, then the v6 fwmark rule. Guarded by the per-step flags so
+	// a rollback from a partially-applied setup doesn't generate spurious errors.
+	if state.v6UnreachAdded {
+		if err := netlink.RouteDel(buildV6UnreachableRoute()); err != nil {
+			errs = append(errs, fmt.Errorf("del IPv6 unreachable default route: %w", err))
+		}
+	}
+	for i := range state.origDefaultsV6 {
+		tableRoute := state.origDefaultsV6[i]
+		tableRoute.Table = tableID
+		if err := netlink.RouteDel(&tableRoute); err != nil {
+			errs = append(errs, fmt.Errorf("del IPv6 route from table %d: %w", tableID, err))
+		}
+	}
+	if state.v6RuleAdded {
+		if err := netlink.RuleDel(buildFwmarkRuleV6(state.fwmark)); err != nil {
+			errs = append(errs, fmt.Errorf("del IPv6 ip rule: %w", err))
+		}
+	}
+
 	if len(errs) > 0 {
 		return fmt.Errorf("teardown gateway routes: %w", errors.Join(errs...))
 	}
@@ -220,6 +357,19 @@ func buildFwmarkRule(fwmark uint32) *netlink.Rule {
 	return r
 }
 
+// buildFwmarkRuleV6 is the IPv6 counterpart of buildFwmarkRule: it steers
+// fwmark-tagged IPv6 packets (libp2p sockets) into tableID so they reach the
+// physical NIC instead of hitting the `unreachable ::/0` fence. SO_MARK is set
+// on every socket regardless of family, so the same fwmark value applies.
+func buildFwmarkRuleV6(fwmark uint32) *netlink.Rule {
+	r := netlink.NewRule()
+	r.Mark = fwmark
+	r.Table = tableID
+	r.Priority = rulePriority
+	r.Family = netlink.FAMILY_V6
+	return r
+}
+
 // buildTunDefaultRoute constructs the default route via the TUN. Scope is
 // SCOPE_LINK because the TUN is a point-to-point device with no gateway —
 // this matches what `ip route add default dev awl0` would produce, and using
@@ -241,6 +391,26 @@ func buildTunDefaultRoute(tunLinkIndex int) *netlink.Route {
 	}
 }
 
+// buildV6UnreachableRoute constructs the `unreachable ::/0` fence installed
+// while the gateway is on. RTN_UNREACHABLE (not RTN_BLACKHOLE) so locally
+// generated IPv6 connect()s fail fast with EHOSTUNREACH and apps fall back to
+// IPv4 through the tunnel (Happy Eyeballs, RFC 8305) instead of timing out.
+// Same low metric as the IPv4 TUN default so it wins LPM over the host's
+// RA/DHCPv6 default. The identical shape is used for Add, stale-cleanup Del and
+// teardown Del so they can't drift. No LinkIndex: an unreachable route is not
+// attached to any interface.
+func buildV6UnreachableRoute() *netlink.Route {
+	return &netlink.Route{
+		Type: unix.RTN_UNREACHABLE,
+		Dst: &net.IPNet{
+			IP:   net.IPv6zero,
+			Mask: net.CIDRMask(0, 128),
+		},
+		Priority: tunRouteMetric,
+		Family:   netlink.FAMILY_V6,
+	}
+}
+
 // getDefaultRoutes returns every IPv4 default route currently in the main
 // routing table. Hosts with multiple uplinks (Wi-Fi + Ethernet) typically
 // have several; we copy all of them into the policy-routing table.
@@ -273,3 +443,38 @@ func isIPv4DefaultDst(dst *net.IPNet) bool {
 	bits, _ := dst.Mask.Size()
 	return bits == 0
 }
+
+// getDefaultRoutesV6 returns every IPv6 default route (::/0) currently in the
+// main routing table, to be copied into tableID as the libp2p exemption path.
+// Unlike getDefaultRoutes, an empty result is NOT an error: the gateway installs
+// the `unreachable ::/0` fence unconditionally, so a host with no IPv6 uplink is
+// simply fenced against IPv6 that may appear later via RA.
+func getDefaultRoutesV6() ([]netlink.Route, error) {
+	allRoutes, err := netlink.RouteList(nil, netlink.FAMILY_V6)
+	if err != nil {
+		return nil, fmt.Errorf("list IPv6 routes: %w", err)
+	}
+
+	var defaults []netlink.Route
+	for i := range allRoutes {
+		r := allRoutes[i]
+		if !isIPv6DefaultDst(r.Dst) {
+			continue
+		}
+		defaults = append(defaults, r)
+	}
+	return defaults, nil
+}
+
+// isIPv6DefaultDst reports whether dst represents the IPv6 default route
+// (nil, or ::/0 expressed as a *net.IPNet with a /0 mask).
+func isIPv6DefaultDst(dst *net.IPNet) bool {
+	if dst == nil {
+		return true
+	}
+	if !dst.IP.Equal(net.IPv6zero) {
+		return false
+	}
+	bits, _ := dst.Mask.Size()
+	return bits == 0
+}

vpn/routes/vpn_hostnet_integration_test.go

@@ -35,6 +35,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -252,6 +253,23 @@ func assertRoutesApplied(t *testing.T) {
 
 	table := strings.TrimSpace(cmdOut(t, "ip", "-4", "route", "show", "table", strconv.Itoa(tableID)))
 	require.NotEmpty(t, table, "original default(s) must be copied into the awl table")
+
+	// IPv6 fail-closed fence. Installed unconditionally — even with IPv6 disabled
+	// via sysctl (disable_ipv6=1 blocks addresses, not routes). It is skipped
+	// only when the IPv6 stack is absent entirely (kernel ipv6.disable=1 →
+	// /proc/sys/net/ipv6 missing), matching setupIPv6Fence's EAFNOSUPPORT path.
+	if _, err := os.Stat("/proc/sys/net/ipv6"); os.IsNotExist(err) {
+		return
+	}
+	rules6 := cmdOut(t, "ip", "-6", "rule", "show")
+	require.Contains(t, rules6, fmt.Sprintf("fwmark 0x%x", testFWMark()), "v6 fwmark ip rule")
+	require.Contains(t, rules6, fmt.Sprintf("lookup %d", tableID), "v6 ip rule must steer to the awl table")
+
+	// Anchor the metric to the unreachable line so an unrelated host route can't
+	// satisfy it.
+	main6 := cmdOut(t, "ip", "-6", "route", "show")
+	require.Regexp(t, fmt.Sprintf(`unreachable default.*metric %d`, tunRouteMetric), main6,
+		"IPv6 unreachable fence present at the expected metric")
 }
 
 // ---------------------------------------------------------------------------
@@ -275,11 +293,25 @@ func snapshotNet(t *testing.T) string {
 	section("ip rule", cmdOut(t, "ip", "rule", "show"))
 	section("route main", cmdOut(t, "ip", "-4", "route", "show"))
 	section("route awl-table", routeTableDump(t, tableID))
+	// v6 route dumps are sanitized: RA-originated defaults carry an `expires
+	// Nsec` countdown that ticks between snapshots and would make before/after
+	// equality flaky on a dual-stack host.
+	section("ip -6 rule", cmdOut(t, "ip", "-6", "rule", "show"))
+	section("route6 main", stripVolatile(cmdOut(t, "ip", "-6", "route", "show")))
+	section("route6 awl-table", stripVolatile(route6TableDump(t, tableID)))
 	section("iptables filter", cmdOut(t, "iptables", "-S"))
 	section("iptables nat", cmdOut(t, "iptables", "-t", "nat", "-S"))
 	return b.String()
 }
 
+// volatileExpires matches the `expires Nsec` attribute that the kernel prints
+// for RA-learned IPv6 routes; its countdown changes between snapshots.
+var volatileExpires = regexp.MustCompile(`expires \d+sec`)
+
+func stripVolatile(s string) string {
+	return volatileExpires.ReplaceAllString(s, "expires")
+}
+
 // ---------------------------------------------------------------------------
 // helpers
 // ---------------------------------------------------------------------------
@@ -369,6 +401,19 @@ func routeTableDump(t *testing.T, table int) string {
 	return string(out)
 }
 
+// route6TableDump is the IPv6 counterpart of routeTableDump.
+func route6TableDump(t *testing.T, table int) string {
+	t.Helper()
+	out, err := exec.Command("ip", "-6", "route", "show", "table", strconv.Itoa(table)).CombinedOutput()
+	if err != nil {
+		if strings.Contains(string(out), "does not exist") {
+			return ""
+		}
+		require.NoErrorf(t, err, "ip -6 route show table %d: %s", table, out)
+	}
+	return string(out)
+}
+
 func lines(s string) []string {
 	var out []string
 	for _, l := range strings.Split(s, "\n") {