WIP 13 vpn: add gateway support

Author: pymq

application.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/anywherelan/ts-dns/control/controlknobs"
@@ -161,7 +162,7 @@ func (a *Application) Init(ctx context.Context, tunDevice tun.Device) error {
 		}, a.Eventbus, new(awlevent.KnownPeerChanged))
 	}
 
-	a.VPNGateway = service.NewVPNGateway(a.Conf, a.Tunnel, a.vpnDevice, a.P2p, a.SockMarker, a.DisableGatewayOSSetup)
+	a.VPNGateway = service.NewVPNGateway(a.Conf, a.Tunnel, a.vpnDevice, a.P2p, a.SockMarker, a.Dns, a.DisableGatewayOSSetup)
 
 	handler := api.NewHandler(a.Conf, a.P2p, a.AuthStatus, a.Tunnel, a.SOCKS5, a.LogBuffer, a.Dns, a.VPNGateway)
 	a.Api = handler
@@ -353,33 +354,60 @@ type DNSService struct {
 	ctx      context.Context
 	logger   *log.ZapEventLogger
 
+	mu                  sync.Mutex
+	dnsHost             string
+	dnsFQDN             dnsname.FQDN
 	dnsOsConfigurator   dns.OSConfigurator
 	dnsResolver         *awldns.Resolver
 	upstreamDNS         string
 	isAwlDNSSetAsSystem bool
+	// forceUpstream forces the awl resolver to capture all queries
+	// (MatchDomains=nil) and forward them to the configured public upstream so
+	// DNS traverses the tunnel instead of leaking to the system resolver. Set
+	// in VPN gateway client mode.
+	forceUpstream bool
 }
 
 func NewDNSService(conf *config.Config, eventbus awlevent.Bus, ctx context.Context, logger *log.ZapEventLogger) *DNSService {
 	return &DNSService{conf: conf, eventbus: eventbus, ctx: ctx, logger: logger}
 }
 
 func (a *DNSService) initDNS(interfaceName string) {
-	var err error
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
 	dnsAddr := a.conf.DNS.ListenAddress
 	dnsHost, _, err := net.SplitHostPort(dnsAddr)
 	if err != nil {
 		a.logger.Errorf("invalid dns listen address %s: %v", dnsAddr, err)
 		return
 	}
+	a.dnsHost = dnsHost
 
+	fqdn, err := dnsname.ToFQDN(awldns.LocalDomain)
+	if err != nil {
+		panic(err)
+	}
+	a.dnsFQDN = fqdn
+
+	// TODO(android awldns): on Android this NewResolver cannot bind :53 (needs
+	// root) and dnsOsConfigurator.SetDNS below fails (no writable resolv.conf),
+	// so awldns is effectively inert there and .awl names do not resolve. The
+	// Android host instead points VpnService at DNS.UpstreamDNSAddress directly
+	// (see awl-flutter MainActivity.establishTun), which prevents leaks but
+	// gives no .awl resolution. A full fix would intercept :53 to a magic awl
+	// IP inside the tunnel read-path (userspace netstack),
+	// rather than binding an OS socket.
 	a.dnsResolver = awldns.NewResolver(dnsAddr)
-	a.upstreamDNS = awldns.DefaultUpstreamDNSAddress
-	a.refreshDNSConfig()
+	a.upstreamDNS = a.conf.DNS.UpstreamDNSAddress
+	a.forceUpstream = a.conf.VPNGateway.ClientEnabled
+	a.refreshDNSConfigLocked()
 
 	awlevent.WrapSubscriptionToCallback(a.ctx, func(_ interface{}) {
-		a.refreshDNSConfig()
+		a.mu.Lock()
+		defer a.mu.Unlock()
+		a.refreshDNSConfigLocked()
 	}, a.eventbus, new(awlevent.KnownPeerChanged))
-	defer a.refreshDNSConfig()
 
 	tsLogger := log.Logger("ts/dnsconf")
 	a.dnsOsConfigurator, err = dns.NewOSConfigurator(func(format string, args ...interface{}) {
@@ -390,52 +418,110 @@ func (a *DNSService) initDNS(interfaceName string) {
 		return
 	}
 
-	fqdn, err := dnsname.ToFQDN(awldns.LocalDomain)
-	if err != nil {
-		panic(err)
-	}
-	newOSConfig := dns.OSConfig{
-		Nameservers:  []netip.Addr{netip.MustParseAddr(dnsHost)},
-		MatchDomains: []dnsname.FQDN{fqdn},
-	}
+	a.applyOSDNSConfigLocked()
+}
+
+// applyOSDNSConfigLocked (re)computes the OS DNS takeover config from the
+// current state (split-DNS support, base config, forceUpstream) and pushes it
+// to the OS, then refreshes the awl resolver. Caller must hold a.mu and have a
+// non-nil dnsOsConfigurator. Safe to call repeatedly.
+func (a *DNSService) applyOSDNSConfigLocked() {
+	supportsSplitDNS := a.dnsOsConfigurator.SupportsSplitDNS()
 
-	if !a.dnsOsConfigurator.SupportsSplitDNS() {
-		newOSConfig.MatchDomains = nil
+	var baseNameservers []netip.Addr
+	if !supportsSplitDNS {
 		baseOSConfig, err := a.dnsOsConfigurator.GetBaseConfig()
 		if err != nil {
 			a.logger.Errorf("get base config from os configurator, abort setting os dns: %v", err)
 			return
 		}
-
 		a.logger.Infof("os does not support split dns. base config: %v", baseOSConfig)
-		if len(baseOSConfig.Nameservers) == 0 {
-			a.logger.Errorf("got zero nameservers from os configurator, use %s as default", awldns.DefaultUpstreamDNSAddress)
-			a.upstreamDNS = awldns.DefaultUpstreamDNSAddress
-		} else {
-			// TODO: use all nameservers in awldns resolver proxy
-			a.upstreamDNS = net.JoinHostPort(baseOSConfig.Nameservers[0].String(), awldns.DefaultDNSPort)
-		}
+		baseNameservers = baseOSConfig.Nameservers
+	}
+
+	matchDomains, upstream := chooseDNSPolicy(a.forceUpstream, supportsSplitDNS, baseNameservers, a.conf.DNS.UpstreamDNSAddress)
+	a.upstreamDNS = upstream
+	a.refreshDNSConfigLocked()
+
+	// TODO: consider setting SearchDomains = ["awl."] so peers resolve by bare
+	// short name (e.g. "mypeer" -> "mypeer.awl") instead of requiring the full
+	// .awl suffix. SearchDomains expands single-label queries into FQDNs and is
+	// additive to the OS's existing search list (distinct from MatchDomains,
+	// which only routes which zones reach the awl resolver). Would likely want a
+	// config toggle to gate it.
+	//
+	// TODO: consider pushing admin.awl into Hosts (a static FQDN->IP map applied
+	// to /etc/hosts) instead of (or in addition to) injecting
+	// AdminHttpServerDomainName into the resolver name mapping in
+	// refreshDNSConfigLocked — that would make the admin UI name resolvable even
+	// when the awl :53 resolver itself is not reachable.
+	newOSConfig := dns.OSConfig{
+		Nameservers:  []netip.Addr{netip.MustParseAddr(a.dnsHost)},
+		MatchDomains: matchDomains,
 	}
-
-	// In VPN gateway client mode, force upstream DNS to a public resolver to
-	// prevent DNS leaks. All traffic (including DNS) routes through the TUN
-	// and the gateway peer. This must come after the split-DNS logic so it
-	// takes final precedence.
-	if a.conf.VPNGateway.ClientEnabled {
-		a.upstreamDNS = awldns.DefaultUpstreamDNSAddress
-		a.logger.Infof("VPN gateway client mode: using public upstream DNS %s to prevent DNS leaks", a.upstreamDNS)
+	if err := a.dnsOsConfigurator.SetDNS(newOSConfig); err != nil {
+		a.logger.Errorf("set dns config to os configurator: %v", err)
+		return
 	}
+	a.logger.Infof("successfully set dns config to os (forceUpstream=%v, upstream=%s, matchDomains=%v)",
+		a.forceUpstream, a.upstreamDNS, matchDomains)
+	a.isAwlDNSSetAsSystem = true
+}
 
-	err = a.dnsOsConfigurator.SetDNS(newOSConfig)
+// chooseDNSPolicy decides which domains the awl resolver should capture and
+// which upstream it forwards non-.awl queries to.
+//
+//   - forceUpstream (VPN gateway client mode): capture everything
+//     (MatchDomains=nil) and forward to the configured public upstream so DNS
+//     goes through the tunnel — no leak.
+//   - split-DNS supported: capture only .awl; other queries are handled by the
+//     OS resolver directly, so the awl upstream is unused (kept as the
+//     configured default for completeness).
+//   - split-DNS unsupported: capture everything and forward to the system's
+//     first base nameserver, falling back to the configured default when the OS
+//     reports none.
+func chooseDNSPolicy(forceUpstream, supportsSplitDNS bool, base []netip.Addr, upstreamCfg string) (matchDomains []dnsname.FQDN, upstream string) {
+	awlFQDN, err := dnsname.ToFQDN(awldns.LocalDomain)
 	if err != nil {
-		a.logger.Errorf("set dns config to os configurator: %v", err)
-	} else {
-		a.logger.Info("successfully set dns config to os")
-		a.isAwlDNSSetAsSystem = true
+		panic(err)
+	}
+
+	if forceUpstream {
+		return nil, upstreamCfg
 	}
+	if supportsSplitDNS {
+		return []dnsname.FQDN{awlFQDN}, upstreamCfg
+	}
+	// no split DNS: capture everything, forward to the system's base resolver
+	if len(base) == 0 {
+		return nil, upstreamCfg
+	}
+	// TODO: use all nameservers in awldns resolver proxy
+	return nil, net.JoinHostPort(base[0].String(), awldns.DefaultDNSPort)
+}
+
+// ForceUpstreamDNS toggles full-capture mode where the awl resolver intercepts
+// all DNS (not just .awl) and forwards it to the configured public upstream, so
+// queries traverse the tunnel and do not leak. Driven by the VPN gateway client
+// apply/teardown. No-op (returns nil) when DNS was never set up as the system
+// resolver (DNS disabled, or Android where the OS DNS takeover does not apply).
+// Idempotent.
+func (a *DNSService) ForceUpstreamDNS(enabled bool) error {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	if a.dnsOsConfigurator == nil || !a.isAwlDNSSetAsSystem {
+		return nil
+	}
+	if a.forceUpstream == enabled {
+		return nil
+	}
+	a.forceUpstream = enabled
+	a.applyOSDNSConfigLocked()
+	return nil
 }
 
-func (a *DNSService) refreshDNSConfig() {
+func (a *DNSService) refreshDNSConfigLocked() {
 	if a.dnsResolver == nil {
 		a.logger.DPanicf("called refreshDNSConfig with nil resolver %v", a.dnsResolver)
 		return
@@ -446,6 +532,8 @@ func (a *DNSService) refreshDNSConfig() {
 }
 
 func (a *DNSService) Close() {
+	a.mu.Lock()
+	defer a.mu.Unlock()
 	if a.dnsOsConfigurator != nil {
 		err := a.dnsOsConfigurator.Close()
 		if err != nil {
@@ -458,13 +546,17 @@ func (a *DNSService) Close() {
 }
 
 func (a *DNSService) AwlDNSAddress() string {
+	a.mu.Lock()
+	defer a.mu.Unlock()
 	if a.dnsResolver != nil {
 		return a.dnsResolver.DNSAddress()
 	}
 	return ""
 }
 
 func (a *DNSService) IsAwlDNSSetAsSystem() bool {
+	a.mu.Lock()
+	defer a.mu.Unlock()
 	return a.isAwlDNSSetAsSystem
 }
 

application_test.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"runtime"
 	"sort"
@@ -18,6 +19,7 @@ import (
 	"golang.org/x/net/proxy"
 
 	"github.com/anywherelan/awl/api"
+	"github.com/anywherelan/awl/awldns"
 	"github.com/anywherelan/awl/config"
 	"github.com/anywherelan/awl/entity"
 	"github.com/anywherelan/awl/protocol"
@@ -82,9 +84,13 @@ func TestRemovePeer(t *testing.T) {
 	ts.Len(peer1.app.AuthStatus.GetIngoingAuthRequests(), 0)
 	ts.Len(peer2.app.AuthStatus.GetIngoingAuthRequests(), 0)
 
-	// test ping
-	p1Ping := peer1.app.P2p.GetPeerLatency(peer2.app.P2p.PeerID())
-	ts.NotEmpty(p1Ping)
+	// test ping. Latency is recorded asynchronously by the status/auth exchange
+	// the re-add above kicks off in a goroutine (RecordPeerLatency); on slow CI
+	// the fixed sleeps aren't always enough for it to land, so poll for it rather
+	// than asserting once.
+	ts.Eventually(func() bool {
+		return peer1.app.P2p.GetPeerLatency(peer2.app.P2p.PeerID()) != 0
+	}, 15*time.Second, 100*time.Millisecond, "peer latency should be recorded after the status exchange")
 }
 
 func TestDeclinePeerFriendRequest(t *testing.T) {
@@ -992,3 +998,78 @@ func TestMetricsEndpoint(t *testing.T) {
 	// Verify libp2p built-in metrics are present
 	ts.Contains(metricsOutput, "libp2p_")
 }
+
+func TestChooseDNSPolicy(t *testing.T) {
+	const upstreamCfg = "9.9.9.9:53"
+	base := []netip.Addr{netip.MustParseAddr("192.168.0.1"), netip.MustParseAddr("8.8.8.8")}
+
+	tests := []struct {
+		name             string
+		forceUpstream    bool
+		supportsSplitDNS bool
+		base             []netip.Addr
+		wantCapturesAll  bool // MatchDomains == nil
+		wantUpstream     string
+	}{
+		{
+			name:             "gateway on captures everything to configured upstream",
+			forceUpstream:    true,
+			supportsSplitDNS: true,
+			base:             base,
+			wantCapturesAll:  true,
+			wantUpstream:     upstreamCfg,
+		},
+		{
+			name:             "gateway on ignores split dns support",
+			forceUpstream:    true,
+			supportsSplitDNS: false,
+			base:             base,
+			wantCapturesAll:  true,
+			wantUpstream:     upstreamCfg,
+		},
+		{
+			name:             "split dns captures only .awl",
+			forceUpstream:    false,
+			supportsSplitDNS: true,
+			base:             base,
+			wantCapturesAll:  false,
+			wantUpstream:     upstreamCfg,
+		},
+		{
+			name:             "no split dns forwards to system base resolver",
+			forceUpstream:    false,
+			supportsSplitDNS: false,
+			base:             base,
+			wantCapturesAll:  true,
+			wantUpstream:     "192.168.0.1:" + awldns.DefaultDNSPort,
+		},
+		{
+			name:             "no split dns with empty base falls back to configured upstream",
+			forceUpstream:    false,
+			supportsSplitDNS: false,
+			base:             nil,
+			wantCapturesAll:  true,
+			wantUpstream:     upstreamCfg,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			matchDomains, upstream := chooseDNSPolicy(tt.forceUpstream, tt.supportsSplitDNS, tt.base, upstreamCfg)
+
+			gotCapturesAll := matchDomains == nil
+			if gotCapturesAll != tt.wantCapturesAll {
+				t.Errorf("captures all queries = %v, want %v (matchDomains=%v)", gotCapturesAll, tt.wantCapturesAll, matchDomains)
+			}
+			if !tt.wantCapturesAll {
+				// split-DNS: must capture exactly the .awl zone
+				if len(matchDomains) != 1 || matchDomains[0].WithoutTrailingDot() != awldns.LocalDomain {
+					t.Errorf("split-DNS match domains = %v, want [%s]", matchDomains, awldns.LocalDomain)
+				}
+			}
+			if upstream != tt.wantUpstream {
+				t.Errorf("upstream = %q, want %q", upstream, tt.wantUpstream)
+			}
+		})
+	}
+}

config/config.go

@@ -108,6 +108,12 @@ type (
 	DNSConfig struct {
 		DisableDNS    bool   `json:"disableDNS"`
 		ListenAddress string `json:"listenAddress"`
+		// UpstreamDNSAddress is the public resolver (host:port) that the awl
+		// DNS resolver forwards non-.awl queries to. Used as a fallback when
+		// the OS exposes no base nameserver, and forced as the sole upstream in
+		// VPN gateway client mode so DNS traverses the tunnel and does not leak.
+		// On Android the host reads this value to configure VpnService DNS.
+		UpstreamDNSAddress string `json:"upstreamDNSAddress"`
 	}
 	KnownPeer struct {
 		// Hex-encoded multihash representing a peer ID

config/other.go

@@ -140,6 +140,7 @@ func ImportConfig(data []byte, directory string) error {
 	return nil
 }
 
+//nolint:gocyclo
 func setDefaults(conf *Config, bus awlevent.Bus) {
 	isEmptyConfig := conf.Version == ""
 
@@ -195,6 +196,9 @@ func setDefaults(conf *Config, bus awlevent.Bus) {
 	if conf.DNS.ListenAddress == "" {
 		conf.DNS.ListenAddress = awldns.DefaultDNSAddress
 	}
+	if conf.DNS.UpstreamDNSAddress == "" {
+		conf.DNS.UpstreamDNSAddress = awldns.DefaultUpstreamDNSAddress
+	}
 
 	uniqAliases := make(map[string]struct{}, len(conf.KnownPeers))
 	if conf.KnownPeers == nil {