Merge pull request #259 from anywherelan/fix/win-gateway-winnat-errors #883
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: [ push, pull_request ] | |
| jobs: | |
| test: | |
| # run job on all pushes OR external PR, not both | |
| if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macos-latest ] | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| - name: Install Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: 1.26.x | |
| cache: true | |
| - name: Create stub static/ | |
| run: mkdir static && touch static/index.html | |
| - name: Check deps | |
| shell: bash | |
| run: ./build.sh deps | |
| - name: gofmt && go mod tidy | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| go mod tidy -compat=1.26 | |
| cd cmd/awl-tray && go mod tidy -compat=1.26 && cd ../.. | |
| test -z "$(gofmt -d .)" || (gofmt -d . && false) | |
| test -z "$(git status --porcelain)" || (git status; git diff && false) | |
| - name: Test | |
| run: go test -count=1 -v ./... | |
| - name: Test with -race | |
| run: go test -race -count=1 -v ./... | |
| - name: VPN gateway host-network integration test (Linux, root) | |
| # Hidden behind the `vpn_hostnet` build tag and excluded from `go test ./...`. | |
| # Exercises the real netlink/iptables plumbing (SetupNAT/SetupGatewayRoutes | |
| # + teardown/stale-recovery) against the runner's own network, so it needs | |
| # root and only runs on Linux. Compile as the normal user, then run the | |
| # binary under sudo so root never pollutes the Go build cache. | |
| # TODO: if a hard failure ever leaves the runner's egress black-holed | |
| # (default route via a dead awl0) and breaks log upload, add a separate | |
| # `if: always()` cleanup step | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| go test -c -tags vpn_hostnet -o gw-hostnet.test ./vpn/netstate/ | |
| sudo ./gw-hostnet.test -test.run '^TestGatewayHostNet' -test.v | |
| - name: VPN gateway host-network integration test (Windows, admin) | |
| # Windows counterpart of the step above: exercises WinNAT + WFP + | |
| # per-interface forwarding (SetupNAT/TeardownNAT) against a real NIC | |
| # and the /1 client routes against a real Wintun adapter. GitHub | |
| # Windows runners execute as Administrator, so no sudo equivalent is | |
| # needed. The diagnostic Get-NetNat shows what the runner image holds | |
| # in WinNAT (a Docker/HNS instance would conflict — decide about | |
| # pre-cleaning based on what this prints). | |
| if: matrix.os == 'windows-latest' | |
| shell: pwsh | |
| run: | | |
| Get-NetNat | Format-List | |
| go test -c -tags vpn_hostnet -o gw-hostnet.test.exe ./vpn/netstate/ | |
| # Flags are quoted: pwsh splits an unquoted `-test.run` at the dot | |
| # into `-test` + `.run`, which the test binary rejects. | |
| ./gw-hostnet.test.exe '-test.run' '^TestGatewayHostNet' '-test.v' | |
| - name: Build cmd/awl | |
| run: go build github.com/anywherelan/awl/cmd/awl | |
| - name: Upload cmd/awl build | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: awl-build-${{ runner.os }} | |
| path: | | |
| awl | |
| awl.exe | |
| if-no-files-found: error | |
| end-to-end-test: | |
| # run only on pushes because we use repository secrets which are unavailable to forks | |
| if: ${{ github.event_name == 'push' }} | |
| needs: [ test ] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macos-latest ] | |
| runs-on: ${{ matrix.os }} | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Retrieve saved cmd/awl build | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: awl-build-${{ runner.os }} | |
| - name: Download librespeed-cli | |
| env: | |
| CONFIG_AWL_LINUX: ${{ secrets.CONFIG_AWL_LINUX }} | |
| CONFIG_AWL_MACOS: ${{ secrets.CONFIG_AWL_MACOS }} | |
| CONFIG_AWL_WINDOWS: ${{ secrets.CONFIG_AWL_WINDOWS }} | |
| CONFIG_LIBRESPEED: ${{ secrets.CONFIG_LIBRESPEED }} | |
| run: | | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| curl -sSL https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_amd64.tar.gz | tar -xzf - | |
| echo $CONFIG_AWL_LINUX > config_awl.json | |
| elif [ "$RUNNER_OS" == "macOS" ]; then | |
| curl -sSL https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_darwin_arm64.tar.gz | tar -xzf - | |
| echo $CONFIG_AWL_MACOS > config_awl.json | |
| elif [ "$RUNNER_OS" == "Windows" ]; then | |
| curl -sSL https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_windows_amd64.zip > download.zip | |
| unzip download.zip | |
| echo $CONFIG_AWL_WINDOWS > config_awl.json | |
| else | |
| echo "$RUNNER_OS not supported" | |
| exit 1 | |
| fi | |
| echo $CONFIG_LIBRESPEED > config_librespeed.json | |
| - name: Run librespeed-cli, awl, librespeed-cli through awl | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| chmod +x awl | |
| sudo ./awl & | |
| ./librespeed-cli --local-json config_librespeed.json --server 1 --json --share --telemetry-level disabled | python3 -m json.tool | |
| sleep 10 | |
| awl_pid=`jobs -l | grep './awl' | awk '{print $2}'` | |
| ./awl cli peers status | |
| ./librespeed-cli --local-json config_librespeed.json --server 2 --json --share --telemetry-level disabled | python3 -m json.tool | |
| ping 10.66.0.2 -w 20 -c 10 | |
| # TODO: remove this temporal hack for linux | |
| ping awl-tester.awl -w 20 -c 10 || true | |
| # ---- VPN gateway server (exit-node) mode: runtime enable/disable round-trips OS state ---- | |
| # awl runs as root here, so enabling the server installs real NAT | |
| # (iptables MASQUERADE + AWL-FORWARD chain) in this netns. We assert the | |
| # commands succeed AND that they actually mutate then fully restore the | |
| # netfilter state. `iptables -S` (no packet/byte counters, unlike | |
| # iptables-save) makes the before/after comparison stable. | |
| nat_state() { sudo iptables -S; sudo iptables -t nat -S; } | |
| NAT_BEFORE=$(nat_state) | |
| ./awl cli gateway server enable | |
| test "$(nat_state)" != "$NAT_BEFORE" # enable actually changed netfilter | |
| sudo iptables -S | grep -q AWL-FORWARD # exit-node chain installed | |
| ./awl cli gateway server disable | |
| test "$(nat_state)" = "$NAT_BEFORE" # disable restored netfilter exactly | |
| # ---- VPN gateway full-tunnel e2e: client = this runner, exit node = awl-tester ---- | |
| # Requires the awl-tester peer to have VPN gateway server enabled AND to | |
| # permit this CI peer as an exit node (WeAllowUsingAsExitNode); both are | |
| # advertised to us via the status protocol. Wait until it shows up as a | |
| # connected, available gateway. | |
| for i in $(seq 1 15); do | |
| if ./awl cli gateway list | grep -E 'awl-tester.*\[connected\]'; then break; fi | |
| sleep 2 | |
| done | |
| ./awl cli gateway list | grep -E 'awl-tester.*\[connected\]' # fail if never became available | |
| # Probe the egress IP via https://ifconfig.me | |
| curl4() { curl -4 -s --max-time 20 --retry 3 --retry-delay 2 https://ifconfig.me/; } | |
| IP_DIRECT=$(curl4); echo "egress IP (direct): $IP_DIRECT" | |
| # Source IP of the physical uplink, used below to force a leak probe | |
| # out of the NIC (bypassing the fwmark policy route). | |
| NIC_IP=$(ip -4 route get 1.1.1.1 | grep -oP 'src \K\S+'); echo "uplink src IP: $NIC_IP" | |
| ./awl cli gateway client use --name awl-tester | |
| ./awl cli gateway status | |
| RULE_ON=$(ip rule show); ROUTE_ON=$(ip route show) | |
| # Tolerate a failed curl here so we always reach `client stop` below | |
| # (a broken tunnel must not leave the runner black-holed); the | |
| # `test -n "$IP_GW"` assertion after teardown turns it into a clean failure. | |
| IP_GW=$(curl4 || true); echo "egress IP (via gateway): $IP_GW" | |
| ./awl cli gateway client stop | |
| RULE_OFF=$(ip rule show); ROUTE_OFF=$(ip route show) | |
| IP_REVERTED=$(curl4); echo "egress IP (reverted): $IP_REVERTED" | |
| # Assertions run with the gateway already disabled, so a failure here | |
| # can't leave the runner's egress black-holed. | |
| echo "$RULE_ON" | grep -q fwmark # client policy route installed | |
| echo "$ROUTE_ON" | grep -q 'default dev awl0' # full-tunnel default via TUN | |
| test -n "$IP_GW" # traffic really flowed through the exit node | |
| test "$IP_GW" != "$IP_DIRECT" # egress changed => full-tunnel works | |
| test "$IP_REVERTED" = "$IP_DIRECT" # runtime disable reverted egress | |
| ! echo "$RULE_OFF" | grep -q fwmark # policy route removed on disable | |
| ! echo "$ROUTE_OFF" | grep -q 'default dev awl0' # TUN default removed on disable | |
| # Second enable/disable cycle: a runtime re-enable after a disable is | |
| # the exact flow that looked broken in manual Windows testing (it was | |
| # not — browser keep-alive pools were lying). Assert the egress flips | |
| # again and reverts again. Also probe for a leak: a socket forced out | |
| # of the physical NIC (--interface) bypasses the fwmark policy route, | |
| # but on Linux those packets are still routed by destination into the | |
| # TUN and die there (their NIC source is not NATed by the exit node), | |
| # so a NIC-bound curl must fail while the gateway is on. This is the | |
| # weak-host-model counterpart of the Windows WFP leak fence. | |
| ./awl cli gateway client use --name awl-tester | |
| IP_GW2=$(curl4 || true); echo "egress IP (via gateway, cycle 2): $IP_GW2" | |
| LEAK=$(curl -4 -s --max-time 8 --interface "$NIC_IP" https://ifconfig.me/ || true) | |
| echo "NIC-bound egress while gateway on (must be empty): '$LEAK'" | |
| ./awl cli gateway client stop | |
| IP_REVERTED2=$(curl4); echo "egress IP (reverted, cycle 2): $IP_REVERTED2" | |
| test -n "$IP_GW2" # re-enable tunnelled again | |
| test "$IP_GW2" != "$IP_DIRECT" # egress changed on 2nd enable too | |
| test -z "$LEAK" # NIC-bound egress did NOT bypass the tunnel | |
| test "$IP_REVERTED2" = "$IP_DIRECT" # 2nd disable reverted egress | |
| # Re-enable so the SIGINT shutdown path (teardownGatewayAtShutdown) is exercised. | |
| ./awl cli gateway client use --name awl-tester | |
| sleep 1 | |
| sudo kill -SIGINT $awl_pid | |
| sleep 2 | |
| # awl is gone: its TUN and the default route via it vanished with the | |
| # process, but the fwmark ip rule is interface-independent — its absence | |
| # proves the shutdown teardown ran. (NAT/ip_forward are server-side only.) | |
| ! ip rule show | grep -q fwmark | |
| - name: Run librespeed-cli, awl, librespeed-cli through awl | |
| if: matrix.os == 'macos-latest' | |
| run: | | |
| chmod +x awl | |
| sudo ./awl & | |
| ./librespeed-cli --local-json config_librespeed.json --server 1 --json --share --telemetry-level disabled | python3 -m json.tool | |
| sleep 10 | |
| awl_pid=`jobs -l | grep './awl' | awk '{print $2}'` | |
| ./awl cli peers status | |
| ./librespeed-cli --local-json config_librespeed.json --server 2 --json --share --telemetry-level disabled | python3 -m json.tool | |
| ping 10.66.0.2 -c 10 | |
| ping awl-tester.awl -c 10 | |
| sleep 1 | |
| sudo kill -SIGINT $awl_pid | |
| sleep 1 | |
| - name: Run librespeed-cli, awl, librespeed-cli through awl | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| chmod +x awl.exe | |
| ./awl.exe & | |
| ./librespeed-cli.exe --local-json config_librespeed.json --server 1 --json --share --telemetry-level disabled | python3 -m json.tool | |
| sleep 10 | |
| awl_pid=`jobs -l | grep './awl' | awk '{print $2}'` | |
| ./awl.exe cli peers status | |
| ./librespeed-cli.exe --local-json config_librespeed.json --server 2 --json --share --telemetry-level disabled | python3 -m json.tool | |
| ping -w 20000 -n 10 10.66.0.2 | |
| ping -w 20000 -n 10 -a awl-tester.awl | |
| # ---- VPN gateway server (exit-node) mode: runtime enable/disable round-trips OS state ---- | |
| # Diagnostic first: what the runner already holds in WinNAT (a | |
| # Docker/HNS instance here would explain a New-NetNat conflict). | |
| powershell -NoProfile -Command 'Get-NetNat | Format-List' | |
| ./awl.exe cli gateway server enable | |
| powershell -NoProfile -Command 'if (-not (Get-NetNat -Name awl-gateway -ErrorAction SilentlyContinue)) { exit 1 }' | |
| # WinNAT must not break p2p traffic inside the TUN | |
| ping -w 20000 -n 4 10.66.0.2 | |
| ./awl.exe cli gateway server disable | |
| powershell -NoProfile -Command 'if (Get-NetNat -Name awl-gateway -ErrorAction SilentlyContinue) { exit 1 }' | |
| # ---- VPN gateway full-tunnel e2e: client = this runner, exit node = awl-tester ---- | |
| # Requires awl-tester to permit this CI peer as an exit node | |
| # (WeAllowUsingAsExitNode) — the Windows CI peer id differs from the | |
| # Linux one. | |
| for i in $(seq 1 15); do | |
| if ./awl.exe cli gateway list | grep -E 'awl-tester.*\[connected\]'; then break; fi | |
| sleep 2 | |
| done | |
| ./awl.exe cli gateway list | grep -E 'awl-tester.*\[connected\]' # fail if never became available | |
| curl4() { curl -4 -s --max-time 20 --retry 3 --retry-delay 2 https://ifconfig.me/; } | |
| IP_DIRECT=$(curl4); echo "egress IP (direct): $IP_DIRECT" | |
| # Source IP of the physical uplink (captured before enable, so the | |
| # best route still points at the NIC, not the TUN). Used to force a | |
| # leak probe out of the NIC past the /1 routes. | |
| NIC_IP=$(powershell -NoProfile -Command '(Find-NetRoute -RemoteIPAddress 1.1.1.1 | Select-Object -First 1).IPAddress' | tr -d '\r'); echo "uplink src IP: $NIC_IP" | |
| # NRPT diagnostics before enable (split-DNS rules for the awl zone) | |
| powershell -NoProfile -Command 'Get-DnsClientNrptPolicy | Format-List Namespace,NameServers' | |
| ./awl.exe cli gateway client use --name awl-tester | |
| ./awl.exe cli gateway status | |
| ROUTES_ON=$(powershell -NoProfile -Command '(Get-NetRoute -DestinationPrefix 0.0.0.0/1,128.0.0.0/1 -ErrorAction SilentlyContinue | Measure-Object).Count' | tr -d '\r') | |
| NRPT_ON=$(powershell -NoProfile -Command '(Get-DnsClientNrptPolicy | Where-Object { $_.Namespace -eq "." } | Measure-Object).Count' | tr -d '\r') | |
| powershell -NoProfile -Command 'Get-DnsClientNrptPolicy | Format-List Namespace,NameServers' | |
| # Route diagnostics incl. lifetimes/protocol: a route created without | |
| # InitializeIpForwardEntry shows zero lifetimes and is ignored by the | |
| # forwarding path while still being listed here. | |
| powershell -NoProfile -Command 'Get-NetRoute -DestinationPrefix 0.0.0.0/1,128.0.0.0/1,::/1,8000::/1 -ErrorAction SilentlyContinue | Format-List DestinationPrefix,InterfaceAlias,RouteMetric,Protocol,ValidLifetime,PreferredLifetime' | |
| # Tolerate a failed curl here so we always reach `client stop` below | |
| # (a broken tunnel must not leave the runner black-holed); the | |
| # `test -n "$IP_GW"` assertion after teardown turns it into a clean failure. | |
| IP_GW=$(curl4 || true); echo "egress IP (via gateway): $IP_GW" | |
| # Leak probe: a socket forced out of the physical NIC (--interface) | |
| # bypasses the /1 routes via the strong host model — exactly the leak | |
| # the WFP fence exists to close. Must fail while the gateway is on. | |
| LEAK=$(curl.exe --interface "$NIC_IP" -4 -s --max-time 8 https://ifconfig.me/ || true) | |
| echo "NIC-bound egress while gateway on (must be empty): '$LEAK'" | |
| ./awl.exe cli gateway client stop | |
| ROUTES_OFF=$(powershell -NoProfile -Command '(Get-NetRoute -DestinationPrefix 0.0.0.0/1,128.0.0.0/1 -ErrorAction SilentlyContinue | Measure-Object).Count' | tr -d '\r') | |
| NRPT_OFF=$(powershell -NoProfile -Command '(Get-DnsClientNrptPolicy | Where-Object { $_.Namespace -eq "." } | Measure-Object).Count' | tr -d '\r') | |
| IP_REVERTED=$(curl4); echo "egress IP (reverted): $IP_REVERTED" | |
| # Assertions run with the gateway already disabled, so a failure here | |
| # can't leave the runner's egress black-holed. Values are echoed | |
| # first so a failed assertion is attributable from the log alone. | |
| echo "ROUTES_ON=$ROUTES_ON NRPT_ON=$NRPT_ON ROUTES_OFF=$ROUTES_OFF NRPT_OFF=$NRPT_OFF LEAK='$LEAK'" | |
| test "$ROUTES_ON" -eq 2 # /1 pair installed while gateway is on | |
| test "$NRPT_ON" -ge 1 # full-capture NRPT rule while gateway is on | |
| test -n "$IP_GW" # traffic really flowed through the exit node | |
| test "$IP_GW" != "$IP_DIRECT" # egress changed => full-tunnel works | |
| test "$IP_REVERTED" = "$IP_DIRECT" # runtime disable reverted egress | |
| test "$ROUTES_OFF" -eq 0 # /1 routes removed on disable | |
| test "$NRPT_OFF" -eq 0 # full-capture NRPT rule removed on disable | |
| test -z "$LEAK" # WFP fence blocked NIC-bound egress bypass | |
| # Second enable/disable cycle: a runtime re-enable after a disable is | |
| # the exact flow that looked broken in manual testing (it was not — | |
| # browser keep-alive pools were lying). Assert egress flips again and | |
| # reverts again; the gateway is already off at each assertion so a | |
| # failure can't black-hole the runner. | |
| ./awl.exe cli gateway client use --name awl-tester | |
| IP_GW2=$(curl4 || true); echo "egress IP (via gateway, cycle 2): $IP_GW2" | |
| ./awl.exe cli gateway client stop | |
| IP_REVERTED2=$(curl4); echo "egress IP (reverted, cycle 2): $IP_REVERTED2" | |
| test -n "$IP_GW2" # re-enable tunnelled again | |
| test "$IP_GW2" != "$IP_DIRECT" # egress changed on 2nd enable too | |
| test "$IP_REVERTED2" = "$IP_DIRECT" # 2nd disable reverted egress | |
| # Re-enable the gateway and hard-kill awl. A graceful-shutdown test is | |
| # not possible here: Git-Bash `kill -SIGINT` cannot deliver a console | |
| # ctrl event to a native Windows process on a console-less CI runner | |
| # (verified: awl kept logging after the kill). Graceful teardown is | |
| # already covered by the `client stop` assertions above and by the | |
| # Linux branch; what a hard kill lets us assert is the documented | |
| # CRASH semantics instead. | |
| ./awl.exe cli gateway client use --name awl-tester | |
| sleep 1 | |
| taskkill //F //IM awl.exe | |
| # Crash fail-open: the /1 routes (v4 + v6 fence) are bound to the | |
| # Wintun LUID and must disappear together with the adapter. Driver | |
| # cleanup after a hard kill is asynchronous — poll. | |
| DANGLING=-1 | |
| for i in $(seq 1 30); do | |
| DANGLING=$(powershell -NoProfile -Command '(Get-NetRoute -DestinationPrefix 0.0.0.0/1,128.0.0.0/1,::/1,8000::/1 -ErrorAction SilentlyContinue | Measure-Object).Count' | tr -d '\r') | |
| if [ "$DANGLING" -eq 0 ]; then break; fi | |
| sleep 1 | |
| done | |
| echo "DANGLING=$DANGLING" | |
| test "$DANGLING" -eq 0 | |
| # The NRPT catch-all rule lives in the registry and must SURVIVE a | |
| # hard kill — that's the documented crash behaviour (the next awl | |
| # start reconfigures DNS and cleans it up). | |
| NRPT_CRASH=$(powershell -NoProfile -Command '(Get-DnsClientNrptPolicy | Where-Object { $_.Namespace -eq "." } | Measure-Object).Count' | tr -d '\r') | |
| echo "NRPT_CRASH=$NRPT_CRASH" | |
| test "$NRPT_CRASH" -ge 1 | |
| # Runner hygiene: with awl dead, the leftover catch-all rule points | |
| # all DNS at a resolver that no longer exists — remove it so the | |
| # runner's post-job steps keep working DNS. | |
| powershell -NoProfile -Command 'Get-DnsClientNrptRule | Where-Object { $_.Namespace -eq "." } | Remove-DnsClientNrptRule -Force' |