Skip to content

Commit 76da2c7

Browse files
tjjh89017tjjh89017
committed
test(e2e): add three-platform WireGuard + opendht pipeline test
Stands up two WireGuard interfaces that are each other's peer, runs `stunmesh --oneshot` on both against the public opendht proxies, and checks that the whole publish -> store -> establish path moved each side's STUN-discovered endpoint to the other. Three assertions, in assert.sh so every platform shares them: 1. neither run logged an error (JSON logs, keyed on .level) 2. the discovered IP matches api.ipify.org, an independent oracle 3. each interface's peer endpoint equals what the other discovered -- the cross-equality that proves the data path regardless of NAT type Only the interface creation differs per OS (ip link / ifconfig wg / wireguard-go utun); wg set and wg show are identical, so run.sh branches on uname just for setup. stunmesh attaches to an existing device, so the keys, port and peer are configured before it runs. Kept off pull_request: it depends on the public dhtproxy endpoints and outbound STUN, and that flakiness should not block PRs. Runs on merge to main, on demand, and weekly. The linux path is validated end to end locally; macos and freebsd are first cuts to iterate on real runners. Signed-off-by: Date Huang <tjjh89017@hotmail.com>
1 parent 1f6b8c8 commit 76da2c7

3 files changed

Lines changed: 274 additions & 0 deletions

File tree

‎.github/workflows/e2e.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: E2E
2+
3+
# Exercises the real publish -> opendht -> establish pipeline on every OS with
4+
# two live WireGuard interfaces. It depends on the public dhtproxy endpoints and
5+
# outbound STUN, so it is kept off pull_request (external flakiness must not
6+
# block PRs) and runs on merges to main, on demand, and weekly.
7+
on:
8+
push:
9+
branches: [main, feat/e2e-workflow] # TEMP: drop before merge
10+
paths:
11+
- '**.go'
12+
- 'test/e2e/**'
13+
- '.github/workflows/e2e.yml'
14+
workflow_dispatch:
15+
schedule:
16+
- cron: '0 6 * * 1'
17+
18+
env:
19+
GO_VERSION: "stable"
20+
21+
jobs:
22+
linux:
23+
name: E2E (linux)
24+
runs-on: ubuntu-24.04
25+
steps:
26+
- uses: actions/checkout@v7
27+
- uses: actions/setup-go@v7
28+
with:
29+
go-version: ${{ env.GO_VERSION }}
30+
- name: Install tools
31+
run: sudo apt-get update && sudo apt-get install -y wireguard-tools jq
32+
- name: Build
33+
run: make build BUILTIN=builtin_opendht APP=stunmesh
34+
- name: Run e2e
35+
run: sh test/e2e/run.sh ./stunmesh
36+
37+
macos:
38+
name: E2E (macos)
39+
runs-on: macos-15
40+
steps:
41+
- uses: actions/checkout@v7
42+
- uses: actions/setup-go@v7
43+
with:
44+
go-version: ${{ env.GO_VERSION }}
45+
- name: Install tools
46+
run: brew install wireguard-tools wireguard-go jq
47+
- name: Build
48+
run: make build BUILTIN=builtin_opendht APP=stunmesh
49+
- name: Run e2e
50+
run: sh test/e2e/run.sh ./stunmesh
51+
52+
freebsd:
53+
name: E2E (freebsd)
54+
runs-on: ubuntu-latest
55+
steps:
56+
- uses: actions/checkout@v7
57+
- name: Run e2e in FreeBSD VM
58+
uses: vmactions/freebsd-vm@v1
59+
with:
60+
arch: amd64
61+
sync: sshfs
62+
prepare: |
63+
pkg update -f
64+
pkg install -y gmake go wireguard-tools jq
65+
kldload if_wg
66+
run: |
67+
cd $GITHUB_WORKSPACE
68+
gmake build BUILTIN=builtin_opendht APP=stunmesh
69+
sh test/e2e/run.sh ./stunmesh

‎test/e2e/assert.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/sh
2+
# Assertions for the two-interface stunmesh e2e, shared by every platform.
3+
#
4+
# Given two stunmesh --oneshot runs (interfaces IF0 and IF1, each the other's
5+
# peer, publishing to the same store), verify the whole publish -> store ->
6+
# establish pipeline actually moved each side's discovered endpoint to the
7+
# other. The runner sits behind NAT, so only the public IP is stable across
8+
# protocols; ports differ per mapping. Assertions are therefore:
9+
#
10+
# 1. neither run logged an error (JSON logs, keyed on .level)
11+
# 2. the STUN-discovered IP matches an independent oracle (api.ipify.org)
12+
# 3. each interface's peer endpoint equals what the other side discovered
13+
# -- the cross-equality that proves the data path, independent of NAT type
14+
#
15+
# Args: IF0 LOG0 IF1 LOG1
16+
set -eu
17+
18+
IF0=$1; LOG0=$2; IF1=$3; LOG1=$4
19+
fail=0
20+
SUDO=${SUDO-sudo} # inherited from run.sh; default sudo for standalone use
21+
22+
discovered() { # LOG -> "ip:port" of the last IPv4 endpoint it found
23+
jq -rc 'select(.message=="discovered IPv4 endpoint")|.ipv4' "$1" | tail -1
24+
}
25+
peer_endpoint() { # IF -> the single peer's endpoint per `wg show`
26+
$SUDO wg show "$1" endpoints | awk 'NR==1{print $2}'
27+
}
28+
29+
echo "== 1. no error-level log lines =="
30+
for pair in "$IF0:$LOG0" "$IF1:$LOG1"; do
31+
log=${pair#*:}
32+
n=$(jq -c 'select(.level=="error")' "$log" | wc -l | tr -d ' ')
33+
if [ "$n" != "0" ]; then
34+
echo "FAIL: ${pair%%:*} logged $n error(s):"
35+
jq -c 'select(.level=="error")|{message,error}' "$log"
36+
fail=1
37+
else
38+
echo "ok: ${pair%%:*} clean"
39+
fi
40+
done
41+
42+
echo "== 2. STUN IP matches api.ipify.org =="
43+
oracle=$(curl -4 -sS -m 15 https://api.ipify.org)
44+
d0=$(discovered "$LOG0"); d1=$(discovered "$LOG1")
45+
for pair in "$IF0:$d0" "$IF1:$d1"; do
46+
ip=${pair#*:}; ip=${ip%:*}
47+
if [ "$ip" = "$oracle" ]; then
48+
echo "ok: ${pair%%:*} discovered $ip == oracle"
49+
else
50+
echo "FAIL: ${pair%%:*} discovered '$ip', oracle '$oracle'"
51+
fail=1
52+
fi
53+
done
54+
55+
echo "== 3. cross-equality of endpoints =="
56+
e0=$(peer_endpoint "$IF0"); e1=$(peer_endpoint "$IF1")
57+
# IF0's peer is IF1, so IF0's peer endpoint must equal what IF1 discovered.
58+
if [ -n "$d1" ] && [ "$e0" = "$d1" ]; then
59+
echo "ok: $IF0 peer endpoint $e0 == $IF1 discovered"
60+
else
61+
echo "FAIL: $IF0 peer endpoint '$e0' != $IF1 discovered '$d1'"
62+
fail=1
63+
fi
64+
if [ -n "$d0" ] && [ "$e1" = "$d0" ]; then
65+
echo "ok: $IF1 peer endpoint $e1 == $IF0 discovered"
66+
else
67+
echo "FAIL: $IF1 peer endpoint '$e1' != $IF0 discovered '$d0'"
68+
fail=1
69+
fi
70+
71+
if [ "$fail" = "0" ]; then
72+
echo "PASS"
73+
else
74+
echo "e2e assertions failed"
75+
exit 1
76+
fi

‎test/e2e/run.sh‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/bin/sh
2+
# Two-interface stunmesh e2e. Creates two WireGuard interfaces that are each
3+
# other's peer, runs `stunmesh --oneshot` on both against a shared opendht
4+
# store, and hands the logs to assert.sh.
5+
#
6+
# stunmesh attaches to an existing device (it never creates one), so the wg
7+
# interface, keys, listen port and peer must be in place before it runs. Only
8+
# the interface *creation* differs per OS; `wg set`/`wg show` are identical
9+
# everywhere.
10+
#
11+
# Usage: run.sh /path/to/stunmesh
12+
# Env: ENDPOINTS newline/space list of opendht proxy URLs
13+
# (default: dhtproxy2 then dhtproxy3, dhtproxy3 as failover)
14+
set -eu
15+
umask 077 # keep the generated private keys off world-readable
16+
17+
BIN=${1:?usage: run.sh /path/to/stunmesh}
18+
HERE=$(CDPATH='' cd "$(dirname "$0")" && pwd)
19+
WORK=$(mktemp -d)
20+
OS=$(uname -s)
21+
# GitHub's Linux/macOS runners are non-root with passwordless sudo; the FreeBSD
22+
# VM runs as root and may lack sudo. Resolve once and share with assert.sh.
23+
[ "$(id -u)" = 0 ] && SUDO='' || SUDO='sudo'
24+
export SUDO
25+
# Space- or newline-separated opendht proxy URLs; the second is failover.
26+
ENDPOINTS=${ENDPOINTS:-'https://dhtproxy2.jami.net https://dhtproxy3.jami.net'}
27+
28+
PORT0=51820; PORT1=51821
29+
ADDR0=10.66.0.1; ADDR1=10.66.0.2
30+
IF0=""; IF1="" # resolved names (utunN on macOS)
31+
32+
log() { echo "[e2e] $*"; }
33+
34+
cleanup() {
35+
for slot in 0 1; do
36+
eval "name=\$IF$slot"
37+
[ -n "$name" ] || continue
38+
case "$OS" in
39+
Linux) $SUDO ip link del "$name" 2>/dev/null || true ;;
40+
FreeBSD) $SUDO ifconfig "$name" destroy 2>/dev/null || true ;;
41+
Darwin) eval "pid=\$PID$slot"; [ -n "${pid:-}" ] && $SUDO kill "$pid" 2>/dev/null || true ;;
42+
esac
43+
done
44+
rm -rf "$WORK"
45+
}
46+
trap cleanup EXIT INT TERM
47+
48+
# create_iface SLOT PRIVKEY_FILE PORT ADDR PEER_PUB PEER_ALLOWED
49+
# Sets IF$SLOT (and, on Darwin, PID$SLOT) to the resolved interface.
50+
create_iface() {
51+
slot=$1; keyfile=$2; port=$3; addr=$4; peer=$5; allowed=$6
52+
case "$OS" in
53+
Linux)
54+
name=wg$slot
55+
$SUDO ip link add "$name" type wireguard
56+
$SUDO ip addr add "$addr/24" dev "$name"
57+
;;
58+
FreeBSD)
59+
name=$($SUDO ifconfig wg create)
60+
$SUDO ifconfig "$name" inet "$addr/24"
61+
;;
62+
Darwin)
63+
namefile=$WORK/tun$slot.name
64+
$SUDO sh -c "WG_TUN_NAME_FILE=$namefile wireguard-go utun >$WORK/wggo$slot.log 2>&1 &"
65+
# wireguard-go writes the chosen utunN into namefile once it is up.
66+
for _ in $(seq 1 20); do [ -s "$namefile" ] && break; sleep 0.5; done
67+
name=$(cat "$namefile")
68+
eval "PID$slot=\$(pgrep -f \"wireguard-go $name\")"
69+
$SUDO ifconfig "$name" inet "$addr" "$addr" alias
70+
;;
71+
*) echo "unsupported OS: $OS" >&2; exit 1 ;;
72+
esac
73+
$SUDO wg set "$name" private-key "$keyfile" listen-port "$port" \
74+
peer "$peer" allowed-ips "$allowed/32"
75+
case "$OS" in
76+
Linux) $SUDO ip link set "$name" up ;;
77+
FreeBSD) $SUDO ifconfig "$name" up ;;
78+
Darwin) $SUDO ifconfig "$name" up ;;
79+
esac
80+
eval "IF$slot=\$name"
81+
}
82+
83+
write_config() { # IF PEER_PUB > FILE
84+
name=$1; peer=$2; out=$3
85+
{
86+
echo "log: {level: info, format: json}"
87+
echo "stun: {addresses: [\"stun.l.google.com:19302\"]}"
88+
echo "plugins:"
89+
echo " dht:"
90+
echo " type: builtin"
91+
echo " name: opendht"
92+
echo " endpoints:"
93+
for url in $ENDPOINTS; do echo " - $url"; done
94+
echo "interfaces:"
95+
echo " $name:"
96+
echo " protocol: ipv4"
97+
echo " peers:"
98+
echo " peer:"
99+
echo " public_key: \"$peer\""
100+
echo " plugin: dht"
101+
echo " protocol: ipv4"
102+
} > "$out"
103+
}
104+
105+
log "OS=$OS work=$WORK"
106+
Apriv=$WORK/a.key; Bpriv=$WORK/b.key
107+
wg genkey > "$Apriv"; wg genkey > "$Bpriv"
108+
Apub=$(wg pubkey < "$Apriv"); Bpub=$(wg pubkey < "$Bpriv")
109+
110+
create_iface 0 "$Apriv" "$PORT0" "$ADDR0" "$Bpub" "$ADDR1"
111+
create_iface 1 "$Bpriv" "$PORT1" "$ADDR1" "$Apub" "$ADDR0"
112+
log "interfaces up: $IF0 (peer $Bpub), $IF1 (peer $Apub)"
113+
114+
write_config "$IF0" "$Bpub" "$WORK/cfg0.yaml"
115+
write_config "$IF1" "$Apub" "$WORK/cfg1.yaml"
116+
117+
log "running --oneshot on both"
118+
# $WORK is created by mktemp as the current user, so redirecting the root
119+
# process's output into it as the current user is intended and correct.
120+
# shellcheck disable=SC2024
121+
$SUDO "$BIN" --oneshot -c "$WORK/cfg0.yaml" > "$WORK/if0.log" 2>&1 &
122+
j0=$!
123+
# shellcheck disable=SC2024
124+
$SUDO "$BIN" --oneshot -c "$WORK/cfg1.yaml" > "$WORK/if1.log" 2>&1 &
125+
j1=$!
126+
wait "$j0"; wait "$j1"
127+
log "both finished; asserting"
128+
129+
sh "$HERE/assert.sh" "$IF0" "$WORK/if0.log" "$IF1" "$WORK/if1.log"

0 commit comments

Comments
 (0)