Windows e2e tests fail on deadlock

[brad-defined]

Reported stack trace on test run https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:133

time="me   13:54:20.454163" level=info msg="Tunnel status" certName=them tunnelCheck="map[method:active state:dead]" vpnIp=10.128.0.2
panic: test timed out after 10m0s

goroutine 61 [running]:
testing.(*M).startAlarm.func1()
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1788 +0x8e
created by time.goFunc
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/time/sleep.go:180 +0x31

goroutine 1 [chan receive, 9 minutes]:
testing.(*T).Run(0xc000053520, {0xe2ebb6, 0x71e153}, 0xe5ac50)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1307 +0x375
testing.runTests.func1(0xc0000407b0)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1598 +0x6e
testing.tRunner(0xc000053520, 0xc0000bdd18)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1259 +0x102
testing.runTests(0xc00006e100, {0x1408400, 0x3, 0x3}, {0x744ced, 0xe273f2, 0x1412ec0})
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1596 +0x43f
testing.(*M).Run(0xc00006e100)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1504 +0x51d
main.main()
	_testmain.go:47 +0x14b

goroutine 22 [chan receive, 9 minutes]:
github.com/slackhq/nebula/overlay.(*TestTun).Get(0xc0004230e0, 0x40)
	D:/a/nebula/nebula/overlay/tun_tester.go:61 +0x85
github.com/slackhq/nebula.(*Control).GetFromTun(...)
	D:/a/nebula/nebula/control_tester.go:68
github.com/slackhq/nebula/e2e.Test_Case1_Stage1Race(0xc0001f11e0)
	D:/a/nebula/nebula/e2e/handshakes_test.go:167 +0x5ba
testing.tRunner(0xc0001f11e0, 0xe5ac50)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1259 +0x102
created by testing.(*T).Run
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/testing/testing.go:1306 +0x35a

goroutine 7 [syscall, 9 minutes]:
os/signal.signal_recv()
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/runtime/sigqueue.go:169 +0x98
os/signal.loop()
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/os/signal/signal_unix.go:24 +0x19
created by os/signal.Notify.func1.1
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/os/signal/signal.go:151 +0x2c

goroutine 33 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc000092c60, 0xc000501fa0, 0xc000292340, 0xc00006ed00, 0xc700c0)
	D:/a/nebula/nebula/udp/udp_tester.go:117 +0x1e9
github.com/slackhq/nebula.(*Interface).listenOut(0xc000420ea0, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 68 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/overlay.(*TestTun).Read(0xc0002dd400, {0xc000300000, 0x2329, 0x0})
	D:/a/nebula/nebula/overlay/tun_tester.go:110 +0x3b
github.com/slackhq/nebula.(*Interface).listenIn(0xc0004598c0, {0x20f6466d9b8, 0xc000080cd0}, 0xc00029c7e0)
	D:/a/nebula/nebula/interface.go:213 +0x173
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:182 +0xa5

goroutine 67 [chan send, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).WriteTo(0xc0000939e0, {0xc000640000, 0x48, 0xc0002926c0}, 0xc000092ce0)
	D:/a/nebula/nebula/udp/udp_tester.go:105 +0x34d
github.com/slackhq/nebula.(*Interface).sendNoMetrics(0xc0004598c0, 0x1, 0xff, 0xc0000d90e0, 0xc00022b5e0, 0xc000092ce0, {0xc0000be5a0, 0x28, 0x28}, {0xc0002926c0, ...}, ...)
	D:/a/nebula/nebula/inside.go:226 +0x2fa
github.com/slackhq/nebula.(*Interface).sendMessageNow(0xc0004598c0, 0xd8, 0x45, 0xc00022b5e0, {0xc0000be5a0, 0x28, 0x28}, {0xc0002926c0, 0xc, 0xc}, ...)
	D:/a/nebula/nebula/inside.go:149 +0x3a5
github.com/slackhq/nebula.(*HostInfo).handshakeComplete(0xc00022b5e0, 0xc00022b5e0, 0xc000093d40)
	D:/a/nebula/nebula/hostmap.go:477 +0x298
github.com/slackhq/nebula.ixHandshakeStage2(0xc0004598c0, 0xc000092420, 0xc00022b5e0, {0xc000150480, 0x10f, 0x10f}, 0xc00009e0a8)
	D:/a/nebula/nebula/handshake_ix.go:462 +0x1d5b
github.com/slackhq/nebula.HandleIncomingHandshake(0xc0004598c0, 0xc000092420, {0xc000150480, 0x10f, 0x10f}, 0xc00009e0a8, 0x0)
	D:/a/nebula/nebula/handshake.go:22 +0xf6
github.com/slackhq/nebula.(*Interface).readOutsidePackets(0xc0004598c0, 0xc000092420, {0xc000316f00, 0x0, 0x2329}, {0xc000150480, 0x10f, 0x10f}, 0xc00009e0a8, 0xc000292010, ...)
	D:/a/nebula/nebula/outside.go:108 +0x365
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc0000939e0, 0xc000637fa0, 0x0, 0x100000001, 0xc000079720)
	D:/a/nebula/nebula/udp/udp_tester.go:120 +0x1c4
github.com/slackhq/nebula.(*Interface).listenOut(0xc0004598c0, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 66 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/overlay.(*TestTun).Read(0xc000250000, {0xc000516f00, 0x2329, 0x0})
	D:/a/nebula/nebula/overlay/tun_tester.go:110 +0x3b
github.com/slackhq/nebula.(*Interface).listenIn(0xc000420ea0, {0x20f6466d9b8, 0xc0000805a0}, 0x0)
	D:/a/nebula/nebula/interface.go:213 +0x173
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:182 +0xa5

goroutine 35 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc00005e920, 0xc000063fa0, 0x0, 0x0, 0x0)
	D:/a/nebula/nebula/udp/udp_tester.go:117 +0x1e9
github.com/slackhq/nebula.(*Interface).listenOut(0xc0002825a0, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 36 [runnable, locked to thread]:
github.com/sirupsen/logrus.(*Entry).WithFields(0xc0003928c0, 0xc0004a58a0)
	C:/Users/runneradmin/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/entry.go:131 +0x18f
github.com/sirupsen/logrus.(*Entry).WithField(...)
	C:/Users/runneradmin/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/entry.go:121
github.com/sirupsen/logrus.(*Logger).WithField(0xc00021ebd0, {0xe16992, 0x6}, {0xd1d1c0, 0xc0005de450})
	C:/Users/runneradmin/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/logger.go:115 +0x16c
github.com/slackhq/nebula.(*Interface).consumeInsidePacket(0xc0002825a0, {0xc000309400, 0x0, 0x2329}, 0xc00009b4d0, {0xc00009b4e0, 0xc, 0xc}, {0xc00030b900, 0x2329, ...}, ...)
	D:/a/nebula/nebula/inside.go:17 +0x792
github.com/slackhq/nebula.(*Interface).listenIn(0xc0002825a0, {0x20f6466d9b8, 0xc000078e60}, 0x0)
	D:/a/nebula/nebula/interface.go:224 +0x14d
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:182 +0xa5

goroutine 37 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc00005f460, 0xc0004e3fa0, 0x0, 0x0, 0x0)
	D:/a/nebula/nebula/udp/udp_tester.go:117 +0x1e9
github.com/slackhq/nebula.(*Interface).listenOut(0xc00029c7e0, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 38 [runnable, locked to thread]:
fmt.(*pp).printArg(0xc0000421a0, {0xd26c80, 0xf2c860}, 0x76)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/fmt/print.go:634 +0x7b1
fmt.(*pp).doPrintf(0xc0000421a0, {0xe36d68, 0x1c}, {0xc0004df9b8, 0x0, 0x0})
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/fmt/print.go:1026 +0x288
fmt.Errorf({0xe36d68, 0x1c}, {0xc0004df9b8, 0x1, 0x1})
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/fmt/errors.go:20 +0x6c
github.com/slackhq/nebula.newPacket({0xc00000a500, 0xe16992, 0xe44213}, 0xc0, 0xc0004dfb00)
	D:/a/nebula/nebula/outside.go:197 +0x33f
github.com/slackhq/nebula.(*Interface).consumeInsidePacket(0xc00029c7e0, {0xc00000a500, 0x0, 0x2329}, 0xc000292320, {0xc000292330, 0xc, 0xc}, {0xc00000ca00, 0x2329, ...}, ...)
	D:/a/nebula/nebula/inside.go:15 +0x85
github.com/slackhq/nebula.(*Interface).listenIn(0xc00029c7e0, {0x20f6466d9b8, 0xc000079450}, 0x0)
	D:/a/nebula/nebula/interface.go:224 +0x14d
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:182 +0xa5

goroutine 28 [select, 9 minutes]:
github.com/slackhq/nebula/config.(*C).CatchHUP.func1()
	D:/a/nebula/nebula/config/config.go:122 +0xe5
created by github.com/slackhq/nebula/config.(*C).CatchHUP
	D:/a/nebula/nebula/config/config.go:120 +0xef

goroutine 27 [select]:
github.com/slackhq/nebula.(*Interface).emitStats(0xc000420ea0, {0xf44cb0, 0xc0000cadc0}, 0xc000069a80)
	D:/a/nebula/nebula/interface.go:316 +0x108
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:322 +0x1e45

goroutine 31 [select]:
github.com/slackhq/nebula.(*LightHouse).LhUpdateWorker(0xc00015a640, {0xf44cb0, 0xc0000cb2c0}, {0xf35940, 0xc0004598c0})
	D:/a/nebula/nebula/lighthouse.go:503 +0x14c
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:306 +0x1d29

goroutine 30 [select]:
github.com/slackhq/nebula.(*HandshakeManager).Run(0xc0000cc180, {0xf44cb0, 0xc0000cb2c0}, {0xf35940, 0xc0004598c0})
	D:/a/nebula/nebula/handshake_manager.go:78 +0x14f
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:305 +0x1c85

goroutine 29 [semacquire, 9 minutes]:
sync.runtime_SemacquireMutex(0x68, 0x70, 0x20f5f210598)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc00022b5e0)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/sync/mutex.go:[138](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:138) +0x165
sync.(*Mutex).Lock(...)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0xc000493810)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/sync/rwmutex.go:111 +0x36
github.com/slackhq/nebula.(*Interface).getOrHandshake(0xc0004598c0, 0xa800001)
	D:/a/nebula/nebula/inside.go:96 +0x12b
github.com/slackhq/nebula.(*Interface).SendMessageToVpnIp(0xc0004598c0, 0x0, 0x0, 0x0, {0x1994c88, 0x0, 0x0}, {0xc000034c10, 0xc, 0xc}, ...)
	D:/a/nebula/nebula/inside.go:154 +0x59
github.com/slackhq/nebula.(*connectionManager).HandleMonitorTick(0xc0000cc700, {0xc084447d99603290, 0xa61b5b4d, 0x[141](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:141)2ec0}, {0x1994c88, 0x0, 0x0}, {0xc000034c10, 0xc, 0xc}, ...)
	D:/a/nebula/nebula/connection_manager.go:213 +0x658
github.com/slackhq/nebula.(*connectionManager).Run(0xf44cb0, {0xf44cb0, 0xc0000cb2c0})
	D:/a/nebula/nebula/connection_manager.go:160 +0x15a
created by github.com/slackhq/nebula.(*connectionManager).Start
	D:/a/nebula/nebula/connection_manager.go:[144](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:144) +0x8f

goroutine 24 [select]:
github.com/slackhq/nebula.(*connectionManager).Run(0x0, {0xf44cb0, 0xc0000cadc0})
	D:/a/nebula/nebula/connection_manager.go:156 +0x20b
created by github.com/slackhq/nebula.(*connectionManager).Start
	D:/a/nebula/nebula/connection_manager.go:144 +0x8f

goroutine 25 [select]:
github.com/slackhq/nebula.(*HandshakeManager).Run(0xc0000cc100, {0xf44cb0, 0xc0000cadc0}, {0xf35940, 0xc000420ea0})
	D:/a/nebula/nebula/handshake_manager.go:78 +0x14f
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:305 +0x1c85

goroutine 32 [select]:
github.com/slackhq/nebula.(*Interface).emitStats(0xc0004598c0, {0xf44cb0, 0xc0000cb2c0}, 0xc0003a9e60)
	D:/a/nebula/nebula/interface.go:316 +0x108
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:322 +0x1e45

goroutine 26 [select]:
github.com/slackhq/nebula.(*LightHouse).LhUpdateWorker(0xc00015a500, {0xf44cb0, 0xc0000cadc0}, {0xf35940, 0xc000420ea0})
	D:/a/nebula/nebula/lighthouse.go:503 +0x14c
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:306 +0x1d29

goroutine 23 [select, 9 minutes]:
github.com/slackhq/nebula/config.(*C).CatchHUP.func1()
	D:/a/nebula/nebula/config/config.go:122 +0xe5
created by github.com/slackhq/nebula/config.(*C).CatchHUP
	D:/a/nebula/nebula/config/config.go:120 +0xef

goroutine 50 [select, 9 minutes]:
github.com/slackhq/nebula/config.(*C).CatchHUP.func1()
	D:/a/nebula/nebula/config/config.go:122 +0xe5
created by github.com/slackhq/nebula/config.(*C).CatchHUP
	D:/a/nebula/nebula/config/config.go:120 +0xef

goroutine 51 [select]:
github.com/slackhq/nebula.(*connectionManager).Run(0x0, {0xf44cb0, 0xc0003a6800})
	D:/a/nebula/nebula/connection_manager.go:156 +0x20b
created by github.com/slackhq/nebula.(*connectionManager).Start
	D:/a/nebula/nebula/connection_manager.go:144 +0x8f

goroutine 52 [select]:
github.com/slackhq/nebula.(*HandshakeManager).Run(0xc00006f680, {0xf44cb0, 0xc0003a6800}, {0xf35940, 0xc0003f0ea0})
	D:/a/nebula/nebula/handshake_manager.go:78 +0x14f
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:305 +0x1c85

goroutine 53 [select]:
github.com/slackhq/nebula.(*LightHouse).LhUpdateWorker(0xc00020cfa0, {0xf44cb0, 0xc0003a6800}, {0xf35940, 0xc0003f0ea0})
	D:/a/nebula/nebula/lighthouse.go:503 +0x14c
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:306 +0x1d29

goroutine 54 [select]:
github.com/slackhq/nebula.(*Interface).emitStats(0xc0003f0ea0, {0xf44cb0, 0xc0003a6800}, 0x0)
	D:/a/nebula/nebula/interface.go:316 +0x108
created by github.com/slackhq/nebula.Main
	D:/a/nebula/nebula/main.go:322 +0x1e45

goroutine 55 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc00039a320, 0xc0002c1fa0, 0x0, 0x0, 0x0)
	D:/a/nebula/nebula/udp/udp_tester.go:117 +0x1e9
github.com/slackhq/nebula.(*Interface).listenOut(0xc00038bc20, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 56 [runnable, locked to thread]:
errors.New(...)
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/errors/errors.go:59
fmt.Errorf({0xe36d68, 0x1c}, {0xc0003b99b8, 0x1, 0x1})
	C:/hostedtoolcache/windows/go/1.17.7/x64/src/fmt/errors.go:24 +0x114
github.com/slackhq/nebula.newPacket({0xc0002d8a00, 0xe16992, 0xe44213}, 0xc0, 0xc0003b9b00)
	D:/a/nebula/nebula/outside.go:197 +0x33f
github.com/slackhq/nebula.(*Interface).consumeInsidePacket(0xc00038bc20, {0xc0002d8a00, 0x0, 0x2329}, 0xc00009a290, {0xc00009a2f0, 0xc, 0xc}, {0xc0002daf00, 0x2329, ...}, ...)
	D:/a/nebula/nebula/inside.go:15 +0x85
github.com/slackhq/nebula.(*Interface).listenIn(0xc00038bc20, {0x20f6466d9b8, 0xc0003a2000}, 0x0)
	D:/a/nebula/nebula/interface.go:224 +0x14d
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:182 +0xa5

goroutine 57 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc00039ae60, 0xc0002cdfa0, 0x0, 0x0, 0x0)
	D:/a/nebula/nebula/udp/udp_tester.go:117 +0x1e9
github.com/slackhq/nebula.(*Interface).listenOut(0xc0003a9e60, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 58 [runnable, locked to thread]:
github.com/sirupsen/logrus.(*Entry).WithFields(0xc000268cb0, 0xc00047f8a0)
	C:/Users/runneradmin/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/entry.go:[147](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:147) +0x385
github.com/sirupsen/logrus.(*Entry).WithField(...)
	C:/Users/runneradmin/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/entry.go:121
github.com/sirupsen/logrus.(*Logger).WithField(0xc000397650, {0xe[169](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:169)92, 0x6}, {0xd1d1c0, 0xc00033f0c8})
	C:/Users/runneradmin/go/pkg/mod/github.com/sirupsen/logrus@v1.8.1/logger.go:115 +0x16c
github.com/slackhq/nebula.(*Interface).consumeInsidePacket(0xc0003a9e60, {0xc00031b900, 0x0, 0x2329}, 0xc00009a040, {0xc00009a050, 0xc, 0xc}, {0xc000592500, 0x2329, ...}, ...)
	D:/a/nebula/nebula/inside.go:17 +0x792
github.com/slackhq/nebula.(*Interface).listenIn(0xc0003a9e60, {0x20f6466d9b8, 0xc0003a25f0}, 0x0)
	D:/a/nebula/nebula/interface.go:224 +0x14d
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:182 +0xa5

goroutine 59 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/udp.(*Conn).ListenOut(0xc00039b9c0, 0xc0004e1fa0, 0x0, 0x0, 0x0)
	D:/a/nebula/nebula/udp/udp_tester.go:117 +0x1e9
github.com/slackhq/nebula.(*Interface).listenOut(0xc0003f0ea0, 0x0)
	D:/a/nebula/nebula/interface.go:199 +0x294
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:177 +0x2c

goroutine 60 [chan receive, 9 minutes, locked to thread]:
github.com/slackhq/nebula/overlay.(*TestTun).Read(0x0, {0xc0002df900, 0x2329, 0x0})
	D:/a/nebula/nebula/overlay/tun_tester.go:110 +0x3b
github.com/slackhq/nebula.(*Interface).listenIn(0xc0003f0ea0, {0x20f6466d9b8, 0xc0003a2c30}, 0x0)
	D:/a/nebula/nebula/interface.go:213 +0x[173](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:173)
created by github.com/slackhq/nebula.(*Interface).run
	D:/a/nebula/nebula/interface.go:[182](https://github.com/DefinedNet/nebula/runs/5554733633?check_suite_focus=true#step:8:182) +0xa5
FAIL	github.com/slackhq/nebula/e2e	600.060s
FAIL
mingw32-make: *** [Makefile:54: e2e] Error 1
Error: Process completed with exit code 1.

[brad-defined]

I think this issue is related to the ordering of operations and the test's simulated UDP and TUN devices using channels.
I can repro this deadlock with the following:

for i in {0..1000}
do
go test -v -tags=e2e_testing -count=1 ./e2e -run Test_Case1_Stage1Race -timeout 2s
if [ $? -ne 0 ]; then
break
fi

It'll reproduce within the 1000 runs, and get a stack trace similar to the above.
If I edit the udp_tester.go's channels to be unbuffered (line 51 and 52 of udp/udp_tester.go), I can repro the same stack trace immediately.
If I edit the udp_tester.go's channels to have a buffer size of 10, I can't repro the issue (or at least as easily.)