poll: do not cancel ephemeral GC until Connect succeeds

Fixes #3382
The issue
---------
With node.ephemeral.inactivity_timeout set (e.g. 15m), ephemeral nodes
are usually deleted after they go offline, but under reconnect churn
some departed nodes stay in the node list as disconnected indefinitely
until they are removed manually or Headscale is restarted.
The root cause
--------------
Ephemeral cleanup is timer-based via EphemeralGarbageCollector, not a
periodic LastSeen scan. serveLongPoll cancelled any pending GC timer at
the very start of a long-poll attempt, and only rescheduled on a clean
disconnect after Connect.
If a reconnect cancelled the timer and then failed before Connect
(for example UpdateNodeFromMapRequest error), the deferred cleanup saw
connectGen == 0 and returned without Schedule. The node remained
offline with no deletion timer and no reconciler to recover it.
The fix
-------
Move beforeServeLongPoll (GC Cancel) to after a successful Connect so
a failed reconnect leaves an already-armed inactivity timer intact.
Successful reconnects still cancel GC once the node is online; a later
disconnect reschedules as before.
Add TestFailedReconnectDoesNotCancelEphemeralGC to lock the ordering,
plus IsScheduled and DeleteNodeFromStoreForTest helpers for the test.

Author: iSerganov

hscontrol/db/node.go

@@ -498,6 +498,16 @@ func (e *EphemeralGarbageCollector) Cancel(nodeID types.NodeID) {
 	}
 }
 
+// IsScheduled reports whether a deletion timer is currently armed for nodeID.
+func (e *EphemeralGarbageCollector) IsScheduled(nodeID types.NodeID) bool {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	_, ok := e.toBeDeleted[nodeID]
+
+	return ok
+}
+
 // Start starts the garbage collector.
 func (e *EphemeralGarbageCollector) Start() {
 	for {

hscontrol/poll.go

@@ -144,8 +144,6 @@ func (m *mapSession) serve() {
 //
 //nolint:gocyclo
 func (m *mapSession) serveLongPoll() {
-	m.beforeServeLongPoll()
-
 	m.log.Trace().Caller().Msg("long poll session started")
 
 	// connectGen is set by [state.State.Connect] below and captured by the deferred cleanup closure.
@@ -248,6 +246,11 @@ func (m *mapSession) serveLongPoll() {
 
 	connectChanges, connectGen = m.h.state.Connect(m.node.ID)
 
+	// Cancel ephemeral GC only after Connect succeeds. Cancelling at the start
+	// of serveLongPoll left departed nodes without a deletion timer when a
+	// reconnect attempt failed before Connect (issue #3382).
+	m.beforeServeLongPoll()
+
 	m.log.Info().Caller().Str(zf.Chan, fmt.Sprintf("%p", m.ch)).Msg("node has connected")
 
 	// TODO(kradalby): Redo the comments here

hscontrol/poll_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
 )
 
 type delayedSuccessResponseWriter struct {
@@ -216,6 +217,67 @@ func TestServeLongPollWritesErrorWhenInitialMapFails(t *testing.T) {
 		"serveLongPoll must write an HTTP error response when the initial map cannot be built, not an empty 200")
 }
 
+// TestFailedReconnectDoesNotCancelEphemeralGC proves that a
+// long-poll reconnect attempt which fails before [state.State.Connect] must
+// not cancel a previously armed ephemeral GC timer. Cancelling at the start of
+// [mapSession.serveLongPoll] left departed ephemeral nodes stuck offline with
+// no deletion scheduled (https://github.com/juanfont/headscale/issues/3382).
+func TestFailedReconnectDoesNotCancelEphemeralGC(t *testing.T) {
+	t.Parallel()
+
+	app := createTestApp(t)
+	app.StartEphemeralGCForTest(t)
+
+	user := app.state.CreateUserForTest("eph-gc-cancel-user")
+	pak, err := app.state.CreatePreAuthKey(user.TypedID(), false, true, nil, nil)
+	require.NoError(t, err)
+
+	machineKey := key.NewMachine()
+	nodeKey := key.NewNode()
+
+	_, err = app.handleRegister(context.Background(), tailcfg.RegisterRequest{
+		Auth: &tailcfg.RegisterResponseAuth{
+			AuthKey: pak.Key,
+		},
+		NodeKey: nodeKey.Public(),
+		Hostinfo: &tailcfg.Hostinfo{
+			Hostname: "eph-gc-cancel-node",
+		},
+		Expiry: time.Now().Add(24 * time.Hour),
+	}, machineKey.Public())
+	require.NoError(t, err)
+
+	nodeView, ok := app.state.GetNodeByNodeKey(nodeKey.Public())
+	require.True(t, ok)
+	require.True(t, nodeView.IsEphemeral(), "node must be ephemeral so Cancel would arm on long-poll")
+
+	node := nodeView.AsStruct()
+
+	// Arm a long-lived deletion timer — the state after a normal disconnect
+	// has called afterServeLongPoll. A long expiry avoids racing the
+	// fail-before-Connect path below.
+	app.ephemeralGC.Schedule(node.ID, time.Hour)
+	require.True(t, app.ephemeralGC.IsScheduled(node.ID), "test sanity: GC timer must be armed")
+
+	// Drop the node from the NodeStore so UpdateNodeFromMapRequest fails before
+	// Connect, while the session still carries an ephemeral AuthKey (so the
+	// old Cancel-on-entry path would clear the timer).
+	app.state.DeleteNodeFromStoreForTest(node.ID)
+
+	writer := &recordingResponseWriter{}
+	session := app.newMapSession(context.Background(), tailcfg.MapRequest{
+		Stream:  true,
+		Version: tailcfg.CapabilityVersion(100),
+	}, writer, node)
+
+	session.serveLongPoll()
+
+	assert.GreaterOrEqual(t, writer.statusCode(), http.StatusInternalServerError,
+		"failed reconnect must write an HTTP error before Connect")
+	assert.True(t, app.ephemeralGC.IsScheduled(node.ID),
+		"failed reconnect must not cancel the ephemeral GC timer (issue #3382)")
+}
+
 // TestGitHubIssue3129_TransientlyBlockedWriteDoesNotLeaveLiveStaleSession
 // tests the scenario reported in
 // https://github.com/juanfont/headscale/issues/3129.

hscontrol/state/state.go

@@ -1458,6 +1458,13 @@ func (s *State) PutNodeInStoreForTest(node types.Node) types.NodeView {
 	return s.nodeStore.PutNode(node)
 }
 
+// DeleteNodeFromStoreForTest removes a node from the in-memory [NodeStore]
+// without touching the database. Used to force [State.UpdateNodeFromMapRequest]
+// failures in poll-session tests while keeping the DB row intact for later restore.
+func (s *State) DeleteNodeFromStoreForTest(id types.NodeID) {
+	s.nodeStore.DeleteNode(id)
+}
+
 // CreateRegisteredNodeForTest creates a test node with allocated IPs. This is a convenience wrapper around the database layer.
 func (s *State) CreateRegisteredNodeForTest(user *types.User, hostname ...string) *types.Node {
 	return s.db.CreateRegisteredNodeForTest(user, hostname...)