fix(daemon): dispatcher skips idle dogs with leaked tmux sessions (gt-o24)

dispatchPlugins was calling mgr.GetIdleDog(), which returns the first idle
dog from mgr.List() in directory order. When a dog's registry state was
marked idle before its tmux session fully terminated (the window between
'gt dog done' setting StateIdle and tmux actually killing the session), the
daemon would pick that dog, send it plugin mail, then hit sm.Start failing
with "session already running: hq-dog-<name>". The failed assignment was
rolled back and the next heartbeat picked the same dog again — infinite loop
on one dog while the rest of the pack sat idle.

Fix: dispatchPlugins now iterates the kennel itself, skipping any dog whose
registry state is non-idle OR whose tmux session is already running per
sm.IsRunning. A transient termination race on one dog no longer blocks
dispatch to the other three; worst case the dispatcher defers one tick.

Extracted the filter into findDispatchableDog so it's unit-testable and
the intent is explicit at the call site.

Tests cover: idle-with-working-peer picks the idle one; all-working returns
nil; empty kennel returns nil; two idle dogs (no live sessions) pick one.
The "skip idle-with-live-session" regression path is exercised at runtime
via sm.IsRunning, whose tmux correctness is covered in session_manager tests.

Observed in production 2026-04-19: alpha locked the dispatcher for 20+
minutes while bravo/charlie/delta idle, ~14 plugin beads queued with
assignees but none pulled. Manual workaround was 'tmux kill-session -t
hq-dog-alpha'; with this fix the dispatcher advances on its own.

Closes gt-o24.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Executed-By: gastown/crew/woodhouse

Author: ckumar1

internal/daemon/handler.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"fmt"
+	"log"
 	"path/filepath"
 	"time"
 
@@ -256,14 +257,16 @@ func (d *Daemon) dispatchPlugins(mgr *dog.Manager, sm *dog.SessionManager, rigsC
 			}
 		}
 
-		// Find an idle dog.
-		idleDog, err := mgr.GetIdleDog()
-		if err != nil {
-			d.logger.Printf("Handler: error finding idle dog: %v", err)
-			return // No point continuing if we can't list dogs
-		}
+		// Find an idle dog that doesn't already have a live tmux session.
+		// A leaked session (dog marked idle before its tmux terminated) would
+		// cause sm.Start to fail with "session already running", and since
+		// mgr.List() returns dogs in directory order, GetIdleDog would always
+		// pick the same first idle dog — infinite-looping the same failed
+		// dispatch instead of advancing to the next idle dog in the pack.
+		// See gt-o24.
+		idleDog := findDispatchableDog(mgr, sm, d.logger)
 		if idleDog == nil {
-			d.logger.Printf("Handler: no idle dogs available, deferring remaining plugins")
+			d.logger.Printf("Handler: no dispatchable idle dogs available, deferring remaining plugins")
 			return
 		}
 
@@ -320,6 +323,41 @@ func (d *Daemon) dispatchPlugins(mgr *dog.Manager, sm *dog.SessionManager, rigsC
 	}
 }
 
+// findDispatchableDog returns the first dog in the kennel whose registry
+// state is idle AND whose tmux session is NOT currently running. Returns nil
+// when no dog satisfies both conditions.
+//
+// This exists because a dog can be marked idle (via gt dog done or the reaper)
+// before its tmux session fully terminates, producing a transient window where
+// sm.Start would fail with "session already running". Picking that dog every
+// dispatch tick infinite-loops the same failed dispatch instead of advancing
+// to another genuinely-free dog in the pack. See gt-o24.
+//
+// IsRunning errors are logged and treated as "not dispatchable" so a flaky
+// tmux check can't wedge the whole dispatch cycle.
+func findDispatchableDog(mgr *dog.Manager, sm *dog.SessionManager, logger *log.Logger) *dog.Dog {
+	dogs, err := mgr.List()
+	if err != nil {
+		logger.Printf("Handler: failed to list dogs while picking dispatch target: %v", err)
+		return nil
+	}
+	for _, d := range dogs {
+		if d.State != dog.StateIdle {
+			continue
+		}
+		running, err := sm.IsRunning(d.Name)
+		if err != nil {
+			logger.Printf("Handler: IsRunning check failed for dog %s: %v; skipping", d.Name, err)
+			continue
+		}
+		if running {
+			continue
+		}
+		return d
+	}
+	return nil
+}
+
 // loadRigsConfig loads the rigs configuration from mayor/rigs.json.
 func (d *Daemon) loadRigsConfig() (*config.RigsConfig, error) {
 	rigsPath := filepath.Join(d.config.TownRoot, "mayor", "rigs.json")

internal/daemon/handler_test.go

@@ -395,3 +395,80 @@ func TestReapIdleDogs_Constants(t *testing.T) {
 		t.Errorf("maxDogPoolSize = %d, want 4", maxDogPoolSize)
 	}
 }
+
+// TestFindDispatchableDog covers the idle-pool filter used by dispatchPlugins.
+// The critical behavior is that dogs with a live tmux session are skipped
+// even when their registry state is idle, preventing the infinite-loop seen
+// in gt-o24 where GetIdleDog kept picking the same dog during a termination race.
+func TestFindDispatchableDog_SkipsWorkingDogs(t *testing.T) {
+	townRoot := t.TempDir()
+	d := testHandlerDaemon(t, townRoot)
+
+	testSetupWorkingDogState(t, townRoot, "alpha", "plugin:x", time.Now())
+	testSetupDogState(t, townRoot, "bravo", dog.StateIdle, time.Now())
+
+	mgr := dog.NewManager(townRoot, nil)
+	sm := dog.NewSessionManager(tmux.NewTmux(), townRoot, mgr)
+
+	got := findDispatchableDog(mgr, sm, d.logger)
+	if got == nil {
+		t.Fatal("findDispatchableDog returned nil, want bravo")
+	}
+	if got.Name != "bravo" {
+		t.Errorf("findDispatchableDog = %q, want bravo (working alpha must be skipped)", got.Name)
+	}
+}
+
+func TestFindDispatchableDog_AllWorkingReturnsNil(t *testing.T) {
+	townRoot := t.TempDir()
+	d := testHandlerDaemon(t, townRoot)
+
+	testSetupWorkingDogState(t, townRoot, "alpha", "plugin:x", time.Now())
+	testSetupWorkingDogState(t, townRoot, "bravo", "plugin:y", time.Now())
+
+	mgr := dog.NewManager(townRoot, nil)
+	sm := dog.NewSessionManager(tmux.NewTmux(), townRoot, mgr)
+
+	got := findDispatchableDog(mgr, sm, d.logger)
+	if got != nil {
+		t.Errorf("findDispatchableDog = %q, want nil (all working)", got.Name)
+	}
+}
+
+func TestFindDispatchableDog_EmptyKennelReturnsNil(t *testing.T) {
+	townRoot := t.TempDir()
+	d := testHandlerDaemon(t, townRoot)
+
+	mgr := dog.NewManager(townRoot, nil)
+	sm := dog.NewSessionManager(tmux.NewTmux(), townRoot, mgr)
+
+	got := findDispatchableDog(mgr, sm, d.logger)
+	if got != nil {
+		t.Errorf("findDispatchableDog = %q, want nil (empty kennel)", got.Name)
+	}
+}
+
+// TestFindDispatchableDog_PicksFirstIdleWhenNoSessionsLive verifies the
+// default path (no tmux sessions exist for any dog): the first idle dog
+// from mgr.List is returned. The "skip idle dogs with live sessions"
+// behavior — the actual gt-o24 regression — is exercised at runtime via
+// sm.IsRunning, whose tmux-backed correctness is covered in session_manager
+// tests.
+func TestFindDispatchableDog_PicksFirstIdleWhenNoSessionsLive(t *testing.T) {
+	townRoot := t.TempDir()
+	d := testHandlerDaemon(t, townRoot)
+
+	testSetupDogState(t, townRoot, "alpha", dog.StateIdle, time.Now())
+	testSetupDogState(t, townRoot, "bravo", dog.StateIdle, time.Now())
+
+	mgr := dog.NewManager(townRoot, nil)
+	sm := dog.NewSessionManager(tmux.NewTmux(), townRoot, mgr)
+
+	got := findDispatchableDog(mgr, sm, d.logger)
+	if got == nil {
+		t.Fatal("findDispatchableDog returned nil; expected an idle dog")
+	}
+	if got.Name != "alpha" && got.Name != "bravo" {
+		t.Errorf("findDispatchableDog = %q, want alpha or bravo", got.Name)
+	}
+}