feat(rotation): Phase 4 retired-key cleaner (#72)

Adds a background sweeper that walks every enabled box on a tick and
removes any keys whose retired_at + overlap window has passed —
completing the install→use→remove three-phase rotation cycle without
operator intervention.

Originally Phase 4 in the issue plan also covered auto-rotation on a
schedule (off by default, configurable cadence). That needs a new
global-settings surface in the dashboard (no app-level config table
exists today; only user_preferences), which is its own design pass.
Deferring the auto-rotate scheduler to a follow-up so this PR stays
focused on the piece that's actually needed regardless of whether
auto-rotation is enabled.

What's here
-----------

`services/agent_keyring/cleaner.go`

- `RetiredKeyCleaner` runs as a goroutine off the dashboard's
  process-lifetime context.
- Hourly tick (`CleanerInterval`) — short enough that a 24h
  rotation cleans up within a few hours of its target, long enough
  that the sweep is cheap.
- Immediate sweep on start so a freshly-deployed dashboard catches
  up on any retired keys left from manual rotations done while the
  prior instance was down.
- Per-box failures are logged but don't halt the sweep; one
  unreachable agent shouldn't block cleanup on the others.

`cmd/server/main.go`

- Wires the cleaner into startup alongside the existing alert
  evaluator. Cancelled when main returns.

Tests
-----

- `TestCleaner_RemovesRetiredKeyOnTick` — rotates a box, then runs
  the cleaner with a 1ms overlap and 20ms interval; verifies the
  retired key is removed from both the mock agent and the DB.
- `TestCleaner_NoopWhenNothingRetired` — no rotation happens, so
  the cleaner finds nothing to do; verifies the seeded entry
  survives a sweep.

Refs: Phase 4 (lite) of the implementation plan posted to #72.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: sarg3nt

gearbox/cmd/server/main.go

@@ -25,6 +25,7 @@ import (
 	gbmiddleware "github.com/sarg3nt/gearbox/internal/framework/middleware"
 	"github.com/sarg3nt/gearbox/internal/framework/models"
 	"github.com/sarg3nt/gearbox/internal/framework/services"
+	"github.com/sarg3nt/gearbox/internal/framework/services/agent_keyring"
 	"github.com/sarg3nt/gearbox/internal/framework/services/alerts"
 	"github.com/sarg3nt/gearbox/internal/framework/services/crypto"
 	"github.com/sarg3nt/gearbox/internal/framework/services/email"
@@ -344,6 +345,24 @@ func main() {
 	alertEvaluator.Start(30 * time.Second) // Evaluate alerts every 30 seconds
 	logger.Info("alert evaluator initialized")
 
+	// Initialize retired-key cleaner. Each manual or scheduled rotation
+	// leaves the demoted key in box_agent_keys with retired_at stamped
+	// but role=secondary so the overlap window can preserve recovery.
+	// The cleaner walks every box every CleanerInterval and removes
+	// keys whose retired_at + overlap window has passed — both on the
+	// agent and in the DB. See issue #72 Phase 4.
+	keyringCleaner := agent_keyring.NewCleaner(
+		agent_keyring.New(db, encryptor, logger),
+		db,
+		agent_keyring.DefaultOverlapWindow,
+		agent_keyring.CleanerInterval,
+		logger,
+	)
+	keyringCleanerCtx, cancelKeyringCleaner := context.WithCancel(context.Background())
+	defer cancelKeyringCleaner()
+	go keyringCleaner.Run(keyringCleanerCtx)
+	logger.Info("retired-key cleaner initialized")
+
 	// Initialize WebSocket manager for Agent connections
 	wsManager := collector.NewWebSocketManager(eventHub, registry, logger)
 	logger.Info("WebSocket manager initialized")

gearbox/internal/framework/services/agent_keyring/cleaner.go

@@ -0,0 +1,111 @@
+package agent_keyring
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"github.com/sarg3nt/gearbox/internal/framework/database"
+)
+
+// CleanerInterval is the cadence at which RetiredKeyCleaner walks the
+// fleet looking for keys whose retired_at + overlap window has passed.
+// One hour is a sensible default: short enough that a 24-hour overlap
+// rotation cleans up within a few hours of its target, long enough that
+// the sweep is cheap (one DB query per box, one DELETE per agent).
+const CleanerInterval = 1 * time.Hour
+
+// RetiredKeyCleaner is a background service that periodically walks
+// every box and asks the Rotator to remove any keys whose retired_at
+// is older than the overlap window. The Phase 3 manual-rotate buttons
+// stamp retired_at but don't remove the old key — the cleaner does, so
+// retired keys don't linger forever after a rotation.
+//
+// Phase 4 in the original plan also covered auto-rotation on a
+// schedule; that needs a new global-settings surface in the dashboard
+// (the dashboard doesn't have one yet for app-level config) and is
+// intentionally deferred to a follow-up. The cleaner is the smaller
+// piece that's genuinely needed regardless of whether auto-rotation
+// is enabled.
+type RetiredKeyCleaner struct {
+	rotator       *Rotator
+	db            *database.DB
+	overlapWindow time.Duration
+	interval      time.Duration
+	logger        *slog.Logger
+}
+
+// NewCleaner builds a cleaner around an existing rotator. Pass
+// overlapWindow=0 to use DefaultOverlapWindow; interval=0 → CleanerInterval.
+func NewCleaner(rotator *Rotator, db *database.DB, overlapWindow, interval time.Duration, logger *slog.Logger) *RetiredKeyCleaner {
+	if overlapWindow <= 0 {
+		overlapWindow = DefaultOverlapWindow
+	}
+	if interval <= 0 {
+		interval = CleanerInterval
+	}
+	return &RetiredKeyCleaner{
+		rotator:       rotator,
+		db:            db,
+		overlapWindow: overlapWindow,
+		interval:      interval,
+		logger:        logger,
+	}
+}
+
+// Run blocks until ctx is cancelled. Sweeps once immediately on start
+// so a freshly-deployed dashboard catches up on any retired keys left
+// over from manual rotations done while the previous instance was
+// down, then ticks every interval.
+//
+// Errors are logged but not propagated — one bad box shouldn't break
+// the fleet sweep, and we don't want to spam fatal errors at startup
+// for a transient agent outage.
+func (c *RetiredKeyCleaner) Run(ctx context.Context) {
+	c.logger.Info("retired-key cleaner started",
+		"interval", c.interval,
+		"overlap_window", c.overlapWindow)
+
+	c.sweepOnce(ctx)
+
+	ticker := time.NewTicker(c.interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			c.logger.Info("retired-key cleaner stopped")
+			return
+		case <-ticker.C:
+			c.sweepOnce(ctx)
+		}
+	}
+}
+
+func (c *RetiredKeyCleaner) sweepOnce(ctx context.Context) {
+	boxes, err := c.db.GetEnabledBoxes()
+	if err != nil {
+		c.logger.Warn("cleaner: failed to list boxes", "error", err)
+		return
+	}
+
+	total := 0
+	for _, box := range boxes {
+		if ctx.Err() != nil {
+			return
+		}
+		removed, err := c.rotator.CleanupRetiredKeys(box.ID, c.overlapWindow)
+		if err != nil {
+			c.logger.Warn("cleaner: box sweep failed",
+				"box_id", box.ID, "name", box.Name, "error", err)
+			continue
+		}
+		if removed > 0 {
+			c.logger.Info("cleaner: swept retired keys",
+				"box_id", box.ID, "name", box.Name, "removed", removed)
+			total += removed
+		}
+	}
+	if total > 0 {
+		c.logger.Info("cleaner: sweep complete", "total_removed", total)
+	}
+}

gearbox/internal/framework/services/agent_keyring/cleaner_test.go

@@ -0,0 +1,67 @@
+package agent_keyring
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestCleaner_RemovesRetiredKeyOnTick(t *testing.T) {
+	rotator, mock, db, box := setupRotator(t)
+	if _, err := rotator.RotateBox(box.ID, 24*time.Hour); err != nil {
+		t.Fatalf("RotateBox: %v", err)
+	}
+	if mock.entryCount() != 2 {
+		t.Fatalf("post-rotation agent entries = %d, want 2", mock.entryCount())
+	}
+
+	// Tiny overlap + tiny interval so the cleaner sweeps quickly and
+	// considers the just-retired legacy entry eligible.
+	cleaner := NewCleaner(rotator, db, 1*time.Millisecond, 20*time.Millisecond, rotator.logger)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	done := make(chan struct{})
+	go func() {
+		cleaner.Run(ctx)
+		close(done)
+	}()
+
+	// Give it time for the immediate-on-start sweep to fire.
+	time.Sleep(50 * time.Millisecond)
+	cancel()
+	<-done
+
+	if got := mock.entryCount(); got != 1 {
+		t.Errorf("agent entries after cleaner sweep = %d, want 1", got)
+	}
+	keys, _ := db.GetBoxAgentKeys(box.ID)
+	if len(keys) != 1 {
+		t.Errorf("db keys after cleaner sweep = %d, want 1", len(keys))
+	}
+}
+
+func TestCleaner_NoopWhenNothingRetired(t *testing.T) {
+	rotator, mock, db, box := setupRotator(t)
+	// No rotation = no retired keys. Cleaner sweep should leave the
+	// single legacy primary alone.
+
+	cleaner := NewCleaner(rotator, db, 1*time.Millisecond, 1*time.Hour, rotator.logger)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		cleaner.Run(ctx)
+		close(done)
+	}()
+	time.Sleep(20 * time.Millisecond)
+	cancel()
+	<-done
+
+	if got := mock.entryCount(); got != 1 {
+		t.Errorf("mock entries = %d, want 1", got)
+	}
+	keys, _ := db.GetBoxAgentKeys(box.ID)
+	if len(keys) != 1 {
+		t.Errorf("db keys = %d, want 1", len(keys))
+	}
+}