From 7645c9b864e0ab848cb69ef44c5b44d064a05bb3 Mon Sep 17 00:00:00 2001 From: Dave Sargent Date: Sun, 17 May 2026 09:20:31 -0700 Subject: [PATCH] feat(rotation): Phase 4 retired-key cleaner (#72) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a background sweeper that walks every enabled box on a tick and removes any keys whose retired_at + overlap window has passed — completing the install→use→remove three-phase rotation cycle without operator intervention. Originally Phase 4 in the issue plan also covered auto-rotation on a schedule (off by default, configurable cadence). That needs a new global-settings surface in the dashboard (no app-level config table exists today; only user_preferences), which is its own design pass. Deferring the auto-rotate scheduler to a follow-up so this PR stays focused on the piece that's actually needed regardless of whether auto-rotation is enabled. What's here ----------- `services/agent_keyring/cleaner.go` - `RetiredKeyCleaner` runs as a goroutine off the dashboard's process-lifetime context. - Hourly tick (`CleanerInterval`) — short enough that a 24h rotation cleans up within a few hours of its target, long enough that the sweep is cheap. - Immediate sweep on start so a freshly-deployed dashboard catches up on any retired keys left from manual rotations done while the prior instance was down. - Per-box failures are logged but don't halt the sweep; one unreachable agent shouldn't block cleanup on the others. `cmd/server/main.go` - Wires the cleaner into startup alongside the existing alert evaluator. Cancelled when main returns. Tests ----- - `TestCleaner_RemovesRetiredKeyOnTick` — rotates a box, then runs the cleaner with a 1ms overlap and 20ms interval; verifies the retired key is removed from both the mock agent and the DB. - `TestCleaner_NoopWhenNothingRetired` — no rotation happens, so the cleaner finds nothing to do; verifies the seeded entry survives a sweep. Refs: Phase 4 (lite) of the implementation plan posted to #72. Co-Authored-By: Claude Opus 4.7 (1M context) --- gearbox/cmd/server/main.go | 19 +++ .../services/agent_keyring/cleaner.go | 111 ++++++++++++++++++ .../services/agent_keyring/cleaner_test.go | 67 +++++++++++ 3 files changed, 197 insertions(+) create mode 100644 gearbox/internal/framework/services/agent_keyring/cleaner.go create mode 100644 gearbox/internal/framework/services/agent_keyring/cleaner_test.go diff --git a/gearbox/cmd/server/main.go b/gearbox/cmd/server/main.go index 475449f..6a79022 100644 --- a/gearbox/cmd/server/main.go +++ b/gearbox/cmd/server/main.go @@ -25,6 +25,7 @@ import ( gbmiddleware "github.com/sarg3nt/gearbox/internal/framework/middleware" "github.com/sarg3nt/gearbox/internal/framework/models" "github.com/sarg3nt/gearbox/internal/framework/services" + "github.com/sarg3nt/gearbox/internal/framework/services/agent_keyring" "github.com/sarg3nt/gearbox/internal/framework/services/alerts" "github.com/sarg3nt/gearbox/internal/framework/services/crypto" "github.com/sarg3nt/gearbox/internal/framework/services/email" @@ -344,6 +345,24 @@ func main() { alertEvaluator.Start(30 * time.Second) // Evaluate alerts every 30 seconds logger.Info("alert evaluator initialized") + // Initialize retired-key cleaner. Each manual or scheduled rotation + // leaves the demoted key in box_agent_keys with retired_at stamped + // but role=secondary so the overlap window can preserve recovery. + // The cleaner walks every box every CleanerInterval and removes + // keys whose retired_at + overlap window has passed — both on the + // agent and in the DB. See issue #72 Phase 4. + keyringCleaner := agent_keyring.NewCleaner( + agent_keyring.New(db, encryptor, logger), + db, + agent_keyring.DefaultOverlapWindow, + agent_keyring.CleanerInterval, + logger, + ) + keyringCleanerCtx, cancelKeyringCleaner := context.WithCancel(context.Background()) + defer cancelKeyringCleaner() + go keyringCleaner.Run(keyringCleanerCtx) + logger.Info("retired-key cleaner initialized") + // Initialize WebSocket manager for Agent connections wsManager := collector.NewWebSocketManager(eventHub, registry, logger) logger.Info("WebSocket manager initialized") diff --git a/gearbox/internal/framework/services/agent_keyring/cleaner.go b/gearbox/internal/framework/services/agent_keyring/cleaner.go new file mode 100644 index 0000000..27616b7 --- /dev/null +++ b/gearbox/internal/framework/services/agent_keyring/cleaner.go @@ -0,0 +1,111 @@ +package agent_keyring + +import ( + "context" + "log/slog" + "time" + + "github.com/sarg3nt/gearbox/internal/framework/database" +) + +// CleanerInterval is the cadence at which RetiredKeyCleaner walks the +// fleet looking for keys whose retired_at + overlap window has passed. +// One hour is a sensible default: short enough that a 24-hour overlap +// rotation cleans up within a few hours of its target, long enough that +// the sweep is cheap (one DB query per box, one DELETE per agent). +const CleanerInterval = 1 * time.Hour + +// RetiredKeyCleaner is a background service that periodically walks +// every box and asks the Rotator to remove any keys whose retired_at +// is older than the overlap window. The Phase 3 manual-rotate buttons +// stamp retired_at but don't remove the old key — the cleaner does, so +// retired keys don't linger forever after a rotation. +// +// Phase 4 in the original plan also covered auto-rotation on a +// schedule; that needs a new global-settings surface in the dashboard +// (the dashboard doesn't have one yet for app-level config) and is +// intentionally deferred to a follow-up. The cleaner is the smaller +// piece that's genuinely needed regardless of whether auto-rotation +// is enabled. +type RetiredKeyCleaner struct { + rotator *Rotator + db *database.DB + overlapWindow time.Duration + interval time.Duration + logger *slog.Logger +} + +// NewCleaner builds a cleaner around an existing rotator. Pass +// overlapWindow=0 to use DefaultOverlapWindow; interval=0 → CleanerInterval. +func NewCleaner(rotator *Rotator, db *database.DB, overlapWindow, interval time.Duration, logger *slog.Logger) *RetiredKeyCleaner { + if overlapWindow <= 0 { + overlapWindow = DefaultOverlapWindow + } + if interval <= 0 { + interval = CleanerInterval + } + return &RetiredKeyCleaner{ + rotator: rotator, + db: db, + overlapWindow: overlapWindow, + interval: interval, + logger: logger, + } +} + +// Run blocks until ctx is cancelled. Sweeps once immediately on start +// so a freshly-deployed dashboard catches up on any retired keys left +// over from manual rotations done while the previous instance was +// down, then ticks every interval. +// +// Errors are logged but not propagated — one bad box shouldn't break +// the fleet sweep, and we don't want to spam fatal errors at startup +// for a transient agent outage. +func (c *RetiredKeyCleaner) Run(ctx context.Context) { + c.logger.Info("retired-key cleaner started", + "interval", c.interval, + "overlap_window", c.overlapWindow) + + c.sweepOnce(ctx) + + ticker := time.NewTicker(c.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + c.logger.Info("retired-key cleaner stopped") + return + case <-ticker.C: + c.sweepOnce(ctx) + } + } +} + +func (c *RetiredKeyCleaner) sweepOnce(ctx context.Context) { + boxes, err := c.db.GetEnabledBoxes() + if err != nil { + c.logger.Warn("cleaner: failed to list boxes", "error", err) + return + } + + total := 0 + for _, box := range boxes { + if ctx.Err() != nil { + return + } + removed, err := c.rotator.CleanupRetiredKeys(box.ID, c.overlapWindow) + if err != nil { + c.logger.Warn("cleaner: box sweep failed", + "box_id", box.ID, "name", box.Name, "error", err) + continue + } + if removed > 0 { + c.logger.Info("cleaner: swept retired keys", + "box_id", box.ID, "name", box.Name, "removed", removed) + total += removed + } + } + if total > 0 { + c.logger.Info("cleaner: sweep complete", "total_removed", total) + } +} diff --git a/gearbox/internal/framework/services/agent_keyring/cleaner_test.go b/gearbox/internal/framework/services/agent_keyring/cleaner_test.go new file mode 100644 index 0000000..4da3080 --- /dev/null +++ b/gearbox/internal/framework/services/agent_keyring/cleaner_test.go @@ -0,0 +1,67 @@ +package agent_keyring + +import ( + "context" + "testing" + "time" +) + +func TestCleaner_RemovesRetiredKeyOnTick(t *testing.T) { + rotator, mock, db, box := setupRotator(t) + if _, err := rotator.RotateBox(box.ID, 24*time.Hour); err != nil { + t.Fatalf("RotateBox: %v", err) + } + if mock.entryCount() != 2 { + t.Fatalf("post-rotation agent entries = %d, want 2", mock.entryCount()) + } + + // Tiny overlap + tiny interval so the cleaner sweeps quickly and + // considers the just-retired legacy entry eligible. + cleaner := NewCleaner(rotator, db, 1*time.Millisecond, 20*time.Millisecond, rotator.logger) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + done := make(chan struct{}) + go func() { + cleaner.Run(ctx) + close(done) + }() + + // Give it time for the immediate-on-start sweep to fire. + time.Sleep(50 * time.Millisecond) + cancel() + <-done + + if got := mock.entryCount(); got != 1 { + t.Errorf("agent entries after cleaner sweep = %d, want 1", got) + } + keys, _ := db.GetBoxAgentKeys(box.ID) + if len(keys) != 1 { + t.Errorf("db keys after cleaner sweep = %d, want 1", len(keys)) + } +} + +func TestCleaner_NoopWhenNothingRetired(t *testing.T) { + rotator, mock, db, box := setupRotator(t) + // No rotation = no retired keys. Cleaner sweep should leave the + // single legacy primary alone. + + cleaner := NewCleaner(rotator, db, 1*time.Millisecond, 1*time.Hour, rotator.logger) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + cleaner.Run(ctx) + close(done) + }() + time.Sleep(20 * time.Millisecond) + cancel() + <-done + + if got := mock.entryCount(); got != 1 { + t.Errorf("mock entries = %d, want 1", got) + } + keys, _ := db.GetBoxAgentKeys(box.ID) + if len(keys) != 1 { + t.Errorf("db keys = %d, want 1", len(keys)) + } +}