Process Invalid API key events from Remote Agents

Add a hook to trigger API key refresh if Remote Agents inform us that
they get Invalid API Key errors.

When users enable secrets to handle their API keys, Remote Agents need a
mechanism to ask Agent to refresh those keys when endpoints return 403s
on submissions, because in this case a 403 is not a hard fault, but
rather a signal that the key has become stale and we need a new one.
Here we add this trigger to the existing RAR event mechanism.

Signed-off-by: Mark Kirichenko <mark.kirichenko@datadoghq.com>

Author: atanzu

comp/core/remoteagentregistry/impl/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//comp/core/remoteagentregistry/def",
         "//comp/core/remoteagentregistry/status",
         "//comp/core/remoteagentregistry/util",
+        "//comp/core/secrets/def",
         "//comp/core/status",
         "//comp/core/telemetry/def",
         "//comp/def",
@@ -51,6 +52,7 @@ dd_agent_go_test(
         "//comp/core/ipc/def",
         "//comp/core/ipc/mock",
         "//comp/core/remoteagentregistry/def",
+        "//comp/core/secrets/mock",
         "//comp/core/telemetry/def",
         "//comp/core/telemetry/impl",
         "//comp/def",

comp/core/remoteagentregistry/impl/registry.go

@@ -21,6 +21,7 @@ import (
 	ipc "github.com/DataDog/datadog-agent/comp/core/ipc/def"
 	remoteagentregistry "github.com/DataDog/datadog-agent/comp/core/remoteagentregistry/def"
 	remoteagentregistryStatus "github.com/DataDog/datadog-agent/comp/core/remoteagentregistry/status"
+	secrets "github.com/DataDog/datadog-agent/comp/core/secrets/def"
 	"github.com/DataDog/datadog-agent/comp/core/status"
 	telemetry "github.com/DataDog/datadog-agent/comp/core/telemetry/def"
 	compdef "github.com/DataDog/datadog-agent/comp/def"
@@ -33,6 +34,7 @@ type Requires struct {
 	Ipc              ipc.Component
 	Lifecycle        compdef.Lifecycle
 	Telemetry        telemetry.Component
+	Secrets          secrets.Component
 	EventSubscribers []*remoteagentregistry.EventSubscriber `group:"remoteAgentEventSubscriber"`
 }
 
@@ -61,6 +63,8 @@ func NewComponent(reqs Requires) Provides {
 
 func newRegistry(reqs Requires) *remoteAgentRegistry {
 	shutdownChan := make(chan struct{})
+	eventSubscribers := append([]*remoteagentregistry.EventSubscriber{}, reqs.EventSubscribers...)
+	eventSubscribers = append(eventSubscribers, newSecretsRefreshEventSubscriber(reqs.Secrets))
 	registry := &remoteAgentRegistry{
 		conf:           reqs.Config,
 		ipc:            reqs.Ipc,
@@ -74,7 +78,7 @@ func newRegistry(reqs Requires) *remoteAgentRegistry {
 			FlareServiceName:     {},
 			TelemetryServiceName: {},
 		},
-		eventSubscribers: reqs.EventSubscribers,
+		eventSubscribers: eventSubscribers,
 	}
 
 	reqs.Lifecycle.Append(compdef.Hook{
@@ -91,6 +95,23 @@ func newRegistry(reqs Requires) *remoteAgentRegistry {
 	return registry
 }
 
+// newSecretsRefreshEventSubscriber creates the subscriber that asks the secrets component to refresh when a remote
+// agent reports that its API key was rejected. Refresh is asynchronous and applies its own configured throttle.
+func newSecretsRefreshEventSubscriber(resolver secrets.Component) *remoteagentregistry.EventSubscriber {
+	return &remoteagentregistry.EventSubscriber{
+		Name: "secrets-refresh",
+		Callback: func(_ remoteagentregistry.RegisteredAgent, events []remoteagentregistry.RemoteAgentEvent) {
+			for _, event := range events {
+				if _, ok := event.Details.(*remoteagentregistry.InvalidAPIKey); ok {
+					// One refresh is sufficient for the whole report; the resolver coalesces and throttles requests.
+					resolver.Refresh()
+					return
+				}
+			}
+		},
+	}
+}
+
 type telemetryStore struct {
 	// remoteAgentRegistered tracks how many remote agents are registered.
 	remoteAgentRegistered telemetry.Gauge

comp/core/remoteagentregistry/impl/registry_test.go

@@ -28,6 +28,7 @@ import (
 	ipc "github.com/DataDog/datadog-agent/comp/core/ipc/def"
 	ipcmock "github.com/DataDog/datadog-agent/comp/core/ipc/mock"
 	remoteagent "github.com/DataDog/datadog-agent/comp/core/remoteagentregistry/def"
+	secretsmock "github.com/DataDog/datadog-agent/comp/core/secrets/mock"
 	"github.com/DataDog/datadog-agent/comp/core/telemetry/def"
 	"github.com/DataDog/datadog-agent/comp/core/telemetry/impl"
 	compdef "github.com/DataDog/datadog-agent/comp/def"
@@ -89,6 +90,34 @@ func TestReportRemoteAgentEvent(t *testing.T) {
 	require.Error(t, component.ReportRemoteAgentEvent("does-not-exist", events))
 }
 
+func TestReportRemoteAgentEventRefreshesSecrets(t *testing.T) {
+	cfg := configmock.New(t)
+	cfg.SetInTest("remote_agent.registry.enabled", true)
+	ipcComp := ipcmock.New(t)
+	resolver := secretsmock.New(t)
+	refreshes := 0
+	resolver.SetRefreshHook(func() bool {
+		refreshes++
+		return true
+	})
+
+	component := NewComponent(Requires{
+		Config:    cfg,
+		Ipc:       ipcComp,
+		Lifecycle: compdef.NewTestLifecycle(t),
+		Telemetry: telemetryimpl.NewMock(t),
+		Secrets:   resolver,
+	}).Comp.(*remoteAgentRegistry)
+	remoteAgent := buildAndRegisterRemoteAgent(t, ipcComp, component, "test-agent", "Test Agent", "1234")
+
+	require.NoError(t, component.ReportRemoteAgentEvent(remoteAgent.registeredSessionID, []remoteagent.RemoteAgentEvent{
+		{Message: "ordinary event"},
+		{Message: "API key rejected", Details: &remoteagent.InvalidAPIKey{}},
+		{Message: "duplicate API key rejection", Details: &remoteagent.InvalidAPIKey{}},
+	}))
+	require.Equal(t, 1, refreshes)
+}
+
 func TestReportRemoteAgentEventBroadcast(t *testing.T) {
 	cfg := configmock.New(t)
 	cfg.SetInTest("remote_agent.registry.enabled", true)
@@ -124,6 +153,7 @@ func TestReportRemoteAgentEventBroadcast(t *testing.T) {
 		Ipc:              ipcComp,
 		Lifecycle:        lc,
 		Telemetry:        tel,
+		Secrets:          secretsmock.New(t),
 		EventSubscribers: []*remoteagent.EventSubscriber{panicker, nil, recorder},
 	}
 	component := NewComponent(reqs).Comp.(*remoteAgentRegistry)
@@ -262,6 +292,7 @@ func buildComponentWithConfig(t *testing.T, config configmodel.Config) (Provides
 		Ipc:       ipc,
 		Lifecycle: lc,
 		Telemetry: telemetry,
+		Secrets:   secretsmock.New(t),
 	}
 
 	return NewComponent(reqs), lc, telemetry, ipc

releasenotes/notes/refresh-secrets-on-remote-agent-api-key-failure-bb3f941bc1f3c7f2.yaml

@@ -0,0 +1,4 @@
+---
+enhancements:
+  - |
+    The Agent can now refresh secrets-managed API keys when a Remote Agent reports an Invalid API Kay event.