agent/proxy: cancel the previous auto-auth token when re-authenticating

changelog/32060.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+agent/proxy: cancel the context of the previous auto-auth token when re-authenticating, so lease renewals derived from it stop instead of repeatedly retrying against a token Vault has already expired
+```

command/agentproxyshared/cache/lease_cache.go

@@ -114,6 +114,14 @@ type LeaseCache struct {
 	// capabilityManager is used when static secrets are enabled to
 	// manage the capabilities of cached tokens.
 	capabilityManager *StaticSecretCapabilityManager
+
+	// autoAuthTokenLock guards autoAuthToken.
+	autoAuthTokenLock sync.Mutex
+
+	// autoAuthToken is the most recent token handed to RegisterAutoAuthToken.
+	// It is tracked so that the context of the one it replaces can be
+	// cancelled when auto-auth re-authenticates.
+	autoAuthToken string
 }
 
 // LeaseCacheConfig is the configuration for initializing a new
@@ -1761,6 +1769,18 @@ func deriveNamespaceAndRevocationPath(req *SendRequest) (string, string) {
 // primarily used to register the auto-auth token and should only be called
 // within a sink's WriteToken func.
 func (c *LeaseCache) RegisterAutoAuthToken(token string) error {
+	if token == "" {
+		return nil
+	}
+
+	// Receiving a different token means auto-auth has re-authenticated. Cancel
+	// the context of the token being replaced before anything else, including
+	// the "already cached" short-circuit below: after a restart with a
+	// persistent cache, restoreTokens has already put the previous token back
+	// into the cache, so that short-circuit is exactly the path on which the
+	// current token needs to be recorded.
+	c.cancelPreviousAutoAuthToken(token)
+
 	// Get the token from the cache
 	oldIndex, err := c.db.Get(cachememdb.IndexNameToken, token)
 	if err != nil && err != cachememdb.ErrCacheItemNotFound {
@@ -1818,6 +1838,54 @@ func (c *LeaseCache) RegisterAutoAuthToken(token string) error {
 	return nil
 }
 
+// cancelPreviousAutoAuthToken records newToken as the current auto-auth token
+// and cancels the context of the one it replaces, if any. Because every lease
+// obtained with a token derives its renewal context from that token's context,
+// cancelling it stops those lease renewals and lets startRenewing evict them.
+//
+// Without this, re-authentication leaves the previous token's lease watchers
+// running against a token Vault has already expired. Each one retries
+// sys/leases/renew until it gives up on its own, which can take minutes and
+// generate a large volume of requests that can only ever be denied.
+//
+// Note that this cancels on any token change, not only on expiry. Auto-auth
+// also re-authenticates when the auth method reports new credentials, in which
+// case the previous token may still be valid; its leases are then dropped from
+// the cache and re-fetched on next use rather than kept renewed.
+func (c *LeaseCache) cancelPreviousAutoAuthToken(newToken string) {
+	c.autoAuthTokenLock.Lock()
+	defer c.autoAuthTokenLock.Unlock()
+
+	previous := c.autoAuthToken
+	c.autoAuthToken = newToken
+
+	switch {
+	case previous == "":
+		// Nothing registered yet, so there is nothing to cancel. Reached on
+		// first login, and on the first registration after a restore.
+		return
+	case previous == newToken:
+		return
+	}
+
+	index, err := c.db.Get(cachememdb.IndexNameToken, previous)
+	if errors.Is(err, cachememdb.ErrCacheItemNotFound) {
+		c.logger.Trace("previous auto-auth token is no longer cached; nothing to cancel")
+		return
+	}
+	if err != nil {
+		c.logger.Error("failed to look up previous auto-auth token in the cache", "error", err)
+		return
+	}
+
+	if index.RenewCtxInfo == nil || index.RenewCtxInfo.CancelFunc == nil {
+		return
+	}
+
+	c.logger.Debug("canceling context of the previous auto-auth token and the leases derived from it")
+	index.RenewCtxInfo.CancelFunc()
+}
+
 type cacheClearInput struct {
 	Type string
 

command/agentproxyshared/cache/lease_cache_test.go

@@ -1674,3 +1674,115 @@ func TestLeaseCacheRestore_expired(t *testing.T) {
 	assert.Equal(t, "autoauthtoken", afterDB[0].Token)
 	assert.Equal(t, cacheboltdb.TokenType, afterDB[0].Type)
 }
+
+func TestLeaseCache_RegisterAutoAuthToken_CancelsPreviousToken(t *testing.T) {
+	// A lease obtained with the previous auto-auth token must stop being
+	// renewed once auto-auth re-authenticates. Vault revokes the lease along
+	// with its parent token, so every later renewal can only be denied.
+	responses := []*SendResponse{
+		newTestSendResponse(http.StatusOK, `{"lease_id": "foo", "renewable": true, "lease_duration": 600, "data": {"value": "foo"}}`),
+	}
+
+	lc := testNewLeaseCache(t, responses)
+	require.NoError(t, lc.RegisterAutoAuthToken("firsttoken"))
+
+	sendReq := &SendRequest{
+		Token:   "firsttoken",
+		Request: httptest.NewRequest("GET", "http://example.com/v1/sample/api", strings.NewReader(`{"value": "input"}`)),
+	}
+	if _, err := lc.Send(context.Background(), sendReq); err != nil {
+		t.Fatal(err)
+	}
+
+	firstIndex, err := lc.db.Get(cachememdb.IndexNameToken, "firsttoken")
+	require.NoError(t, err)
+	require.NotNil(t, firstIndex.RenewCtxInfo)
+	previousCtx := firstIndex.RenewCtxInfo.Ctx
+
+	leaseIndex, err := lc.db.Get(cachememdb.IndexNameLease, "foo")
+	require.NoError(t, err)
+	require.NotNil(t, leaseIndex.RenewCtxInfo)
+	leaseCtx := leaseIndex.RenewCtxInfo.Ctx
+
+	// Auto-auth re-authenticates and hands the cache a different token.
+	require.NoError(t, lc.RegisterAutoAuthToken("secondtoken"))
+
+	select {
+	case <-previousCtx.Done():
+	case <-time.After(2 * time.Second):
+		t.Fatal("context of the previous auto-auth token was not cancelled")
+	}
+
+	// Cancelling the token's context must cascade to the leases derived from
+	// it, which is what stops them being renewed against a dead token.
+	select {
+	case <-leaseCtx.Done():
+	case <-time.After(2 * time.Second):
+		t.Fatal("context of the lease derived from the previous auto-auth token was not cancelled")
+	}
+
+	// The replacement token is registered and left alone.
+	secondIndex, err := lc.db.Get(cachememdb.IndexNameToken, "secondtoken")
+	require.NoError(t, err)
+	require.NotNil(t, secondIndex)
+	require.NoError(t, secondIndex.RenewCtxInfo.Ctx.Err())
+}
+
+func TestLeaseCache_RegisterAutoAuthToken_CancelsAfterRestore(t *testing.T) {
+	// After a restart with a persistent cache, restoreTokens puts the previous
+	// auto-auth token back into the cache before auto-auth registers it again.
+	// That registration takes the "already cached" path, which must still
+	// record the token, otherwise the next re-authentication has nothing to
+	// cancel and the stale lease watchers survive.
+	lc := testNewLeaseCache(t, nil)
+
+	restored := &cachememdb.Index{
+		ID:          "restored-index",
+		Token:       "restoredtoken",
+		Namespace:   "root/",
+		RequestPath: "/v1/auth/token/lookup-self",
+		Type:        cacheboltdb.TokenType,
+	}
+	restored.RenewCtxInfo = lc.createCtxInfo(nil)
+	require.NoError(t, lc.db.Set(restored))
+
+	// Auto-auth hands back the token that was just restored.
+	require.NoError(t, lc.RegisterAutoAuthToken("restoredtoken"))
+
+	index, err := lc.db.Get(cachememdb.IndexNameToken, "restoredtoken")
+	require.NoError(t, err)
+	restoredCtx := index.RenewCtxInfo.Ctx
+	require.NoError(t, restoredCtx.Err(), "restored token must not be cancelled by its own registration")
+
+	// The first re-authentication after the restart must cancel it.
+	require.NoError(t, lc.RegisterAutoAuthToken("freshtoken"))
+
+	select {
+	case <-restoredCtx.Done():
+	case <-time.After(2 * time.Second):
+		t.Fatal("context of the restored auto-auth token was not cancelled on re-authentication")
+	}
+}
+
+func TestLeaseCache_RegisterAutoAuthToken_IgnoresEmptyToken(t *testing.T) {
+	// The sink can be handed an empty token while auto-auth is shutting down.
+	// That must not cancel the live token's context.
+	lc := testNewLeaseCache(t, nil)
+	require.NoError(t, lc.RegisterAutoAuthToken("livetoken"))
+
+	index, err := lc.db.Get(cachememdb.IndexNameToken, "livetoken")
+	require.NoError(t, err)
+	liveCtx := index.RenewCtxInfo.Ctx
+
+	require.NoError(t, lc.RegisterAutoAuthToken(""))
+	require.NoError(t, liveCtx.Err(), "an empty token write must not cancel the live auto-auth token")
+
+	// It must also leave the tracked token in place, otherwise the next real
+	// re-authentication would find nothing to cancel.
+	require.NoError(t, lc.RegisterAutoAuthToken("nexttoken"))
+	select {
+	case <-liveCtx.Done():
+	case <-time.After(2 * time.Second):
+		t.Fatal("empty token write cleared the tracked auto-auth token")
+	}
+}