Retry Claude requests on Anthropic 529/5xx overload with bounded backoff

Users hit "Repeated 529 Overloaded errors" force-fails during a live Anthropic
capacity outage (verifier counted 22-71 upstream 5xx per 5-min window).
subrouter passed every 529 straight through, so brief blips consumed the
client's whole retry budget.

The retry transport now absorbs brief overloads: on a Claude 5xx it retries the
SAME account up to 2 times, honoring Retry-After (capped at 10s) else 1s/2s
backoff, via a context-cancellable wait (injectable for tests). No account
rotation (overload is API-wide, rotating burns the failover budget for nothing),
no exhaustion-marking, and overload retries do not consume the account-failover
budget. After the small budget the 5xx passes through and the client's own
backoff takes over, so a sustained outage is not amplified.

Also fixes a latent header bug the new path exposed: an overload retry after an
earlier account failover must keep the failover account's auth headers rather
than reverting to the original account (covered by
TestClaudeOverloadRetryPreservesFailoverAccount).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: lawrencecchen

internal/proxy/claude_ratelimit_routing_test.go

@@ -722,3 +722,156 @@ func TestClientRemoteIP(t *testing.T) {
 		}
 	}
 }
+
+// recordSleep returns a sleep fn that records waits without actually sleeping.
+func recordSleep(waits *[]time.Duration) func(context.Context, time.Duration) error {
+	return func(ctx context.Context, d time.Duration) error {
+		*waits = append(*waits, d)
+		return ctx.Err()
+	}
+}
+
+// TestClaudeOverloadRetrySucceeds: a brief 529 blip is absorbed on the SAME
+// account with backoff; the client sees the eventual 200 and the account is
+// never marked exhausted.
+func TestClaudeOverloadRetrySucceeds(t *testing.T) {
+	server, store := claudeFailoverServer(t)
+	if _, err := store.Put("claude", "session-529", "cooked@example.com", ""); err != nil {
+		t.Fatal(err)
+	}
+	var calls int
+	stub := &stubRoundTripper{responses: func(req *http.Request) *http.Response {
+		calls++
+		if calls <= 2 {
+			return &http.Response{StatusCode: 529, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"type":"error","error":{"type":"overloaded_error"}}`))}
+		}
+		return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"id":"msg_ok"}`))}
+	}}
+	var waits []time.Duration
+	transport := usageLimitRetryTransport{
+		base: stub, server: &server, provider: accounts.ProviderClaude,
+		agent: "claude", session: "session-529", account: "cooked@example.com",
+		method: http.MethodPost, path: "/v1/messages", maxAttempts: 6,
+		sleep: recordSleep(&waits),
+	}
+	req, _ := http.NewRequest(http.MethodPost, "https://api.anthropic.com/v1/messages", bytes.NewReader([]byte(`{"model":"claude-opus-4-8"}`)))
+	req.Header.Set("Authorization", "Bearer tok-cooked")
+	response, err := transport.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200 after overload retries", response.StatusCode)
+	}
+	if calls != 3 {
+		t.Fatalf("upstream calls = %d, want 3 (529, 529, 200)", calls)
+	}
+	if len(waits) != 2 || waits[0] != time.Second || waits[1] != 2*time.Second {
+		t.Fatalf("backoff waits = %v, want [1s 2s]", waits)
+	}
+	if server.SchedulerRef.Get().Exhausted(accounts.ProviderClaude, "cooked@example.com") {
+		t.Fatal("an overload must NOT mark the account exhausted")
+	}
+}
+
+// TestClaudeOverloadRetryGivesUpAfterBudget: a sustained outage passes the 5xx
+// through after the small retry budget so the client's own backoff takes over.
+func TestClaudeOverloadRetryGivesUpAfterBudget(t *testing.T) {
+	server, _ := claudeFailoverServer(t)
+	var calls int
+	stub := &stubRoundTripper{responses: func(req *http.Request) *http.Response {
+		calls++
+		return &http.Response{StatusCode: 529, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"type":"error","error":{"type":"overloaded_error"}}`))}
+	}}
+	var waits []time.Duration
+	transport := usageLimitRetryTransport{
+		base: stub, server: &server, provider: accounts.ProviderClaude,
+		agent: "claude", session: "s", account: "cooked@example.com",
+		method: http.MethodPost, path: "/v1/messages", maxAttempts: 6,
+		sleep: recordSleep(&waits),
+	}
+	req, _ := http.NewRequest(http.MethodPost, "https://api.anthropic.com/v1/messages", bytes.NewReader([]byte(`{}`)))
+	req.Header.Set("Authorization", "Bearer tok-cooked")
+	response, err := transport.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != 529 {
+		t.Fatalf("status = %d, want 529 passed through after budget", response.StatusCode)
+	}
+	if calls != 1+claudeOverloadMaxRetries {
+		t.Fatalf("upstream calls = %d, want %d", calls, 1+claudeOverloadMaxRetries)
+	}
+}
+
+// TestClaudeOverloadRetryPreservesFailoverAccount is the header-revert
+// regression: after failover moved the request to a second account, an overload
+// retry must keep that account's auth, not silently revert to the first.
+func TestClaudeOverloadRetryPreservesFailoverAccount(t *testing.T) {
+	server, store := claudeFailoverServer(t)
+	if _, err := store.Put("claude", "session-mix", "cooked@example.com", ""); err != nil {
+		t.Fatal(err)
+	}
+	var freshCalls int
+	var authsSeen []string
+	stub := &stubRoundTripper{responses: func(req *http.Request) *http.Response {
+		auth := req.Header.Get("Authorization")
+		authsSeen = append(authsSeen, auth)
+		if strings.Contains(auth, "tok-cooked") {
+			h := http.Header{}
+			h.Set("Anthropic-Ratelimit-Unified-Status", "rejected")
+			return &http.Response{StatusCode: http.StatusTooManyRequests, Header: h, Body: io.NopCloser(strings.NewReader(`{"type":"error","error":{"type":"rate_limit_error"}}`))}
+		}
+		freshCalls++
+		if freshCalls == 1 {
+			return &http.Response{StatusCode: 529, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"type":"error","error":{"type":"overloaded_error"}}`))}
+		}
+		return &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"id":"msg_fresh"}`))}
+	}}
+	var waits []time.Duration
+	transport := usageLimitRetryTransport{
+		base: stub, server: &server, provider: accounts.ProviderClaude,
+		agent: "claude", session: "session-mix", account: "cooked@example.com",
+		method: http.MethodPost, path: "/v1/messages", maxAttempts: 6,
+		sleep: recordSleep(&waits),
+	}
+	req, _ := http.NewRequest(http.MethodPost, "https://api.anthropic.com/v1/messages", bytes.NewReader([]byte(`{}`)))
+	req.Header.Set("Authorization", "Bearer tok-cooked")
+	response, err := transport.RoundTrip(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+	body, _ := io.ReadAll(response.Body)
+	if response.StatusCode != http.StatusOK || !strings.Contains(string(body), "msg_fresh") {
+		t.Fatalf("status=%d body=%s, want fresh 200 (429 -> failover -> 529 -> retry same fresh account)", response.StatusCode, string(body))
+	}
+	// Sequence must be cooked, fresh (529), fresh (200): the overload retry
+	// stays on the failover account.
+	if len(authsSeen) != 3 || !strings.Contains(authsSeen[1], "tok-fresh") || !strings.Contains(authsSeen[2], "tok-fresh") {
+		t.Fatalf("auth sequence = %v, want cooked then fresh twice", authsSeen)
+	}
+}
+
+func TestClaudeOverloadBackoff(t *testing.T) {
+	h := http.Header{}
+	if d := claudeOverloadBackoff(h, 0); d != time.Second {
+		t.Fatalf("retry0 = %v, want 1s", d)
+	}
+	if d := claudeOverloadBackoff(h, 1); d != 2*time.Second {
+		t.Fatalf("retry1 = %v, want 2s", d)
+	}
+	h.Set("Retry-After", "3")
+	if d := claudeOverloadBackoff(h, 0); d != 3*time.Second {
+		t.Fatalf("retry-after 3 = %v, want 3s", d)
+	}
+	h.Set("Retry-After", "9999")
+	if d := claudeOverloadBackoff(h, 0); d != claudeOverloadMaxWait {
+		t.Fatalf("retry-after 9999 = %v, want cap %v", d, claudeOverloadMaxWait)
+	}
+	if !claudeOverloadStatus(529) || !claudeOverloadStatus(500) || claudeOverloadStatus(429) || claudeOverloadStatus(200) {
+		t.Fatal("claudeOverloadStatus classification wrong")
+	}
+}

internal/proxy/proxy.go

@@ -19,6 +19,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -2666,6 +2667,64 @@ type usageLimitRetryTransport struct {
 	path        string
 	upstream    string
 	maxAttempts int
+	// sleep waits for the backoff duration or until the context is cancelled.
+	// Injectable for tests; nil means a real timer wait.
+	sleep func(context.Context, time.Duration) error
+}
+
+// claudeOverloadMaxRetries bounds the same-account retries subrouter itself
+// performs on an Anthropic 5xx/529 overload before passing the error through to
+// the client (which retries further with its own backoff). Small on purpose:
+// subrouter absorbs brief blips without stacking long waits on top of the
+// client's retry budget or amplifying load during a sustained outage.
+const claudeOverloadMaxRetries = 2
+
+// claudeOverloadMaxWait caps a single overload backoff wait, including one
+// requested via Retry-After, so a pathological header cannot hold a proxied
+// request hostage.
+const claudeOverloadMaxWait = 10 * time.Second
+
+// claudeOverloadStatus reports whether the upstream response is an
+// Anthropic-side server failure (529 overloaded_error or another 5xx) worth
+// retrying on the SAME account: it is not account-specific, so rotating
+// accounts cannot help and would only burn the failover budget.
+func claudeOverloadStatus(code int) bool {
+	return code >= 500 && code <= 599
+}
+
+// claudeOverloadBackoff picks the wait before an overload retry: Retry-After
+// when the upstream sent one (capped), else 1s << retry (1s, 2s, ...).
+func claudeOverloadBackoff(header http.Header, retry int) time.Duration {
+	if ra := strings.TrimSpace(claudeHeaderGet(header, "Retry-After")); ra != "" {
+		if secs, err := strconv.Atoi(ra); err == nil && secs > 0 {
+			wait := time.Duration(secs) * time.Second
+			if wait > claudeOverloadMaxWait {
+				return claudeOverloadMaxWait
+			}
+			return wait
+		}
+	}
+	wait := time.Second << retry
+	if wait > claudeOverloadMaxWait {
+		return claudeOverloadMaxWait
+	}
+	return wait
+}
+
+// sleepCtx waits for d or until ctx is cancelled, using the injected sleep when
+// present (tests) and a real timer otherwise.
+func (t usageLimitRetryTransport) sleepCtx(ctx context.Context, d time.Duration) error {
+	if t.sleep != nil {
+		return t.sleep(ctx, d)
+	}
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
 }
 
 // responseUsageLimited reports whether the upstream response means the current
@@ -2778,11 +2837,48 @@ func (t usageLimitRetryTransport) RoundTrip(req *http.Request) (*http.Response,
 	if accountID != "" {
 		tried[accountID] = struct{}{}
 	}
+	overloadRetries := 0
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
 		response, err := base.RoundTrip(attemptReq)
 		if err != nil || req.GetBody == nil || req.Context().Err() != nil {
 			return response, err
 		}
+		// Anthropic overload (529/5xx): retry the SAME account after a bounded
+		// backoff. Overload is API-wide, not account-specific, so no failover, no
+		// exhaustion-marking, and no failover-budget consumption. Once the small
+		// overload budget is spent the 5xx passes through and the client's own
+		// backoff takes over.
+		if t.provider == accounts.ProviderClaude && claudeOverloadStatus(response.StatusCode) {
+			if overloadRetries >= claudeOverloadMaxRetries {
+				return response, nil
+			}
+			wait := claudeOverloadBackoff(response.Header, overloadRetries)
+			overloadRetries++
+			if t.logger != nil {
+				t.logger.Warn("retrying after anthropic overload", "agent", t.agent, "session", t.session, "account", accountID, "method", t.method, "path", t.path, "upstream", t.upstream, "status", response.StatusCode, "wait", wait.String(), "overload_retry", overloadRetries, "max_overload_retries", claudeOverloadMaxRetries)
+			}
+			if response.Body != nil {
+				_ = response.Body.Close()
+			}
+			if sleepErr := t.sleepCtx(req.Context(), wait); sleepErr != nil {
+				return nil, sleepErr
+			}
+			body, bodyErr := req.GetBody()
+			if bodyErr != nil {
+				return nil, bodyErr
+			}
+			// Preserve the CURRENT attempt's headers: after an earlier account
+			// failover attemptReq carries that account's auth, and cloning from the
+			// original req would silently revert to the first account.
+			currentHeader := attemptReq.Header.Clone()
+			attemptReq = req.Clone(req.Context())
+			attemptReq.Body = body
+			attemptReq.GetBody = req.GetBody
+			attemptReq.ContentLength = req.ContentLength
+			attemptReq.Header = currentHeader
+			attempt-- // overload retries do not consume the account-failover budget
+			continue
+		}
 		usageLimited, inspectErr := t.responseUsageLimited(response)
 		if inspectErr != nil {
 			if t.logger != nil {