feat: add HTTP 429 rate limit handling with retry support

- Automatically retry on HTTP 429 (Too Many Requests) errors
- Respect Retry-After header when provided by the API
- Fall back to exponential backoff if no Retry-After header
- Add tests to verify rate limit handling works correctly
- Prevents provider failures when hitting API rate limits

Author: Trozz

internal/client/client.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -21,6 +22,20 @@ type Client struct {
 	httpClient *http.Client
 }
 
+// RateLimitError represents a 429 rate limit error with optional Retry-After information
+type RateLimitError struct {
+	StatusCode int
+	Message    string
+	RetryAfter string // Can be seconds or HTTP-date
+}
+
+func (e *RateLimitError) Error() string {
+	if e.RetryAfter != "" {
+		return fmt.Sprintf("HTTP %d: %s (Retry-After: %s)", e.StatusCode, e.Message, e.RetryAfter)
+	}
+	return fmt.Sprintf("HTTP %d: %s", e.StatusCode, e.Message)
+}
+
 // NewClient creates a new Pocket-ID API client
 func NewClient(baseURL, apiToken string, skipTLSVerify bool, timeout int64) (*Client, error) {
 	if baseURL == "" {
@@ -58,9 +73,24 @@ func (c *Client) doRequestWithContext(ctx context.Context, method, endpoint stri
 	var lastErr error
 
 	for attempt := 0; attempt <= maxRetries; attempt++ {
+		var backoff time.Duration
+
 		if attempt > 0 {
 			// Exponential backoff: 1s, 2s, 4s
-			backoff := time.Duration(1<<(attempt-1)) * time.Second
+			backoff = time.Duration(1<<(attempt-1)) * time.Second
+
+			// Special handling for rate limit errors to respect Retry-After header
+			if rateLimitErr, ok := lastErr.(*RateLimitError); ok && rateLimitErr.RetryAfter != "" {
+				retryAfterSeconds := parseRetryAfter(rateLimitErr.RetryAfter)
+				if retryAfterSeconds > 0 {
+					backoff = time.Duration(retryAfterSeconds) * time.Second
+					tflog.Info(ctx, "Rate limited, using Retry-After header", map[string]interface{}{
+						"retry_after":     rateLimitErr.RetryAfter,
+						"backoff_seconds": retryAfterSeconds,
+					})
+				}
+			}
+
 			tflog.Debug(ctx, "Retrying request after backoff", map[string]interface{}{
 				"attempt": attempt,
 				"backoff": backoff.String(),
@@ -120,6 +150,16 @@ func isRetryableError(err error) bool {
 		return true
 	}
 
+	// 429 Too Many Requests is retryable (rate limiting)
+	if strings.Contains(errStr, "HTTP 429") {
+		return true
+	}
+
+	// Check if it's a RateLimitError
+	if _, ok := err.(*RateLimitError); ok {
+		return true
+	}
+
 	return false
 }
 
@@ -199,18 +239,56 @@ func (c *Client) doSingleRequest(ctx context.Context, method, endpoint string, b
 				"status_code": resp.StatusCode,
 				"raw_body":    string(respBody),
 			})
+			// Handle rate limit errors with Retry-After header
+			if resp.StatusCode == 429 {
+				retryAfter := resp.Header.Get("Retry-After")
+				return nil, &RateLimitError{
+					StatusCode: resp.StatusCode,
+					Message:    string(respBody),
+					RetryAfter: retryAfter,
+				}
+			}
 			return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody))
 		}
 		tflog.Error(ctx, "API Error", map[string]interface{}{
 			"status_code": resp.StatusCode,
 			"error":       errResp.Error,
 		})
+		// Handle rate limit errors with Retry-After header
+		if resp.StatusCode == 429 {
+			retryAfter := resp.Header.Get("Retry-After")
+			return nil, &RateLimitError{
+				StatusCode: resp.StatusCode,
+				Message:    errResp.Error,
+				RetryAfter: retryAfter,
+			}
+		}
 		return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, errResp.Error)
 	}
 
 	return respBody, nil
 }
 
+// parseRetryAfter parses the Retry-After header value
+// It can be either a delay in seconds or an HTTP-date
+func parseRetryAfter(retryAfter string) int {
+	// First try to parse as integer seconds
+	if seconds, err := strconv.Atoi(retryAfter); err == nil && seconds > 0 {
+		return seconds
+	}
+
+	// Try to parse as HTTP-date
+	if t, err := http.ParseTime(retryAfter); err == nil {
+		delay := time.Until(t).Seconds()
+		if delay > 0 {
+			return int(delay)
+		}
+	}
+
+	// Default to 60 seconds if we can't parse
+	return 60
+}
+
 // OIDC Client methods
 
 // CreateClient creates a new OIDC client

internal/client/client_test.go

@@ -518,3 +518,71 @@ func TestClient_UpdateClientAllowedUserGroups(t *testing.T) {
 	err = c.UpdateClientAllowedUserGroups("test-client-id", []string{"group1", "group2"})
 	assert.NoError(t, err)
 }
+
+func TestClient_RateLimitHandling(t *testing.T) {
+	attempts := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts++
+		if attempts < 3 {
+			// Simulate rate limit for first 2 attempts
+			w.Header().Set("Content-Type", "application/json")
+			w.Header().Set("Retry-After", "1") // 1 second retry
+			w.WriteHeader(http.StatusTooManyRequests)
+			fmt.Fprint(w, `{"error": "Rate limit exceeded"}`)
+			return
+		}
+		// Success on third attempt
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(&client.OIDCClient{
+			ID:   "test-client-id",
+			Name: "Test Client",
+		})
+	}))
+	defer server.Close()
+
+	c, err := client.NewClient(server.URL, "test-token", false, 30)
+	require.NoError(t, err)
+
+	start := time.Now()
+	result, err := c.GetClient("test-client-id")
+	elapsed := time.Since(start)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+	assert.Equal(t, "test-client-id", result.ID)
+	assert.Equal(t, "Test Client", result.Name)
+	assert.Equal(t, 3, attempts)
+	// Should have waited at least 2 seconds (2 retries with 1 second each)
+	assert.True(t, elapsed >= 2*time.Second, "Expected at least 2 seconds elapsed, got %v", elapsed)
+}
+
+func TestClient_RateLimitWithoutRetryAfter(t *testing.T) {
+	attempts := 0
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts++
+		if attempts < 2 {
+			// Simulate rate limit without Retry-After header
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusTooManyRequests)
+			fmt.Fprint(w, `{"error": "Rate limit exceeded"}`)
+			return
+		}
+		// Success on second attempt
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(&client.OIDCClient{
+			ID:   "test-client-id",
+			Name: "Test Client",
+		})
+	}))
+	defer server.Close()
+
+	c, err := client.NewClient(server.URL, "test-token", false, 30)
+	require.NoError(t, err)
+
+	result, err := c.GetClient("test-client-id")
+
+	assert.NoError(t, err)
+	assert.NotNil(t, result)
+	assert.Equal(t, "test-client-id", result.ID)
+	assert.Equal(t, 2, attempts)
+}