feat: retry IBM Cloud Global Catalog calls on HTTP 429

Add a shared DoWithRetry helper to handle IBM Cloud API rate
limiting (HTTP 429) using exponential backoff and honoring the
Retry-After header when present.

Apply the retry wrapper to GlobalCatalog client call sites:
- GetCatalogEntryWithContext
- ListCatalogEntriesWithContext
- GetPricing

Add unit tests covering:
- successful call without retry
- non-429 passthrough
- retry then success
- retry exhaustion
- context cancellation
- Retry-After handling without real sleep

Signed-off-by: Anand Nekkunti <anand.nekkunti@ibm.com>

Author: nekkunti

pkg/cloudprovider/ibm/catalog.go

@@ -85,7 +85,9 @@ func (c *GlobalCatalogClient) GetInstanceType(ctx context.Context, id string) (*
 		Include: core.StringPtr("*"), // Include all fields
 	}
 
-	entry, _, err := c.client.GetCatalogEntryWithContext(ctx, options)
+	entry, err := DoWithRetry(ctx, func() (*globalcatalogv1.CatalogEntry, *core.DetailedResponse, error) {
+		return c.client.GetCatalogEntryWithContext(ctx, options)
+	})
 	if err != nil {
 		return nil, fmt.Errorf("getting catalog entry: %w", err)
 	}
@@ -114,7 +116,9 @@ func (c *GlobalCatalogClient) ListInstanceTypes(ctx context.Context) ([]globalca
 			Limit:    &limit,
 		}
 
-		result, _, err := c.client.ListCatalogEntriesWithContext(ctx, options)
+		result, err := DoWithRetry(ctx, func() (*globalcatalogv1.EntrySearchResult, *core.DetailedResponse, error) {
+			return c.client.ListCatalogEntriesWithContext(ctx, options)
+		})
 		if err != nil {
 			return nil, fmt.Errorf("listing catalog entries: %w", err)
 		}
@@ -147,7 +151,9 @@ func (c *GlobalCatalogClient) GetPricing(ctx context.Context, catalogEntryID str
 			ID: &catalogEntryID,
 		}
 
-		pricingData, _, err := sdkClient.GetPricing(pricingOptions)
+		pricingData, err := DoWithRetry(ctx, func() (*globalcatalogv1.PricingGet, *core.DetailedResponse, error) {
+			return sdkClient.GetPricing(pricingOptions)
+		})
 		if err != nil {
 			return nil, fmt.Errorf("calling GetPricing API: %w", err)
 		}

pkg/cloudprovider/ibm/retry.go

@@ -0,0 +1,62 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package ibm
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/IBM/go-sdk-core/v5/core"
+)
+
+// DoWithRetry handles HTTP 429 rate limiting with exponential backoff.
+// It retries up to 5 times, respecting the Retry-After header if present.
+func DoWithRetry[T any](ctx context.Context, fn func() (T, *core.DetailedResponse, error)) (T, error) {
+	var zero T
+	backoff := 100 * time.Millisecond // initialBackoff
+	maxRetries := 5
+
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		result, response, err := fn()
+
+		// Success or non-rate-limit error
+		if response == nil || response.StatusCode != http.StatusTooManyRequests {
+			return result, err
+		}
+
+		// Use Retry-After for this sleep only; keep exponential backoff progression separate.
+		delay := backoff
+		if response.Headers != nil {
+			if ra := response.Headers.Get("Retry-After"); ra != "" {
+				if secs, parseErr := strconv.Atoi(ra); parseErr == nil && secs > 0 {
+					delay = time.Duration(secs) * time.Second
+				}
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return zero, ctx.Err()
+		case <-time.After(delay):
+			backoff = min(backoff*2, 30*time.Second) // maxBackoff 30 Sec
+		}
+	}
+
+	return zero, fmt.Errorf("rate limited after %d retries", maxRetries)
+}

pkg/cloudprovider/ibm/retry_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package ibm
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/IBM/go-sdk-core/v5/core"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDoWithRetry_SuccessFirstTry(t *testing.T) {
+	ctx := context.Background()
+	calls := 0
+	result, err := DoWithRetry(ctx, func() (string, *core.DetailedResponse, error) {
+		calls++
+		return "ok", &core.DetailedResponse{StatusCode: 200}, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "ok", result)
+	assert.Equal(t, 1, calls)
+}
+
+func TestDoWithRetry_Non429ErrorReturnsImmediately(t *testing.T) {
+	ctx := context.Background()
+	calls := 0
+	apiErr := assert.AnError
+	result, err := DoWithRetry(ctx, func() (string, *core.DetailedResponse, error) {
+		calls++
+		return "", &core.DetailedResponse{StatusCode: 500}, apiErr
+	})
+	require.Error(t, err)
+	assert.ErrorIs(t, err, apiErr)
+	assert.Equal(t, "", result)
+	assert.Equal(t, 1, calls)
+}
+
+func TestDoWithRetry_NilResponseReturnsImmediately(t *testing.T) {
+	ctx := context.Background()
+	calls := 0
+	result, err := DoWithRetry(ctx, func() (string, *core.DetailedResponse, error) {
+		calls++
+		return "ok", nil, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "ok", result)
+	assert.Equal(t, 1, calls)
+}
+
+func TestDoWithRetry_429ThenSuccess(t *testing.T) {
+	ctx := context.Background()
+	calls := 0
+	result, err := DoWithRetry(ctx, func() (int, *core.DetailedResponse, error) {
+		calls++
+		if calls == 1 {
+			return 0, &core.DetailedResponse{StatusCode: 429}, nil
+		}
+		return 42, &core.DetailedResponse{StatusCode: 200}, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, 42, result)
+	assert.Equal(t, 2, calls)
+}
+
+func TestDoWithRetry_ExhaustedRetries(t *testing.T) {
+	ctx := context.Background()
+	calls := 0
+	result, err := DoWithRetry(ctx, func() (string, *core.DetailedResponse, error) {
+		calls++
+		return "", &core.DetailedResponse{StatusCode: 429}, nil
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "rate limited after 5 retries")
+	assert.Equal(t, "", result)
+	assert.Equal(t, 5, calls)
+}
+
+func TestDoWithRetry_ContextCanceledDuringBackoff(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	calls := 0
+	done := make(chan struct{})
+	var result string
+	var err error
+	go func() {
+		result, err = DoWithRetry(ctx, func() (string, *core.DetailedResponse, error) {
+			calls++
+			return "", &core.DetailedResponse{StatusCode: 429}, nil
+		})
+		close(done)
+	}()
+	cancel()
+	<-done
+	require.Error(t, err)
+	assert.ErrorIs(t, err, context.Canceled)
+	assert.Equal(t, "", result)
+	assert.Equal(t, 1, calls)
+}
+
+func TestDoWithRetry_RespectsRetryAfterHeader(t *testing.T) {
+	ctx := context.Background()
+	calls := 0
+	headers := http.Header{}
+	headers.Set("Retry-After", "1")
+	result, err := DoWithRetry(ctx, func() (string, *core.DetailedResponse, error) {
+		calls++
+		if calls == 1 {
+			return "", &core.DetailedResponse{StatusCode: 429, Headers: headers}, nil
+		}
+		return "ok", &core.DetailedResponse{StatusCode: 200}, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "ok", result)
+	assert.Equal(t, 2, calls)
+}