[resty] [retry] Retry-After w/ Jitter Support and Slower Status Checks

kaleido/kaleidobase/providerdata.go

@@ -17,9 +17,11 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"math/rand/v2"
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/go-resty/resty/v2"
@@ -121,9 +123,16 @@ func NewProviderData(logCtx context.Context, conf *ProviderModel) *ProviderData
 			}
 			return false
 		}).
-		SetRetryCount(3).
-		SetRetryWaitTime(1 * time.Second).
-		SetRetryMaxWaitTime(10 * time.Second).
+		SetRetryCount(5).
+		SetRetryAfter(func(c *resty.Client, r *resty.Response) (time.Duration, error) {
+			tflog.Debug(logCtx, fmt.Sprintf("retryAfter: %s", r.Header().Get("Retry-After")))
+			retryAfter, err := strconv.ParseFloat(r.Header().Get("Retry-After"), 64) // decimal seconds
+			if err != nil {
+				retryAfter = 1.0
+			}
+			jitter := time.Duration(rand.NormFloat64() * float64(5*time.Second)) // up to 5 second random jitter to account for 5 concurrent connections all retrying up to 5 times
+			return time.Duration(retryAfter*float64(time.Second)) + jitter, nil
+		}).
 		SetBaseURL(platformAPI)
 	if platformUsername != "" && platformPassword != "" {
 		platform = platform.SetBasicAuth(platformUsername, platformPassword)

kaleido/kaleidobase/retry.go

@@ -29,8 +29,8 @@ type CustomRetry struct {
 }
 
 var Retry = &CustomRetry{
-	InitialDelay: 500 * time.Millisecond,
-	MaximumDelay: 5 * time.Second,
+	InitialDelay: 5 * time.Second,
+	MaximumDelay: 30 * time.Second,
 	Factor:       2.0,
 }
 

kaleido/platform/cms_action_base.go

@@ -68,8 +68,11 @@ func (r *cms_action_baseResource) waitForActionStatus(ctx context.Context, data
 	path := r.apiPath(data)
 	cancelInfo := APICancelInfo()
 	_ = kaleidobase.Retry.Do(ctx, fmt.Sprintf("build-check %s", path), func(attempt int) (retry bool, err error) {
-		ok, _ := r.apiRequest(ctx, http.MethodGet, path, nil, &api, diagnostics, cancelInfo)
+		ok, statusCode := r.apiRequest(ctx, http.MethodGet, path, nil, &api, diagnostics, cancelInfo)
 		if !ok {
+			if statusCode == 429 {
+				return true, fmt.Errorf("rate limit exceeded")
+			}
 			return false, fmt.Errorf("action-check failed") // already set in diag
 		}
 		cancelInfo.CancelInfo = fmt.Sprintf("waiting for completion - status: %s", api.OutputBase().Status)

kaleido/platform/cms_build.go

@@ -413,8 +413,11 @@ func (r *cms_buildResource) waitForBuildStatus(ctx context.Context, data *CMSBui
 	path := r.apiPath(data)
 	cancelInfo := APICancelInfo()
 	_ = kaleidobase.Retry.Do(ctx, fmt.Sprintf("build-check %s", path), func(attempt int) (retry bool, err error) {
-		ok, _ := r.apiRequest(ctx, http.MethodGet, path, nil, &api, diagnostics, cancelInfo)
+		ok, statusCode := r.apiRequest(ctx, http.MethodGet, path, nil, &api, diagnostics, cancelInfo)
 		if !ok {
+			if statusCode == 429 {
+				return true, fmt.Errorf("rate limit exceeded")
+			}
 			return false, fmt.Errorf("build-check failed") // already set in diag
 		}
 		cancelInfo.CancelInfo = fmt.Sprintf("(waiting for completion - status: %s)", api.Status)

kaleido/platform/common.go

@@ -310,8 +310,11 @@ func (r *commonResource) waitForReadyStatus(ctx context.Context, path string, di
 	cancelInfo := APICancelInfo()
 	_ = kaleidobase.Retry.Do(ctx, fmt.Sprintf("ready-check %s", path), func(attempt int) (retry bool, err error) {
 		var res statusResponse
-		ok, _ := r.apiRequest(ctx, http.MethodGet, path, nil, &res, diagnostics, cancelInfo)
+		ok, statusCode := r.apiRequest(ctx, http.MethodGet, path, nil, &res, diagnostics, cancelInfo)
 		if !ok {
+			if statusCode == 429 {
+				return true, fmt.Errorf("rate limit exceeded")
+			}
 			return false, fmt.Errorf("ready-check failed") // already set in diag
 		}
 		cancelInfo.CancelInfo = fmt.Sprintf("(waiting for ready - status: %s)", res.Status)

kaleido/platform/firefly_registration.go

@@ -131,8 +131,11 @@ func (r *firefly_registrationResource) ensureRegistered(ctx context.Context, dat
 	orgSubmitted := false
 	cancelInfo := APICancelInfo()
 	_ = kaleidobase.Retry.Do(ctx, "register", func(attempt int) (retry bool, err error) {
-		ok, _ := r.apiRequest(ctx, http.MethodGet, r.apiPath(data, "status"), nil, &status, diagnostics, cancelInfo)
+		ok, statusCode := r.apiRequest(ctx, http.MethodGet, r.apiPath(data, "status"), nil, &status, diagnostics, cancelInfo)
 		if !ok {
+			if statusCode == 429 {
+				return true, fmt.Errorf("rate limit exceeded")
+			}
 			return false, fmt.Errorf("status-check failed") // already set in diag
 		}
 		if registered := status.toData(data, diagnostics); registered {