Introduced Rate Limiter and Retry Options (429 response) for all platform clients

Author: Reinhard-Pilz-Dynatrace

dynatrace/api/automation/business_calendars/service.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/url"
+	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api"
 	tfrest "github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest"
@@ -51,7 +52,8 @@ func (me *service) client(ctx context.Context) *automation.Client {
 		TokenURL:     me.credentials.OAuth.TokenURL,
 	})
 	u, _ := url.Parse(me.credentials.OAuth.EnvironmentURL)
-	restClient := rest.NewClient(u, httpClient)
+
+	restClient := rest.NewClient(u, httpClient, rest.WithRateLimiter(), rest.WithRetryOptions(&rest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: rest.RetryIfTooManyRequests}))
 	restClient.SetHeader("User-Agent", "Dynatrace Terraform Provider")
 	return automation.NewClient(restClient)
 }

dynatrace/api/automation/scheduling_rules/service.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/url"
+	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api"
 	tfrest "github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest"
@@ -51,7 +52,7 @@ func (me *service) client(ctx context.Context) *automation.Client {
 		TokenURL:     me.credentials.OAuth.TokenURL,
 	})
 	u, _ := url.Parse(me.credentials.OAuth.EnvironmentURL)
-	restClient := rest.NewClient(u, httpClient)
+	restClient := rest.NewClient(u, httpClient, rest.WithRateLimiter(), rest.WithRetryOptions(&rest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: rest.RetryIfTooManyRequests}))
 	restClient.SetHeader("User-Agent", "Dynatrace Terraform Provider")
 	return automation.NewClient(restClient)
 }

dynatrace/api/automation/workflows/service.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"net/url"
+	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api"
 	tfrest "github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest"
@@ -53,7 +54,7 @@ func (me *service) client(ctx context.Context) *automation.Client {
 		TokenURL:     me.credentials.OAuth.TokenURL,
 	})
 	u, _ := url.Parse(me.credentials.OAuth.EnvironmentURL)
-	restClient := rest.NewClient(u, httpClient)
+	restClient := rest.NewClient(u, httpClient, rest.WithRateLimiter(), rest.WithRetryOptions(&rest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: rest.RetryIfTooManyRequests}))
 	restClient.SetHeader("User-Agent", "Dynatrace Terraform Provider")
 	return automation.NewClient(restClient)
 }

dynatrace/api/grail/segments/service.go

@@ -20,11 +20,13 @@ package segments
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/settings"
 	segmentsapi "github.com/dynatrace/dynatrace-configuration-as-code-core/api"
+	crest "github.com/dynatrace/dynatrace-configuration-as-code-core/api/rest"
 	"github.com/dynatrace/dynatrace-configuration-as-code-core/clients"
 	segmentsclient "github.com/dynatrace/dynatrace-configuration-as-code-core/clients/segments"
 	"golang.org/x/oauth2/clientcredentials"
@@ -51,7 +53,9 @@ func (me *service) client(ctx context.Context) *segmentsclient.Client {
 			ClientSecret: me.credentials.OAuth.ClientSecret,
 			TokenURL:     me.credentials.OAuth.TokenURL,
 		}).
-		WithUserAgent("Dynatrace Terraform Provider")
+		WithUserAgent("Dynatrace Terraform Provider").
+		WithRetryOptions(&crest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: crest.RetryIfTooManyRequests}).
+		WithRateLimiter(true)
 
 	segmentClient, _ := clientsFactory.SegmentsClient(ctx)
 	return segmentClient

dynatrace/api/openpipeline/service.go

@@ -20,12 +20,14 @@ package openpipeline
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api/automation/httplog"
 	openpipeline "github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api/openpipeline/settings"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/settings"
+	crest "github.com/dynatrace/dynatrace-configuration-as-code-core/api/rest"
 	"github.com/dynatrace/dynatrace-configuration-as-code-core/clients"
 	caclib "github.com/dynatrace/dynatrace-configuration-as-code-core/clients/openpipeline"
 	"golang.org/x/oauth2/clientcredentials"
@@ -66,7 +68,9 @@ func (s *service) createClient(ctx context.Context) (*caclib.Client, error) {
 			ClientSecret: s.credentials.OAuth.ClientSecret,
 			TokenURL:     s.credentials.OAuth.TokenURL,
 		}).
-		WithHTTPListener(httplog.HTTPListener)
+		WithHTTPListener(httplog.HTTPListener).
+		WithRetryOptions(&crest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: crest.RetryIfTooManyRequests}).
+		WithRateLimiter(true)
 
 	return factory.OpenPipelineClient(ctx)
 }

dynatrace/api/platform/buckets/service.go

@@ -26,13 +26,13 @@ import (
 	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api"
+	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api/automation/httplog"
+	buckets "github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api/platform/buckets/settings"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/settings"
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/shutdown"
-
-	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api/automation/httplog"
-	buckets "github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/api/platform/buckets/settings"
 	coreapi "github.com/dynatrace/dynatrace-configuration-as-code-core/api"
+	crest "github.com/dynatrace/dynatrace-configuration-as-code-core/api/rest"
 	"github.com/dynatrace/dynatrace-configuration-as-code-core/clients"
 	bucket "github.com/dynatrace/dynatrace-configuration-as-code-core/clients/buckets"
 	"golang.org/x/oauth2/clientcredentials"
@@ -55,7 +55,10 @@ func (me *service) client(ctx context.Context) *bucket.Client {
 			ClientSecret: me.credentials.OAuth.ClientSecret,
 			TokenURL:     me.credentials.OAuth.TokenURL,
 		}).
-		WithHTTPListener(httplog.HTTPListener)
+		WithHTTPListener(httplog.HTTPListener).
+		WithRetryOptions(&crest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: crest.RetryIfTooManyRequests}).
+		WithRateLimiter(true)
+
 	bucketClient, _ := factory.BucketClient(ctx)
 	return bucketClient
 }

dynatrace/rest/platform_request.go

@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/dynatrace-oss/terraform-provider-dynatrace/dynatrace/rest/logging"
 	"github.com/dynatrace/dynatrace-configuration-as-code-core/api/rest"
@@ -47,15 +48,19 @@ func createPlatformRestClient(u string, httpClient *http.Client) (*rest.Client,
 		return nil, fmt.Errorf("failed to parse URL %q: %w", u, err)
 	}
 
-	opts := []rest.Option{rest.WithHTTPListener(logging.HTTPListener("plat/tok"))}
+	opts := []rest.Option{
+		rest.WithHTTPListener(logging.HTTPListener("plat/tok")),
+		rest.WithRateLimiter(),
+		rest.WithRetryOptions(&rest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: rest.RetryIfTooManyRequests}),
+	}
 
 	restClient := rest.NewClient(parsedURL, httpClient, opts...)
 	restClient.SetHeader("User-Agent", "Dynatrace Terraform Provider")
 
 	return restClient, nil
 }
 
-func platformClient(platformURL string, credentials *Credentials) (*rest.Client, error) {
+func platformClient(ctx context.Context, platformURL string, credentials *Credentials) (*rest.Client, error) {
 	clientID := credentials.OAuth.ClientID
 	clientSecret := credentials.OAuth.ClientSecret
 
@@ -72,11 +77,13 @@ func platformClient(platformURL string, credentials *Credentials) (*rest.Client,
 		client, err = createPlatformRestClient(platformURL, NewBearerTokenBasedClient(credentials.OAuth.PlatformToken))
 	} else if credentials.ContainsOAuth() {
 		factory := clients.Factory()
+		factory = factory.WithRetryOptions(&rest.RetryOptions{MaxRetries: 30, DelayAfterRetry: 10 * time.Second, ShouldRetryFunc: rest.RetryIfTooManyRequests})
+		factory = factory.WithRateLimiter(true)
 		factory = factory.WithUserAgent("Dynatrace Terraform Provider")
 		factory = factory.WithPlatformURL(platformURL)
 		factory = factory.WithHTTPListener(logging.HTTPListener("platform"))
 		factory = factory.WithOAuthCredentials(clientcredentials.Config{ClientID: clientID, ClientSecret: clientSecret, TokenURL: evalTokenURL(platformURL)})
-		client, err = factory.CreatePlatformClient(context.Background())
+		client, err = factory.CreatePlatformClient(ctx)
 	} else {
 		return nil, errors.New("neither oauth credentials nor platform token present")
 	}
@@ -98,7 +105,7 @@ func (me *platform_request) Finish(optionalTarget ...any) error {
 
 	platformURL := me.evalPlatformURL()
 
-	client, err := platformClient(platformURL, me.client.Credentials())
+	client, err := platformClient(me.ctx, platformURL, me.client.Credentials())
 	if err != nil {
 		return err
 	}