fix(tests): handle autoscaler policy version conflicts with wait.Backoff retry

 When node template and autoscaler policies are updated concurrently
 in the same terraform apply, the API rejects updates with:
"node template has changed since the policies have been retrieved, refetch the policies and perform the update again"

Author: kasia-kujawa

castai/resource_autoscaler.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	jsonpatch "github.com/evanphx/json-patch"
@@ -17,6 +18,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 	"github.com/mitchellh/mapstructure"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/castai/terraform-provider-castai/castai/sdk"
 	"github.com/castai/terraform-provider-castai/castai/types"
@@ -805,23 +807,73 @@ func updateAutoscalerPolicies(ctx context.Context, data *schema.ResourceData, me
 		return nil
 	}
 
-	policies, err := getChangedPolicies(ctx, data, meta, clusterId)
-	if err != nil {
-		return err
+	// Define the update operation that will be executed with retry logic
+	updatePolicies := func() error {
+		policies, err := getChangedPolicies(ctx, data, meta, clusterId)
+		if err != nil {
+			return err
+		}
+
+		if policies == nil {
+			log.Printf("[DEBUG] changed policies json not calculated. Skipping autoscaler policies changes")
+			return nil
+		}
+
+		changedPoliciesJSON := string(policies)
+		if changedPoliciesJSON == "" {
+			log.Printf("[DEBUG] changed policies json not found. Skipping autoscaler policies changes")
+			return nil
+		}
+
+		return upsertPolicies(ctx, meta, clusterId, changedPoliciesJSON)
 	}
 
-	if policies == nil {
-		log.Printf("[DEBUG] changed policies json not calculated. Skipping autoscaler policies changes")
+	// Try to update policies immediately without any delay.
+	err := updatePolicies()
+	if err == nil {
 		return nil
 	}
 
-	changedPoliciesJSON := string(policies)
-	if changedPoliciesJSON == "" {
-		log.Printf("[DEBUG] changed policies json not found. Skipping autoscaler policies changes")
-		return nil
+	// Check if error is retryable (node template version conflict)
+	if !isNodeTemplateVersionConflict(err) {
+		return err // Non-retryable error
+	}
+
+	// Fall back to exponential backoff retry only if version conflict occurred.
+	log.Printf("[INFO] Node template version conflict detected, will retry with exponential backoff: %v", err)
+
+	// Exponential backoff configuration
+	backoff := wait.Backoff{
+		Duration: 100 * time.Millisecond,
+		Factor:   2.0,
+		Jitter:   0.1,
+		Steps:    5,
+		Cap:      2 * time.Second,
 	}
 
-	return upsertPolicies(ctx, meta, clusterId, changedPoliciesJSON)
+	retryErr := wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (done bool, err error) {
+		err = updatePolicies()
+		if err == nil {
+			return true, nil // Success - stop retrying
+		}
+
+		// Check if error is retryable
+		if !isNodeTemplateVersionConflict(err) {
+			return false, err // Non-retryable error - stop with error
+		}
+
+		log.Printf("[DEBUG] Retry failed with version conflict: %v", err)
+		return false, nil // Retryable error - continue retrying
+	})
+
+	if retryErr != nil {
+		if wait.Interrupted(retryErr) {
+			return fmt.Errorf("timeout waiting for autoscaler policy update after version conflicts: %w", err)
+		}
+		return retryErr
+	}
+
+	return nil
 }
 
 func upsertPolicies(ctx context.Context, meta interface{}, clusterId string, changedPoliciesJSON string) error {
@@ -835,6 +887,15 @@ func upsertPolicies(ctx context.Context, meta interface{}, clusterId string, cha
 	return nil
 }
 
+// isNodeTemplateVersionConflict checks if the error is due to version mismatch
+func isNodeTemplateVersionConflict(err error) bool {
+	if err == nil {
+		return false
+	}
+	errMsg := err.Error()
+	return strings.Contains(errMsg, "template has changed") || strings.Contains(errMsg, "refetch the policies")
+}
+
 func readAutoscalerPolicies(ctx context.Context, data *schema.ResourceData, meta interface{}) error {
 	log.Printf("[INFO] AUTOSCALER policies get call start")
 	defer log.Printf("[INFO] AUTOSCALER policies get call end")

go.mod

@@ -4,9 +4,8 @@ go 1.24.0
 
 require (
 	github.com/cenkalti/backoff/v4 v4.1.3
-	github.com/evanphx/json-patch v4.9.0+incompatible
+	github.com/evanphx/json-patch v4.12.0+incompatible
 	github.com/golang/mock v1.5.0
-	github.com/google/martian v2.1.0+incompatible
 	github.com/google/uuid v1.6.0
 	github.com/gruntwork-io/terratest v0.40.18
 	github.com/hashicorp/go-cty v1.5.0
@@ -22,9 +21,10 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/oapi-codegen/runtime v1.1.1
 	github.com/samber/lo v1.49.1
-	github.com/stretchr/testify v1.10.0
-	golang.org/x/crypto v0.41.0
+	github.com/stretchr/testify v1.11.1
+	golang.org/x/crypto v0.44.0
 	golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df
+	k8s.io/apimachinery v0.30.0
 )
 
 require (
@@ -39,6 +39,7 @@ require (
 	github.com/cloudflare/circl v1.6.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fatih/color v1.16.0 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
@@ -85,19 +86,22 @@ require (
 	github.com/zclconf/go-cty v1.16.3 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
-	golang.org/x/mod v0.26.0 // indirect
-	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
 	golang.org/x/oauth2 v0.30.0 // indirect
-	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/sys v0.35.0 // indirect
-	golang.org/x/text v0.28.0 // indirect
-	golang.org/x/tools v0.35.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect
+	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
 	google.golang.org/api v0.47.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
 	google.golang.org/grpc v1.75.1 // indirect
 	google.golang.org/protobuf v1.36.9 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
 )
 
 exclude github.com/satori/go.uuid v1.2.0