fix: add retry logic for one time instruction

Author: pratikjagrut

Diff for: pkg/applyinator/applyinator.go

@@ -23,6 +23,8 @@ import (
 	"github.com/rancher/system-agent/pkg/prober"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
 )
 
 type Applyinator struct {
@@ -266,20 +268,51 @@ func (a *Applyinator) Apply(ctx context.Context, input ApplyInput) (ApplyOutput,
 		}
 
 		oneTimeApplySucceeded := true
+		customRetry := wait.Backoff{
+			Steps:    5,
+			Duration: 2 * time.Second,
+			Factor:   2.0,
+			Jitter:   0.1,
+		}
+
 		for index, instruction := range input.CalculatedPlan.Plan.OneTimeInstructions {
-			logrus.Debugf("[Applyinator] Executing instruction %d attempt %d for plan %s", index, input.OneTimeInstructionAttempts, input.CalculatedPlan.Checksum)
+			logrus.Debugf("[Applyinator] Executing instruction %d for plan %s", index, input.CalculatedPlan.Checksum)
+
 			executionInstructionDir := filepath.Join(executionDir, input.CalculatedPlan.Checksum+"_"+strconv.Itoa(index))
 			prefix := input.CalculatedPlan.Checksum + "_" + strconv.Itoa(index)
-			executeOutput, _, exitCode, err := a.execute(ctx, prefix, executionInstructionDir, instruction.CommonInstruction, true, input.OneTimeInstructionAttempts)
+
+			var executeOutput []byte
+			var exitCode, retryCount int
+
+			err := retry.OnError(customRetry, func(err error) bool {
+				retryCount++
+				logrus.Debugf("[Applyinator] Execution failed: %v", err)
+				logrus.Debugf("[Applyinator] (attempt %d) Retrying instruction %d for plan %s", retryCount, index, input.CalculatedPlan.Checksum)
+
+				// Always retry regardless of error type
+				return true
+			}, func() error {
+				var retryErr error
+				executeOutput, _, exitCode, retryErr = a.execute(ctx, prefix, executionInstructionDir, instruction.CommonInstruction, true, input.OneTimeInstructionAttempts)
+				if retryErr != nil || exitCode != 0 {
+					logrus.Debugf("[Applyinator] Execution failed for instruction %d: %v (exit code: %d)", index, retryErr, exitCode)
+					return retryErr
+				}
+				return nil
+			})
+
+			// Final evaluation after retries
 			if err != nil || exitCode != 0 {
 				logrus.Errorf("error executing instruction %d: %v", index, err)
 				oneTimeApplySucceeded = false
 			}
+
 			if instruction.Name == "" && instruction.SaveOutput {
 				logrus.Errorf("instruction does not have a name set, cannot save output data")
 			} else if instruction.SaveOutput {
 				executionOutputs[instruction.Name] = executeOutput
 			}
+
 			// If we have failed to apply our one-time instructions, we need to break in order to stop subsequent instructions from executing.
 			if !oneTimeApplySucceeded {
 				break