fix: add retry logic for one time instruction

Author: pratikjagrut

pkg/applyinator/applyinator.go

@@ -23,6 +23,8 @@ import (
 	"github.com/rancher/system-agent/pkg/prober"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/retry"
 )
 
 type Applyinator struct {
@@ -100,6 +102,13 @@ const restartPendingInterlockFile = "restart-pending"
 const applyinatorActiveInterlockFile = "applyinator-active"
 const restartPendingTimeout = 5 * time.Minute // Wait a maximum of 5 minutes before force-applying a plan if a restart is pending.
 
+var customRetry = wait.Backoff{
+	Steps:    5,
+	Duration: 3 * time.Second,
+	Factor:   2.0,
+	Jitter:   0.1,
+}
+
 func NewApplyinator(workDir string, preserveWorkDir bool, appliedPlanDir, interlockDir string, imageUtil *image.Utility) *Applyinator {
 	return &Applyinator{
 		mu:              &sync.Mutex{},
@@ -266,20 +275,46 @@ func (a *Applyinator) Apply(ctx context.Context, input ApplyInput) (ApplyOutput,
 		}
 
 		oneTimeApplySucceeded := true
+
 		for index, instruction := range input.CalculatedPlan.Plan.OneTimeInstructions {
-			logrus.Debugf("[Applyinator] Executing instruction %d attempt %d for plan %s", index, input.OneTimeInstructionAttempts, input.CalculatedPlan.Checksum)
+			logrus.Debugf("[Applyinator] Executing instruction %d for plan %s", index, input.CalculatedPlan.Checksum)
+
 			executionInstructionDir := filepath.Join(executionDir, input.CalculatedPlan.Checksum+"_"+strconv.Itoa(index))
 			prefix := input.CalculatedPlan.Checksum + "_" + strconv.Itoa(index)
-			executeOutput, _, exitCode, err := a.execute(ctx, prefix, executionInstructionDir, instruction.CommonInstruction, true, input.OneTimeInstructionAttempts)
+
+			var executeOutput []byte
+			var exitCode, retryCount int
+
+			err := retry.OnError(customRetry, func(err error) bool {
+				retryCount++
+				logrus.Errorf("[Applyinator] Execution failed: %v", err)
+				logrus.Infof("[Applyinator] (attempt #%d) Applying one-time instructions for plan with checksum %s", retryCount, input.CalculatedPlan.Checksum)
+				logrus.Infof("[Applyinator] (attempt #%d) Retrying instruction %d for plan %s", retryCount, index, input.CalculatedPlan.Checksum)
+
+				// Always retry regardless of error type
+				return true
+			}, func() error {
+				var retryErr error
+				executeOutput, _, exitCode, retryErr = a.execute(ctx, prefix, executionInstructionDir, instruction.CommonInstruction, true, input.OneTimeInstructionAttempts)
+				if retryErr != nil || exitCode != 0 {
+					logrus.Debugf("[Applyinator] Execution failed for instruction %d: %v (exit code: %d)", index, retryErr, exitCode)
+					return retryErr
+				}
+				return nil
+			})
+
+			// Final evaluation after retries
 			if err != nil || exitCode != 0 {
 				logrus.Errorf("error executing instruction %d: %v", index, err)
 				oneTimeApplySucceeded = false
 			}
+
 			if instruction.Name == "" && instruction.SaveOutput {
 				logrus.Errorf("instruction does not have a name set, cannot save output data")
 			} else if instruction.SaveOutput {
 				executionOutputs[instruction.Name] = executeOutput
 			}
+
 			// If we have failed to apply our one-time instructions, we need to break in order to stop subsequent instructions from executing.
 			if !oneTimeApplySucceeded {
 				break