Merge pull request #1 from hrathina/fix/rhoaieng-59039-rhai-e2e-v3

Fix/rhoaieng 59039 rhai e2e v3

Author: robert-bell

pkg/controller/trainjob_controller.go

@@ -144,22 +144,23 @@ func (r *TrainJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 		err = errors.Join(err, statusErr)
 	}
 
-	if deadlineResult, deadlineErr := r.reconcileDeadline(ctx, &trainJob); deadlineErr != nil || deadlineResult.RequeueAfter > 0 {
-		if !equality.Semantic.DeepEqual(&trainJob.Status, &prevTrainJob.Status) {
-			return deadlineResult, errors.Join(err, r.client.Status().Patch(ctx, &trainJob, client.MergeFrom(prevTrainJob)))
-		}
-		return deadlineResult, errors.Join(err, deadlineErr)
-	}
+	deadlineResult, deadlineErr := r.reconcileDeadline(ctx, &trainJob)
+	err = errors.Join(err, deadlineErr)
 
 	if !equality.Semantic.DeepEqual(&trainJob.Status, prevTrainJob.Status) {
 		// TODO(astefanutti): Consider using SSA once controller-runtime client has SSA support
 		// for sub-resources. See: https://github.com/kubernetes-sigs/controller-runtime/issues/3183
-		return ctrl.Result{}, errors.Join(err, r.client.Status().Patch(ctx, &trainJob, client.MergeFrom(prevTrainJob)))
+		if statusErr := r.client.Status().Patch(ctx, &trainJob, client.MergeFrom(prevTrainJob)); statusErr != nil {
+			return ctrl.Result{}, errors.Join(err, statusErr)
+		}
 	}
 
-	// RHAI progression tracking (use APIReader to avoid pod watches)
-	result, progressionErr := progression.ReconcileProgression(ctx, r.client, r.apiReader, log, &trainJob)
-	return result, errors.Join(err, progressionErr)
+	// RHAI progression tracking
+	result, _ := progression.ReconcileProgression(ctx, r.client, r.apiReader, log, &trainJob)
+	if deadlineResult.RequeueAfter > 0 && (result.RequeueAfter == 0 || deadlineResult.RequeueAfter < result.RequeueAfter) {
+		return deadlineResult, err
+	}
+	return result, err
 }
 
 func (r *TrainJobReconciler) reconcileObjects(ctx context.Context, runtime jobruntimes.Runtime, trainJob *trainer.TrainJob) error {

pkg/rhai/progression/progression.go

@@ -721,6 +721,11 @@ func ReconcileProgression(ctx context.Context, c client.Client, reader client.Re
 		return ctrl.Result{}, nil
 	}
 
+	// Re-fetch from API server to get latest status before patching annotations
+	if err := reader.Get(ctx, client.ObjectKeyFromObject(trainJob), trainJob); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
 	isRunning := !meta.IsStatusConditionTrue(trainJob.Status.Conditions, trainer.TrainJobSuspended) &&
 		!meta.IsStatusConditionTrue(trainJob.Status.Conditions, trainer.TrainJobComplete) &&
 		!meta.IsStatusConditionTrue(trainJob.Status.Conditions, trainer.TrainJobFailed)