boundary issue

Signed-off-by: yuteng <a08h0283@gmail.com>

Author: 0yukali0

flytepropeller/pkg/controller/nodes/dynamic/handler.go

@@ -96,9 +96,10 @@ func (d dynamicNodeTaskNodeHandler) produceDynamicWorkflow(ctx context.Context,
 	dCtx, err := d.buildContextualDynamicWorkflow(ctx, nCtx)
 	if err != nil {
 		if stdErrors.IsCausedBy(err, utils.ErrorCodeUser) {
+			isPermanent := !stdErrors.IsCausedBy(err, utils.ErrorCodeSystem)
 			return handler.DoTransition(handler.TransitionTypeEphemeral,
 				handler.PhaseInfoFailure(core.ExecutionError_USER, "DynamicWorkflowBuildFailed", err.Error(), nil),
-			), handler.DynamicNodeState{Phase: v1alpha1.DynamicNodePhaseFailing, Reason: err.Error(), IsFailurePermanent: true}, nil
+			), handler.DynamicNodeState{Phase: v1alpha1.DynamicNodePhaseFailing, Reason: err.Error(), IsFailurePermanent: isPermanent}, nil
 		}
 		return handler.Transition{}, handler.DynamicNodeState{}, err
 	}
@@ -123,9 +124,10 @@ func (d dynamicNodeTaskNodeHandler) handleDynamicSubNodes(ctx context.Context, n
 	dCtx, err := d.buildContextualDynamicWorkflow(ctx, nCtx)
 	if err != nil {
 		if stdErrors.IsCausedBy(err, utils.ErrorCodeUser) {
+			isPermanent := !stdErrors.IsCausedBy(err, utils.ErrorCodeSystem)
 			return handler.DoTransition(handler.TransitionTypeEphemeral,
 				handler.PhaseInfoFailure(core.ExecutionError_USER, "DynamicWorkflowBuildFailed", err.Error(), nil),
-			), handler.DynamicNodeState{Phase: v1alpha1.DynamicNodePhaseFailing, Reason: err.Error(), IsFailurePermanent: true}, nil
+			), handler.DynamicNodeState{Phase: v1alpha1.DynamicNodePhaseFailing, Reason: err.Error(), IsFailurePermanent: isPermanent}, nil
 		}
 		// Mostly a system error or unknown
 		return handler.Transition{}, handler.DynamicNodeState{}, err

flytepropeller/pkg/controller/workflow/executor.go

@@ -494,7 +494,10 @@ func (c *workflowExecutor) HandleAbortedWorkflow(ctx context.Context, w *v1alpha
 			return err
 		}
 
-		if w.Status.FailedAttempts > maxRetries {
+		if w.Status.FailedAttempts >= maxRetries {
+			if err != nil {
+				logger.Warningf(ctx, "Discarding cleanup error because system retries are exhausted: %v", err)
+			}
 			err = errors.Errorf(errors.RuntimeExecutionError, w.GetID(), "max number of system retry attempts [%d/%d] exhausted. Last known status message: %v", w.Status.FailedAttempts, maxRetries, w.Status.Message)
 		}