fix: cron, retry on fail submission

Signed-off-by: Tianchu Zhao <[email protected]>

Author: tczhao

workflow/cron/operator.go

@@ -100,7 +100,21 @@ func (woc *cronWfOperationCtx) run(ctx context.Context, scheduledRuntime time.Ti
 
 	wf := common.ConvertCronWorkflowToWorkflowWithProperties(woc.cronWf, getChildWorkflowName(woc.cronWf.Name, scheduledRuntime), scheduledRuntime)
 
-	runWf, err := util.SubmitWorkflow(ctx, woc.wfClient, woc.wfClientset, woc.cronWf.Namespace, wf, &v1alpha1.SubmitOpts{})
+	var runWf *v1alpha1.Workflow
+	// DefaultRetry is k8s default retry, also used by k8s client, retry 5 times
+	err = waitutil.Backoff(retry.DefaultRetry, func() (bool, error) {
+		var err error
+		runWf, err = util.SubmitWorkflow(ctx, woc.wfClient, woc.wfClientset, woc.cronWf.Namespace, wf, &v1alpha1.SubmitOpts{})
+		if err != nil {
+			if errors.IsAlreadyExists(err) {
+				return true, err
+			}
+			woc.log.Warnf("Failed to submit Workflow, intermittent: %s", err)
+			// return false will trigger retry
+			return false, err
+		}
+		return true, nil
+	})
 	if err != nil {
 		// If the workflow already exists (i.e. this is a duplicate submission), do not report an error
 		if errors.IsAlreadyExists(err) {