feat: add delay time to prevent premature exit before supply chain is ready when waiting (#652)

Author: anibmurthy

docs/command-reference/tanzu_apps_workload_apply.md

@@ -29,6 +29,7 @@ tanzu apps workload apply --file workload.yaml
   -a, --app name                       application name the workload is a part of
       --build-env "key=value" pair     build environment variables represented as a "key=value" pair ("key-" to remove, flag can be used multiple times)
       --debug                          put the workload in debug mode (--debug=false to deactivate)
+      --delay duration                 delay set to prevent premature exit before supply chain step completion when waiting/tailing (default 30s)
       --dry-run                        print kubernetes resources to stdout rather than apply them to the cluster, messages normally on stdout will be sent to stderr
   -e, --env "key=value" pair           environment variables represented as a "key=value" pair ("key-" to remove, flag can be used multiple times)
   -f, --file file path                 file path containing the description of a single workload, other flags are layered on top of this resource. Use value "-" to read from stdin

docs/command-reference/tanzu_apps_workload_create.md

@@ -31,6 +31,7 @@ tanzu apps workload create --file workload.yaml
   -a, --app name                       application name the workload is a part of
       --build-env "key=value" pair     build environment variables represented as a "key=value" pair ("key-" to remove, flag can be used multiple times)
       --debug                          put the workload in debug mode (--debug=false to deactivate)
+      --delay duration                 delay set to prevent premature exit before supply chain step completion when waiting/tailing (default 30s)
       --dry-run                        print kubernetes resources to stdout rather than apply them to the cluster, messages normally on stdout will be sent to stderr
   -e, --env "key=value" pair           environment variables represented as a "key=value" pair ("key-" to remove, flag can be used multiple times)
   -f, --file file path                 file path containing the description of a single workload, other flags are layered on top of this resource. Use value "-" to read from stdin

pkg/cli-runtime/wait/wait.go

@@ -32,12 +32,15 @@ var (
 
 type ConditionFunc = func(client.Object) (bool, error)
 
-func UntilCondition(ctx context.Context, watchClient client.WithWatch, target types.NamespacedName, listType client.ObjectList, condition ConditionFunc) error {
+func UntilCondition(ctx context.Context, watchClient client.WithWatch, target types.NamespacedName, listType client.ObjectList, condition ConditionFunc, delayTime time.Duration) error {
+	readyStatus := false
+	timer := time.NewTimer(delayTime)
 	eventWatcher, err := watchClient.Watch(ctx, listType, &client.ListOptions{Namespace: target.Namespace})
 	if err != nil {
 		return err
 	}
 	defer eventWatcher.Stop()
+	defer timer.Stop()
 	for {
 		select {
 		case event := <-eventWatcher.ResultChan():
@@ -53,10 +56,25 @@ func UntilCondition(ctx context.Context, watchClient client.WithWatch, target ty
 						return err
 					}
 					if cond {
-						return nil
+						// Timer is started/reset to track ready status change.
+						timer.Reset(delayTime)
+						readyStatus = true
+					} else {
+						// This is to capture 'unknown' state to avoid incorrect exit from tailing
+						readyStatus = false
 					}
 				}
 			}
+		case <-timer.C:
+			// Wait until the delay time is met before stopping the tail. This is done to address the use case where
+			// the workload apply flows through supply chain steps to rerun, which may result in the workload status
+			// switching between Ready - unknown - Ready - unknown, and so on.
+			// The delay timer provides an option to allow the supply chain to parse through the steps before exiting the tail.
+			if readyStatus {
+				return nil
+			} else {
+				timer.Reset(delayTime)
+			}
 		case <-ctx.Done():
 			return ctx.Err()
 		}

pkg/cli-runtime/wait/wait_test.go

@@ -133,7 +133,7 @@ func TestUntilReady(t *testing.T) {
 			done := make(chan error, 1)
 			defer close(done)
 			go func() {
-				done <- UntilCondition(ctx, fakeWithWatcher, types.NamespacedName{Name: test.resource.Name, Namespace: test.resource.Namespace}, &cartov1alpha1.WorkloadList{}, test.condFunc)
+				done <- UntilCondition(ctx, fakeWithWatcher, types.NamespacedName{Name: test.resource.Name, Namespace: test.resource.Namespace}, &cartov1alpha1.WorkloadList{}, test.condFunc, 0*time.Second)
 			}()
 
 			for _, r := range objs {

pkg/commands/workload.go

@@ -147,6 +147,7 @@ type WorkloadOptions struct {
 
 	Wait           bool
 	WaitTimeout    time.Duration
+	DelayTime      time.Duration
 	Tail           bool
 	TailTimestamps bool
 	DryRun         bool
@@ -896,18 +897,18 @@ func getStatusChangeWorker(c *cli.Config, workload *cartov1alpha1.Workload) wait
 				}
 			}
 			return false, nil
-		})
+		}, 0*time.Second)
 	})
 	return worker
 }
 
-func getReadyConditionWorker(c *cli.Config, workload *cartov1alpha1.Workload) wait.Worker {
+func getReadyConditionWorker(c *cli.Config, workload *cartov1alpha1.Workload, delayTime time.Duration) wait.Worker {
 	worker := wait.Worker(func(ctx context.Context) error {
 		clientWithWatch, err := watch.GetWatcher(ctx, c)
 		if err != nil {
 			return err
 		}
-		return wait.UntilCondition(ctx, clientWithWatch, types.NamespacedName{Name: workload.Name, Namespace: workload.Namespace}, &cartov1alpha1.WorkloadList{}, cartov1alpha1.WorkloadReadyConditionFunc)
+		return wait.UntilCondition(ctx, clientWithWatch, types.NamespacedName{Name: workload.Name, Namespace: workload.Namespace}, &cartov1alpha1.WorkloadList{}, cartov1alpha1.WorkloadReadyConditionFunc, delayTime)
 	})
 
 	return worker
@@ -1018,6 +1019,8 @@ func (opts *WorkloadOptions) DefineFlags(ctx context.Context, c *cli.Config, cmd
 	cmd.MarkFlagFilename(cli.StripDash(flags.FilePathFlagName), ".yaml", ".yml")
 	cmd.Flags().BoolVar(&opts.DryRun, cli.StripDash(flags.DryRunFlagName), false, "print kubernetes resources to stdout rather than apply them to the cluster, messages normally on stdout will be sent to stderr")
 	cmd.Flags().BoolVarP(&opts.Yes, cli.StripDash(flags.YesFlagName), "y", false, "accept all prompts")
+	cmd.Flags().DurationVar(&opts.DelayTime, cli.StripDash(flags.DelayTimeFlagName), 30*time.Second, "delay set to prevent premature exit before supply chain step completion when waiting/tailing")
+	cmd.RegisterFlagCompletionFunc(cli.StripDash(flags.DelayTimeFlagName), completion.SuggestDurationUnits(ctx, completion.CommonDurationUnits))
 }
 
 func (opts *WorkloadOptions) DefineEnvVars(ctx context.Context, c *cli.Config, cmd *cobra.Command) {

pkg/commands/workload_apply.go

@@ -225,7 +225,7 @@ func (opts *WorkloadApplyOptions) Exec(ctx context.Context, c *cli.Config) error
 				}
 			}
 
-			workers = append(workers, getReadyConditionWorker(c, workload))
+			workers = append(workers, getReadyConditionWorker(c, workload, opts.DelayTime))
 
 			if anyTail {
 				workers = append(workers, getTailWorker(c, workload, opts.TailTimestamps))

pkg/commands/workload_apply_test.go

@@ -350,7 +350,7 @@ status:
 		{
 			Name: "create - output yaml with wait",
 			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch,
-				flags.OutputFlagName, printer.OutputFormatYaml, flags.WaitFlagName, flags.YesFlagName},
+				flags.OutputFlagName, printer.OutputFormatYaml, flags.WaitFlagName, flags.YesFlagName, flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: givenNamespaceDefault,
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
@@ -936,7 +936,7 @@ To get status: "tanzu apps workload get my-workload"
 		{
 			Name: "wait with timeout error",
 			Skip: runtm.GOOS == "windows",
-			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.WaitTimeoutFlagName, "1ns"},
+			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.WaitTimeoutFlagName, "1ns", flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
 					ObjectMeta: metav1.ObjectMeta{
@@ -1011,7 +1011,7 @@ Error waiting for ready condition: timeout after 1ns waiting for "my-workload" t
 		},
 		{
 			Name: "create - successful wait for ready cond",
-			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName},
+			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
 					ObjectMeta: metav1.ObjectMeta{
@@ -1203,7 +1203,7 @@ Workload "my-workload" is ready
 		},
 		{
 			Name: "create - watcher error",
-			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName},
+			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				fakewatch := watchfakes.NewFakeWithWatch(true, config.Client, []watch.Event{})
 				ctx = watchhelper.WithWatcher(ctx, fakewatch)
@@ -1990,7 +1990,7 @@ Error: conflict updating workload, the object was modified by another user; plea
 		{
 			Name: "update - wait for ready condition - error with timeout",
 			Skip: runtm.GOOS == "windows",
-			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns"},
+			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns", flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: []client.Object{
 				parent.
 					SpecDie(func(d *diecartov1alpha1.WorkloadSpecDie) {
@@ -2102,7 +2102,7 @@ Error waiting for ready condition: timeout after 1ns waiting for "my-workload" t
 		{
 			Name: "update - wait timeout when there is no transition time",
 			Skip: runtm.GOOS == "windows",
-			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns"},
+			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns", flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: []client.Object{
 				parent.
 					SpecDie(func(d *diecartov1alpha1.WorkloadSpecDie) {
@@ -2209,7 +2209,7 @@ Error waiting for status change: timeout after 1ns waiting for "my-workload" to
 		{
 			Name: "update - wait timeout when there is no ready cond",
 			Skip: runtm.GOOS == "windows",
-			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns"},
+			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns", flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: []client.Object{
 				parent.
 					SpecDie(func(d *diecartov1alpha1.WorkloadSpecDie) {
@@ -2302,7 +2302,7 @@ Error waiting for status change: timeout after 1ns waiting for "my-workload" to
 		{
 			Name: "update - wait for timestamp change error with timeout",
 			Skip: runtm.GOOS == "windows",
-			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns"},
+			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.WaitTimeoutFlagName, "1ns", flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: []client.Object{
 				parent.
 					SpecDie(func(d *diecartov1alpha1.WorkloadSpecDie) {
@@ -2412,7 +2412,7 @@ Error waiting for status change: timeout after 1ns waiting for "my-workload" to
 		},
 		{
 			Name: "update - wait error for false condition",
-			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName},
+			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: []client.Object{
 				parent.
 					SpecDie(func(d *diecartov1alpha1.WorkloadSpecDie) {
@@ -2523,7 +2523,7 @@ Error waiting for ready condition: Failed to become ready: a hopefully informati
 		},
 		{
 			Name: "update - successful wait for ready condition",
-			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName},
+			Args: []string{workloadName, flags.ServiceRefFlagName, "database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db", flags.WaitFlagName, flags.YesFlagName, flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: []client.Object{
 				parent.
 					SpecDie(func(d *diecartov1alpha1.WorkloadSpecDie) {
@@ -5356,7 +5356,7 @@ status:
 			Name: "output workload after update in yaml format with wait error",
 			Args: []string{workloadName, flags.ServiceRefFlagName,
 				"database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db",
-				flags.OutputFlagName, printer.OutputFormatYml, flags.WaitFlagName},
+				flags.OutputFlagName, printer.OutputFormatYml, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
 					ObjectMeta: metav1.ObjectMeta{
@@ -5499,7 +5499,7 @@ status:
 			Name: "console interaction - output workload after update in yaml format with wait",
 			Args: []string{workloadName, flags.ServiceRefFlagName,
 				"database=services.tanzu.vmware.com/v1alpha1:PostgreSQL:my-prod-db",
-				flags.OutputFlagName, printer.OutputFormatYml, flags.WaitFlagName},
+				flags.OutputFlagName, printer.OutputFormatYml, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
 					ObjectMeta: metav1.ObjectMeta{

pkg/commands/workload_create.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
@@ -144,7 +145,7 @@ func (opts *WorkloadCreateOptions) Exec(ctx context.Context, c *cli.Config) erro
 		if opts.Wait || anyTail {
 			cli.PrintPrompt(shouldPrint, c.Infof, "Waiting for workload %q to become ready...\n", opts.Name)
 
-			workers = append(workers, getReadyConditionWorker(c, workload))
+			workers = append(workers, getReadyConditionWorker(c, workload, 0*time.Second))
 
 			if anyTail {
 				workers = append(workers, getTailWorker(c, workload, opts.TailTimestamps))

pkg/commands/workload_create_test.go

@@ -185,7 +185,7 @@ status:
 		{
 			Name: "create - output yaml with wait",
 			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch,
-				flags.OutputFlagName, printer.OutputFormatYaml, flags.WaitFlagName, flags.YesFlagName},
+				flags.OutputFlagName, printer.OutputFormatYaml, flags.WaitFlagName, flags.YesFlagName, flags.DelayTimeFlagName, "0ns"},
 			GivenObjects: givenNamespaceDefault,
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
@@ -310,7 +310,7 @@ status:
 		},
 		{
 			Name: "wait error for false condition",
-			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName},
+			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
 					ObjectMeta: metav1.ObjectMeta{
@@ -463,7 +463,7 @@ Error waiting for ready condition: timeout after 1ns waiting for "my-workload" t
 		},
 		{
 			Name: "successful wait for ready cond",
-			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName},
+			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				workload := &cartov1alpha1.Workload{
 					ObjectMeta: metav1.ObjectMeta{
@@ -756,7 +756,7 @@ Error: workload "default/my-workload" already exists
 		},
 		{
 			Name: "watcher error",
-			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName},
+			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch, flags.YesFlagName, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			Prepare: func(t *testing.T, ctx context.Context, config *cli.Config, tc *clitesting.CommandTestCase) (context.Context, error) {
 				fakewatch := watchfakes.NewFakeWithWatch(true, config.Client, []watch.Event{})
 				ctx = watchhelper.WithWatcher(ctx, fakewatch)
@@ -1901,7 +1901,7 @@ Error waiting for ready condition: failed to create watcher
 			Name:         "output workload after create in json format with wait error",
 			GivenObjects: givenNamespaceDefault,
 			Args: []string{workloadName, flags.GitRepoFlagName, gitRepo, flags.GitBranchFlagName, gitBranch,
-				flags.TypeFlagName, "web", flags.OutputFlagName, printer.OutputFormatJson, flags.WaitFlagName},
+				flags.TypeFlagName, "web", flags.OutputFlagName, printer.OutputFormatJson, flags.WaitFlagName, flags.DelayTimeFlagName, "0ns"},
 			WithConsoleInteractions: func(t *testing.T, c *expect.Console) {
 				c.ExpectString(clitesting.ToInteractTerminal("Do you want to create this workload? [yN]: "))
 				c.Send(clitesting.InteractInputLine("y"))

pkg/flags/flags.go

@@ -30,6 +30,7 @@ const (
 	ConfigFlagName           = "--config"
 	ContextFlagName          = cli.ContextFlagName
 	DebugFlagName            = "--debug"
+	DelayTimeFlagName        = "--delay"
 	DryRunFlagName           = "--dry-run"
 	EnvFlagName              = "--env"
 	ExportFlagName           = "--export"