Fixed ns termination status by reconciling (#91)

Fixes #88
Fixed ns termination status by reconciling

Signed-off-by: nitishm <[email protected]>

Co-authored-by: nitishm <[email protected]>

Author: nitishm

controllers/appgroup_controller.go

@@ -60,6 +60,8 @@ type ApplicationGroupReconciler struct {
 
 	// Recorder generates kubernetes events
 	Recorder record.EventRecorder
+
+	lastSuccessfulApplicationGroup *orkestrav1alpha1.ApplicationGroup
 }
 
 // +kubebuilder:rbac:groups=orkestra.azure.microsoft.com,resources=applicationgroups,verbs=get;list;watch;create;update;patch;delete
@@ -83,16 +85,17 @@ func (r *ApplicationGroupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	}
 
 	if appGroup.GetAnnotations() != nil {
-		// TODO (nitishm) Use this in error remediation by reapplying last successful appgroup spec
-		// lastSuccessfulApplicationGroup := appGroup.Annotations[lastSuccessfulApplicationGroupKey]
-		_ = appGroup.Annotations[lastSuccessfulApplicationGroupKey]
+		last := &orkestrav1alpha1.ApplicationGroup{}
+		s := appGroup.Annotations[lastSuccessfulApplicationGroupKey]
+		_ = json.Unmarshal([]byte(s), last)
+		r.lastSuccessfulApplicationGroup = last
 	}
 
-	// handle DELETE if deletion timestamp is non-zero
+	// handle deletes if deletion timestamp is non-zero
 	if !appGroup.DeletionTimestamp.IsZero() {
 		// If finalizer is found, remove it and requeue
 		if appGroup.Finalizers != nil {
-			logr.Info("Cleaning up")
+			logr.Info("cleaning up the applicationgroup resource")
 			// TODO: Take remediation action
 			// Reverse the entire workflow to remove all the Helm Releases
 			appGroup.Finalizers = nil
@@ -113,34 +116,34 @@ func (r *ApplicationGroupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 		return ctrl.Result{Requeue: true}, nil
 	}
 
-	// handle UPDATE if checksum mismatched
+	// handle first time install and subsequent updates
 	checksums, err := pkg.Checksum(&appGroup)
 	if err != nil {
 		// TODO (nitishm) Handle different error types here to decide remediation action
 		if errors.Is(err, pkg.ErrChecksumAppGroupSpecMismatch) {
 			if appGroup.Status.Checksums != nil {
 				appGroup.Status.Update = true
 			}
-			appGroup.Status.Checksums = checksums
 			requeue, err = r.reconcile(ctx, logr, r.WorkflowNS, &appGroup)
 			if err != nil {
 				logr.Error(err, "failed to reconcile ApplicationGroup instance")
-				r.updateStatusAndEvent(ctx, appGroup, requeue, err)
+				r.handleResponseAndEvent(ctx, appGroup, requeue, err)
 				return ctrl.Result{Requeue: requeue}, err
 			}
 
+			appGroup.Status.Checksums = checksums
+
 			if appGroup.Status.Phase != v1alpha12.NodeSucceeded {
-				r.updateStatusAndEvent(ctx, appGroup, requeue, err)
+				r.handleResponseAndEvent(ctx, appGroup, requeue, err)
 				return ctrl.Result{Requeue: true, RequeueAfter: requeueAfter}, nil
 			}
 
-			r.updateStatusAndEvent(ctx, appGroup, requeue, err)
-			return ctrl.Result{Requeue: false}, nil
+			r.handleResponseAndEvent(ctx, appGroup, requeue, err)
+			return ctrl.Result{Requeue: requeue}, err
 		}
 
-		appGroup.Status.Error = err.Error()
-		_ = r.Status().Update(ctx, &appGroup)
 		logr.Error(err, "failed to calculate checksum annotations for application group specs")
+		r.handleResponseAndEvent(ctx, appGroup, false, err)
 		return ctrl.Result{Requeue: false}, err
 	}
 
@@ -155,8 +158,7 @@ func (r *ApplicationGroupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	err = r.List(ctx, &wfs, listOption)
 	if err != nil {
 		logr.Error(err, "failed to find generate workflow instance")
-		appGroup.Status.Error = err.Error()
-		_ = r.Status().Update(ctx, &appGroup)
+		r.handleResponseAndEvent(ctx, appGroup, false, err)
 		return ctrl.Result{Requeue: false}, err
 	}
 
@@ -169,18 +171,16 @@ func (r *ApplicationGroupReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e
 	switch appGroup.Status.Phase {
 	case v1alpha12.NodeRunning, v1alpha12.NodePending:
 		logr.V(1).Info("workflow in pending/running state. requeue and reconcile after a short period")
-		_ = r.Status().Update(ctx, &appGroup)
+		r.handleResponseAndEvent(ctx, appGroup, true, nil)
 		return ctrl.Result{Requeue: true, RequeueAfter: requeueAfter}, nil
 	case v1alpha12.NodeSucceeded:
 		logr.V(1).Info("workflow ran to completion and succeeded")
-		appGroup.Status.Error = ""
-		r.updateStatusAndEvent(ctx, appGroup, false, nil)
+		r.handleResponseAndEvent(ctx, appGroup, false, nil)
 		return ctrl.Result{Requeue: false}, nil
 	case v1alpha12.NodeError, v1alpha12.NodeFailed:
 		err = fmt.Errorf("workflow in failure/error condition")
 		logr.Error(err, "workflow in failure/error condition")
-		appGroup.Status.Error = err.Error()
-		_ = r.Status().Update(ctx, &appGroup)
+		r.handleResponseAndEvent(ctx, appGroup, false, err)
 		return ctrl.Result{Requeue: false}, err
 	}
 
@@ -195,9 +195,11 @@ func (r *ApplicationGroupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *ApplicationGroupReconciler) updateStatusAndEvent(ctx context.Context, grp orkestrav1alpha1.ApplicationGroup, requeue bool, err error) {
+func (r *ApplicationGroupReconciler) handleResponseAndEvent(ctx context.Context, grp orkestrav1alpha1.ApplicationGroup, requeue bool, err error) {
 	errStr := ""
 	if err != nil {
+		// Handle the error by remediating the workflow
+		r.handleRemediation(ctx, err)
 		errStr = err.Error()
 	}
 
@@ -206,9 +208,11 @@ func (r *ApplicationGroupReconciler) updateStatusAndEvent(ctx context.Context, g
 	_ = r.Status().Update(ctx, &grp)
 
 	if grp.Status.Phase == v1alpha12.NodeSucceeded {
+		// Annotate the resource with the last successful ApplicationGroup spec
 		b, _ := json.Marshal(&grp)
 		grp.SetAnnotations(map[string]string{lastSuccessfulApplicationGroupKey: string(b)})
 		_ = r.Update(ctx, &grp)
+
 		r.Recorder.Event(&grp, "Normal", "ReconcileSuccess", fmt.Sprintf("Successfully reconciled ApplicationGroup %s", grp.Name))
 	}
 
@@ -264,3 +268,10 @@ func initApplications(appGroup *orkestrav1alpha1.ApplicationGroup) {
 	}
 	appGroup.Spec.Applications = v.DeepCopy().Spec.Applications
 }
+
+func (r *ApplicationGroupReconciler) handleRemediation(ctx context.Context, err error) {
+	if r.lastSuccessfulApplicationGroup != nil {
+		r.lastSuccessfulApplicationGroup.Status.Checksums = nil
+		_ = r.Update(ctx, r.lastSuccessfulApplicationGroup)
+	}
+}

controllers/appgroup_reconciler.go

@@ -2,13 +2,15 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"os"
 
 	"fmt"
 
 	"github.com/Azure/Orkestra/api/v1alpha1"
 	orkestrav1alpha1 "github.com/Azure/Orkestra/api/v1alpha1"
 	"github.com/Azure/Orkestra/pkg/registry"
+	"github.com/Azure/Orkestra/pkg/workflow"
 	v1alpha12 "github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
 	"github.com/go-logr/logr"
 )
@@ -184,6 +186,10 @@ func (r *ApplicationGroupReconciler) generateWorkflow(ctx context.Context, logr
 
 	err = r.Engine.Submit(ctx, logr, g)
 	if err != nil {
+		if errors.Is(err, workflow.ErrNamespaceTerminating) {
+			logr.V(1).Info("namespace is in terminating state")
+			return true, err
+		}
 		logr.Error(err, "engine failed to submit workflow")
 		return false, err
 	}

pkg/workflow/argo.go

@@ -2,6 +2,7 @@ package workflow
 
 import (
 	"context"
+	kerrors "errors"
 	"fmt"
 	"os"
 	"strings"
@@ -31,6 +32,10 @@ const (
 	valuesKeyGlobal = "global"
 )
 
+var (
+	ErrNamespaceTerminating = kerrors.New("namespace is in terminating phase")
+)
+
 type argo struct {
 	scheme *runtime.Scheme
 	cli    client.Client
@@ -122,7 +127,6 @@ func (a *argo) Submit(ctx context.Context, l logr.Logger, g *v1alpha1.Applicatio
 		// FIXME (nitishm) Handle namespace in termination state by requeueing
 		err := a.cli.Get(ctx, types.NamespacedName{Name: ns.Name}, &ns)
 		if err != nil {
-			fmt.Printf("\n\n\n\n\nGET %s\n\n\n\n", err)
 			if errors.IsNotFound(err) {
 				// Add OwnershipReference
 				err = controllerutil.SetControllerReference(g, &ns, a.scheme)
@@ -132,12 +136,14 @@ func (a *argo) Submit(ctx context.Context, l logr.Logger, g *v1alpha1.Applicatio
 
 				err = a.cli.Create(ctx, &ns)
 				if err != nil {
-					fmt.Printf("\n\n\n\n\nCREATE %s\n\n\n\n", err)
-
 					return fmt.Errorf("failed to CREATE namespace %s object : %w", ns.Name, err)
 				}
 			}
 		}
+
+		if ns.Status.Phase == corev1.NamespaceTerminating {
+			return ErrNamespaceTerminating
+		}
 	}
 
 	err := a.cli.Get(ctx, types.NamespacedName{Namespace: a.wf.Namespace, Name: a.wf.Name}, obj)