Add reconciliation logic to update deployments

Author: christinaexyou

controllers/gorch/guardrailsorchestrator_controller.go

@@ -19,6 +19,9 @@ package gorch
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"time"
+
 	"github.com/go-logr/logr"
 	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -28,8 +31,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
-	"strconv"
-	"time"
 
 	gorchv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/gorch/v1alpha1"
 	appsv1 "k8s.io/api/apps/v1"
@@ -323,6 +324,41 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct
 	} else if err != nil {
 		r.handleReconciliationError(ctx, log, orchestrator, err, utils.ReconcileFailed, "Failed to get Deployment")
 		return ctrl.Result{}, err
+	} else {
+		// Deployment exists, check if it needs to be updated
+		newDeployment, err := r.createDeployment(ctx, orchestrator)
+		if err != nil {
+			r.handleReconciliationError(ctx, log, orchestrator, err, utils.ReconcileFailed, "Failed to create updated Deployment spec")
+			return ctrl.Result{}, err
+		}
+
+		// add TLS mounts to the new deployment spec
+		err = r.addTLSMounts(ctx, orchestrator, newDeployment, tlsMounts)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				log.Info("Could not find required TLS serving secrets, will try again.")
+				return ctrl.Result{}, nil
+			}
+			r.handleReconciliationError(ctx, log, orchestrator, err, utils.ReconcileFailed, "Failed to add TLS Mounts")
+			return ctrl.Result{}, err
+		}
+
+		// Preserve existing annotations and add configmap hashes
+		annotations := existingDeployment.Spec.Template.Annotations
+		if annotations == nil {
+			annotations = map[string]string{}
+		}
+		r.setConfigMapHashAnnotations(ctx, orchestrator, annotations)
+		newDeployment.Spec.Template.Annotations = annotations
+
+		// Patch the deployment if there are changes
+		if patchDeployment(existingDeployment, newDeployment) {
+			log.Info("Updating Deployment", "Deployment.Namespace", existingDeployment.Namespace, "Deployment.Name", existingDeployment.Name)
+			if updateErr := r.Update(ctx, existingDeployment); updateErr != nil {
+				r.handleReconciliationError(ctx, log, orchestrator, updateErr, utils.ReconcileFailed, "Failed to update Deployment")
+				return ctrl.Result{}, updateErr
+			}
+		}
 	}
 
 	// monitor the orchestrator or gateway config for changes

controllers/gorch/guardrailsorchestrator_controller_test.go

@@ -47,7 +47,6 @@ func createGuardrailsOrchestrator(ctx context.Context, orchestratorConfigMap str
 				Namespace: typedNamespacedName.Namespace,
 			},
 			Spec: gorchv1alpha1.GuardrailsOrchestratorSpec{
-				Replicas:           1,
 				OrchestratorConfig: &orchestratorConfigMap,
 			},
 		}
@@ -505,6 +504,93 @@ func testCreateGuardrailsOrchestratorWithCustomReplicas(namespaceName string) {
 	})
 }
 
+func testCreateGuardrailsOrchestratorWithScaledUpReplicas(namespaceName string) {
+	It("Should successfully scale up the deployment from 1 to 3 replicas", func() {
+		By("Creating a custom resource for the GuardrailsOrchestrator with 1 replica")
+		ctx := context.Background()
+		typedNamespacedName := types.NamespacedName{Name: orchestratorName, Namespace: namespaceName}
+
+		orchConfig := &corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "ConfigMap",
+				APIVersion: "v1",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      orchestratorName + "-config",
+				Namespace: namespaceName,
+			},
+		}
+		err := k8sClient.Create(ctx, orchConfig)
+		if err != nil && !errors.IsAlreadyExists(err) {
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = createGuardrailsOrchestratorWithReplicas(ctx, orchestratorName+"-config", 1)
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Checking if the custom resource was successfully created")
+		err = k8sClient.Get(ctx, typedNamespacedName, &gorchv1alpha1.GuardrailsOrchestrator{})
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Reconciling the custom resource that was created")
+		reconciler := &GuardrailsOrchestratorReconciler{
+			Client:    k8sClient,
+			Scheme:    k8sClient.Scheme(),
+			Namespace: namespaceName,
+		}
+
+		_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typedNamespacedName})
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Checking if the deployment was created with 1 replica")
+		Eventually(func() error {
+			deployment := &appsv1.Deployment{}
+			if err = k8sClient.Get(ctx, types.NamespacedName{Name: orchestratorName, Namespace: namespaceName}, deployment); err != nil {
+				return err
+			}
+			Expect(*deployment.Spec.Replicas).Should(Equal(int32(1)))
+			return nil
+		}, time.Second*10, time.Millisecond*10).Should(Succeed())
+
+		By("Updating the GuardrailsOrchestrator to 3 replicas")
+		orchestrator := &gorchv1alpha1.GuardrailsOrchestrator{}
+		err = k8sClient.Get(ctx, typedNamespacedName, orchestrator)
+		Expect(err).ToNot(HaveOccurred())
+
+		orchestrator.Spec.Replicas = 3
+		err = k8sClient.Update(ctx, orchestrator)
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Reconciling the updated custom resource")
+		_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typedNamespacedName})
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Checking if the deployment was scaled up to 3 replicas")
+		Eventually(func() error {
+			deployment := &appsv1.Deployment{}
+			if err = k8sClient.Get(ctx, types.NamespacedName{Name: orchestratorName, Namespace: namespaceName}, deployment); err != nil {
+				return err
+			}
+			Expect(*deployment.Spec.Replicas).Should(Equal(int32(3)))
+			Expect(deployment.Namespace).Should(Equal(namespaceName))
+			Expect(deployment.Name).Should(Equal(orchestratorName))
+			return nil
+		}, time.Second*10, time.Millisecond*10).Should(Succeed())
+
+		By("Deleting the custom resource for the GuardrailsOrchestrator")
+		err = deleteGuardrailsOrchestrator(ctx, orchestratorName, namespaceName)
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Deleting the orchestrator configmap")
+		err = k8sClient.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: orchestratorName + "-config", Namespace: namespaceName}})
+		Expect(err).ToNot(HaveOccurred())
+
+		By("Reconciling the custom resource that was deleted")
+		_, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typedNamespacedName})
+		Expect(err).ToNot(HaveOccurred())
+	})
+}
+
 func testCreateTwoGuardrailsOrchestratorsInSameNamespace(namespaceName string) {
 	It("Should successfully reconcile two custom resources for the GuardrailsOrchestrator in the same namespace", func() {
 		By("Creating the first custom resource for the GuardrailsOrchestrator")
@@ -897,6 +983,7 @@ var _ = Describe("GuardrailsOrchestrator Controller", func() {
 		testCreateDeleteGuardrailsOrchestratorSidecar(namespaceName)
 		testCreateDeleteGuardrailsOrchestratorOtelExporter(namespaceName)
 		testCreateGuardrailsOrchestratorWithCustomReplicas(namespaceName)
+		testCreateGuardrailsOrchestratorWithScaledUpReplicas(namespaceName)
 		testCreateTwoGuardrailsOrchestratorsInSameNamespace(namespaceName)
 		testCreateTwoGuardrailsOrchestratorsInDifferentNamespaces(namespaceName, secondNamespaceName)
 	})