test: Added upgrade tests (feast-dev#5072)

Added upgrade tests

Signed-off-by: Theodor Mihalache <[email protected]>

Author: tmihalac

.github/workflows/operator-e2e-integration-tests.yml

@@ -56,6 +56,30 @@ jobs:
           cd infra/feast-operator/
           make test-e2e
 
+      - name: Clean up docker images
+        if: always()
+        run: |
+          docker images --format '{{.Repository}}:{{.Tag}}' | grep 'feast' | xargs -r docker rmi -f
+          docker system prune -a -f
+
+      - name: Run Previous version tests
+        run: |
+          # Run the previous version tests
+          cd infra/feast-operator/
+          make test-previous-version
+
+      - name: Clean up docker images
+        if: always()
+        run: |
+          docker images --format '{{.Repository}}:{{.Tag}}' | grep 'feast' | xargs -r docker rmi -f
+          docker system prune -a -f
+
+      - name: Run Upgrade tests
+        run: |
+          # Run the upgrade tests
+          cd infra/feast-operator/
+          make test-upgrade
+
       - name: Debug KIND Cluster when there is a failure
         if: failure()
         run: |
@@ -66,6 +90,4 @@ jobs:
         if: always()
         run: |
           # Delete the KIND cluster after tests
-          kind delete cluster --name kind-$KIND_CLUSTER
-
-      
+          kind delete cluster --name kind-$KIND_CLUSTER

infra/feast-operator/.golangci.yml

@@ -19,6 +19,12 @@ issues:
     - path: "test/*"
       linters:
         - lll
+    - path: "upgrade/*"
+      linters:
+        - lll
+    - path: "previous-version/*"
+      linters:
+        - lll
 linters:
   disable-all: true
   enable:

infra/feast-operator/Makefile

@@ -113,12 +113,20 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: build-installer vet lint envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v test/e2e | grep -v test/data-source-types) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v test/e2e | grep -v test/data-source-types | grep -v test/upgrade | grep -v test/previous-version) -coverprofile cover.out
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
 .PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
 test-e2e:
-	go test -timeout 30m ./test/e2e/ -v -ginkgo.v
+	go test -timeout 60m ./test/e2e/ -v -ginkgo.v
+
+.PHONY: test-upgrade  # Run the upgrade tests against a Kind k8s instance that is spun up.
+test-upgrade:
+	go test -timeout 60m ./test/upgrade/ -v -ginkgo.v
+
+.PHONY: test-previous-version  # Run e2e tests against previous version in a Kind k8s instance that is spun up.
+test-previous-version:
+	go test -timeout 60m ./test/previous-version/ -v -ginkgo.v
 
 # Requires python3
 .PHONY: test-datasources

infra/feast-operator/test/e2e/e2e_test.go

@@ -17,217 +17,38 @@ limitations under the License.
 package e2e
 
 import (
-	"fmt"
-	"os"
-	"os/exec"
-	"time"
-
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-
 	"github.com/feast-dev/feast/infra/feast-operator/test/utils"
-)
-
-const (
-	feastControllerNamespace = "feast-operator-system"
-	timeout                  = 2 * time.Minute
-	controllerDeploymentName = "feast-operator-controller-manager"
-	feastPrefix              = "feast-"
+	. "github.com/onsi/ginkgo/v2"
 )
 
 var _ = Describe("controller", Ordered, func() {
-	BeforeAll(func() {
-		_, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI")
-		if !isRunOnOpenShiftCI {
-			By("creating manager namespace")
-			cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace)
-			_, _ = utils.Run(cmd)
-
-			var err error
-			// projectimage stores the name of the image used in the example
-			var projectimage = "localhost/feast-operator:v0.0.1"
-
-			By("building the manager(Operator) image")
-			cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("loading the the manager(Operator) image on Kind")
-			err = utils.LoadImageToKindClusterWithName(projectimage)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			// this image will be built in above make target.
-			var feastImage = "feastdev/feature-server:dev"
-			var feastLocalImage = "localhost/feastdev/feature-server:dev"
-
-			By("building the feast image")
-			cmd = exec.Command("make", "feast-ci-dev-docker-img")
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("Tag the local feast image for the integration tests")
-			cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage)
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-
-			By("loading the the feast image on Kind cluster")
-			err = utils.LoadImageToKindClusterWithName(feastLocalImage)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+	featureStoreName := "simple-feast-setup"
+	feastResourceName := utils.FeastPrefix + featureStoreName
+	feastK8sResourceNames := []string{
+		feastResourceName + "-online",
+		feastResourceName + "-offline",
+		feastResourceName + "-ui",
+	}
 
-			By("installing CRDs")
-			cmd = exec.Command("make", "install")
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+	runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/e2e",
+		"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml",
+		featureStoreName, feastResourceName, feastK8sResourceNames)
 
-			By("deploying the controller-manager")
-			cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage), fmt.Sprintf("FS_IMG=%s", feastLocalImage))
-			_, err = utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+	runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/e2e",
+		"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml",
+		"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml",
+		featureStoreName, feastResourceName, feastK8sResourceNames)
 
-			By("Validating that the controller-manager deployment is in available state")
-			err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout)
-			Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-				"Deployment %s is not available but expected to be available. \nError: %v\n",
-				controllerDeploymentName, err,
-			))
-			fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName)
-		}
+	BeforeAll(func() {
+		utils.DeployOperatorFromCode("/test/e2e")
 	})
 
 	AfterAll(func() {
-		// Add any post clean up code here.
-		_, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI")
-		if !isRunOnOpenShiftCI {
-			By("Uninstalling the feast CRD")
-			cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace)
-			_, err := utils.Run(cmd)
-			ExpectWithOffset(1, err).NotTo(HaveOccurred())
-		}
+		utils.DeleteOperatorDeployment("/test/e2e")
 	})
 
 	Context("Operator E2E Tests", func() {
-		It("Should be able to deploy and run a default feature store CR successfully", func() {
-			By("deploying the Simple Feast Custom Resource to Kubernetes")
-			namespace := "default"
-
-			cmd := exec.Command("kubectl", "apply", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
-			_, cmdOutputerr := utils.Run(cmd)
-			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
-
-			featureStoreName := "simple-feast-setup"
-			validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout)
-
-			By("deleting the feast deployment")
-			cmd = exec.Command("kubectl", "delete", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml")
-			_, cmdOutputerr = utils.Run(cmd)
-			ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred())
-		})
-
-		It("Should be able to deploy and run a feature store with remote registry CR successfully", func() {
-			By("deploying the Simple Feast Custom Resource to Kubernetes")
-			namespace := "default"
-			cmd := exec.Command("kubectl", "apply", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
-			_, cmdOutputErr := utils.Run(cmd)
-			ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred())
-
-			featureStoreName := "simple-feast-setup"
-			validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout)
-
-			var remoteRegistryNs = "remote-registry"
-			By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs))
-			cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs)
-			_, _ = utils.Run(cmd)
-
-			By("deploying the Simple Feast remote registry Custom Resource on Kubernetes")
-			cmd = exec.Command("kubectl", "apply", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs)
-			_, cmdOutputErr = utils.Run(cmd)
-			ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred())
-
-			remoteFeatureStoreName := "simple-feast-remote-setup"
-
-			validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout)
-
-			By("deleting the feast remote registry deployment")
-			cmd = exec.Command("kubectl", "delete", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs)
-			_, cmdOutputErr = utils.Run(cmd)
-			ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred())
-
-			By("deleting the feast deployment")
-			cmd = exec.Command("kubectl", "delete", "-f",
-				"test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace)
-			_, cmdOutputErr = utils.Run(cmd)
-			ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred())
-		})
+		It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc)
+		It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction)
 	})
 })
-
-func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) {
-	hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName)
-	Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-		"Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n",
-		featureStoreName, err))
-
-	feastResourceName := feastPrefix + featureStoreName
-	k8sResourceNames := []string{feastResourceName}
-	feastK8sResourceNames := []string{
-		feastResourceName + "-online",
-		feastResourceName + "-offline",
-		feastResourceName + "-ui",
-	}
-
-	if !hasRemoteRegistry {
-		feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry")
-	}
-
-	for _, deploymentName := range k8sResourceNames {
-		By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName))
-		err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout)
-		Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-			"Deployment %s is not available but expected to be available. \nError: %v\n",
-			deploymentName, err,
-		))
-		fmt.Printf("Feast Deployment %s is available\n", deploymentName)
-	}
-
-	By("Check if the feast client - kubernetes config map exists.")
-	configMapName := feastResourceName + "-client"
-	err = checkIfConfigMapExists(namespace, configMapName)
-	Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-		"config map %s is not available but expected to be available. \nError: %v\n",
-		configMapName, err,
-	))
-	fmt.Printf("Feast Deployment client config map %s is available\n", configMapName)
-
-	for _, serviceAccountName := range k8sResourceNames {
-		By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName))
-		err = checkIfServiceAccountExists(namespace, serviceAccountName)
-		Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-			"Service account %s does not exist in namespace %s. Error: %v",
-			serviceAccountName, namespace, err,
-		))
-		fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace)
-	}
-
-	for _, serviceName := range feastK8sResourceNames {
-		By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName))
-		err = checkIfKubernetesServiceExists(namespace, serviceName)
-		Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-			"kubernetes service %s is not available but expected to be available. \nError: %v\n",
-			serviceName, err,
-		))
-		fmt.Printf("kubernetes service %s is available\n", serviceName)
-	}
-
-	By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName))
-	err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace)
-	Expect(err).ToNot(HaveOccurred(), fmt.Sprintf(
-		"FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n",
-		featureStoreName, err,
-	))
-	fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName)
-}