[YUNIKORN-3204] Quota Preemption E2E testing(#1009)

Closes: #1009

Signed-off-by: Manikandan R <manirajv06@gmail.com>

Author: adityadtu5

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/looplab/fsm v1.0.3
 	github.com/onsi/ginkgo/v2 v2.27.2
-	github.com/onsi/gomega v1.38.2
+	github.com/onsi/gomega v1.40.0
 	github.com/prometheus/client_golang v1.23.2
 	github.com/sasha-s/go-deadlock v0.3.9
 	go.uber.org/zap v1.27.1

go.sum

@@ -153,6 +153,8 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns
 github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
 github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
 github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
+github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc=
+github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=

test/e2e/framework/helpers/k8s/k8s_utils.go

@@ -795,6 +795,18 @@ func GetPodObj(yamlPath string) (*v1.Pod, error) {
 	return pod, nil
 }
 
+func GetDeploymentObj(yamlPath string) (*appsv1.Deployment, error) {
+	o, err := common.Yaml2Obj(yamlPath)
+	if err != nil {
+		return nil, err
+	}
+	deployment, ok := o.(*appsv1.Deployment)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert object to Deployment")
+	}
+	return deployment, nil
+}
+
 func (k *KubeCtl) CreateDeployment(deployment *appsv1.Deployment, namespace string) (*appsv1.Deployment, error) {
 	return k.clientSet.AppsV1().Deployments(namespace).Create(context.TODO(), deployment, metav1.CreateOptions{})
 }
@@ -1113,6 +1125,24 @@ func (k *KubeCtl) WaitForPodBySelectorRunning(namespace string, selector string,
 	return nil
 }
 
+func (k *KubeCtl) WaitForNPodsBySelectorRunning(namespace string, selector string, expectedCount int, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(context.TODO(), time.Millisecond*100, timeout, false, func(ctx context.Context) (bool, error) {
+		podList, err := k.ListPods(namespace, selector)
+		if err != nil {
+			return false, err
+		}
+		if len(podList.Items) != expectedCount {
+			return false, nil
+		}
+		for _, pod := range podList.Items {
+			if pod.Status.Phase != v1.PodRunning {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+}
+
 // Wait for all pods in 'namespace' with given 'selector' to enter succeeded state.
 // Returns an error if no pods are found or not all discovered pods enter succeeded state.
 func (k *KubeCtl) WaitForPodBySelectorSucceeded(namespace string, selector string, timeout time.Duration) error {

test/e2e/quota-preemption/quota_preemption_suite_test.go

@@ -0,0 +1,77 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package quota_preemption_test
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2/reporters"
+	"github.com/onsi/gomega"
+
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/configmanager"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/common"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/k8s"
+	"github.com/apache/yunikorn-k8shim/test/e2e/framework/helpers/yunikorn"
+)
+
+func init() {
+	configmanager.YuniKornTestConfig.ParseFlags()
+}
+
+func TestQuotaPreemption(t *testing.T) {
+	ginkgo.ReportAfterSuite("TestQuotaPreemption", func(report ginkgo.Report) {
+		err := reporters.GenerateJUnitReportWithConfig(
+			report,
+			filepath.Join(configmanager.YuniKornTestConfig.LogDir, "TEST-quotapreemption_junit.xml"),
+			reporters.JunitReportConfig{OmitSpecLabels: true},
+		)
+		Ω(err).NotTo(HaveOccurred())
+	})
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "TestQuotaPreemption", ginkgo.Label("TestQuotaPreemption"))
+}
+
+var Ω = gomega.Ω
+var HaveOccurred = gomega.HaveOccurred
+
+var _ = ginkgo.BeforeSuite(func() {
+	_, filename, _, _ := runtime.Caller(0)
+	suiteName = common.GetSuiteName(filename)
+	// Initializing kubectl client
+	kClient = k8s.KubeCtl{}
+	Ω(kClient.SetClient()).To(gomega.Succeed())
+	// Initializing rest client
+	restClient = yunikorn.RClient{}
+	Ω(restClient).NotTo(gomega.BeNil())
+
+	yunikorn.EnsureYuniKornConfigsPresent()
+
+	ginkgo.By("Port-forward the scheduler pod")
+	var err = kClient.PortForwardYkSchedulerPod()
+	Ω(err).NotTo(gomega.HaveOccurred())
+})
+
+var _ = ginkgo.AfterSuite(func() {
+	ginkgo.By("Check Yunikorn's health")
+	checks, err := yunikorn.GetFailedHealthChecks()
+	Ω(err).NotTo(gomega.HaveOccurred())
+	Ω(checks).To(gomega.Equal(""), checks)
+})