integration test on actual cluster

Author: ksraj123

Makefile

@@ -5,7 +5,7 @@ ifeq (${KUBECONFIG}, )
   export KUBECONFIG
 endif
 
-# Requires KUBECONFIG env and Ginkgo binary
+# Requires KUBECONFIG env
 .PHONY: integration-test
 integration-test:
-	@cd tests && ginkgo -v
+	@cd tests && go test .

deploy/sc.yaml

@@ -1,7 +1,7 @@
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: test-storage-class-1
+  name: test-storage-class
   annotations:
     openebs.io/delete-dangling-pvc: "true"
     openebs.io/cas-type: local

tests/suite_test.go tests/integration_test.go

@@ -2,8 +2,8 @@ package tests
 
 import (
 	"context"
-	"fmt"
-	"strings"
+	"flag"
+	"os"
 	"testing"
 	"time"
 
@@ -12,30 +12,31 @@ import (
 	CoreV1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	envtest "sigs.k8s.io/controller-runtime/pkg/envtest"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
-func startCluster() (*kubernetes.Clientset, *envtest.Environment) {
-	testenv := &envtest.Environment{}
-	cfg, err := testenv.Start()
+var (
+	kubeConfigPath                  string
+	openebsNamespace                string
+	clientSet                       *kubernetes.Clientset
+	LocalPVProvisionerLabelSelector = "openebs.io/component-name=openebs-localpv-provisioner"
+)
+
+func init() {
+	flag.StringVar(&kubeConfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "path to kubeconfig to invoke kubernetes API calls")
+	flag.StringVar(&openebsNamespace, "openebs-namespace", "openebs", "kubernetes namespace where the OpenEBS components are present")
+	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	if err != nil {
 		panic(err.Error())
 	}
-	clientset, err := kubernetes.NewForConfig(cfg)
+	clientSet, err = kubernetes.NewForConfig(config)
 	if err != nil {
 		panic(err.Error())
 	}
-	return clientset, testenv
-}
-
-func stopCluster(testenv *envtest.Environment) {
-	testenv.Stop()
 }
 
 func TestMain(t *testing.T) {
 	ctx := context.Background()
-	clientSet, clusterTestEnv := startCluster()
-	defer stopCluster(clusterTestEnv)
 
 	// Ensure openebs-localpv-provisioner pod running
 	setup.TestOpenEbsPodRunningState(t, clientSet, ctx, 90)
@@ -75,15 +76,15 @@ func TestMain(t *testing.T) {
 
 	time.Sleep(5 * time.Second)
 
-	pvcs, err := clientSet.CoreV1().PersistentVolumeClaims("default").List(ctx, metav1.ListOptions{})
+	setup.TestDanglingPVCDeleted(t, clientSet, ctx, statefulsetName, 90)
+
+	err = clientSet.BatchV1().Jobs("default").Delete(ctx, "test-job", metav1.DeleteOptions{})
 	if err != nil {
-		fmt.Printf("error %s, getting PVCs\n", err.Error())
-	}
-	for _, pvc := range pvcs.Items {
-		if strings.Contains(pvc.Name, statefulsetName) {
-			t.Fatalf("Dangling PVCs %v not deleted by Job", pvc.Name)
-		}
+		panic(err.Error())
 	}
 
-	// clientSet.BatchV1().Jobs("default").Delete(ctx, "test-job", metav1.DeleteOptions{})
+	err = clientSet.StorageV1().StorageClasses().Delete(ctx, "test-storage-class", metav1.DeleteOptions{})
+	if err != nil {
+		panic(err.Error())
+	}
 }

tests/setup/setup.go

@@ -2,6 +2,8 @@ package setup
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -45,6 +47,32 @@ func TestOpenEbsPodRunningState(t *testing.T, clientset *kubernetes.Clientset, c
 	}
 }
 
+func TestDanglingPVCDeleted(t *testing.T, clientset *kubernetes.Clientset, ctx context.Context, statefulsetName string, maxRetry int) {
+	isSuccessful := false
+	var danglingPvcName string
+	for i := 0; i < maxRetry; i++ {
+		pvcs, err := clientset.CoreV1().PersistentVolumeClaims("default").List(ctx, metav1.ListOptions{})
+		if err != nil {
+			fmt.Printf("error %s, getting PVCs\n", err.Error())
+		}
+		count := 0
+		for _, pvc := range pvcs.Items {
+			if strings.Contains(pvc.Name, statefulsetName) {
+				count++
+				danglingPvcName = pvc.Name
+			}
+		}
+		if count == 0 {
+			isSuccessful = true
+			break
+		}
+		time.Sleep(5 * time.Second)
+	}
+	if !isSuccessful {
+		t.Fatalf("Dangling PVCs %v not deleted by Job", danglingPvcName)
+	}
+}
+
 func CreateStorageClass(t *testing.T, clientset *kubernetes.Clientset, ctx context.Context, maxRetry int) {
 	storageClass := generators.GenerateStorageClass(
 		"test-storage-class",
@@ -68,7 +96,7 @@ func CreateStorageClass(t *testing.T, clientset *kubernetes.Clientset, ctx conte
 }
 
 func CreateStatefulSet(t *testing.T, clientset *kubernetes.Clientset, ctx context.Context, name string, maxRetry int) {
-	statefulset := generators.GenerateStatefulSet(name, "default", 2, map[string]string{"sts-pvc": "true"}, "test-storage-class")
+	statefulset := generators.GenerateStatefulSet(name, "default", 1, map[string]string{"sts-pvc": "true"}, "test-storage-class")
 	_, err := clientset.AppsV1().StatefulSets("default").Create(ctx, statefulset, metav1.CreateOptions{})
 	if err != nil {
 		panic(err.Error())