Integration tests and Job

Author: ksraj123

Dockerfile

@@ -7,6 +7,9 @@ COPY go.sum ./
 RUN go mod download
 
 COPY *.go ./
+COPY pkg ./pkg
+COPY Makefile ./
+COPY tests ./tests
 
 RUN go build -o /lister-sa
 

deploy/job.yaml

@@ -15,6 +15,11 @@ spec:
       automountServiceAccountToken: true
       containers:
       - name: command-demo-container
-        image: lister-sa:0.1
-        imagePullPolicy: Never
+        image: ksraj123/stale-sts-pvc-cleaner:0.1
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: PROVISIONERS
+          value: "openebs.io/local"
+        - name: NAMESPACES
+          value: "default"
       restartPolicy: Never

deploy/mongodb/pv.yaml

@@ -12,7 +12,7 @@ spec:
  clusterIP: None
  selector:
    role: mongo
-   sts-pvc-selector: sts-pvc
+   sts-pvc: "true"
 ---
 apiVersion: apps/v1
 kind: StatefulSet
@@ -29,11 +29,8 @@ spec:
    metadata:
      labels:
        role: mongo
-       sts-pvc-selector: sts-pvc
+       sts-pvc: "true"
        environment: test
-       #This label will be used by openebs to place in replica
-       # pod anti-affinity to make sure data of different mongo
-       # instances are not co-located on the same node
        openebs.io/replica-anti-affinity: vehicle-db
    spec:
      terminationGracePeriodSeconds: 10

deploy/mongodb/pvc.yaml

@@ -7,8 +7,13 @@ spec:
   automountServiceAccountToken: true
   containers:
   - name: command-demo-container
-    image: ksraj123/sts-pv:0.1.1
-    imagePullPolicy: Never
+    image: ksraj123/stale-sts-pvc-cleaner:0.1
+    imagePullPolicy: IfNotPresent
     command: ["sleep"]
     args: ["infinity"]
+    env:
+    - name: PROVISIONERS
+      value: "openebs.io/local"
+    - name: NAMESPACES
+      value: "default"
   restartPolicy: Never

deploy/mongodb/sts.yaml

@@ -1,7 +1,7 @@
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: test-storage-class
+  name: test-storage-class-1
   annotations:
     openebs.io/delete-dangling-pvc: "true"
     openebs.io/cas-type: local

deploy/pod-sa.yaml

@@ -6,7 +6,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.17.0 // indirect
-	k8s.io/api v0.22.3
+	k8s.io/api v0.22.4
 	k8s.io/apimachinery v0.22.4
 	k8s.io/client-go v0.22.3
 	sigs.k8s.io/controller-runtime v0.10.3 // indirect

deploy/sc.yaml

@@ -727,6 +727,8 @@ k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw=
 k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8=
 k8s.io/api v0.22.3 h1:wOoES2GoSkUsdped2RB4zYypPqWtvprGoKCENTOOjP4=
 k8s.io/api v0.22.3/go.mod h1:azgiXFiXqiWyLCfI62/eYBOu19rj2LKmIhFPP4+33fs=
+k8s.io/api v0.22.4 h1:UvyHW0ezB2oIgHAxlYoo6UJQObYXU7awuNarwoHEOjw=
+k8s.io/api v0.22.4/go.mod h1:Rgs+9gIGYC5laXQSZZ9JqT5NevNgoGiOdVWi1BAB3qk=
 k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4=
 k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA=
 k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk=

go.mod

@@ -6,4 +6,5 @@ const (
 	PROVISIONERS_ENV_VAR     = "PROVISIONERS"
 	STORAGE_CLASS_ANNOTATION = "openebs.io/delete-dangling-pvc"
 	STS_PVC_SELECTOR         = "sts-pvc-selector"
+	OPENEBS_NAMESPACe        = "openebs"
 )

go.sum

@@ -2,6 +2,7 @@ package generators
 
 import (
 	AppsV1 "k8s.io/api/apps/v1"
+	BatchV1 "k8s.io/api/batch/v1"
 	CoreV1 "k8s.io/api/core/v1"
 	StorageV1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -77,13 +78,48 @@ func GeneratePersistentVolumeClaim(name string, namespace string, storageClassNa
 
 func GenerateStorageClass(name string, annotations map[string]string, parameters map[string]string, provisioner string) *StorageV1.StorageClass {
 	var deletePolicy CoreV1.PersistentVolumeReclaimPolicy = "Delete"
+	mode := StorageV1.VolumeBindingWaitForFirstConsumer
 	return &StorageV1.StorageClass{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        name,
 			Annotations: annotations,
 		},
-		Parameters:    parameters,
-		Provisioner:   provisioner,
-		ReclaimPolicy: &deletePolicy,
+		Parameters:        parameters,
+		Provisioner:       provisioner,
+		ReclaimPolicy:     &deletePolicy,
+		VolumeBindingMode: &mode,
+	}
+}
+
+func GenerateJob(name string, labels map[string]string, serviceAccountName string, image string, env []CoreV1.EnvVar) *BatchV1.Job {
+	automountServiceAccountToken := true
+	imagePullPolicy := CoreV1.PullIfNotPresent
+	restartPolicy := CoreV1.RestartPolicyNever
+	return &BatchV1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   name,
+			Labels: labels,
+		},
+		Spec: BatchV1.JobSpec{
+			Template: CoreV1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   name,
+					Labels: labels,
+				},
+				Spec: CoreV1.PodSpec{
+					ServiceAccountName:           serviceAccountName,
+					AutomountServiceAccountToken: &automountServiceAccountToken,
+					Containers: []CoreV1.Container{
+						{
+							Name:            "container",
+							Image:           image,
+							ImagePullPolicy: imagePullPolicy,
+							Env:             env,
+						},
+					},
+					RestartPolicy: restartPolicy,
+				},
+			},
+		},
 	}
 }

pkg/constants/constants.go

@@ -0,0 +1,77 @@
+package setup
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/ksraj123/lister-sa/pkg/constants"
+	"github.com/ksraj123/lister-sa/tests/generators"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func TestOpenEbsPodRunningState(t *testing.T, clientset *kubernetes.Clientset, ctx context.Context, maxRetry int) {
+	LocalPVProvisionerLabelSelector := "openebs.io/component-name=openebs-localpv-provisioner"
+
+	openebsNamespace := constants.OPENEBS_NAMESPACe
+
+	tests := map[string]struct {
+		expected int
+	}{
+		"Testing openebs-localpv-provisioner pod state": {
+			expected: 1,
+		},
+	}
+
+	testPassed := false
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			for i := 0; i < maxRetry; i++ {
+				pods, _ := clientset.CoreV1().Pods(openebsNamespace).List(ctx, metav1.ListOptions{
+					LabelSelector: LocalPVProvisionerLabelSelector,
+				})
+				if len(pods.Items) == test.expected {
+					testPassed = true
+					break
+				}
+				time.Sleep(5 * time.Second)
+			}
+			if !testPassed {
+				t.Fatalf("openebs-localpv-provisioner pod not running in namespace, %v", openebsNamespace)
+			}
+		})
+	}
+}
+
+func CreateStorageClass(t *testing.T, clientset *kubernetes.Clientset, ctx context.Context, maxRetry int) {
+	storageClass := generators.GenerateStorageClass(
+		"test-storage-class",
+		map[string]string{
+			constants.STORAGE_CLASS_ANNOTATION: "true",
+			"openebs.io/cas-type":              "local",
+			"cas.openebs.io/config": `- name: StorageType
+  value: "hostpath"
+- name: BasePath
+  value: "/var/openebs/local/"`,
+		},
+		map[string]string{
+			constants.STS_PVC_SELECTOR: "sts-pvc",
+		},
+		"openebs.io/local")
+	_, err := clientset.StorageV1().StorageClasses().Create(ctx, storageClass, metav1.CreateOptions{})
+	if err != nil {
+		panic(err.Error())
+	}
+	time.Sleep(5 * time.Second)
+}
+
+func CreateStatefulSet(t *testing.T, clientset *kubernetes.Clientset, ctx context.Context, name string, maxRetry int) {
+	statefulset := generators.GenerateStatefulSet(name, "default", 2, map[string]string{"sts-pvc": "true"}, "test-storage-class")
+	_, err := clientset.AppsV1().StatefulSets("default").Create(ctx, statefulset, metav1.CreateOptions{})
+	if err != nil {
+		panic(err.Error())
+	}
+	time.Sleep(5 * time.Second)
+}

tests/generators/generators.go

@@ -1,52 +1,89 @@
 package tests
 
 import (
-	"flag"
-	"os"
+	"context"
+	"fmt"
+	"strings"
 	"testing"
+	"time"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	"github.com/ksraj123/lister-sa/tests/generators"
+	"github.com/ksraj123/lister-sa/tests/setup"
+	CoreV1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
+	envtest "sigs.k8s.io/controller-runtime/pkg/envtest"
 )
 
-var (
-	kubeConfigPath                  string
-	openebsNamespace                string
-	clientset                       *kubernetes.Clientset
-	LocalPVProvisionerLabelSelector = "openebs.io/component-name=openebs-localpv-provisioner"
-)
-
-func init() {
-	flag.StringVar(&kubeConfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "path to kubeconfig to invoke kubernetes API calls")
-	flag.StringVar(&openebsNamespace, "openebs-namespace", "openebs", "kubernetes namespace where the OpenEBS components are present")
-	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+func startCluster() (*kubernetes.Clientset, *envtest.Environment) {
+	testenv := &envtest.Environment{}
+	cfg, err := testenv.Start()
 	if err != nil {
 		panic(err.Error())
 	}
-	clientset, err = kubernetes.NewForConfig(config)
+	clientset, err := kubernetes.NewForConfig(cfg)
 	if err != nil {
 		panic(err.Error())
 	}
+	return clientset, testenv
 }
 
-func TestSource(t *testing.T) {
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Test application deployment")
+func stopCluster(testenv *envtest.Environment) {
+	testenv.Stop()
 }
 
-var _ = BeforeSuite(func() {
-	By("waiting for openebs-localpv-provisioner pod to come into running state")
-	provPodCount := GetPodRunningCountEventually(
-		openebsNamespace,
-		LocalPVProvisionerLabelSelector,
-		1,
-		clientset,
-	)
-	Expect(provPodCount).To(Equal(1))
+func TestMain(t *testing.T) {
+	ctx := context.Background()
+	clientSet, clusterTestEnv := startCluster()
+	defer stopCluster(clusterTestEnv)
+
+	// Ensure openebs-localpv-provisioner pod running
+	setup.TestOpenEbsPodRunningState(t, clientSet, ctx, 90)
+	setup.CreateStorageClass(t, clientSet, ctx, 10)
+	statefulsetName := "test-sts"
+	setup.CreateStatefulSet(t, clientSet, ctx, statefulsetName, 10)
 
-	// run our job
-})
+	// Delete the statefulset
+
+	err := clientSet.AppsV1().StatefulSets("default").Delete(ctx, statefulsetName, metav1.DeleteOptions{})
+	if err != nil {
+		panic(err.Error())
+	}
+
+	time.Sleep(5 * time.Second)
+
+	// now the statefulset PVCs are in dangling state
+
+	serviceAccountName := "openebs-maya-operator"
+	image := "ksraj123/stale-sts-pvc-cleaner:0.1"
+	env := []CoreV1.EnvVar{
+		{
+			Name:  "PROVISIONERS",
+			Value: "openebs.io/local",
+		},
+		{
+			Name:  "NAMESPACES",
+			Value: "default",
+		},
+	}
 
-var _ = Describe("Shopping cart", func() {})
+	job := generators.GenerateJob("test-job", map[string]string{"jobGroup": "test"}, serviceAccountName, image, env)
+	_, err = clientSet.BatchV1().Jobs("default").Create(ctx, job, metav1.CreateOptions{})
+	if err != nil {
+		panic(err.Error())
+	}
+
+	time.Sleep(5 * time.Second)
+
+	pvcs, err := clientSet.CoreV1().PersistentVolumeClaims("default").List(ctx, metav1.ListOptions{})
+	if err != nil {
+		fmt.Printf("error %s, getting PVCs\n", err.Error())
+	}
+	for _, pvc := range pvcs.Items {
+		if strings.Contains(pvc.Name, statefulsetName) {
+			t.Fatalf("Dangling PVCs %v not deleted by Job", pvc.Name)
+		}
+	}
+
+	// clientSet.BatchV1().Jobs("default").Delete(ctx, "test-job", metav1.DeleteOptions{})
+}