[no-relnote] E2E Create test objects from file

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>

Author: ArangoGutierrez

tests/e2e/data/job-1.yaml

@@ -0,0 +1,25 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: j-e2e-1
+  labels:
+    app.nvidia.com: k8s-device-plugin-test-app
+spec:
+  template:
+    metadata:
+      name: gpu-pod
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: cuda-container
+          image: nvcr.io/nvidia/k8s/cuda-sample:nbody-cuda11.7.1-ubuntu18.04
+          args:
+            - "--benchmark"
+            - "--numbodies=10000"
+          resources:
+            limits:
+              nvidia.com/gpu: "1"
+      tolerations:
+        - key: "nvidia.com/gpu"
+          operator: "Exists"
+          effect: "NoSchedule"

tests/e2e/device-plugin_test.go

@@ -133,15 +133,12 @@ var _ = Describe("GPU Device Plugin", Ordered, func() {
 		})
 		It("it should run GPU jobs", func(ctx context.Context) {
 			By("Creating a GPU job")
-			job := newGPUJob(testNamespace.Name)
-			job.Namespace = testNamespace.Name
-
-			_, err := clientSet.BatchV1().Jobs(testNamespace.Name).Create(ctx, job, metav1.CreateOptions{})
+			job, err := CreateOrUpdateJobsFromFile(ctx, clientSet, "job-1.yaml", testNamespace.Name)
 			Expect(err).NotTo(HaveOccurred())
 
 			By("Waiting for job to complete")
 			Eventually(func() error {
-				job, err := clientSet.BatchV1().Jobs(testNamespace.Name).Get(ctx, job.Name, metav1.GetOptions{})
+				job, err := clientSet.BatchV1().Jobs(testNamespace.Name).Get(ctx, job[0], metav1.GetOptions{})
 				if err != nil {
 					return err
 				}

tests/e2e/e2e_test.go

@@ -23,7 +23,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"testing"
 	"time"
@@ -34,11 +33,9 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	extclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -56,7 +53,6 @@ const (
 )
 
 var (
-	packagePath           string
 	Kubeconfig            string
 	Timeout               time.Duration
 	HelmChart             string
@@ -186,9 +182,6 @@ func getTestEnv() {
 	defer GinkgoRecover()
 	var err error
 
-	_, thisFile, _, _ := runtime.Caller(0)
-	packagePath = filepath.Dir(thisFile)
-
 	Kubeconfig = os.Getenv("KUBECONFIG")
 	Expect(Kubeconfig).NotTo(BeEmpty(), "KUBECONFIG must be set")
 
@@ -284,44 +277,3 @@ func CreateTestingNS(baseName string, c clientset.Interface, labels map[string]s
 
 	return got, nil
 }
-
-func newGPUJob(namespace string) *batchv1.Job {
-	return &batchv1.Job{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "gpu-job",
-			Namespace: namespace,
-			Labels: map[string]string{
-				"app.nvidia.com": "k8s-device-plugin-test-app",
-			},
-		},
-		Spec: batchv1.JobSpec{
-			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Name: "gpu-pod",
-				},
-				Spec: corev1.PodSpec{
-					RestartPolicy: corev1.RestartPolicyNever,
-					Containers: []corev1.Container{
-						{
-							Name:  "cuda-container",
-							Image: "nvcr.io/nvidia/k8s/cuda-sample:nbody-cuda11.7.1-ubuntu18.04",
-							Args:  []string{"--benchmark", "--numbodies=10000"},
-							Resources: corev1.ResourceRequirements{
-								Limits: corev1.ResourceList{
-									"nvidia.com/gpu": resource.MustParse("1"),
-								},
-							},
-						},
-					},
-					Tolerations: []corev1.Toleration{
-						{
-							Key:      "nvidia.com/gpu",
-							Operator: corev1.TolerationOpExists,
-							Effect:   corev1.TaintEffectNoSchedule,
-						},
-					},
-				},
-			},
-		},
-	}
-}

tests/e2e/utils.go

@@ -18,22 +18,32 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"math/rand"
+	"os"
+	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
 	. "github.com/onsi/gomega"
 	gomegatypes "github.com/onsi/gomega/types"
 
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiruntime "k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
+	k8sscheme "k8s.io/client-go/kubernetes/scheme"
 )
 
+var packagePath string
+
 type k8sLabels map[string]string
 
 // eventuallyNonControlPlaneNodes is a helper for asserting node properties
@@ -254,3 +264,80 @@ func getNode(nodes []corev1.Node, nodeName string) corev1.Node {
 	}
 	return corev1.Node{}
 }
+
+// CreateOrUpdateJobsFromFile creates or updates jobs from a file
+func CreateOrUpdateJobsFromFile(ctx context.Context, cli clientset.Interface, filename, namespace string) ([]string, error) {
+	jobs, err := newJobFromfile(filepath.Join(packagePath, "data", filename))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Job from file: %w", err)
+	}
+
+	names := make([]string, len(jobs))
+	for i, job := range jobs {
+		job.Namespace = namespace
+
+		names[i] = job.Name
+
+		// create or update the job
+		_, err := cli.BatchV1().Jobs(namespace).Get(ctx, job.Name, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				_, err = cli.BatchV1().Jobs(namespace).Create(ctx, job, metav1.CreateOptions{})
+				if err != nil {
+					return nil, fmt.Errorf("failed to create Job %s: %w", job.Name, err)
+				}
+			} else {
+				return nil, fmt.Errorf("failed to get Job %s: %w", job.Name, err)
+			}
+		}
+	}
+
+	return names, nil
+}
+
+func newJobFromfile(path string) ([]*batchv1.Job, error) {
+	objs, err := apiObjsFromFile(path, k8sscheme.Codecs.UniversalDeserializer())
+	if err != nil {
+		return nil, err
+	}
+
+	jobs := make([]*batchv1.Job, len(objs))
+
+	for i, obj := range objs {
+		var ok bool
+		jobs[i], ok = obj.(*batchv1.Job)
+		if !ok {
+			return nil, fmt.Errorf("unexpected type %t when reading %q", obj, path)
+		}
+	}
+
+	return jobs, nil
+}
+func apiObjsFromFile(path string, decoder apiruntime.Decoder) ([]apiruntime.Object, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO: find out a nicer way to decode multiple api objects from a single
+	// file (K8s must have that somewhere)
+	split := bytes.Split(data, []byte("---"))
+	objs := []apiruntime.Object{}
+
+	for _, slice := range split {
+		if len(slice) == 0 {
+			continue
+		}
+		obj, _, err := decoder.Decode(slice, nil, nil)
+		if err != nil {
+			return nil, err
+		}
+		objs = append(objs, obj)
+	}
+	return objs, err
+}
+
+func init() {
+	_, thisFile, _, _ := runtime.Caller(0)
+	packagePath = filepath.Dir(thisFile)
+}