feat(test): testing is never easy (#58)

This commit is a step to make testing simple.

- This is all about simplified testing
- This caters to integration as well as e2e testing
- Tests are implemented as custom resources
- No shell or bash scripting
- No grep, awk or sed
- No timing, retries, teardown
- Knowledge of kubectl will help

mage testprep
cd test
go run suite.go

- Build the binary of your code
- Copy the binary to test/bin
- Implement test cases as YAMLs inside test/experiments
- Create test manifests i.e. kubernetes yamls required to start testing
  - Put them under test/manifests
- Which kind of kubernetes setup do you need for these tests to run?
  - Specify that in test/setup/controlplane
  - controlplane setup is supported currently
  - k3s, kind, k3d, etc setups will be supported in near future

Q/ What is a control plane setup?
A/ This is a kubernetes setup with kube apiserver, etcd & kubectl binaries
that talk to each other.

Following is an example, where I want to test my binary against a kubernetes
control plane setup.

apiVersion: v1.0.0
kind: TestOnControlPlane
metadata:
  name: test-dope-on-ctrl-plane
spec:
  target:
    binary:
      path: bin
      name: dope
      args:
      - --logtostderr
      - --alsologtostderr
      - --run-as-local
      - --v=2
      - --discovery-interval=40s
      - --cache-flush-interval=240s
      - --metac-config-path=../config/
      kubeAPIServerURLFlag: --kube-apiserver-url
    deploy:
      path: "../manifests"
  test:
    deploy:
      path: manifests
    experiments:
      path: experiments
    inference:
      experimentName: inference
      experimentNamespace: d-testing
      displaySelector:
        matchLabels:
          d-testing.metacontroller.app/enabled: "true"

Signed-off-by: AmitKumarDas <[email protected]>

Author: Amit Kumar Das

pkg/job/job.go

@@ -45,8 +45,8 @@ type Runner struct {
 // NewRunner returns a new instance of Runner
 func NewRunner(config RunnerConfig) *Runner {
 	var isTearDown bool
-	if config.Job.JobSpec.Teardown != nil {
-		isTearDown = *config.Job.JobSpec.Teardown
+	if config.Job.Spec.Teardown != nil {
+		isTearDown = *config.Job.Spec.Teardown
 	}
 	return &Runner{
 		isTearDown: isTearDown,
@@ -58,10 +58,10 @@ func NewRunner(config RunnerConfig) *Runner {
 }
 
 func (r *Runner) init() {
-	if r.Job.JobSpec.Enabled == nil {
+	if r.Job.Spec.Enabled == nil {
 		r.when = types.Once
 	} else {
-		r.when = r.Job.JobSpec.Enabled.When
+		r.when = r.Job.Spec.Enabled.When
 	}
 }
 
@@ -151,13 +151,13 @@ func (r *Runner) buildLockRunner() *LockRunner {
 		Task:               lock,
 		LockForever:        isLockForever,
 		Retry:              NewRetry(RetryConfig{}),
-		ProtectedTaskCount: len(r.Job.JobSpec.Tasks),
+		ProtectedTaskCount: len(r.Job.Spec.Tasks),
 	}
 }
 
 // evalAll evaluates all tasks
 func (r *Runner) evalAll() error {
-	for _, task := range r.Job.JobSpec.Tasks {
+	for _, task := range r.Job.Spec.Tasks {
 		err := r.eval(task)
 		if err != nil {
 			return err
@@ -197,7 +197,7 @@ func (r *Runner) runAll() (status *types.JobStatus, err error) {
 		r.fixture.TearDown()
 	}()
 	var failedTasks int
-	for idx, task := range r.Job.JobSpec.Tasks {
+	for idx, task := range r.Job.Spec.Tasks {
 		tr := &TaskRunner{
 			Fixture:   r.fixture,
 			TaskIndex: idx + 1,
@@ -225,7 +225,7 @@ func (r *Runner) runAll() (status *types.JobStatus, err error) {
 	} else {
 		r.JobStatus.Phase = r.mayBePassedOrCompletedStatus()
 	}
-	r.JobStatus.TaskCount = len(r.Job.JobSpec.Tasks)
+	r.JobStatus.TaskCount = len(r.Job.Spec.Tasks)
 	return r.JobStatus, nil
 }
 

test/testfiles/assert-job-teardown.yaml renamed to test/experiments/assert-job-teardown.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: d-testing

test/testfiles/assert-lock-persists-for-job-that-runs-once.yaml renamed to test/experiments/assert-lock-persists-for-job-that-runs-once.yaml

@@ -0,0 +1,35 @@
+package common
+
+import (
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// CMDArgs defines a command along with its arguments
+type CMDArgs struct {
+	CMD  string
+	Args []string
+}
+
+// UnstructList defines an array of unstructured instances
+type UnstructList []unstructured.Unstructured
+
+// ToTyped transforms the provided unstruct instance
+// to target type
+func ToTyped(src *unstructured.Unstructured, target interface{}) error {
+	if src == nil || src.Object == nil {
+		return errors.Errorf(
+			"Can't transform unstruct to typed: Nil unstruct content",
+		)
+	}
+	if target == nil {
+		return errors.Errorf(
+			"Can't transform unstruct to typed: Nil target",
+		)
+	}
+	return runtime.DefaultUnstructuredConverter.FromUnstructured(
+		src.UnstructuredContent(),
+		target,
+	)
+}

test/testfiles/crud-ops-on-pod.yaml renamed to test/experiments/crud-ops-on-pod.yaml

@@ -0,0 +1,89 @@
+package common
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"k8s.io/klog/v2"
+)
+
+// SetupLoader loads platform setup files
+type SetupLoader struct {
+	Path string
+}
+
+// Load loads all platform setup files and uses populator
+// as a callback to operate against these loaded files.
+//
+// NOTE:
+//	Argument 'populator' is invoked for every valid file
+// found in the configured path
+func (l *SetupLoader) Load(populator func([]byte) error) error {
+	klog.V(3).Infof(
+		"Will load & populate setup file(s) at path %q", l.Path,
+	)
+
+	files, err := ioutil.ReadDir(l.Path)
+	if err != nil {
+		return err
+	}
+
+	if len(files) == 0 {
+		return errors.Errorf(
+			"No setup files(s) were found at %q", l.Path,
+		)
+	}
+
+	// there can be multiple config files
+	for _, file := range files {
+		fileName := file.Name()
+		if file.IsDir() || file.Mode().IsDir() {
+			klog.V(4).Infof(
+				"Will skip setup file %q at path %q: Not a file",
+				fileName,
+				l.Path,
+			)
+			// we don't want to load directory
+			continue
+		}
+		if !strings.HasSuffix(fileName, ".yaml") &&
+			!strings.HasSuffix(fileName, ".json") {
+			klog.V(4).Infof(
+				"Will skip setup file %q at path %q: Not yaml or json",
+				fileName,
+				l.Path,
+			)
+			continue
+		}
+		// load the file
+		fileNameWithPath := filepath.Join(l.Path, fileName)
+		content, err := ioutil.ReadFile(fileNameWithPath)
+		if err != nil {
+			return errors.Wrapf(
+				err,
+				"Failed to load setup file %q",
+				fileNameWithPath,
+			)
+		}
+		// poluate the loaded content using populator callback
+		err = populator(content)
+		if err != nil {
+			return errors.Wrapf(
+				err,
+				"Failed to populate from setup file %q",
+				fileNameWithPath,
+			)
+		}
+		klog.V(3).Infof(
+			"Setup file %q was loaded & populated successfully", fileNameWithPath,
+		)
+	}
+
+	klog.V(3).Infof(
+		"Setup files(s) at path %q were loaded & populated successfully",
+		l.Path,
+	)
+	return nil
+}