feat(assertions): support custom assertions

Signed-off-by: GitHub <noreply@github.com>

Author: lowlighter

README.md

@@ -39,6 +39,7 @@ Venom is a CLI (Command Line Interface) that aims to create, manage and run your
   - [Assertions](#assertions)
     - [Keywords](#keywords)
       - [`Must` keywords](#must-keywords)
+      - [Custom assertions](#custom-assertions)
     - [Using logical operators](#using-logical-operators)
 - [Write and run your first test suite](#write-and-run-your-first-test-suite)
 - [Export tests report](#export-tests-report)
@@ -695,6 +696,45 @@ Example:
   # Remaining steps in this context will not be executed
 ```
 
+#### Custom assertions
+
+It is possible to use the [user defined executors syntax](#user-defined-executors) and prefixing your executor name with `Should` to create a new custom assertion keyword. 
+
+Example:
+```yml
+# lib/ShouldBeHttp2XX.yml
+executor: ShouldBeHttp2XX
+steps:
+  - assertions:
+    - a ShouldBeGreaterThanOrEqualTo 200
+    - a ShouldBeLessThan 300
+```
+
+You may also include additional steps like a regular user defined executor.
+
+Custom assertions are executed in an entirely clean context, containing only the following variables:
+- `a`: the left operand
+- `b`: the (first) right operand
+- `argv`: the rights operands
+
+If you need to be compatible with the `input` syntax of user defined executors, you could use the `argv` as the default value and access these through the regular `input.*` syntax. 
+```yaml
+input:
+  test: "{{.argv.argv1}}"
+```
+
+To call a registered custom assertions, just use its registered name in the `assertions` array, like any built-in assertion keyword.
+
+```yml
+# test.yml
+testcases:
+- steps:
+  - type: http
+    url: https://example.com
+    assertions:
+    - result.statuscode ShouldBeHttp2XX
+```
+
 ### Using logical operators
 
 While assertions use `and` operator implicitly, it is possible to use other logical operators to perform complex assertions.

assertions/assertions.go

@@ -1508,3 +1508,13 @@ func getTimeFromString(in interface{}) (time.Time, error) {
 
 	return t, nil
 }
+
+// RegisterCustomAssertFunc registers a custom assertion function
+func RegisterCustomAssertFunc(name string, ux AssertFunc) error {
+	if _, ok := assertMap[name]; ok {
+		return errors.Errorf("cannot redefine existing assertion %q", name)
+	}
+
+	assertMap[name] = ux
+	return nil
+}

assertions/assertions_test.go

@@ -2438,3 +2438,37 @@ func TestShouldNotJSONEqual(t *testing.T) {
 		})
 	}
 }
+
+func TestRegisterCustomAssertFunc(t *testing.T) {
+	var assertFunc AssertFunc
+
+	tests := []struct {
+		name    string
+		actual  string
+		wantErr bool
+	}{
+		{
+			name:   "register custom assertion",
+			actual: "ShouldCustom",
+		},
+		{
+			name:    "redefine custom assertion error",
+			actual:  "ShouldCustom",
+			wantErr: true,
+		},
+		{
+			name:    "redefine builtin assertion error",
+			actual:  "ShouldEqual",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := RegisterCustomAssertFunc(tt.actual, assertFunc)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("RegisterCustomAssertFunc() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

tests/assertions/ShouldCustom.yml

@@ -0,0 +1,8 @@
+name: Assertions testsuite
+testcases:
+- name: test assertion ShouldCustom
+  steps:
+  - script: echo foo
+    assertions:
+    - result.systemout ShouldCustom bar qux
+    - result.systemout MustCustom bar qux

tests/lib/ShouldCustom.yml

@@ -0,0 +1,15 @@
+executor: ShouldCustom
+input:
+  test: "{{.argv.argv1}}"
+steps:
+  - script: echo baz
+    vars: 
+      c:
+        from: result.systemout
+  - assertions:
+    - a ShouldEqual foo
+    - b ShouldEqual bar
+    - c ShouldEqual baz
+    - input.test ShouldEqual qux
+    - argv.argv1 ShouldEqual qux
+    - argv.__Len__ ShouldEqual 2

venom.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/confluentinc/bincover"
 	"github.com/fatih/color"
+	"github.com/ovh/venom/assertions"
 	"github.com/pkg/errors"
 	"github.com/spf13/cast"
 )
@@ -280,6 +281,57 @@ func (v *Venom) registerUserExecutors(ctx context.Context) error {
 			return errors.Wrapf(err, "unable to register user executor %q from file %q", ux.Executor, f)
 		}
 		Info(ctx, "User executor %q registered", ux.Executor)
+
+		if strings.HasPrefix(ux.Executor, "Should") {
+			err = v.registerUserAssertFunc(ctx, ux.Executor)
+			if err != nil {
+				return errors.Wrapf(err, "unable to register user executor %q from file %q as custom assertion", ux.Executor, f)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (v *Venom) registerUserAssertFunc(ctx context.Context, name string) error {
+	_, e, err := v.GetExecutorRunner(ctx, TestStep{"type": name}, H{})
+	if err != nil {
+		return errors.Wrapf(err, "unable to load user executor %q", name)
+	}
+
+	err = assertions.RegisterCustomAssertFunc(name, func(actual interface{}, expected ...interface{}) error {
+		// Prepare a minimal context in which the user executor can be run
+		// We only need to register the operands in the var set
+		// as parent contexts will already have been templated
+		tc := &TestCase{TestCaseInput: TestCaseInput{Name: name}}
+		tc.TestSuiteVars.Add("a", actual)
+		if len(expected) > 0 {
+			tc.TestSuiteVars.Add("b", expected[0])
+		}
+		tc.TestSuiteVars.Add("argv", expected)
+		ts := &TestStepResult{}
+
+		_, err := v.RunUserExecutor(ctx, e, tc, ts, TestStep{})
+
+		// We reformat any sub-assertions errors to avoid redundant texts
+		if len(ts.Errors) > 0 {
+			msg := make([]string, len(ts.Errors))
+			for _, e := range ts.Errors {
+				msg = append(msg, fmt.Sprintf(`  %d: Sub-assertion %q failed. %s`,
+					e.StepNumber,
+					RemoveNotPrintableChar(e.Assertion),
+					RemoveNotPrintableChar(e.Error.Error()),
+				))
+			}
+			return errors.New(strings.Join(msg, "\n"))
+		} else if err != nil {
+			return errors.Wrapf(err, "custom assertion failed during execution")
+		}
+
+		return nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, "unable to register user assertion function %q", name)
 	}
 
 	return nil