fix: user executor failures missing from JUnit/XML/JSON/HTML reports (#907)

Signed-off-by: Ivan Velasco <ivan.velasco@socotra.com>

Author: ivan-velasco

process_testcase.go

@@ -419,13 +419,7 @@ func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestS
 // Print a single step result (if verbosity is enabled)
 func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) {
 	fromUserExecutor := tsIn != nil
-	if fromUserExecutor {
-		// move back up user executor errors to parent test step for later logging
-		tsIn.Errors = append(tsIn.Errors, ts.Errors...)
-
-		// move back up user executor logs to parent test step for later logging
-		tsIn.ComputedInfo = append(tsIn.ComputedInfo, ts.ComputedInfo...)
-	} else if v.Verbose >= 1 {
+	if !fromUserExecutor && v.Verbose >= 1 {
 		if len(ts.Errors) > 0 {
 			v.Println(" %s", Red(StatusFail))
 			for _, i := range ts.ComputedInfo {

tests/lib/user_executor_nested_fail_inner.yml

@@ -0,0 +1,14 @@
+executor: nested_fail_inner
+input:
+  message: "default_inner"
+steps:
+- type: exec
+  script: echo '{{.input.message}}'
+  assertions:
+  - result.code ShouldEqual 0
+- type: exec
+  script: echo 'this assertion will fail'
+  assertions:
+  - result.systemout ShouldEqual "this will never match"
+output:
+  result: "{{.result.systemout}}"

tests/lib/user_executor_nested_fail_outer.yml

@@ -0,0 +1,12 @@
+executor: nested_fail_outer
+input:
+  greeting: "hello"
+steps:
+- type: exec
+  script: echo 'outer step 1 ok'
+  assertions:
+  - result.code ShouldEqual 0
+- type: nested_fail_inner
+  message: '{{.input.greeting}}_from_outer'
+output:
+  result: "{{.result.systemout}}"

tests/lib/user_executor_with_failure.yml

@@ -0,0 +1,17 @@
+executor: user_executor_with_failure
+input:
+  expected_result: "success"
+
+steps:
+- type: exec
+  script: echo "this step succeeds"
+  assertions:
+  - result.code ShouldEqual 0
+
+- type: exec
+  script: echo "failing step"
+  assertions:
+  - result.code ShouldEqual 1
+
+output:
+  result: "{{.result.systemout}}"

tests/user_executor_with_failure.yml

@@ -0,0 +1,10 @@
+name: user executor failure propagation
+testcases:
+- name: simple user executor failure
+  steps:
+  - type: user_executor_with_failure
+
+- name: nested user executor failure (outer calls inner which fails)
+  steps:
+  - type: nested_fail_outer
+    greeting: "world"

types_executor.go

@@ -285,6 +285,23 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn
 
 	v.runTestSteps(ctx, tc, tsIn)
 
+	// Merge inner step results into the parent test step result so they
+	// appear in JUnit XML, JSON, HTML, and other report outputs.
+	for _, innerResult := range tc.TestStepResults {
+		if len(innerResult.Errors) > 0 {
+			tsIn.Errors = append(tsIn.Errors, innerResult.Errors...)
+		}
+		if len(innerResult.ComputedInfo) > 0 {
+			tsIn.ComputedInfo = append(tsIn.ComputedInfo, innerResult.ComputedInfo...)
+		}
+		if strings.TrimSpace(innerResult.Systemout) != "" {
+			tsIn.Systemout += innerResult.Systemout
+		}
+		if strings.TrimSpace(innerResult.Systemerr) != "" {
+			tsIn.Systemerr += innerResult.Systemerr
+		}
+	}
+
 	computedVars, err := DumpString(tc.computedVars)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to dump testcase computedVars")