Return error when rspec exits with non-zero but no failure

internal/runner/rspec.go

@@ -120,8 +120,24 @@ func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) err
 		result.RecordTestResult(mapExampleToTestCase(example), status)
 	}
 
-	if report.Summary.ErrorsOutsideOfExamplesCount > 0 {
-		result.error = fmt.Errorf("RSpec failed with errors outside of examples")
+	if exitError := new(exec.ExitError); errors.As(err, &exitError) {
+		// If rspec exits with a non-zero status and reported test failures,
+		// we can ignore the error. The test failures are handled by the *RunResult.
+		if report.Summary.FailureCount > 0 {
+			return nil
+		}
+
+		// If rspec exits with a non-zero status and reported errors outside of examples,
+		// set the error in *RunResult to fail the job.
+		if report.Summary.ErrorsOutsideOfExamplesCount > 0 {
+			result.error = fmt.Errorf("RSpec failed with errors outside of examples")
+			return nil
+		}
+
+		// Otherwise, the non-zero exit code is unexpected
+		// and we should set the error in *RunResult to fail the job and surface the exit code.
+		result.error = fmt.Errorf("RSpec exited with code %d", exitError.ExitCode())
+		return nil
 	}
 
 	return nil

internal/runner/rspec_test.go

@@ -208,6 +208,36 @@ func TestRspecRun_TestSkipped(t *testing.T) {
 	}
 }
 
+func TestRspecRun_TestExit(t *testing.T) {
+	rspec := NewRspec(RunnerConfig{
+		TestCommand: "rspec --format json --out {{resultPath}} --format progress",
+		ResultPath:  "tmp/rspec.json",
+	})
+
+	t.Cleanup(func() {
+		os.Remove(rspec.ResultPath)
+	})
+
+	testCases := []plan.TestCase{
+		{Path: "./testdata/rspec/spec/exit_spec.rb"},
+	}
+	result := NewRunResult([]plan.TestCase{})
+	err := rspec.Run(result, testCases, false)
+
+	if err != nil {
+		t.Errorf("Rspec.Run(%q) error = %v", testCases, err)
+	}
+
+	if result.Status() != RunStatusError {
+		t.Errorf("Rspec.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusError)
+	}
+
+	wantError := "RSpec exited with code 7"
+	if diff := cmp.Diff(result.error.Error(), wantError); diff != "" {
+		t.Errorf("Rspec.Run(%q) RunResult.error diff (-got +want):\n%s", testCases, diff)
+	}
+}
+
 func TestRspecRun_ErrorOutsideOfExamples(t *testing.T) {
 	rspec := NewRspec(RunnerConfig{
 		TestCommand: "rspec --format json --out {{resultPath}} --format documentation",

internal/runner/testdata/rspec/spec/exit_spec.rb

@@ -0,0 +1,5 @@
+RSpec.describe("Exit") do
+  it("exits") do
+    exit 7
+  end
+end