Return error when rspec exits with non-zero but no failure

Author: nprizal

internal/runner/rspec.go

@@ -124,6 +124,16 @@ func (r Rspec) Run(result *RunResult, testCases []plan.TestCase, retry bool) err
 		result.error = fmt.Errorf("RSpec failed with errors outside of examples")
 	}
 
+	if exitError := new(exec.ExitError); errors.As(err, &exitError) {
+		if report.Summary.FailureCount == 0 && report.Summary.ErrorsOutsideOfExamplesCount == 0 {
+			// If Rspec exits with a non-zero exit code, but the report is parsed successfully and contains no failures or errors,
+			// it may mean that the process was terminated by an explicit call to `exit` in the code or specs.
+			// Rspec does not report this as a test failure or error, but instead exits with the same exit code that terminated the process.
+			// In this case, we should mark the result as an error.
+			result.error = fmt.Errorf("RSpec exited with code %d, but no failed tests were reported. This may be caused by an explicit call to `exit` in the code or specs", exitError.ExitCode())
+		}
+	}
+
 	return nil
 }
 

internal/runner/rspec_test.go

@@ -208,6 +208,36 @@ func TestRspecRun_TestSkipped(t *testing.T) {
 	}
 }
 
+func TestRspecRun_TestExit(t *testing.T) {
+	rspec := NewRspec(RunnerConfig{
+		TestCommand: "rspec --format json --out {{resultPath}} --format progress",
+		ResultPath:  "tmp/rspec.json",
+	})
+
+	t.Cleanup(func() {
+		os.Remove(rspec.ResultPath)
+	})
+
+	testCases := []plan.TestCase{
+		{Path: "./testdata/rspec/spec/exit_spec.rb"},
+	}
+	result := NewRunResult([]plan.TestCase{})
+	err := rspec.Run(result, testCases, false)
+
+	if err != nil {
+		t.Errorf("Rspec.Run(%q) error = %v", testCases, err)
+	}
+
+	if result.Status() != RunStatusError {
+		t.Errorf("Rspec.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusError)
+	}
+
+	wantError := "RSpec exited with code 7, but no failed tests were reported. This may be caused by an explicit call to `exit` in the code or specs."
+	if diff := cmp.Diff(result.error.Error(), wantError); diff != "" {
+		t.Errorf("Rspec.Run(%q) RunResult.error diff (-got +want):\n%s", testCases, diff)
+	}
+}
+
 func TestRspecRun_ErrorOutsideOfExamples(t *testing.T) {
 	rspec := NewRspec(RunnerConfig{
 		TestCommand: "rspec --format json --out {{resultPath}} --format documentation",

internal/runner/testdata/rspec/spec/exit_spec.rb

@@ -0,0 +1,5 @@
+RSpec.describe("Exit") do
+  it("exits") do
+    exit 7
+  end
+end