feat: tests can opt-in to retries

Problem:
Tests have no way to retry without manually managing `before_each` and
`after_each` setup/teardown. This is painful. And busted does a good job
of guarding its internals and makes it impossible to extend it: AFAICT,
consumers have no way to access the current "test context", or the
current list of `before_each`/`after_each` hooks.

Solution:
Introduce `env.set_retries()`. On failure the test is retried up to `n`
more times, rerunning `after_each`/`before_each` around each attempt.
`busted.safe()` gains an optional `deferred` collector so only the final
attempt's failure/error/pending events are published: output handlers
and the exit code never see non-final attempts.

```lua
it('...', function()
  set_retries(2) -- If the test fails, it will be retried up to 2 times.
end)
```

Testing:

    luarocks --local remove busted --force && luarocks --local make && ~/.luarocks/bin/busted --pattern=core

Note: Deferral shifts event timing slightly: a failing test's
failure/error event is now published after its `finally` runs, not
before. Relative event order is unchanged; in-tree handlers can't
observe the difference.

Author: justinmk

.luacheckrc

@@ -10,6 +10,7 @@ globals = {
     "async",
     "done",
     "busted",
+    "set_retries",
     --"ngx.IS_CLI",
 }
 

busted/core.lua

@@ -170,7 +170,9 @@ return function()
     end
   end
 
-  function busted.safe(descriptor, run, element)
+  -- `deferred`: optional list collecting the failure/error/pending event of a failed `run` as
+  --             publish arguments, instead of publishing it. For `set_retries`.
+  function busted.safe(descriptor, run, element, deferred)
     busted.context.push(element)
     local trace, message
     local status = 'success'
@@ -198,7 +200,12 @@ return function()
       -- a test failure, but rather an error outside the test, much like a
       -- failure in a support function (i.e. before_each/after_each or
       -- setup/teardown).
-      busted.publish({ status, element.descriptor }, element, busted.context.parent(element), message, trace)
+      local parent = busted.context.parent(element)
+      if deferred then
+        deferred[#deferred + 1] = { { status, element.descriptor }, element, parent, message, trace }
+      else
+        busted.publish({ status, element.descriptor }, element, parent, message, trace)
+      end
     end
     ret[1] = busted.status(status)
 

busted/init.lua

@@ -20,6 +20,8 @@ local function init(busted)
   local it = function(element)
     local parent = busted.context.parent(element)
     local finally
+    local attempt = 1
+    local max_attempts = 1
 
     if not block.lazySetup(parent) then
       -- skip test if any setup failed
@@ -31,16 +33,42 @@ local function init(busted)
     block.rejectAll(element)
     element.env.finally = function(fn) finally = fn end
     element.env.pending = busted.pending
+    element.env.set_retries = function(n) max_attempts = n + 1 end
 
     local pass, ancestor = block.execAll('before_each', parent, true)
 
     if pass then
       local status = busted.status('success')
       if busted.safe_publish('test', { 'test', 'start' }, element, parent) then
-        status:update(busted.safe('it', element.run, element))
-        if finally then
-          block.reject('pending', element)
-          status:update(busted.safe('finally', finally, element))
+        local deferred = {}
+        while attempt <= max_attempts do
+          if attempt > 1 then
+            -- Retrying: finish the failed attempt, then redo setup.
+            finally = nil
+            block.dexecAll('after_each', ancestor, true)
+            pass, ancestor = block.execAll('before_each', parent, true)
+            if not pass then
+              break
+            end
+          end
+
+          deferred = {}
+          status = busted.status('success')
+          status:update(busted.safe('it', element.run, element, deferred))
+          if finally then
+            block.reject('pending', element)
+            status:update(busted.safe('finally', finally, element, deferred))
+          end
+
+          if not (status:failure() or status:error()) then
+            break
+          end
+          attempt = attempt + 1
+        end
+
+        -- Publish only the final attempt's events (see `deferred` in `busted.safe`).
+        for _, event in ipairs(deferred) do
+          busted.publish(event[1], event[2], event[3], event[4], event[5])
         end
       else
         status = busted.status('error')

spec/cl_retries.lua

@@ -0,0 +1,16 @@
+-- Fixture for cl_spec.lua
+
+describe('Tests retries', function()
+  local attempts = 0
+
+  it('succeeds once retried', function()
+    set_retries(2)
+    attempts = attempts + 1
+    assert.is_equal(3, attempts)
+  end)
+
+  it('reports one error when retries are exhausted', function()
+    set_retries(2)
+    error('never succeeds')
+  end)
+end)

spec/cl_spec.lua

@@ -181,6 +181,13 @@ describe('Tests the busted command-line options', function()
     assert.is_equal(3, errcnt)
   end)
 
+  it('tests running with retries', function()
+    local success, errcnt, out = executeBusted('--pattern=cl_retries.lua$')
+    assert.is_false(success)
+    assert.is_equal(1, errcnt)
+    assert.is_truthy(out:match('1 success'))
+  end)
+
   it('tests running with -l specified', function()
     local _, _, result = executeBusted('-l --pattern=cl_list.lua$')
     local expected = 'spec/cl_list.lua:4: Tests list test 1\n' ..

spec/core_spec.lua

@@ -13,6 +13,26 @@ assert(type(mock) == 'table')
 assert(type(match) == 'table')
 assert(type(assert) == 'table')
 
+describe('Test retries', function()
+  local before_count, after_count = 0, 0
+
+  before_each(function()
+    before_count = before_count + 1
+  end)
+
+  after_each(function()
+    after_count = after_count + 1
+  end)
+
+  it('reruns the each-hooks around every attempt', function()
+    set_retries(2)  -- 3 total attempts
+    assert.is_equal(before_count - 1, after_count)
+    if before_count < 3 then
+      assert.is_true(false, ('failing attempt %d'):format(before_count))
+    end
+  end)
+end)
+
 describe('Before each', function()
   local test_val = false