test: add regression tests for retry config accumulation

Author: smalia2001

test/data/sandbox/configs/retryHooks/codecept.retry.accumulation.conf.js

@@ -0,0 +1,22 @@
+export const config = {
+  tests: './retry_accumulation_test.js',
+  output: './output',
+  helpers: {
+    Playwright: {
+      url: 'http://localhost:8000',
+      manualStart: true,
+    },
+    AccumulationHelper: {
+      require: './helper.accumulation.js',
+    },
+  },
+  plugins: {
+    retryFailedStep: {
+      enabled: true,
+      retries: 2,
+    },
+  },
+  bootstrap: null,
+  mocha: {},
+  name: 'retryAccumulation',
+};

test/data/sandbox/configs/retryHooks/codecept.retry.multipleScenarios.conf.js

@@ -0,0 +1,23 @@
+export const config = {
+  tests: './retry_multiple_scenarios_test.js',
+  output: './output',
+  helpers: {
+    Playwright: {
+      url: 'http://localhost:8000',
+      show: false,
+      restart: false,
+    },
+    AccumulationHelper: {
+      require: './helper.accumulation.js',
+    },
+  },
+  plugins: {
+    retryFailedStep: {
+      enabled: true,
+      retries: 2,
+    },
+  },
+  bootstrap: null,
+  mocha: {},
+  name: 'retryMultipleScenarios',
+};

test/data/sandbox/configs/retryHooks/helper.accumulation.js

@@ -0,0 +1,16 @@
+import Helper from '../../../../../lib/helper.js';
+
+class AccumulationHelper extends Helper {
+  _before() {
+    this._failCount = 0;
+  }
+
+  failingStep() {
+    this._failCount++;
+    if (this._failCount <= 2) {
+      throw new Error('failing step - retry expected');
+    }
+  }
+}
+
+export default AccumulationHelper;

test/data/sandbox/configs/retryHooks/retry_accumulation_test.js

@@ -0,0 +1,9 @@
+Feature('Retry Config Accumulation Test');
+
+Scenario('first scenario', async ({ I }) => {
+  I.failingStep();
+});
+
+Scenario('second scenario', async ({ I }) => {
+  I.failingStep();
+});

test/data/sandbox/configs/retryHooks/retry_multiple_scenarios_test.js

@@ -0,0 +1,13 @@
+Feature('Retry FailedStep - Multiple Consequent Scenarios');
+
+Scenario('first scenario', async ({ I }) => {
+  I.failingStep();
+});
+
+Scenario('second scenario', async ({ I }) => {
+  I.failingStep();
+});
+
+Scenario('third scenario', async ({ I }) => {
+  I.failingStep();
+});

test/runner/retry_hooks_test.js

@@ -66,4 +66,20 @@ describe('CodeceptJS Retry Hooks', function () {
       done()
     })
   })
+
+  it('should prevent retry config accumulation across tests', done => {
+    exec(config_run_config('codecept.retry.accumulation.conf.js', ''), (err, stdout) => {
+      debug_this_test && console.log(stdout)
+      expect(stdout).toContain('2 passed')
+      done()
+    })
+  })
+
+  it('should retryFailedStep on multiple consequent scenarios', done => {
+    exec(config_run_config('codecept.retry.multipleScenarios.conf.js', ''), (err, stdout) => {
+      debug_this_test && console.log(stdout)
+      expect(stdout).toContain('3 passed')
+      done()
+    })
+  })
 })