fix(cypress): support numeric retries with ATR (#9484)

Author: juan-fernandez

integration-tests/cypress-esm-config.mjs

@@ -4,14 +4,15 @@
 // Cypress does not call setupNodeEvents from inline config objects.
 import cypress from 'cypress'
 
+const retries = Number(process.env.CYPRESS_RETRIES || 0)
+
 async function runCypress () {
   const results = await cypress.run({
     config: {
       defaultCommandTimeout: 1000,
-      retries: {
-        runMode: Number(process.env.CYPRESS_RETRIES || 0),
-        openMode: 0,
-      },
+      retries: process.env.CYPRESS_RETRIES_AS_NUMBER === undefined
+        ? { runMode: retries, openMode: 0 }
+        : Number(process.env.CYPRESS_RETRIES_AS_NUMBER),
       e2e: {
         ...(process.env.CYPRESS_TEST_ISOLATION === undefined
           ? {}

integration-tests/cypress.config.js

@@ -2,12 +2,13 @@
 
 const { defineConfig } = require('cypress')
 
+const retries = Number(process.env.CYPRESS_RETRIES || 0)
+
 module.exports = defineConfig({
   defaultCommandTimeout: 1000,
-  retries: {
-    runMode: Number(process.env.CYPRESS_RETRIES || 0),
-    openMode: 0,
-  },
+  retries: process.env.CYPRESS_RETRIES_AS_NUMBER === undefined
+    ? { runMode: retries, openMode: 0 }
+    : Number(process.env.CYPRESS_RETRIES_AS_NUMBER),
   e2e: {
     ...(process.env.CYPRESS_TEST_ISOLATION === undefined
       ? {}

integration-tests/cypress/cypress-atr.spec.js

@@ -125,7 +125,7 @@ moduleTypes.forEach(({
     })
 
     context('flaky test retries', () => {
-      it('retries flaky tests', async () => {
+      it('retries flaky tests with object Cypress retries', async () => {
         receiver.setSettings({
           itr_enabled: false,
           code_coverage: false,
@@ -232,6 +232,64 @@ moduleTypes.forEach(({
         ])
       })
 
+      over12It('retries flaky tests', async () => {
+        receiver.setSettings({
+          itr_enabled: false,
+          code_coverage: false,
+          tests_skipping: false,
+          flaky_test_retries_enabled: true,
+          early_flake_detection: {
+            enabled: false,
+          },
+        })
+
+        const envVars = getCiVisEvpProxyConfig(receiver.port)
+
+        const specToRun = 'cypress/e2e/numeric-retries.cy.js'
+
+        childProcess = exec(
+          version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`,
+          {
+            cwd,
+            env: {
+              ...envVars,
+              CYPRESS_BASE_URL: webAppBaseUrl,
+              CYPRESS_RETRIES_AS_NUMBER: '0',
+              SPEC_PATTERN: specToRun,
+            },
+          }
+        )
+
+        const receiverPromise = receiver
+          .gatherPayloadsUntilChildExit(
+            childProcess,
+            ({ url }) => url.endsWith('/api/v2/citestcycle'),
+            payloads => {
+              const events = payloads.flatMap(({ payload }) => payload.events)
+              const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)
+              assert.strictEqual(testSuites.length, 1)
+              assert.strictEqual(testSuites[0].meta[TEST_STATUS], 'pass')
+
+              const tests = events.filter(event => event.type === 'test').map(event => event.content)
+              assert.strictEqual(tests.length, 3)
+
+              const resource = 'cypress/e2e/numeric-retries.cy.js.numeric Cypress retries eventually passes'
+              assert.deepStrictEqual(tests.map(test => test.resource), [resource, resource, resource])
+              assert.strictEqual(tests[0].meta[TEST_STATUS], 'fail')
+              assert.strictEqual(tests[1].meta[TEST_STATUS], 'fail')
+              assert.strictEqual(tests[2].meta[TEST_STATUS], 'pass')
+              assert.strictEqual(tests[1].meta[TEST_IS_RETRY], 'true')
+              assert.strictEqual(tests[2].meta[TEST_IS_RETRY], 'true')
+              assert.strictEqual(tests[1].meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.atr)
+              assert.strictEqual(tests[2].meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.atr)
+            }, { hardTimeout: 30000 })
+
+        await Promise.all([
+          once(childProcess, 'exit'),
+          receiverPromise,
+        ])
+      })
+
       it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', async () => {
         receiver.setSettings({
           itr_enabled: false,

integration-tests/cypress/e2e/numeric-retries.cy.js

@@ -0,0 +1,10 @@
+/* eslint-disable */
+let attempt = 0
+
+describe('numeric Cypress retries', () => {
+  it('eventually passes', () => {
+    cy.then(() => {
+      expect(attempt++).to.equal(2)
+    })
+  })
+})

packages/datadog-plugin-cypress/src/cypress-plugin.js

@@ -858,6 +858,12 @@ class CypressPlugin {
           if (isFlakyTestRetriesEnabled && this.isTestIsolationEnabled) {
             this.isFlakyTestRetriesEnabled = true
             this.flakyTestRetriesCount = flakyTestRetriesCount ?? 0
+            if (typeof this.cypressConfig.retries === 'number') {
+              this.cypressConfig.retries = {
+                openMode: this.cypressConfig.retries,
+                runMode: this.cypressConfig.retries,
+              }
+            }
             this.cypressConfig.retries.runMode = this.flakyTestRetriesCount
           } else {
             this.flakyTestRetriesCount = 0