Skip to content

Commit 6d8228a

Browse files
juan-fernandezBridgeAR
juan-fernandez
authored and
BridgeAR
committed
test(cypress): rebalance parallel integration specs (#8844)
1 parent 2d92e96 commit 6d8228a

13 files changed

Lines changed: 1806 additions & 2352 deletions

‎.github/workflows/test-optimization.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -362,13 +362,13 @@ jobs:
362362
module-type: ["commonJS", "esm"]
363363
spec:
364364
- cypress-reporting
365+
- cypress-reporting-instrumentation
365366
- cypress-itr
366367
- cypress-tia-code-coverage
367368
- cypress-efd
368369
- cypress-atr
369370
- cypress-test-management
370-
- cypress-impacted-tests
371-
- cypress-final-status
371+
- cypress-final-status-impacted-tests
372372
exclude:
373373
# 6.7.0 only runs on DD_MAJOR <= 5 with Node 16/commonJS.
374374
# On DD_MAJOR >= 6, shouldTestsRun() skips this remaining matrix entry.

‎integration-tests/cypress-double-run.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ const runOptions = {
2222
async function runCypressTwice () {
2323
for (let runNumber = 0; runNumber < 2; runNumber++) {
2424
const results = await cypress.run(runOptions)
25-
if (results.totalFailed !== 0) {
26-
process.exit(1)
25+
const failures = results.totalFailed ?? results.failures ?? 0
26+
if (failures !== 0) {
27+
process.exit(failures)
2728
}
2829
}
2930
}

‎integration-tests/cypress-double-run.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ const runOptions = {
1919

2020
for (let runNumber = 0; runNumber < 2; runNumber++) {
2121
const results = await cypress.run(runOptions)
22-
if (results.totalFailed !== 0) {
23-
process.exit(1)
22+
const failures = results.totalFailed ?? results.failures ?? 0
23+
if (failures !== 0) {
24+
process.exit(failures)
2425
}
2526
}

‎integration-tests/cypress-esm-config.mjs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@ async function runCypress () {
2828
},
2929
})
3030

31-
if (results.totalFailed !== 0) {
32-
process.exit(1)
31+
const failures = results.totalFailed ?? results.failures ?? 0
32+
if (failures !== 0) {
33+
process.exit(failures)
3334
}
3435
}
3536

‎integration-tests/cypress/cypress-efd.spec.js‎

Lines changed: 50 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -124,79 +124,6 @@ moduleTypes.forEach(({
124124
})
125125

126126
context('early flake detection', () => {
127-
it('retries new tests', async () => {
128-
receiver.setSettings({
129-
early_flake_detection: {
130-
enabled: true,
131-
slow_test_retries: {
132-
'5s': NUM_RETRIES_EFD,
133-
},
134-
},
135-
known_tests_enabled: true,
136-
})
137-
138-
receiver.setKnownTests({
139-
cypress: {
140-
'cypress/e2e/spec.cy.js': [
141-
// 'context passes', // This test will be considered new
142-
'other context fails',
143-
],
144-
},
145-
})
146-
147-
const envVars = getCiVisEvpProxyConfig(receiver.port)
148-
149-
const specToRun = 'cypress/e2e/spec.cy.js'
150-
151-
childProcess = exec(
152-
version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`,
153-
{
154-
cwd,
155-
env: {
156-
...envVars,
157-
CYPRESS_BASE_URL: webAppBaseUrl,
158-
SPEC_PATTERN: specToRun,
159-
},
160-
}
161-
)
162-
163-
const receiverPromise = receiver
164-
.gatherPayloadsUntilChildExit(
165-
childProcess,
166-
({ url }) => url.endsWith('/api/v2/citestcycle'),
167-
payloads => {
168-
const events = payloads.flatMap(({ payload }) => payload.events)
169-
const tests = events.filter(event => event.type === 'test').map(event => event.content)
170-
assert.strictEqual(tests.length, 5)
171-
172-
const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
173-
assert.strictEqual(newTests.length, NUM_RETRIES_EFD + 1)
174-
175-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
176-
assert.strictEqual(retriedTests.length, NUM_RETRIES_EFD)
177-
178-
retriedTests.forEach((retriedTest) => {
179-
assert.strictEqual(retriedTest.meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.efd)
180-
})
181-
182-
newTests.forEach(newTest => {
183-
assert.strictEqual(newTest.resource, 'cypress/e2e/spec.cy.js.context passes')
184-
})
185-
186-
const knownTest = tests.filter(test => !test.meta[TEST_IS_NEW])
187-
assert.strictEqual(knownTest.length, 1)
188-
assert.strictEqual(knownTest[0].resource, 'cypress/e2e/spec.cy.js.other context fails')
189-
190-
const testSession = events.find(event => event.type === 'test_session_end').content
191-
assert.strictEqual(testSession.meta[TEST_EARLY_FLAKE_ENABLED], 'true')
192-
}, { hardTimeout: 25000 })
193-
194-
await Promise.all([
195-
once(childProcess, 'exit'),
196-
receiverPromise,
197-
])
198-
})
199-
200127
it('disables manual Cypress retries for new tests retried by EFD', async () => {
201128
receiver.setSettings({
202129
early_flake_detection: {
@@ -303,6 +230,14 @@ moduleTypes.forEach(({
303230
'cypress/e2e/efd-duration.cy.js.efd duration retries instant test'
304231
)
305232
assert.strictEqual(instantTests.length, 3)
233+
assert.strictEqual(
234+
instantTests.filter(test => test.meta[TEST_IS_NEW] === 'true').length,
235+
3
236+
)
237+
assert.strictEqual(
238+
instantTests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length,
239+
2
240+
)
306241
assert.strictEqual(
307242
instantTests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.efd).length,
308243
2
@@ -864,13 +799,14 @@ moduleTypes.forEach(({
864799
const tests = events.filter(event => event.type === 'test').map(event => event.content)
865800

866801
// 1 known test + 1 new test with retries: 1 + (1 + 3) = 5 tests
867-
assert.equal(tests.length, 5)
802+
assert.strictEqual(tests.length, 5)
868803

869-
// Extract test execution order: [testName, isRetry]
870804
const testExecutionOrder = tests.map(test => ({
805+
resource: test.resource,
871806
name: test.meta[TEST_NAME],
872807
isRetry: test.meta[TEST_IS_RETRY] === 'true',
873808
isNew: test.meta[TEST_IS_NEW] === 'true',
809+
retryReason: test.meta[TEST_RETRY_REASON],
874810
}))
875811

876812
// Expected order:
@@ -879,19 +815,51 @@ moduleTypes.forEach(({
879815
// 3. "other context fails" (retry 1)
880816
// 4. "other context fails" (retry 2)
881817
// 5. "other context fails" (retry 3)
882-
883-
assertObjectContains(testExecutionOrder, [
884-
{ name: 'context passes', isRetry: false, isNew: false },
885-
{ name: 'other context fails', isRetry: false, isNew: true },
886-
{ name: 'other context fails', isRetry: true, isNew: true },
887-
{ name: 'other context fails', isRetry: true, isNew: true },
888-
{ name: 'other context fails', isRetry: true, isNew: true },
818+
assert.deepStrictEqual(testExecutionOrder, [
819+
{
820+
resource: 'cypress/e2e/spec.cy.js.context passes',
821+
name: 'context passes',
822+
isRetry: false,
823+
isNew: false,
824+
retryReason: undefined,
825+
},
826+
{
827+
resource: 'cypress/e2e/spec.cy.js.other context fails',
828+
name: 'other context fails',
829+
isRetry: false,
830+
isNew: true,
831+
retryReason: undefined,
832+
},
833+
{
834+
resource: 'cypress/e2e/spec.cy.js.other context fails',
835+
name: 'other context fails',
836+
isRetry: true,
837+
isNew: true,
838+
retryReason: TEST_RETRY_REASON_TYPES.efd,
839+
},
840+
{
841+
resource: 'cypress/e2e/spec.cy.js.other context fails',
842+
name: 'other context fails',
843+
isRetry: true,
844+
isNew: true,
845+
retryReason: TEST_RETRY_REASON_TYPES.efd,
846+
},
847+
{
848+
resource: 'cypress/e2e/spec.cy.js.other context fails',
849+
name: 'other context fails',
850+
isRetry: true,
851+
isNew: true,
852+
retryReason: TEST_RETRY_REASON_TYPES.efd,
853+
},
889854
])
890855

891856
// Verify TEST_HAS_FAILED_ALL_RETRIES is set correctly
892857
const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true')
893858
assert.strictEqual(newTests.length, NUM_RETRIES_EFD + 1)
894859

860+
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
861+
assert.strictEqual(retriedTests.length, NUM_RETRIES_EFD)
862+
895863
const testsWithFailedAllRetries = newTests.filter(
896864
test => test.meta[TEST_HAS_FAILED_ALL_RETRIES] === 'true'
897865
)

0 commit comments

Comments
 (0)