feat(test-optimization): add WebdriverIO optimizations (#9556)

Author: juan-fernandez

.github/workflows/test-optimization.yml

@@ -239,7 +239,7 @@ jobs:
         with:
           version: ${{ matrix.node-version }}
       - uses: ./.github/actions/install
-      - run: npm run test:instrumentations
+      - run: npm run test:instrumentations:ci
         env:
           PLUGINS: webdriverio
       - run: npm run test:integration:webdriverio:coverage

integration-tests/webdriverio/fixtures/atr.e2e.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+let attempts = 0
+
+describe('WebdriverIO ATR', () => {
+  it('passes on retry', () => {
+    assert.strictEqual(attempts++, 1)
+  })
+})

integration-tests/webdriverio/fixtures/disabled-efd.e2e.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+describe('WebdriverIO disabled EFD', () => {
+  it('is new', () => {
+    assert.fail('disabled test ran')
+  })
+})

integration-tests/webdriverio/fixtures/efd-after-each-fail.e2e.js

@@ -0,0 +1,9 @@
+'use strict'
+
+describe('WebdriverIO EFD afterEach failure', () => {
+  afterEach(() => {
+    throw new Error('EFD afterEach failure')
+  })
+
+  it('fails every EFD hook attempt', () => {})
+})

integration-tests/webdriverio/fixtures/efd.e2e.js

@@ -0,0 +1,11 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+let attempts = 0
+
+describe('WebdriverIO EFD', () => {
+  it('retries a new test', () => {
+    assert.strictEqual(attempts++ % 2, 1)
+  })
+})

integration-tests/webdriverio/fixtures/failed-test-replay.e2e.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+const sum = require('./dependency')
+
+let attempts = 0
+
+describe('WebdriverIO Failed Test Replay', () => {
+  it('captures the failure on retry', () => {
+    if (attempts++ === 0) {
+      sum(11, 3)
+    }
+
+    assert.throws(() => sum(11, 3), /a is too big/)
+    assert.strictEqual(sum(1, 3), 4)
+  })
+})

integration-tests/webdriverio/fixtures/impacted.e2e.js

@@ -0,0 +1,5 @@
+'use strict'
+
+describe('WebdriverIO impacted tests', () => {
+  it('marks a modified test', () => {})
+})

integration-tests/webdriverio/fixtures/managed-hook-fail.e2e.js

@@ -0,0 +1,21 @@
+'use strict'
+
+let earlyFlakeDetectionAttempts = 0
+
+describe('WebdriverIO quarantined hook failure', () => {
+  beforeEach(() => {
+    throw new Error('quarantined hook failure')
+  })
+
+  it('is quarantined', () => {})
+})
+
+describe('WebdriverIO EFD hook failure', () => {
+  beforeEach(() => {
+    if (earlyFlakeDetectionAttempts++ === 1) {
+      throw new Error('EFD hook failure')
+    }
+  })
+
+  it('passes an EFD retry', () => {})
+})

integration-tests/webdriverio/fixtures/suite-hook-fail.e2e.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const hookType = process.env.WEBDRIVERIO_SUITE_HOOK
+
+describe('WebdriverIO suite hook failure', () => {
+  if (hookType === 'beforeAll') {
+    before(() => {
+      throw new Error('beforeAll failure')
+    })
+  } else {
+    after(() => {
+      throw new Error('afterAll failure')
+    })
+  }
+
+  it('is quarantined', () => {})
+})

integration-tests/webdriverio/fixtures/test-management.e2e.js

@@ -0,0 +1,15 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+describe('WebdriverIO Test Management', () => {
+  it('is disabled', () => {
+    assert.fail('disabled test ran')
+  })
+
+  it('is quarantined', () => {
+    assert.fail('quarantined failure')
+  })
+
+  it('is attempt to fix', () => {})
+})