feat(test-optimization): support WebdriverIO Jasmine reporting (#9607)

Author: juan-fernandez

integration-tests/ci-visibility-intake.js

@@ -46,6 +46,7 @@ const DEFAULT_TEST_MANAGEMENT_TESTS_RESPONSE_STATUS = 200
 
 class FakeCiVisIntake extends FakeAgent {
   #settings = DEFAULT_SETTINGS
+  #settingsResponseDelayMs = 0
   #settingsResponseStatusCode = 200
   #settingsResponseStatusCodes = []
   #mediaResponseDelayMs = 0
@@ -103,6 +104,16 @@ class FakeCiVisIntake extends FakeAgent {
     this.#settings = newSettings
   }
 
+  /**
+   * Delays settings responses to exercise initialization ordering.
+   *
+   * @param {number} delayMs
+   * @returns {void}
+   */
+  setSettingsResponseDelay (delayMs) {
+    this.#settingsResponseDelayMs = delayMs
+  }
+
   setSettingsResponseCode (statusCode) {
     this.#settingsResponseStatusCode = statusCode
   }
@@ -280,22 +291,30 @@ class FakeCiVisIntake extends FakeAgent {
       '/api/v2/libraries/tests/services/setting',
       '/evp_proxy/:version/api/v2/libraries/tests/services/setting',
     ], (req, res) => {
-      const settingsResponseStatusCode = this.#settingsResponseStatusCodes.shift() ??
-        this.#settingsResponseStatusCode
-      res.status(settingsResponseStatusCode)
-      if (settingsResponseStatusCode >= 200 && settingsResponseStatusCode < 300) {
-        res.send(JSON.stringify({
-          data: {
-            attributes: this.#settings,
-          },
-        }))
+      const respond = () => {
+        const settingsResponseStatusCode = this.#settingsResponseStatusCodes.shift() ??
+          this.#settingsResponseStatusCode
+        res.status(settingsResponseStatusCode)
+        if (settingsResponseStatusCode >= 200 && settingsResponseStatusCode < 300) {
+          res.send(JSON.stringify({
+            data: {
+              attributes: this.#settings,
+            },
+          }))
+        } else {
+          res.send(JSON.stringify({ errors: ['error'] }))
+        }
+        this.emit('message', {
+          headers: req.headers,
+          url: req.url,
+        })
+      }
+
+      if (this.#settingsResponseDelayMs > 0) {
+        setTimeout(respond, this.#settingsResponseDelayMs)
       } else {
-        res.send(JSON.stringify({ errors: ['error'] }))
+        respond()
       }
-      this.emit('message', {
-        headers: req.headers,
-        url: req.url,
-      })
     })
 
     app.post([
@@ -417,6 +436,7 @@ class FakeCiVisIntake extends FakeAgent {
 
   stop () {
     this.#settings = DEFAULT_SETTINGS
+    this.#settingsResponseDelayMs = 0
     this.#settingsResponseStatusCode = 200
     this.#settingsResponseStatusCodes = []
     this.#suitesToSkip = DEFAULT_SUITES_TO_SKIP

integration-tests/webdriverio/fixtures/empty.e2e.js

@@ -0,0 +1 @@
+'use strict'

integration-tests/webdriverio/fixtures/jasmine-after-all-fail.e2e.js

@@ -0,0 +1,16 @@
+'use strict'
+
+const tracer = require('dd-trace')
+
+const assert = require('node:assert/strict')
+
+describe('WebdriverIO Jasmine afterAll failure', () => {
+  // eslint-disable-next-line no-undef -- Jasmine exposes afterAll through WebdriverIO.
+  afterAll(() => {
+    throw new Error('expected WebdriverIO Jasmine afterAll failure')
+  })
+
+  it('passes before afterAll fails', () => {
+    assert.ok(tracer.scope().active())
+  })
+})

integration-tests/webdriverio/fixtures/jasmine-global-after-all-fail.e2e.js

@@ -0,0 +1,10 @@
+'use strict'
+
+// eslint-disable-next-line no-undef -- Jasmine exposes afterAll through WebdriverIO.
+afterAll(() => {
+  throw new Error('expected WebdriverIO Jasmine global afterAll failure')
+})
+
+describe('Jasmine global afterAll failure', () => {
+  it('passes before the global hook fails', () => {})
+})

integration-tests/webdriverio/fixtures/jasmine-hooks.e2e.js

@@ -0,0 +1,27 @@
+'use strict'
+
+const tracer = require('dd-trace')
+
+const assert = require('node:assert/strict')
+
+describe('WebdriverIO Jasmine per-test hooks', () => {
+  let testSpan
+
+  beforeEach(() => {
+    testSpan = tracer.scope().active()
+
+    assert.ok(testSpan)
+    testSpan.setTag('test.webdriverio.jasmine.before-each', 'active')
+  })
+
+  afterEach(() => {
+    const activeSpan = tracer.scope().active()
+
+    assert.strictEqual(activeSpan, testSpan)
+    activeSpan.setTag('test.webdriverio.jasmine.after-each', 'active')
+  })
+
+  it('keeps the test span active', () => {
+    assert.strictEqual(tracer.scope().active(), testSpan)
+  })
+})

integration-tests/webdriverio/fixtures/jasmine-statuses.e2e.js

@@ -0,0 +1,21 @@
+'use strict'
+
+const tracer = require('dd-trace')
+
+const assert = require('node:assert/strict')
+
+describe('WebdriverIO Jasmine statuses', () => {
+  it('reports a passing test with an active span', () => {
+    const activeSpan = tracer.scope().active()
+
+    assert.ok(activeSpan)
+    activeSpan.setTag('test.webdriverio.worker', 'jasmine')
+  })
+
+  it('reports a failing test', () => {
+    assert.fail('expected WebdriverIO Jasmine integration failure')
+  })
+
+  // eslint-disable-next-line mocha/no-pending-tests -- this fixture verifies skipped-test reporting.
+  xit('reports a skipped test', () => {})
+})

integration-tests/webdriverio/fixtures/wdio.conf.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const scenario = process.env.WEBDRIVERIO_SCENARIO || 'parallel'
+const framework = process.env.WEBDRIVERIO_FRAMEWORK || 'mocha'
 
 const baseConfig = {
   runner: 'local',
@@ -18,8 +19,12 @@ const baseConfig = {
   path: '/',
   connectionRetryCount: 0,
   services: [],
-  framework: 'mocha',
+  framework,
   reporters: [],
+  jasmineOpts: {
+    defaultTimeoutInterval: 10_000,
+    random: false,
+  },
   mochaOpts: {
     ui: 'bdd',
     timeout: 10_000,
@@ -100,6 +105,13 @@ const scenarioConfig = {
       './second.e2e.js',
     ]],
   },
+  groupedEmpty: {
+    maxInstances: 1,
+    specs: [[
+      './empty.e2e.js',
+      './first.e2e.js',
+    ]],
+  },
   hookFailure: {
     maxInstances: 1,
     specs: [[
@@ -111,6 +123,29 @@ const scenarioConfig = {
     maxInstances: 1,
     specs: ['./impacted.e2e.js'],
   },
+  jasmineStatuses: {
+    maxInstances: 1,
+    specs: ['./jasmine-statuses.e2e.js'],
+  },
+  jasmineAfterAllFailure: {
+    maxInstances: 1,
+    specs: ['./jasmine-after-all-fail.e2e.js'],
+  },
+  jasmineDelayedSettings: {
+    maxInstances: 1,
+    specs: ['./first.e2e.js'],
+  },
+  jasmineGlobalAfterAllFailure: {
+    maxInstances: 1,
+    specs: [[
+      './jasmine-global-after-all-fail.e2e.js',
+      './first.e2e.js',
+    ]],
+  },
+  jasmineHooks: {
+    maxInstances: 1,
+    specs: ['./jasmine-hooks.e2e.js'],
+  },
   loadFailure: {
     maxInstances: 1,
     specs: ['./load-fail.e2e.js'],