feat(cypress): upload failure screenshots

Author: juan-fernandez

integration-tests/ci-visibility-intake.js

@@ -211,6 +211,31 @@ class FakeCiVisIntake extends FakeAgent {
       })
     })
 
+    app.post([
+      '/api/v2/ci/tests/screenshots',
+      '/evp_proxy/:version/api/v2/ci/tests/screenshots',
+    ], upload.any(), (req, res) => {
+      res.status(200).send('OK')
+
+      const screenshotFile = req.files.find(f => f.fieldname === 'screenshot')
+      const eventFile = req.files.find(f => f.fieldname === 'event')
+
+      this.emit('message', {
+        headers: req.headers,
+        screenshotFile: screenshotFile && {
+          name: screenshotFile.fieldname,
+          filename: screenshotFile.originalname,
+          type: screenshotFile.mimetype,
+          content: screenshotFile.buffer,
+        },
+        eventFile: eventFile && {
+          name: eventFile.fieldname,
+          content: JSON.parse(eventFile.buffer.toString('utf8')),
+        },
+        url: req.url,
+      })
+    })
+
     app.post([
       '/api/v2/libraries/tests/services/setting',
       '/evp_proxy/:version/api/v2/libraries/tests/services/setting',

integration-tests/cypress-esm-config.mjs

@@ -20,7 +20,7 @@ async function runCypress () {
         specPattern: process.env.SPEC_PATTERN || 'cypress/e2e/**/*.cy.js',
       },
       video: false,
-      screenshotOnRunFailure: false,
+      screenshotOnRunFailure: process.env.CYPRESS_SCREENSHOT_ON_RUN_FAILURE === 'true',
     },
   })
 

integration-tests/cypress/cypress-reporting.spec.js

@@ -2074,5 +2074,78 @@ moduleTypes.forEach(({
         receiverPromise,
       ])
     })
+
+    it('uploads screenshots for failed tests using the test trace id', async function () {
+      const envVars = getCiVisAgentlessConfig(receiver.port)
+      const command = version === '6.7.0'
+        ? './node_modules/.bin/cypress run ' +
+          '--config-file cypress-config.json ' +
+          '--config screenshotOnRunFailure=true --spec "cypress/e2e/basic-fail.js"'
+        : `${testCommand} --config screenshotOnRunFailure=true`
+      let testOutput = ''
+
+      childProcess = exec(
+        command,
+        {
+          cwd,
+          env: {
+            ...envVars,
+            CYPRESS_BASE_URL: webAppBaseUrl,
+            CYPRESS_SCREENSHOT_ON_RUN_FAILURE: 'true',
+            SPEC_PATTERN: 'cypress/e2e/basic-fail.js',
+          },
+        }
+      )
+      childProcess.stdout.on('data', data => { testOutput += data.toString() })
+      childProcess.stderr.on('data', data => { testOutput += data.toString() })
+
+      const receiverPromise = receiver.gatherPayloadsUntilChildExit(
+        childProcess,
+        ({ url }) => url === '/api/v2/ci/tests/screenshots' || url === '/api/v2/citestcycle',
+        payloads => {
+          const screenshotPayload = payloads.find(({ url }) => url === '/api/v2/ci/tests/screenshots')
+          const events = payloads
+            .filter(({ url }) => url === '/api/v2/citestcycle')
+            .flatMap(({ payload }) => payload.events)
+          const failedTest = events.find(event =>
+            event.content.resource === 'cypress/e2e/basic-fail.js.basic fail suite can fail'
+          )
+          assert.ok(screenshotPayload)
+          assert.ok(failedTest)
+          const expectedTraceId = failedTest.content.trace_id.toString()
+
+          assertObjectContains(screenshotPayload, {
+            screenshotFile: {
+              name: 'screenshot',
+              filename: `${expectedTraceId}.png`,
+              type: 'image/png',
+            },
+            eventFile: {
+              name: 'event',
+              content: {
+                type: 'test_screenshot',
+                trace_id: expectedTraceId,
+                test_name: 'basic fail suite can fail',
+                test_suite: 'cypress/e2e/basic-fail.js',
+                filename: `${expectedTraceId}.png`,
+                content_type: 'image/png',
+              },
+            },
+          })
+          assert.deepStrictEqual(
+            [...screenshotPayload.screenshotFile.content.subarray(0, 8)],
+            [137, 80, 78, 71, 13, 10, 26, 10]
+          )
+        }, { hardTimeout: 60000 })
+        .catch(error => {
+          error.message += `\nCypress output:\n${testOutput}`
+          throw error
+        })
+
+      await Promise.all([
+        once(childProcess, 'exit'),
+        receiverPromise,
+      ])
+    })
   })
 })

packages/datadog-plugin-cypress/src/cypress-plugin.js

@@ -128,6 +128,55 @@ const CYPRESS_STATUS_TO_TEST_STATUS = {
   skipped: 'skip',
 }
 
+const SCREENSHOT_ATTEMPT_RE = /\(attempt \d+\)/
+
+function getScreenshotFilePath (screenshot) {
+  return typeof screenshot === 'string' ? screenshot : screenshot?.path
+}
+
+function isFailureScreenshot (screenshot) {
+  return !!getScreenshotFilePath(screenshot) && screenshot?.testFailure !== false
+}
+
+function getAttemptScreenshots (cypressTest, attemptIndex) {
+  if (!Array.isArray(cypressTest.attempts)) {
+    return []
+  }
+  const attempt = cypressTest.attempts[attemptIndex]
+  if (!Array.isArray(attempt?.screenshots)) {
+    return []
+  }
+  return attempt.screenshots.filter(isFailureScreenshot)
+}
+
+function isScreenshotForTestAttempt (screenshot, titleParts, attemptIndex) {
+  const screenshotFilePath = getScreenshotFilePath(screenshot)
+  if (!screenshotFilePath || !isFailureScreenshot(screenshot)) {
+    return false
+  }
+  for (const titlePart of titleParts) {
+    if (!screenshotFilePath.includes(titlePart)) {
+      return false
+    }
+  }
+  if (attemptIndex === 0) {
+    return !SCREENSHOT_ATTEMPT_RE.test(screenshotFilePath)
+  }
+  return screenshotFilePath.includes(`(attempt ${attemptIndex + 1})`)
+}
+
+function getTestScreenshots (cypressTest, attemptIndex, specScreenshots) {
+  const attemptScreenshots = getAttemptScreenshots(cypressTest, attemptIndex)
+  if (attemptScreenshots.length > 0) {
+    return attemptScreenshots
+  }
+  if (!Array.isArray(specScreenshots)) {
+    return []
+  }
+  const titleParts = Array.isArray(cypressTest.title) ? cypressTest.title : []
+  return specScreenshots.filter(screenshot => isScreenshotForTestAttempt(screenshot, titleParts, attemptIndex))
+}
+
 function getSessionStatus (summary) {
   if (summary.totalFailed !== undefined && summary.totalFailed > 0) {
     return 'fail'
@@ -1034,9 +1083,11 @@ class CypressPlugin {
   }
 
   afterSpec (spec, results) {
-    const { tests, stats } = results || {}
+    const { tests, stats, screenshots } = results || {}
     const cypressTests = tests || []
+    const specScreenshots = screenshots || []
     const finishedTests = this.finishedTestsByFile[spec.relative] || []
+    const screenshotUploadPromises = []
 
     if (!this.testSuiteSpan) {
       // dd:testSuiteStart hasn't been triggered for whatever reason
@@ -1143,6 +1194,20 @@ class CypressPlugin {
         if (cypressTest.displayError) {
           latestError = new Error(cypressTest.displayError)
         }
+
+        if (cypressTestStatus === 'fail' || finishedTest.testStatus === 'fail' || cypressTest.displayError) {
+          const testScreenshots = getTestScreenshots(cypressTest, attemptIndex, specScreenshots)
+          const screenshotUploadPromise = this.uploadTestScreenshots({
+            screenshots: testScreenshots,
+            traceId: finishedTest.testSpan.context().toTraceId(),
+            testName,
+            testSuite: spec.relative,
+          })
+          if (screenshotUploadPromise) {
+            screenshotUploadPromises.push(screenshotUploadPromise)
+          }
+        }
+
         // Update test status - but NOT for non-ATF quarantined tests where we intentionally
         // report 'fail' to Datadog even though Cypress sees it as 'pass'
         const isQuarantinedTest = finishedTest.testSpan?.context()?._tags?.[TEST_MANAGEMENT_IS_QUARANTINED] === 'true'
@@ -1214,6 +1279,53 @@ class CypressPlugin {
       this.testSuiteSpan = null
       this.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite')
     }
+
+    if (screenshotUploadPromises.length > 0) {
+      return Promise.all(screenshotUploadPromises).then(() => null)
+    }
+  }
+
+  /**
+   * Uploads failure screenshots for a Cypress test attempt.
+   *
+   * @param {object} options - Upload options
+   * @param {Array<object|string>} options.screenshots - Cypress screenshot objects or screenshot paths
+   * @param {string} options.traceId - Test trace id used as the screenshot key
+   * @param {string} options.testName - Test name associated with the screenshots
+   * @param {string} options.testSuite - Test suite associated with the screenshots
+   * @returns {Promise<void>|undefined}
+   */
+  uploadTestScreenshots ({ screenshots, traceId, testName, testSuite }) {
+    const exporter = this.tracer?._tracer?._exporter
+    if (!screenshots.length || !exporter?.canUploadTestScreenshots?.() || !exporter.uploadTestScreenshot) {
+      return
+    }
+
+    const uploadedFilePaths = new Set()
+    const uploadPromises = []
+
+    for (const screenshot of screenshots) {
+      const filePath = getScreenshotFilePath(screenshot)
+      if (!filePath || uploadedFilePaths.has(filePath)) {
+        continue
+      }
+      uploadedFilePaths.add(filePath)
+      uploadPromises.push(new Promise(resolve => {
+        exporter.uploadTestScreenshot({
+          filePath,
+          traceId,
+          testName,
+          testSuite,
+          testEnvironmentMetadata: this.testEnvironmentMetadata,
+        }, () => {
+          resolve()
+        })
+      }))
+    }
+
+    if (uploadPromises.length > 0) {
+      return Promise.all(uploadPromises).then(() => {})
+    }
   }
 
   getTasks () {

packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js

@@ -68,6 +68,7 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
           evpProxyPrefix,
         })
         this._codeCoverageReportUrl = this._url
+        this._testScreenshotUploadUrl = this._url
         if (isTestDynamicInstrumentationEnabled) {
           const canFowardLogs = getCanForwardDebuggerLogs(err, agentInfo)
           if (canFowardLogs) {

packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -22,6 +22,7 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
     this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
 
     this._codeCoverageReportUrl = url || new URL(`https://ci-intake.${site}`)
+    this._testScreenshotUploadUrl = url || new URL(`https://ci-intake.${site}`)
 
     if (isTestDynamicInstrumentationEnabled) {
       const DynamicInstrumentationLogsWriter = require('./di-logs-writer')

packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -8,6 +8,7 @@ const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detectio
 const { getTestManagementTests: getTestManagementTestsRequest } =
   require('../test-management/get-test-management-tests')
 const { uploadCoverageReport: uploadCoverageReportRequest } = require('../requests/upload-coverage-report')
+const { uploadTestScreenshot: uploadTestScreenshotRequest } = require('../requests/upload-test-screenshot')
 const log = require('../../log')
 const BufferingExporter = require('../../exporters/common/buffering-exporter')
 const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
@@ -417,6 +418,43 @@ class CiVisibilityExporter extends BufferingExporter {
       evpProxyPrefix: this.evpProxyPrefix,
     }, callback)
   }
+
+  /**
+   * Returns whether the exporter can upload test screenshots.
+   *
+   * @returns {boolean}
+   */
+  canUploadTestScreenshots () {
+    return !!this._testScreenshotUploadUrl
+  }
+
+  /**
+   * Uploads a single test screenshot to the CI intake.
+   *
+   * @param {object} options - Upload options
+   * @param {string} options.filePath - Path to the screenshot file
+   * @param {string} options.traceId - Test trace id used as the screenshot key
+   * @param {string} options.testName - Test name associated with the screenshot
+   * @param {string} options.testSuite - Test suite associated with the screenshot
+   * @param {object} options.testEnvironmentMetadata - Test environment metadata containing git/CI tags
+   * @param {Function} callback - Callback function (err)
+   */
+  uploadTestScreenshot ({ filePath, traceId, testName, testSuite, testEnvironmentMetadata }, callback) {
+    if (!this._testScreenshotUploadUrl) {
+      return callback(new Error('Test screenshot upload URL not configured'))
+    }
+
+    uploadTestScreenshotRequest({
+      filePath,
+      traceId,
+      testName,
+      testSuite,
+      testEnvironmentMetadata,
+      url: this._testScreenshotUploadUrl,
+      isEvpProxy: !!this._isUsingEvpProxy,
+      evpProxyPrefix: this.evpProxyPrefix,
+    }, callback)
+  }
 }
 
 module.exports = CiVisibilityExporter