ci: reduce system tests execution time, split AppSec ubuntu matrix, rework All Green report uploads

.codecov.yml

@@ -1,3 +1,12 @@
+codecov:
+  notify:
+    # All Green uploads coverage per sibling workflow as each one finishes, so without this,
+    # Codecov would compute and post its status/patch check after the first upload lands, well
+    # before the rest have arrived. `scripts/all-green.mjs` calls `codecovcli send-notifications`
+    # once every workflow's uploads are done, which is the only thing that triggers notification
+    # while this is set.
+    manual_trigger: true
+
 coverage:
   range: 90..100
   round: down

.github/actions/coverage/action.yml

@@ -1,13 +1,13 @@
 name: coverage
 description: >-
   Verify a job's coverage output and stash it as an artifact. All Green collects every
-  `coverage-*` artifact, merges them per integration, and uploads the groups to Codecov and
-  Datadog — so a commit sends ~100 grouped reports instead of one per matrix cell, which kept
+  `coverage-*` artifact and uploads a single combined report to Codecov and Datadog —
+  so a commit sends one upload per backend instead of one per matrix cell, which kept
   Codecov from silently dropping uploads past its per-commit ceiling.
 
 inputs:
   flags:
-    description: "Coverage flags — the grouping key All Green folds matrix cells by"
+    description: "Coverage flags — identifies the artifact so verify-coverage can report failures clearly"
     required: false
   report-dir:
     description: "Coverage report directory (defaults to 'coverage')"

.github/actions/upload-coverage-artifact/action.yml

@@ -13,15 +13,14 @@ inputs:
 runs:
   using: composite
   steps:
-    # The reports live at `<report-dir>/node-<version>-<label>/{lcov.info,coverage-final.json}`, one
-    # level below `report-dir`, so the presence check has to recurse — a top-level-only probe finds
-    # nothing and silently skips the upload, which is what starved the Datadog coverage upload.
-    # Both formats ship: Codecov reads branch/function coverage from the istanbul JSON, Datadog from
-    # the lcov.
+    # The reports live at `<report-dir>/node-<version>-<label>/lcov.info`, one level below
+    # `report-dir`, so the presence check has to recurse — a top-level-only probe finds nothing and
+    # silently skips the upload, which is what starved the Datadog coverage upload. Only lcov ships:
+    # both Datadog and Codecov ingest it (see `scripts/upload-coverage.mjs`).
     - id: check
       shell: bash
       run: |
-        count=$(find "${{ inputs.report-dir }}" \( -name lcov.info -o -name coverage-final.json \) -type f 2>/dev/null | wc -l | tr -d ' ')
+        count=$(find "${{ inputs.report-dir }}" -name lcov.info -type f 2>/dev/null | wc -l | tr -d ' ')
         echo "has_coverage=$([ "$count" -gt 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
     # The artifact name carries the grouping flag plus a per-cell suffix (`__<job>-<matrix-index>`)
     # so matrix cells that share a flag — cypress varies `spec` outside its flag — upload distinct
@@ -32,9 +31,7 @@ runs:
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
       with:
         name: coverage-${{ inputs.flags }}__${{ github.job }}-${{ strategy.job-index }}
-        path: |
-          ${{ inputs.report-dir }}/**/lcov.info
-          ${{ inputs.report-dir }}/**/coverage-final.json
+        path: ${{ inputs.report-dir }}/**/lcov.info
         retention-days: 1
     - if: github.actor != 'dependabot[bot]' && steps.check.outputs.has_coverage == 'true' && steps.upload.outcome == 'failure'
       shell: bash
@@ -43,8 +40,6 @@ runs:
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
       with:
         name: coverage-${{ inputs.flags }}__${{ github.job }}-${{ strategy.job-index }}
-        path: |
-          ${{ inputs.report-dir }}/**/lcov.info
-          ${{ inputs.report-dir }}/**/coverage-final.json
+        path: ${{ inputs.report-dir }}/**/lcov.info
         retention-days: 1
         overwrite: true

.github/workflows/all-green.yml

@@ -32,77 +32,32 @@ jobs:
         with:
           version: active
       - run: yarn add @actions/core @actions/github octokit
-      - run: node scripts/all-green.mjs
-        env:
-          DELAY: ${{ github.run_attempt == 1 && '1' || '0' }} # 1 minute on first attempt, no delay on reruns
-          RUN_ATTEMPT: ${{ github.run_attempt }}
-          GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
-          GITHUB_TOKEN: ${{ steps.octo-sts.outputs.token }}
-          POLLING_INTERVAL: 1
-          RETRIES: 20
-      # Sort every downloaded per-cell `coverage-*` artifact into one directory per integration under
-      # `coverage-upload/<group>/{lcov,json}/`. The reports are not merged here — both backends merge
-      # same-flag uploads server-side — so each cell's report passes through byte-for-byte. Codecov
-      # gets the istanbul JSON (carries branch/function coverage), Datadog gets the lcov.
-      - if: "!cancelled()"
-        run: node scripts/group-coverage.mjs
-      - if: "!cancelled()"
-        uses: ./.github/actions/dd-sts-api-key
+      - uses: ./.github/actions/dd-sts-api-key
         id: dd-sts
-      - if: "!cancelled()"
-        uses: ./.github/actions/datadog-ci
-      - if: "!cancelled()"
-        run: datadog-ci junit upload --service dd-trace-js-tests --auto-discovery junit-results
-        env:
-          DD_API_KEY: ${{ steps.dd-sts.outputs.api_key }}
-      # One Datadog upload per integration, flagged with the integration name so coverage is
-      # attributable in the Datadog UI the same way the Codecov flags attribute it. Datadog reads the
-      # lcov reports; it does not ingest istanbul's JSON.
-      - if: "!cancelled()"
-        run: |
-          set -o pipefail
-          shopt -s nullglob
-          # shellcheck disable=SC2016 # the inner shell must expand "$1", not the template.
-          printf '%s\0' coverage-upload/*/lcov/ | xargs -0 -r -P 20 -I {} \
-            bash -c 'datadog-ci coverage upload "$1" --flags "$(basename "$(dirname "$1")")"' _ {}
+      - uses: ./.github/actions/datadog-ci
+      - run: pip install --quiet codecov-cli
+      # Polls sibling workflows and, as soon as each one reaches a final state, downloads its
+      # artifacts, merges its reports, and uploads its junit/coverage to Datadog and Codecov —
+      # instead of waiting for every workflow to finish before downloading or uploading anything.
+      # Both Datadog and Codecov get the same merged lcov — Codecov's own cross-session merge for a
+      # shared file has been observed overwriting rather than summing when more than one session
+      # reports it, so it's merged into one report per run before upload instead.
+      # `master-coverage` is the flag `.codecov.yml` gates `codecov/patch` on; attached only on PRs
+      # targeting master so release-branch PRs auto-pass. The SHA/PR are passed explicitly because
+      # All Green checks out the merge ref, not the head the other workflows reported coverage for.
+      # No Codecov token: the repo uploads tokenless (a token would have to be an org secret, which
+      # forks can't read).
+      - run: node scripts/all-green.mjs
         env:
+          BASE_REF: ${{ github.base_ref }}
           DD_API_KEY: ${{ steps.dd-sts.outputs.api_key }}
-      # One Codecov upload per integration via the CLI directly — the codecov-action would re-validate
-      # and re-download the uploader once per group (~100×). Codecov reads branch/function coverage
-      # from the istanbul JSON, so the upload points at each group's `json/` subdirectory.
-      # `master-coverage` is the flag `.codecov.yml` gates `codecov/patch` on; attach it only on PRs
-      # targeting master so release-branch PRs auto-pass. The SHA/branch/PR are passed explicitly
-      # because All Green checks out the merge ref, not the head the other workflows reported coverage
-      # for. No token: the repo uploads tokenless (a Codecov token would have to be an org secret,
-      # which forks can't read).
-      - if: "!cancelled()"
-        run: |
-          commit_args=(--sha "$HEAD_SHA" --branch "$HEAD_BRANCH")
-          upload_args=(--sha "$HEAD_SHA")
-          if [ -n "$PR_NUMBER" ]; then
-            commit_args+=(--pr "$PR_NUMBER")
-            upload_args+=(--pr "$PR_NUMBER")
-          fi
-          master_flag=()
-          if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ "$BASE_REF" = "master" ]; then
-            master_flag=(-F master-coverage)
-          fi
-          shopt -s nullglob
-          groups=(coverage-upload/*/json/)
-          [ ${#groups[@]} -gt 0 ] || exit 0
-          pip install --quiet codecov-cli
-          # create-commit and create-report set up the report once and must finish before any upload;
-          # only the do-upload fan-out is safe to parallelize.
-          codecovcli create-commit "${commit_args[@]}"
-          codecovcli create-report --sha "$HEAD_SHA"
-          set -o pipefail
-          # shellcheck disable=SC2016 # the inner shell must expand "$@"/"$dir", not the template.
-          printf '%s\0' "${groups[@]}" | xargs -0 -r -P 20 -I {} \
-            bash -c 'dir="$1"; shift; codecovcli do-upload "$@" --dir "$dir" -F "$(basename "$(dirname "$dir")")"' \
-              _ {} "${upload_args[@]}" "${master_flag[@]}"
-        env:
+          DELAY: ${{ github.run_attempt == 1 && '2.5' || '0' }} # 2.5 minutes on first attempt, no delay on reruns
           GITHUB_EVENT_NAME: ${{ github.event_name }}
-          BASE_REF: ${{ github.base_ref }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+          GITHUB_TOKEN: ${{ steps.octo-sts.outputs.token }}
           HEAD_BRANCH: ${{ github.head_ref || github.ref_name }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          POLLING_INTERVAL: 0.5
           PR_NUMBER: ${{ github.event.pull_request.number }}
+          RETRIES: 30
+          RUN_ATTEMPT: ${{ github.run_attempt }}

.github/workflows/appsec.yml

@@ -40,28 +40,28 @@ jobs:
           id: ${{ github.job }}
 
   ubuntu:
-    name: ${{ github.workflow }} / ubuntu
+    strategy:
+      fail-fast: false
+      matrix:
+        version: [oldest, maintenance, active, latest]
+    name: ${{ github.workflow }} / ubuntu (node-${{ matrix.version }})
     runs-on: ubuntu-latest
     permissions:
       id-token: write
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: ./.github/actions/node/oldest-maintenance-lts
+      - uses: ./.github/actions/node
+        with:
+          version: ${{ matrix.version }}
       - uses: ./.github/actions/install
       - run: npm run test:appsec:ci
-      - uses: ./.github/actions/node/newest-maintenance-lts
-      - run: npm run test:appsec:ci
-      - uses: ./.github/actions/node/active-lts
-      - run: npm run test:appsec:ci
-      - uses: ./.github/actions/node/latest
-      - run: npm run test:appsec:ci
       - uses: ./.github/actions/coverage
         with:
-          flags: appsec-ubuntu
+          flags: appsec-ubuntu-${{ matrix.version }}
       - uses: ./.github/actions/upload-junit-artifacts
         if: "!cancelled()"
         with:
-          id: ${{ github.job }}
+          id: ${{ github.job }}-${{ strategy.job-index }}
 
   windows:
     name: ${{ github.workflow }} / windows

.github/workflows/system-tests.yml

@@ -40,7 +40,7 @@ jobs:
     with:
       library: nodejs
       binaries_artifact: system_tests_binaries
-      desired_execution_time: 300 # 5 minutes
+      desired_execution_time: 180 # 3 minutes
       scenarios_groups: tracer-release
       excluded_scenarios: APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN # require AWS and datadog credentials
       parametric_job_count: 8

.mochamultireporterrc.js

@@ -1,15 +1,34 @@
 'use strict'
 
+// eslint-disable-next-line eslint-rules/eslint-process-env
+const isCI = Boolean(process.env.CI)
+
 const reporterEnabled = ['spec']
+if (isCI) reporterEnabled.push('./scripts/junit-reporter.js')
 
-// eslint-disable-next-line eslint-rules/eslint-process-env
-if (process.env.CI) {
-  reporterEnabled.push('./scripts/junit-reporter.js')
-}
+// datadog-ci junit upload derives the Pipeline/Job UI facets from GITHUB_* env vars in its own
+// process at upload time, which are All Green's own since it uploads every sibling workflow's
+// results in a single call. Stamping them here instead, while this job's own GITHUB_* values are
+// still correct, lets `--xpath-tag` (see scripts/upload-junit.mjs) remap them onto the real
+// ci.pipeline.*/ci.job.* tags per test instead.
+const GITHUB_ENV = process.env // eslint-disable-line eslint-rules/eslint-process-env
+const {
+  GITHUB_JOB, GITHUB_RUN_ID, GITHUB_WORKFLOW, GITHUB_RUN_NUMBER, GITHUB_SERVER_URL, GITHUB_REPOSITORY,
+} = GITHUB_ENV
 
 module.exports = {
   reporterEnabled,
   scriptsJunitReporterJsReporterOptions: {
     mochaFile: `./node-${process.versions.node}-junit.xml`,
+    properties: {
+      node_version: process.versions.node,
+      ...(isCI && {
+        'ci.pipeline.name': GITHUB_WORKFLOW,
+        'ci.pipeline.id': GITHUB_RUN_ID,
+        'ci.pipeline.number': GITHUB_RUN_NUMBER,
+        'ci.pipeline.url': `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`,
+        'ci.job.name': GITHUB_JOB,
+      }),
+    },
   },
 }

package.json

@@ -226,6 +226,7 @@
     "globals": "^17.7.0",
     "graphql": "*",
     "husky": "^9.1.7",
+    "istanbul-lib-coverage": "^3.2.2",
     "istanbul-lib-report": "^3.0.0",
     "istanbul-reports": "^3.0.2",
     "jsonc-parser": "^3.3.1",

scripts/all-green.mjs

@@ -3,14 +3,24 @@ import { Octokit } from 'octokit'
 import { summary } from '@actions/core'
 import { context } from '@actions/github'
 import { downloadArtifacts } from './download-artifacts.mjs'
+import { logUploads } from './run-upload.mjs'
+import { uploadAllJunit } from './upload-junit.mjs'
+import {
+  uploadAllCoverageToDatadog, uploadCoverage, sendCodecovNotifications, hasCodecovCommit,
+} from './upload-coverage.mjs'
 
 /* eslint-disable no-console */
 
 const {
+  BASE_REF,
   DELAY,
+  GITHUB_EVENT_NAME,
   GITHUB_SHA,
   GITHUB_TOKEN,
+  HEAD_BRANCH,
+  HEAD_SHA,
   POLLING_INTERVAL,
+  PR_NUMBER,
   RETRIES,
   RUN_ATTEMPT,
 } = process.env
@@ -94,9 +104,64 @@ async function getRuns () {
   }
 }
 
+// Runs already dispatched to `processRun`, so a run settling across more than one
+// `scheduleProcessing` call (e.g. after a retry) isn't downloaded/uploaded twice.
+const dispatchedRunIds = new Set()
+const processingPromises = []
+
+/**
+ * Download a single finished workflow run's junit and coverage artifacts, merge them, and upload
+ * the coverage merge to Codecov.
+ *
+ * @param {{ id: number, name: string }} run
+ * @returns {Promise<void>}
+ */
+async function processRun (run) {
+  const { downloaded, failed } = await downloadArtifacts(octokit, { owner, repo, token: GITHUB_TOKEN, runs: [run] })
+
+  const coverageResults = await uploadCoverage(run, {
+    sha: HEAD_SHA,
+    branch: HEAD_BRANCH,
+    prNumber: PR_NUMBER,
+    eventName: GITHUB_EVENT_NAME,
+    baseRef: BASE_REF,
+  })
+  const downloadSummary = failed > 0 ? `${downloaded} artifact(s), ${failed} failed` : `${downloaded} artifact(s)`
+  logUploads(`${run.name} (${downloadSummary})`, coverageResults)
+}
+
+/**
+ * Kick off processing for any run that just reached a final state and hasn't been processed yet.
+ * A run is final once it's completed and either isn't retried (its conclusion isn't in
+ * `pollingRetryConclusions`) or already went through a retry attempt.
+ *
+ * @param {Array<{ id: number, name: string, status: string, conclusion: string }>} runs
+ */
+function scheduleProcessing (runs) {
+  if (!process.env.GITHUB_ACTIONS) return
+
+  const settled = runs.filter(r =>
+    r.status === 'completed' &&
+    (!pollingRetryConclusions.has(r.conclusion) || retriedRunIds.has(r.id)) &&
+    !dispatchedRunIds.has(r.id)
+  )
+
+  for (const run of settled) {
+    dispatchedRunIds.add(run.id)
+    processingPromises.push(
+      processRun(run).catch(err => {
+        console.error(`Failed to process workflow run ${run.id} (${run.name}): ${err.message}`)
+        process.exitCode = 1
+      })
+    )
+  }
+}
+
 async function pollUntilDone () {
   const runs = await getRuns()
 
+  scheduleProcessing(runs)
+
   // Check before modifying retriedRunIds to avoid false positives on freshly requeued runs.
   const retryFailed = runs.filter(r =>
     r.status === 'completed' &&
@@ -198,9 +263,11 @@ async function checkAllGreen () {
 
   await printSummary(runs)
 
-  if (process.env.GITHUB_ACTIONS) {
-    await downloadArtifacts(octokit, { owner, repo, token: GITHUB_TOKEN, runs })
-  }
+  console.log(`Waiting for ${processingPromises.length} workflow run report upload(s) to finish.`)
+  await Promise.all(processingPromises)
+
+  const [junitResults, coverageResults] = await Promise.all([uploadAllJunit(), uploadAllCoverageToDatadog()])
+  logUploads('junit + coverage (every run)', [...junitResults, ...coverageResults])
 
   if (!done) {
     console.log(`State is still pending after ${RETRIES} retries.`)
@@ -213,6 +280,16 @@ async function checkAllGreen () {
     failureConclusions.has(r.conclusion) && !staleFailureRunIds.has(r.id)
   )
 
+  // Codecov's `manual_trigger` (`.codecov.yml`) holds off computing/posting the coverage status
+  // until this fires, since uploads land one sibling workflow at a time rather than all at once.
+  // Only notify when every sibling workflow passed — a failing suite's coverage is expected to be
+  // low, and posting it would report a misleadingly low status against an otherwise healthy commit.
+  // Also skip when no run ever registered a commit/report (e.g. Dependabot PRs, whose coverage
+  // artifacts are skipped) — notifying then would target a report that was never created.
+  if (process.env.GITHUB_ACTIONS && failedRuns.length === 0 && hasCodecovCommit()) {
+    logUploads('codecov', [await sendCodecovNotifications(HEAD_SHA)])
+  }
+
   if (failedRuns.length === 0) {
     console.log('All jobs were successful.')
   } else {