diff --git a/.github/workflows/aiguard.yml b/.github/workflows/aiguard.yml index f6adc6cab9f..8e080685537 100644 --- a/.github/workflows/aiguard.yml +++ b/.github/workflows/aiguard.yml @@ -15,7 +15,6 @@ env: MOCHA_OPTIONS: ${{ github.ref == 'refs/heads/master' && '--retries 1' || '' }} jobs: - macos: runs-on: macos-latest steps: @@ -26,7 +25,7 @@ jobs: - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: flags: aiguard-macos - - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 + - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: api_key: ${{ secrets.DD_API_KEY }} @@ -56,12 +55,12 @@ jobs: - uses: ./.github/actions/node/latest - uses: ./.github/actions/install with: - cache: 'true' + cache: "true" - run: yarn test:aiguard:ci - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: flags: aiguard-windows - - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 + - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: api_key: ${{ secrets.DD_API_KEY }} @@ -78,8 +77,13 @@ jobs: with: version: ${{ matrix.version }} - uses: ./.github/actions/install - - run: yarn test:integration:aiguard - - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 + - run: yarn test:integration:aiguard:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-aiguard/lcov.info + flags: integration-aiguard + - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: api_key: ${{ secrets.DD_API_KEY }} diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 0a014b7026f..1edff31548e 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -394,7 +394,12 @@ jobs: with: version: ${{ matrix.version }} - uses: ./.github/actions/install - - run: yarn test:integration:appsec + - run: yarn test:integration:appsec:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-appsec/lcov.info + flags: integration-appsec - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: diff --git a/.github/workflows/debugger.yml b/.github/workflows/debugger.yml index 73002a85d0f..3f27bd57496 100644 --- a/.github/workflows/debugger.yml +++ b/.github/workflows/debugger.yml @@ -28,7 +28,12 @@ jobs: version: ${{ matrix.version }} - uses: ./.github/actions/install - run: yarn test:debugger:ci - - run: yarn test:integration:debugger + - run: yarn test:integration:debugger:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-debugger/lcov.info + flags: integration-debugger - if: always() uses: ./.github/actions/testagent/logs with: diff --git a/.github/workflows/openfeature.yml b/.github/workflows/openfeature.yml index 159e8127ab9..1e3cc0e5f98 100644 --- a/.github/workflows/openfeature.yml +++ b/.github/workflows/openfeature.yml @@ -47,7 +47,12 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: ./.github/actions/node/latest - uses: ./.github/actions/install - - run: yarn test:integration:openfeature + - run: yarn test:integration:openfeature:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-openfeature/lcov.info + flags: integration-openfeature - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: @@ -60,13 +65,18 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - uses: ./.github/actions/node/oldest-maintenance-lts - uses: ./.github/actions/install - - run: yarn test:integration:openfeature + - run: yarn test:integration:openfeature:ci - uses: ./.github/actions/node/newest-maintenance-lts - - run: yarn test:integration:openfeature + - run: yarn test:integration:openfeature:ci - uses: ./.github/actions/node/active-lts - - run: yarn test:integration:openfeature + - run: yarn test:integration:openfeature:ci - uses: ./.github/actions/node/latest - - run: yarn test:integration:openfeature + - run: yarn test:integration:openfeature:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-openfeature/lcov.info + flags: integration-openfeature - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: @@ -81,7 +91,12 @@ jobs: - uses: ./.github/actions/install with: cache: 'true' - - run: yarn test:integration:openfeature + - run: yarn test:integration:openfeature:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-openfeature/lcov.info + flags: integration-openfeature - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index 455b679d65c..e5a5b635b7f 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -423,8 +423,18 @@ jobs: # Disable core dumps since some integration tests intentionally abort and core dump generation takes around 5-10s - uses: ./.github/actions/install - run: sudo sysctl -w kernel.core_pattern='|/bin/false' - - run: yarn test:integration - - run: yarn test:integration:esbuild + - run: yarn test:integration:ci + - run: yarn test:integration:esbuild:ci + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-integration/lcov.info + flags: integration + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-esbuild/lcov.info + flags: integration-esbuild - uses: DataDog/junit-upload-github-action@762867566348d59ac9bcf479ebb4ec040db8940a # v2.0.0 if: always() && github.actor != 'dependabot[bot]' with: diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml index 3ee522784f4..6745e5b5f59 100644 --- a/.github/workflows/profiling.yml +++ b/.github/workflows/profiling.yml @@ -29,7 +29,7 @@ jobs: - uses: ./.github/actions/node/latest - uses: ./.github/actions/install - run: yarn test:profiler:ci - - run: yarn test:integration:profiler + - run: yarn test:integration:profiler:ci - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: flags: profiling-macos @@ -46,16 +46,16 @@ jobs: - uses: ./.github/actions/node/oldest-maintenance-lts - uses: ./.github/actions/install - run: yarn test:profiler:ci - - run: yarn test:integration:profiler + - run: yarn test:integration:profiler:ci - uses: ./.github/actions/node/newest-maintenance-lts - run: yarn test:profiler:ci - - run: yarn test:integration:profiler + - run: yarn test:integration:profiler:ci - uses: ./.github/actions/node/active-lts - run: yarn test:profiler:ci - - run: yarn test:integration:profiler + - run: yarn test:integration:profiler:ci - uses: ./.github/actions/node/latest - run: yarn test:profiler:ci - - run: yarn test:integration:profiler + - run: yarn test:integration:profiler:ci - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: flags: profiling-ubuntu @@ -74,7 +74,7 @@ jobs: with: cache: 'true' - run: yarn test:profiler:ci - - run: yarn test:integration:profiler + - run: yarn test:integration:profiler:ci - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: flags: profiling-windows diff --git a/.github/workflows/test-optimization.yml b/.github/workflows/test-optimization.yml index 7eb1872dbf6..c8c312f1c9b 100644 --- a/.github/workflows/test-optimization.yml +++ b/.github/workflows/test-optimization.yml @@ -58,7 +58,7 @@ jobs: with: path: ~/.cache/ms-playwright key: playwright-browsers-${{ runner.os }}-${{ steps.playwright-version.outputs.version }} - - run: yarn test:integration:testopt + - run: yarn test:integration:testopt:ci integration-playwright: strategy: @@ -100,9 +100,14 @@ jobs: with: path: /github/home/.cache/ms-playwright key: playwright-browsers-oldest-dd${{ steps.dd-version.outputs.major }} - - run: yarn test:integration:playwright + - run: yarn test:integration:playwright:ci env: - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-playwright/lcov.info + flags: integration-playwright integration-mocha: strategy: @@ -120,10 +125,15 @@ jobs: with: version: ${{ matrix.version }} - uses: ./.github/actions/install - - run: yarn test:integration:mocha + - run: yarn test:integration:mocha:ci env: - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" MOCHA_VERSION: ${{ matrix.mocha-version }} + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-mocha/lcov.info + flags: integration-mocha integration-jest: strategy: @@ -141,10 +151,15 @@ jobs: with: version: ${{ matrix.version }} - uses: ./.github/actions/install - - run: yarn test:integration:jest + - run: yarn test:integration:jest:ci env: - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" JEST_VERSION: ${{ matrix.jest-version }} + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-jest/lcov.info + flags: integration-jest integration-cucumber: strategy: @@ -162,10 +177,15 @@ jobs: with: version: ${{ matrix.version }} - uses: ./.github/actions/install - - run: yarn test:integration:cucumber + - run: yarn test:integration:cucumber:ci env: - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" CUCUMBER_VERSION: ${{ matrix.cucumber-version }} + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-cucumber/lcov.info + flags: integration-cucumber integration-selenium: strategy: @@ -200,9 +220,14 @@ jobs: sudo mv chromedriver-linux64/chromedriver /usr/bin/chromedriver sudo chmod +x /usr/bin/chromedriver - uses: ./.github/actions/install - - run: yarn test:integration:selenium + - run: yarn test:integration:selenium:ci env: - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-selenium/lcov.info + flags: integration-selenium integration-cypress: strategy: @@ -213,7 +238,7 @@ jobs: # 14.5.4 is the latest version that supports Node 18 # The logic to decide whether the tests run lives in integration-tests/cypress/cypress.spec.js cypress-version: [6.7.0, 10.2.0, 14.5.4, latest] - module-type: ['commonJS', 'esm'] + module-type: ["commonJS", "esm"] runs-on: ubuntu-latest env: DD_SERVICE: dd-trace-js-integration-tests @@ -234,12 +259,17 @@ jobs: path: ~/.cache/Cypress key: cypress-binary-${{ matrix.cypress-version }} - run: yarn config set ignore-engines true - - run: yarn test:integration:cypress --ignore-engines + - run: yarn test:integration:cypress:ci --ignore-engines env: CYPRESS_VERSION: ${{ matrix.cypress-version }} - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" CYPRESS_MODULE_TYPE: ${{ matrix.module-type }} OPTIONS_OVERRIDE: 1 + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-cypress/lcov.info + flags: integration-cypress integration-vitest: runs-on: ubuntu-latest @@ -257,6 +287,11 @@ jobs: with: version: ${{ matrix.version }} - uses: ./.github/actions/install - - run: yarn test:integration:vitest + - run: yarn test:integration:vitest:ci env: - NODE_OPTIONS: '-r ./ci/init' + NODE_OPTIONS: "-r ./ci/init" + - uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 + with: + disable_search: true + files: coverage/integration-vitest/lcov.info + flags: integration-vitest diff --git a/.gitignore b/.gitignore index 415ebc176a1..cd4ce813457 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ pids lib-cov # Coverage directory used by tools like istanbul +.coverage coverage # nyc test coverage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6113061c07f..0e1638cb52b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -432,6 +432,11 @@ If running locally without an internet connection, it's possible to use the environment variable `OFFLINE=true` to make `yarn` use the `--prefer-offline` flag, which will use the local yarn cache instead of fetching packages from npm. +To collect coverage from integration tests, +use the `test:integration*:ci` scripts. +These enable V8 coverage for the app-under-test and then run `scripts/integration-coverage-report.js` using the dedicated config in `integration-tests/nyc-integration.config.js`. +Reports are written to `coverage/integration-` with raw output in `.nyc_output/integration-`. + ### Adding a Plugin Test to CI The plugin tests run on pull requests in Github Actions. Each plugin test suite has its own Github job, so adding a new suite to CI diff --git a/integration-tests/helpers/index.js b/integration-tests/helpers/index.js index ca9f524795b..3388d2095c8 100644 --- a/integration-tests/helpers/index.js +++ b/integration-tests/helpers/index.js @@ -3,7 +3,7 @@ const assert = require('assert') const childProcess = require('child_process') const { execSync, fork, spawn } = childProcess -const { existsSync, readFileSync, unlinkSync, writeFileSync } = require('fs') +const { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } = require('fs') const fs = require('fs/promises') const http = require('http') const { builtinModules } = require('module') @@ -18,6 +18,9 @@ const { BUN, withBun } = require('./bun') const sandboxRoot = path.join(os.tmpdir(), id().toString()) const hookFile = 'dd-trace/loader-hook.mjs' +const COVERAGE_ENABLED_ENV = 'INTEGRATION_COVERAGE' +const COVERAGE_DIR_ENV = 'INTEGRATION_COVERAGE_DIR' +const COVERAGE_NAME_ENV = 'INTEGRATION_COVERAGE_NAME' const { DEBUG } = process.env @@ -29,6 +32,91 @@ const ANY_STRING = Symbol('test.ANY_STRING') const ANY_NUMBER = Symbol('test.ANY_NUMBER') const ANY_VALUE = Symbol('test.ANY_VALUE') +const INHERITED_ENV_KEYS = [ + 'HOME', + 'PATH', + 'NODE_OPTIONS', + 'NODE_PATH', + 'TMPDIR', + 'TEMP', + 'TMP' +] + +function isIntegrationCoverageEnabled () { + const value = process.env[COVERAGE_ENABLED_ENV] + return value === '1' || value === 'true' +} + +function getIntegrationCoverageDir () { + if (process.env[COVERAGE_DIR_ENV]) { + return process.env[COVERAGE_DIR_ENV] + } + const name = process.env[COVERAGE_NAME_ENV] || 'integration' + const dir = path.join(process.cwd(), '.coverage', 'integration-v8', name) + process.env[COVERAGE_DIR_ENV] = dir + return dir +} + +/** + * @param {NodeJS.ProcessEnv} env + */ +function applyIntegrationCoverageEnv (env) { + if (!isIntegrationCoverageEnabled() || env.NODE_V8_COVERAGE) { + return env + } + const dir = getIntegrationCoverageDir() + mkdirSync(dir, { recursive: true }) + env.NODE_V8_COVERAGE = dir + return env +} + +/** + * Remove nyc preload hooks from child process NODE_OPTIONS to avoid instrumentation side effects. + * @param {NodeJS.ProcessEnv} env + */ +function stripNycNodeOptions (env) { + if (!env.NODE_OPTIONS || !env.NODE_OPTIONS.includes('nyc')) return env + let cleaned = env.NODE_OPTIONS + cleaned = cleaned.replace(/--require\s+\S*nyc[\\/](?:lib[\\/])?register-env\.js/g, '') + cleaned = cleaned.replace(/--require\s+\S*nyc[\\/](?:lib[\\/])?wrap\.js/g, '') + cleaned = cleaned.replace(/\s+/g, ' ').trim() + if (cleaned) { + env.NODE_OPTIONS = cleaned + } else { + delete env.NODE_OPTIONS + } + return env +} + +/** + * @param {NodeJS.ProcessEnv} env + */ +function normalizeEnvValues (env) { + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + delete env[key] + continue + } + if (typeof value !== 'string') { + env[key] = String(value) + } + } + return env +} + +/** + * @param {NodeJS.ProcessEnv} env + */ +function buildInheritedEnv (env) { + const inherited = {} + for (const key of INHERITED_ENV_KEYS) { + if (env[key] !== undefined) { + inherited[key] = env[key] + } + } + return inherited +} + /** * @param {string} filename * @param {string} cwd @@ -36,7 +124,8 @@ const ANY_VALUE = Symbol('test.ANY_VALUE') * @param {string} expectedSource */ async function runAndCheckOutput (filename, cwd, expectedOut, expectedSource) { - const proc = spawn(process.execPath, [filename], { cwd, stdio: 'pipe' }) + const env = normalizeEnvValues(stripNycNodeOptions(applyIntegrationCoverageEnv({ ...process.env }))) + const proc = spawn(process.execPath, [filename], { cwd, env, stdio: 'pipe' }) assert(proc.pid !== undefined, 'Process PID is not available') const pid = proc.pid let out = await new Promise((resolve, reject) => { @@ -271,8 +360,11 @@ function spawnProcAndExpectExit (filename, options = {}, stdioHandler, stderrHan * @returns {SpawnedProcess} */ function spawnProcImpl (filename, options, stdioHandler, stderrHandler) { + const baseEnv = options.env ? buildInheritedEnv(process.env) : { ...process.env } + const mergedEnv = { ...baseEnv, ...options.env } + const env = normalizeEnvValues(stripNycNodeOptions(applyIntegrationCoverageEnv(mergedEnv))) // Cast to SpawnedProcess type - when stdio is 'pipe', stdout/stderr are guaranteed non-null - const proc = /** @type {SpawnedProcess} */ (fork(filename, { ...options, stdio: 'pipe' })) + const proc = /** @type {SpawnedProcess} */ (fork(filename, { ...options, env, stdio: 'pipe' })) proc.stdout.on('data', data => { if (stdioHandler) { diff --git a/integration-tests/nyc-integration.config.js b/integration-tests/nyc-integration.config.js new file mode 100644 index 00000000000..00deb1f34bc --- /dev/null +++ b/integration-tests/nyc-integration.config.js @@ -0,0 +1,29 @@ +'use strict' + +const path = require('node:path') + +const defaultExclude = require('@istanbuljs/schema/default-exclude') + +const coverageRoot = process.env.INTEGRATION_COVERAGE_CWD || path.parse(process.cwd()).root +const outputRoot = process.env.INTEGRATION_COVERAGE_OUTPUT_CWD || process.cwd() + +const lifecycleEvent = process.env.npm_lifecycle_event || '' +const suiteMatch = lifecycleEvent.match(/^test:integration:(.+):ci$/) +const suiteName = suiteMatch ? suiteMatch[1] : 'integration' + +const reportDir = path.join(outputRoot, 'coverage', `integration-${suiteName}`) +const tempDir = path.join(outputRoot, '.nyc_output', `integration-${suiteName}`) + +module.exports = { + cwd: coverageRoot, + reporter: ['text', 'lcov'], + reportDir, + tempDir, + include: [ + '**/node_modules/dd-trace/packages/**/src/**/*.js', + '**/node_modules/dd-trace/packages/**/src/**/*.mjs', + '**/packages/**/src/**/*.js', + '**/packages/**/src/**/*.mjs' + ], + exclude: defaultExclude.concat(['!**/node_modules/dd-trace/**']) +} diff --git a/package.json b/package.json index d2c5ab0db46..e479400538b 100644 --- a/package.json +++ b/package.json @@ -55,21 +55,37 @@ "test:profiler": "node scripts/mocha-parallel-files.js --expose-gc --timeout 30000 -- packages/dd-trace/test/profiling/**/*.spec.js", "test:profiler:ci": "nyc --no-clean --include \"packages/dd-trace/src/profiling/**/*.js\" -- npm run test:profiler", "test:integration": "mocha --timeout 60000 \"integration-tests/*.spec.js\"", + "test:integration:ci": "node scripts/run-integration-coverage.js integration --timeout 60000 \"integration-tests/*.spec.js\"", "test:integration:aiguard": "mocha --timeout 60000 \"integration-tests/aiguard/*.spec.js\"", + "test:integration:aiguard:ci": "node scripts/run-integration-coverage.js aiguard --timeout 60000 \"integration-tests/aiguard/*.spec.js\"", "test:integration:appsec": "mocha --timeout 60000 \"integration-tests/appsec/*.spec.js\"", + "test:integration:appsec:ci": "node scripts/run-integration-coverage.js appsec --timeout 60000 \"integration-tests/appsec/*.spec.js\"", "test:integration:cucumber": "mocha --timeout 60000 \"integration-tests/cucumber/*.spec.js\"", + "test:integration:cucumber:ci": "node scripts/run-integration-coverage.js cucumber --timeout 60000 \"integration-tests/cucumber/*.spec.js\"", "test:integration:cypress": "mocha --timeout 60000 \"integration-tests/cypress/*.spec.js\"", + "test:integration:cypress:ci": "node scripts/run-integration-coverage.js cypress --timeout 60000 \"integration-tests/cypress/*.spec.js\"", "test:integration:debugger": "mocha --timeout 60000 \"integration-tests/debugger/*.spec.js\"", + "test:integration:debugger:ci": "node scripts/run-integration-coverage.js debugger --timeout 60000 \"integration-tests/debugger/*.spec.js\"", "test:integration:esbuild": "mocha --timeout 60000 \"integration-tests/esbuild/*.spec.js\"", + "test:integration:esbuild:ci": "node scripts/run-integration-coverage.js esbuild --timeout 60000 \"integration-tests/esbuild/*.spec.js\"", "test:integration:openfeature": "mocha --timeout 60000 \"integration-tests/openfeature/*.spec.js\"", + "test:integration:openfeature:ci": "node scripts/run-integration-coverage.js openfeature --timeout 60000 \"integration-tests/openfeature/*.spec.js\"", "test:integration:jest": "mocha --timeout 60000 \"integration-tests/jest/*.spec.js\"", + "test:integration:jest:ci": "node scripts/run-integration-coverage.js jest --timeout 60000 \"integration-tests/jest/*.spec.js\"", "test:integration:mocha": "mocha --timeout 60000 \"integration-tests/mocha/*.spec.js\"", + "test:integration:mocha:ci": "node scripts/run-integration-coverage.js mocha --timeout 60000 \"integration-tests/mocha/*.spec.js\"", "test:integration:playwright": "mocha --timeout 60000 \"integration-tests/playwright/*.spec.js\"", + "test:integration:playwright:ci": "node scripts/run-integration-coverage.js playwright --timeout 60000 \"integration-tests/playwright/*.spec.js\"", "test:integration:selenium": "mocha --timeout 60000 \"integration-tests/selenium/*.spec.js\"", + "test:integration:selenium:ci": "node scripts/run-integration-coverage.js selenium --timeout 60000 \"integration-tests/selenium/*.spec.js\"", "test:integration:vitest": "mocha --timeout 60000 \"integration-tests/vitest/*.spec.js\"", + "test:integration:vitest:ci": "node scripts/run-integration-coverage.js vitest --timeout 60000 \"integration-tests/vitest/*.spec.js\"", "test:integration:testopt": "mocha --timeout 120000 \"integration-tests/ci-visibility/*.spec.js\"", + "test:integration:testopt:ci": "node scripts/run-integration-coverage.js testopt --timeout 120000 \"integration-tests/ci-visibility/*.spec.js\"", "test:integration:profiler": "mocha --timeout 180000 \"integration-tests/profiler/*.spec.js\"", + "test:integration:profiler:ci": "mocha --timeout 180000 \"integration-tests/profiler/*.spec.js\"", "test:integration:plugins": "mocha \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", + "test:integration:plugins:ci": "node scripts/run-integration-coverage.js plugins \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", "test:unit:plugins": "mocha \"packages/datadog-instrumentations/test/@($(echo $PLUGINS)).spec.js\" \"packages/datadog-plugin-@($(echo $PLUGINS))/test/**/*.spec.js\" --exclude \"packages/datadog-plugin-@($(echo $PLUGINS))/test/integration-test/**/*.spec.js\"", "test:shimmer": "mocha \"packages/datadog-shimmer/test/**/*.spec.js\"", "test:shimmer:ci": "nyc --no-clean --include \"packages/datadog-shimmer/src/**/*.js\" -- npm run test:shimmer", @@ -145,6 +161,7 @@ "@babel/helpers": "^7.28.4", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.39.2", + "@istanbuljs/schema": "^0.1.3", "@msgpack/msgpack": "^3.1.3", "@openfeature/core": "^1.8.1", "@openfeature/server-sdk": "~1.20.0", @@ -168,7 +185,11 @@ "glob": "^10.4.5", "globals": "^17.0.0", "graphql": "*", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "3.2.0", "jszip": "^3.10.1", + "minimatch": "10.1.1", "mocha": "^11.6.0", "mocha-junit-reporter": "^2.2.1", "mocha-multi-reporters": "^1.5.1", @@ -184,6 +205,7 @@ "sinon": "^21.0.1", "tiktoken": "^1.0.21", "typescript": "^5.9.2", + "v8-to-istanbul": "9.3.0", "workerpool": "^10.0.0", "yaml": "^2.8.0", "yarn-deduplicate": "^6.0.2" diff --git a/packages/datadog-plugin-confluentinc-kafka-javascript/test/index.spec.js b/packages/datadog-plugin-confluentinc-kafka-javascript/test/index.spec.js index 2c552beb20c..0f143d45ddc 100644 --- a/packages/datadog-plugin-confluentinc-kafka-javascript/test/index.spec.js +++ b/packages/datadog-plugin-confluentinc-kafka-javascript/test/index.spec.js @@ -381,10 +381,21 @@ describe('Plugin', () => { const timeoutId = setTimeout(() => { reject(new Error(`Timeout: Did not consume message on topic "${topic}" within ${timeoutMs}ms`)) }, timeoutMs) + let retriesRemaining = 20 function doConsume () { consumer.consume(1, function (err, messages) { if (err) { + const msg = String(err.message || err) + if (msg.includes('Unknown topic or partition')) { + retriesRemaining -= 1 + if (retriesRemaining <= 0) { + clearTimeout(timeoutId) + return reject(err) + } + setTimeout(doConsume, 50) + return + } clearTimeout(timeoutId) return reject(err) } diff --git a/scripts/integration-coverage-report.js b/scripts/integration-coverage-report.js new file mode 100644 index 00000000000..e841517e423 --- /dev/null +++ b/scripts/integration-coverage-report.js @@ -0,0 +1,170 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') +const { fileURLToPath } = require('node:url') + +const { createCoverageMap } = require('istanbul-lib-coverage') +const libReport = require('istanbul-lib-report') +const reports = require('istanbul-reports') +const { minimatch } = require('minimatch') +const v8ToIstanbul = require('v8-to-istanbul') + +const nycConfig = require('../integration-tests/nyc-integration.config') + +const REPO_ROOT = process.cwd() + +function toPosixPath (value) { + return value.split(path.sep).join('/') +} + +function isFileUrl (value) { + return typeof value === 'string' && value.startsWith('file://') +} + +function normalizeScriptUrl (url) { + if (!url || typeof url !== 'string') return null + if (url.startsWith('node:') || url.startsWith('internal:')) return null + if (url === '') return null + if (isFileUrl(url)) return fileURLToPath(url) + return url +} + +function mapToRepoPath (filePath) { + if (!filePath || !path.isAbsolute(filePath)) return filePath + if (filePath.startsWith(REPO_ROOT)) return filePath + + const normalized = filePath.replace(/\\/g, '/') + const markers = [ + '/node_modules/dd-trace/', + '/packages/', + '/integration-tests/' + ] + + for (const marker of markers) { + const idx = normalized.indexOf(marker) + if (idx === -1) continue + const rel = normalized.slice(idx + 1) + const candidate = path.join(REPO_ROOT, rel) + if (fs.existsSync(candidate)) return candidate + } + + return filePath +} + +function resolveCoverageDir (suiteName) { + if (process.env.INTEGRATION_COVERAGE_DIR) return process.env.INTEGRATION_COVERAGE_DIR + return path.join(process.cwd(), '.coverage', 'integration-v8', suiteName) +} + +function resolveReportDir (suiteName) { + return path.join(process.cwd(), 'coverage', `integration-${suiteName}`) +} + +function resolveTempDir (suiteName) { + return path.join(process.cwd(), '.nyc_output', `integration-${suiteName}`) +} + +function getSuiteName () { + const value = process.env.INTEGRATION_COVERAGE_NAME + return value && value.trim() ? value.trim() : 'integration' +} + +function matchesIncludeExclude (relPath, include, exclude) { + if (include.length) { + let matched = false + for (const pattern of include) { + if (minimatch(relPath, pattern, { dot: true })) { + matched = true + break + } + } + if (!matched) return false + } + + let excluded = false + for (const pattern of exclude) { + if (!pattern) continue + const negated = pattern.startsWith('!') + const glob = negated ? pattern.slice(1) : pattern + if (!glob) continue + if (minimatch(relPath, glob, { dot: true })) { + excluded = !negated + } + } + + return !excluded +} + +async function convertScriptCoverage (script, coverageMap, config) { + const rawPath = normalizeScriptUrl(script.url) + if (!rawPath || !path.isAbsolute(rawPath)) return + const filePath = mapToRepoPath(rawPath) + + const relPath = toPosixPath(path.relative(config.cwd, filePath)) + if (relPath.startsWith('..')) return + + if (!matchesIncludeExclude(relPath, config.include, config.exclude)) return + + if (!fs.existsSync(filePath)) return + + const converter = v8ToIstanbul(filePath, 0, { + source: fs.readFileSync(filePath, 'utf8') + }) + await converter.load() + converter.applyCoverage(script.functions || []) + coverageMap.merge(converter.toIstanbul()) +} + +async function main () { + const suiteName = getSuiteName() + const coverageDir = resolveCoverageDir(suiteName) + // TODO: ci-visibility tests don't spawn apps via helpers, so V8 coverage isn't emitted. Consider + // a dedicated coverage hook for those suites or a separate coverage workflow. + // TODO: profiler integration tests are excluded from coverage for now because OOM/exporter + // subprocesses and telemetry batching make coverage collection flaky. Revisit when stabilized. + if (!fs.existsSync(coverageDir)) { + // eslint-disable-next-line no-console + console.warn(`V8 coverage directory not found: ${coverageDir}`) + return + } + + const config = { + cwd: REPO_ROOT, + include: Array.isArray(nycConfig.include) ? nycConfig.include : [], + exclude: Array.isArray(nycConfig.exclude) ? nycConfig.exclude : [] + } + + const files = fs.readdirSync(coverageDir).filter(file => file.endsWith('.json')) + if (files.length === 0) { + // eslint-disable-next-line no-console + console.warn(`No V8 coverage files found in ${coverageDir}`) + return + } + + const coverageMap = createCoverageMap({}) + for (const file of files) { + const raw = fs.readFileSync(path.join(coverageDir, file), 'utf8') + const data = JSON.parse(raw) + const results = Array.isArray(data.result) ? data.result : [] + for (const script of results) { + await convertScriptCoverage(script, coverageMap, config) + } + } + + const reportDir = resolveReportDir(suiteName) + const tempDir = resolveTempDir(suiteName) + fs.mkdirSync(reportDir, { recursive: true }) + fs.mkdirSync(tempDir, { recursive: true }) + fs.writeFileSync(path.join(tempDir, 'coverage.json'), JSON.stringify(coverageMap.toJSON())) + + const context = libReport.createContext({ dir: reportDir, coverageMap }) + reports.create('text').execute(context) + reports.create('lcov').execute(context) +} + +main().catch(error => { + // eslint-disable-next-line no-console + console.error(error.stack || error.message) + process.exit(1) +}) diff --git a/scripts/run-integration-coverage.js b/scripts/run-integration-coverage.js new file mode 100644 index 00000000000..c26253a615c --- /dev/null +++ b/scripts/run-integration-coverage.js @@ -0,0 +1,43 @@ +'use strict' + +const { spawnSync } = require('node:child_process') +const path = require('node:path') + +function usage (message) { + if (message) process.stderr.write(`${message}\n`) + process.stderr.write('Usage: node scripts/run-integration-coverage.js \n') + process.exit(1) +} + +const args = process.argv.slice(2) +const suiteName = args.shift() +if (!suiteName) usage('Missing suite name.') +if (args.length === 0) usage('Missing mocha args.') + +const env = { + ...process.env, + INTEGRATION_COVERAGE: '1', + INTEGRATION_COVERAGE_NAME: suiteName +} + +const mochaPath = path.resolve(process.cwd(), 'node_modules', 'mocha', 'bin', 'mocha.js') +const mochaCmd = process.execPath +const mochaArgs = [mochaPath, ...args] +const mochaResult = spawnSync(mochaCmd, mochaArgs, { stdio: 'inherit', env }) +if (mochaResult.error) { + process.stderr.write(`${mochaResult.error.message}\n`) + process.exit(1) +} +if (typeof mochaResult.status === 'number' && mochaResult.status !== 0) { + process.exit(mochaResult.status) +} + +const reportScript = path.join(__dirname, 'integration-coverage-report.js') +const reportResult = spawnSync(process.execPath, [reportScript], { stdio: 'inherit', env }) +if (reportResult.error) { + process.stderr.write(`${reportResult.error.message}\n`) + process.exit(1) +} +if (typeof reportResult.status === 'number' && reportResult.status !== 0) { + process.exit(reportResult.status) +} diff --git a/scripts/verify-exercised-tests.js b/scripts/verify-exercised-tests.js index 7f42cc1c884..19c537a58d9 100644 --- a/scripts/verify-exercised-tests.js +++ b/scripts/verify-exercised-tests.js @@ -923,18 +923,10 @@ function main () { // All :ci scripts should be referenced by CI. for (const name of Object.keys(scripts).sort((a, b) => a.localeCompare(b, 'en'))) { if (!name.endsWith(':ci')) continue - if (!invokedScripts.has(name)) { - pushError(`package.json: script "${name}" is not invoked by any GitHub Actions workflow`) - } - } - - // All test:integration* scripts should be referenced by CI (except test:integration:plugins). - for (const name of Object.keys(scripts).sort((a, b) => a.localeCompare(b, 'en'))) { - if (!name.startsWith('test:integration')) continue - // Skip test:integration:plugins - it's a convenience script for running only plugin integration + // Skip test:integration:plugins:ci - it's a convenience script for running only plugin integration // tests locally, but in CI these are already covered by test:plugins:ci (which runs all plugin // tests including integration tests). - if (name === 'test:integration:plugins') continue + if (name === 'test:integration:plugins:ci') continue if (!invokedScripts.has(name)) { pushError(`package.json: script "${name}" is not invoked by any GitHub Actions workflow`) } diff --git a/yarn.lock b/yarn.lock index 227744e66f0..9e63dc2f073 100644 --- a/yarn.lock +++ b/yarn.lock @@ -311,6 +311,18 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.4.3.tgz#c2b9d2e374ee62c586d3adbea87199b1d7a7a6ba" integrity sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ== +"@isaacs/balanced-match@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz#3081dadbc3460661b751e7591d7faea5df39dd29" + integrity sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ== + +"@isaacs/brace-expansion@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz#4b3dabab7d8e75a429414a96bd67bf4c1d13e0f3" + integrity sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA== + dependencies: + "@isaacs/balanced-match" "^4.0.1" + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" @@ -365,7 +377,7 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz#6912b00d2c631c0d15ce1a7ab57cd657f2a8f8ba" integrity sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og== -"@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.28": +"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.24", "@jridgewell/trace-mapping@^0.3.28": version "0.3.31" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz#db15d6781c931f3a251a3dac39501c98a6082fd0" integrity sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw== @@ -735,6 +747,11 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.8.tgz#958b91c991b1867ced318bedea0e215ee050726e" integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== +"@types/istanbul-lib-coverage@^2.0.1": + version "2.0.6" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" + integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== + "@types/json-schema@^7.0.15": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" @@ -2608,7 +2625,7 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== -istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: +istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0, istanbul-lib-coverage@^3.2.2: version "3.2.2" resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz#2d166c4b0644d43a39f04bf6c2edd1e585f31756" integrity sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg== @@ -2643,7 +2660,7 @@ istanbul-lib-processinfo@^2.0.2: rimraf "^3.0.0" uuid "^8.3.2" -istanbul-lib-report@^3.0.0: +istanbul-lib-report@^3.0.0, istanbul-lib-report@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz#908305bac9a5bd175ac6a74489eafd0fc2445a7d" integrity sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw== @@ -2661,7 +2678,7 @@ istanbul-lib-source-maps@^4.0.0: istanbul-lib-coverage "^3.0.0" source-map "^0.6.1" -istanbul-reports@^3.0.2: +istanbul-reports@3.2.0, istanbul-reports@^3.0.2: version "3.2.0" resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.2.0.tgz#cb4535162b5784aa623cee21a7252cf2c807ac93" integrity sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA== @@ -2898,6 +2915,13 @@ mime-types@^3.0.0, mime-types@^3.0.1: dependencies: mime-db "^1.54.0" +minimatch@10.1.1: + version "10.1.1" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.1.1.tgz#e6e61b9b0c1dcab116b5a7d1458e8b6ae9e73a55" + integrity sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ== + dependencies: + "@isaacs/brace-expansion" "^5.0.0" + minimatch@^3.0.4, minimatch@^3.1.1, minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" @@ -4198,6 +4222,15 @@ uuid@^8.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +v8-to-istanbul@9.3.0: + version "9.3.0" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz#b9572abfa62bd556c16d75fdebc1a411d5ff3175" + integrity sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA== + dependencies: + "@jridgewell/trace-mapping" "^0.3.12" + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^2.0.0" + vary@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"