diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 17ec1f040b9..f5e2064912a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,13 +1,33 @@ # TODO: Restructure the project by product and clean up this file. -# Default recursive ownership +# Shared repository tooling /.agents/ @DataDog/lang-platform-js /.claude/ @DataDog/lang-platform-js /.github/actions/ @DataDog/lang-platform-js +/.github/ISSUE_TEMPLATE/ @DataDog/lang-platform-js +/.gitlab/ @DataDog/lang-platform-js +/.husky/ @DataDog/lang-platform-js +/.vscode/ @DataDog/lang-platform-js /scripts/ @DataDog/lang-platform-js +/vendor/ @DataDog/lang-platform-js + +# Shared tracer infrastructure; product-specific rules below take precedence. +/packages/dd-trace/src/agent/ @DataDog/lang-platform-js +/packages/dd-trace/src/encode/ @DataDog/lang-platform-js +/packages/dd-trace/src/exporters/agent/ @DataDog/lang-platform-js +/packages/dd-trace/src/exporters/log/ @DataDog/lang-platform-js +/packages/dd-trace/src/exporters/span-stats/ @DataDog/lang-platform-js +/packages/dd-trace/src/external-logger/ @DataDog/lang-platform-js +/packages/dd-trace/src/flare/ @DataDog/lang-platform-js +/packages/dd-trace/src/guardrails/ @DataDog/lang-platform-js +/packages/dd-trace/src/msgpack/ @DataDog/lang-platform-js +/packages/dd-trace/src/noop/ @DataDog/lang-platform-js +/packages/dd-trace/test/encode/ @DataDog/lang-platform-js +/packages/dd-trace/test/fixtures/esm/ @DataDog/lang-platform-js # AppSec /benchmark/sirun/appsec/ @DataDog/asm-js +/benchmark/sirun/appsec-waf/ @DataDog/asm-js /benchmark/sirun/appsec-iast/ @DataDog/asm-js /benchmark/sirun/iast/ @DataDog/asm-js /integration-tests/appsec/ @DataDog/asm-js @@ -15,6 +35,7 @@ /integration-tests/standalone-asm/ @DataDog/asm-js /packages/dd-trace/src/appsec/ @DataDog/asm-js /packages/dd-trace/test/appsec/ @DataDog/asm-js +/packages/dd-trace/test/fixtures/config/appsec-* @DataDog/asm-js /integration-tests/aiguard/ @DataDog/asm-js /packages/dd-trace/src/aiguard/ @DataDog/asm-js @@ -39,6 +60,8 @@ # Serverless /packages/dd-trace/src/lambda/ @DataDog/serverless-aws @DataDog/apm-serverless +/packages/dd-trace/src/azure_metadata.js @DataDog/apm-serverless +/packages/dd-trace/src/serverless.js @DataDog/apm-serverless /packages/dd-trace/test/lambda/ @DataDog/serverless-aws @DataDog/apm-serverless /packages/dd-trace/test/azure_metadata.spec.js @DataDog/apm-serverless /packages/dd-trace/test/serverless.spec.js @DataDog/apm-serverless @@ -83,13 +106,18 @@ /packages/dd-trace/index.electron.js @DataDog/apm-idm-js /packages/dd-trace/src/exporters/electron/ @DataDog/apm-idm-js /packages/dd-trace/src/plugins/ @DataDog/apm-idm-js +/packages/dd-trace/src/payload-tagging/ @DataDog/apm-idm-js +/packages/dd-trace/src/process-tags/ @DataDog/apm-idm-js +/packages/dd-trace/src/propagation-hash/ @DataDog/apm-idm-js /packages/dd-trace/test/plugins/ @DataDog/apm-idm-js +/packages/dd-trace/test/process-tags.spec.js @DataDog/apm-idm-js /packages/dd-trace/src/service-naming/ @DataDog/apm-idm-js /packages/dd-trace/test/service-naming/ @DataDog/apm-idm-js /packages/dd-trace/test/payload_tagging.spec.js @DataDog/apm-idm-js /packages/dd-trace/test/payload-tagging/ @DataDog/apm-idm-js /packages/dd-trace/test/propagation-hash.spec.js @DataDog/apm-idm-js /packages/dd-trace/test/tracer_metadata.spec.js @DataDog/apm-idm-js +/packages/dd-trace/src/tracer_metadata.js @DataDog/apm-idm-js # Test Optimization /ci/ @DataDog/ci-app-libraries @@ -107,9 +135,13 @@ /packages/dd-trace/src/plugins/ci_plugin.js @DataDog/ci-app-libraries /packages/dd-trace/test/ci-visibility/ @DataDog/ci-app-libraries /packages/dd-trace/test/encode/agentless-ci-visibility.spec.js @DataDog/ci-app-libraries +/packages/dd-trace/test/encode/agentless-json.spec.js @DataDog/ci-app-libraries /packages/dd-trace/test/encode/coverage-ci-visibility.spec.js @DataDog/ci-app-libraries +/packages/dd-trace/test/encode/tags-processors.spec.js @DataDog/ci-app-libraries /packages/dd-trace/test/git_metadata.spec.js @DataDog/ci-app-libraries /packages/dd-trace/test/git_metadata_tagger.spec.js @DataDog/ci-app-libraries +/packages/dd-trace/test/fixtures/config/git-folder*/ @DataDog/ci-app-libraries @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/test/fixtures/config/git.properties* @DataDog/ci-app-libraries @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/plugins/util/git.spec.js @DataDog/ci-app-libraries /packages/dd-trace/test/plugins/util/ci-env/ @DataDog/ci-app-libraries /packages/dd-trace/test/plugins/util/fixtures/jest/ @DataDog/ci-app-libraries @@ -210,7 +242,7 @@ /packages/datadog-plugin-anthropic/ @DataDog/ml-observability /packages/datadog-plugin-claude-agent-sdk/ @DataDog/ml-observability /benchmark/sirun/plugin-claude-agent-sdk/ @DataDog/ml-observability -/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime @DataDog/ml-observability +/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/ @DataDog/ml-observability /packages/datadog-plugin-aws-sdk/test/bedrockruntime.spec.js @DataDog/ml-observability /packages/datadog-plugin-google-cloud-vertexai/ @DataDog/ml-observability /packages/datadog-plugin-langchain/ @DataDog/ml-observability @@ -228,6 +260,8 @@ # API SDK Capabilities /eslint-rules/ @DataDog/apm-sdk-capabilities-js +/ext/kinds.* @DataDog/apm-sdk-capabilities-js +/ext/priority.* @DataDog/apm-sdk-capabilities-js /benchmark/sirun/propagation/ @DataDog/apm-sdk-capabilities-js /benchmark/sirun/sampling/ @DataDog/apm-sdk-capabilities-js @@ -262,28 +296,18 @@ /packages/dd-trace/test/baggage.spec.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/src/carrier.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/carrier.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/src/sampler.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/sampler.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/src/priority_sampler.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/priority_sampler.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/ramdom_sampler.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/rate_limiter.spec.js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/src/*sampler.js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/test/*sampler.spec.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/src/sampling_rule.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/sampling_rule.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/src/span_sampler.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/analytics_sampler.spec.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/custom-metrics.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/process-tags.spec.js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/test/custom-metrics-app.js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/test/fixtures/config/span-sampling-rules.json @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/test/helpers/config.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/src/runtime_metrics/ @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/runtime_metrics.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/scope.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/span_format.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/span_processor.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/span_stats.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/span_sampler.spec.js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/src/startup-log.js @DataDog/apm-sdk-capabilities-js /packages/dd-trace/test/startup-log.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/tagger.spec.js @DataDog/apm-sdk-capabilities-js -/packages/dd-trace/test/tracer.spec.js @DataDog/apm-sdk-capabilities-js /scripts/generate-config-types.js @DataDog/apm-sdk-capabilities-js /scripts/generate-supported-integrations.js @DataDog/apm-sdk-capabilities-js @@ -295,7 +319,9 @@ /supported_versions_table.csv @DataDog/apm-sdk-capabilities-js # Feature Flagging +/integration-tests/esbuild/*openfeature* @DataDog/apm-idm-js @DataDog/feature-flagging-and-experimentation-sdk /integration-tests/openfeature/ @DataDog/feature-flagging-and-experimentation-sdk +/integration-tests/webpack/*openfeature* @DataDog/apm-idm-js @DataDog/feature-flagging-and-experimentation-sdk /packages/dd-trace/src/openfeature/ @DataDog/feature-flagging-and-experimentation-sdk /packages/dd-trace/test/openfeature/ @DataDog/feature-flagging-and-experimentation-sdk @@ -310,11 +336,19 @@ /.github/workflows/codeql-analysis.yml @DataDog/sdlc-security /.github/workflows/mirror-image.yml @Datadog/lang-platform-js /.github/workflows/all-green.yml @Datadog/lang-platform-js +/.github/workflows/audit.yml @DataDog/lang-platform-js /.github/workflows/custom-node-version-dispatch.yml @Datadog/lang-platform-js +/.github/workflows/dependabot-automation.yml @DataDog/lang-platform-js @DataDog/apm-sdk-capabilities-js +/.github/workflows/flakiness.yml @DataDog/lang-platform-js +/.github/workflows/platform.yml @DataDog/lang-platform-js /.github/workflows/project.yml @Datadog/lang-platform-js +/.github/workflows/release-proposal.yml @DataDog/lang-platform-js +/.github/workflows/release-validate.yml @DataDog/lang-platform-js /.github/workflows/stale.yml @Datadog/lang-platform-js +/.github/workflows/update-3rdparty-licenses.yml @DataDog/lang-platform-js /.github/workflows/apm-capabilities.yml @DataDog/apm-sdk-capabilities-js +/.github/workflows/eslint-rules.yml @DataDog/apm-sdk-capabilities-js /.github/workflows/apm-integrations.yml @DataDog/apm-idm-js /.github/workflows/aiguard.yml @DataDog/asm-js /.github/workflows/appsec.yml @DataDog/asm-js @@ -336,12 +370,26 @@ /integration-tests/profiler/ @DataDog/profiling-js /packages/dd-trace/*/profiling/ @DataDog/profiling-js +/packages/dd-trace/src/otel-thread-ctx.js @DataDog/profiling-js +/packages/dd-trace/src/profiler.js @DataDog/profiling-js +/packages/dd-trace/src/storage-channels.js @DataDog/profiling-js @DataDog/apm-sdk-capabilities-js +/packages/dd-trace/src/web-tags-cache.js @DataDog/profiling-js /packages/dd-trace/test/exporters/common/form-data.spec.js @DataDog/profiling-js +/packages/dd-trace/test/otel-thread-ctx.spec.js @DataDog/profiling-js +/packages/dd-trace/test/web-tags-cache.spec.js @DataDog/profiling-js # Language Platform /* @DataDog/lang-platform-js +/openfeature.d.ts @DataDog/lang-platform-js @DataDog/feature-flagging-and-experimentation-sdk +/openfeature.js @DataDog/lang-platform-js @DataDog/feature-flagging-and-experimentation-sdk + +/.github/CODEOWNERS @DataDog/lang-platform-js +/.github/dependabot.yml @DataDog/lang-platform-js @DataDog/apm-sdk-capabilities-js +/.github/pull_request_template.md @DataDog/lang-platform-js +/.github/vendored-dependencies.csv @DataDog/lang-platform-js /benchmark/* @DataDog/lang-platform-js +/benchmark/openfeature.js @DataDog/lang-platform-js @DataDog/feature-flagging-and-experimentation-sdk /benchmark/sirun/* @DataDog/lang-platform-js /benchmark/stubs/ @DataDog/lang-platform-js /benchmark/sirun/async_hooks/ @DataDog/lang-platform-js @@ -357,6 +405,8 @@ /benchmark/sirun/startup/ @DataDog/lang-platform-js /benchmark/sirun/tracing-channel/ @DataDog/lang-platform-js +/ext/scopes.* @DataDog/lang-platform-js + /integration-tests/bun/ @DataDog/lang-platform-js /integration-tests/coverage/ @DataDog/lang-platform-js /integration-tests/coverage-child-process.spec.js @DataDog/lang-platform-js @@ -379,16 +429,35 @@ /packages/datadog-core/ @DataDog/lang-platform-js /packages/datadog-shimmer/ @DataDog/lang-platform-js /packages/dd-trace/*/crashtracking/ @DataDog/lang-platform-js +/packages/dd-trace/index.js @DataDog/lang-platform-js /packages/dd-trace/src/bootstrap.js @DataDog/lang-platform-js +/packages/dd-trace/src/constants.js @DataDog/lang-platform-js +/packages/dd-trace/src/dogstatsd.js @DataDog/lang-platform-js +/packages/dd-trace/src/exporter.js @DataDog/lang-platform-js /packages/dd-trace/src/feature-registry.js @DataDog/lang-platform-js /packages/dd-trace/src/exporters/common/ @DataDog/lang-platform-js /packages/dd-trace/src/exporters/common/client-library-headers.js @DataDog/lang-platform-js @DataDog/feature-flagging-and-experimentation-sdk -/packages/dd-trace/src/guardrails/ @DataDog/lang-platform-js +/packages/dd-trace/src/heap_snapshots.js @DataDog/lang-platform-js +/packages/dd-trace/src/histogram.js @DataDog/lang-platform-js +/packages/dd-trace/src/id.js @DataDog/lang-platform-js +/packages/dd-trace/src/iitm.js @DataDog/lang-platform-js +/packages/dd-trace/src/index.js @DataDog/lang-platform-js +/packages/dd-trace/src/pkg.js @DataDog/lang-platform-js /packages/dd-trace/src/proxy.js @DataDog/lang-platform-js +/packages/dd-trace/src/rate_limiter.js @DataDog/lang-platform-js +/packages/dd-trace/src/require-package-json.js @DataDog/lang-platform-js +/packages/dd-trace/src/ritm.js @DataDog/lang-platform-js +/packages/dd-trace/src/scope.js @DataDog/lang-platform-js +/packages/dd-trace/src/span_format.js @DataDog/lang-platform-js +/packages/dd-trace/src/span_processor.js @DataDog/lang-platform-js +/packages/dd-trace/src/span_stats.js @DataDog/lang-platform-js +/packages/dd-trace/src/spanleak.js @DataDog/lang-platform-js +/packages/dd-trace/src/tagger.js @DataDog/lang-platform-js +/packages/dd-trace/src/tracer.js @DataDog/lang-platform-js +/packages/dd-trace/src/util.js @DataDog/lang-platform-js /packages/dd-trace/test/agent/ @DataDog/lang-platform-js /packages/dd-trace/test/dd-trace.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/dogstatsd.spec.js @DataDog/lang-platform-js -/packages/dd-trace/test/encode/ @DataDog/lang-platform-js /packages/dd-trace/test/esm-named-exports.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/exporter.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/exporters/ @DataDog/lang-platform-js @@ -403,19 +472,40 @@ /packages/dd-trace/test/log.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/msgpack/ @DataDog/lang-platform-js /packages/dd-trace/test/mocha-hooks.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/node_modules/ @DataDog/lang-platform-js /packages/dd-trace/test/noop.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/pkg.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/plugin_manager.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/plugins/plugin.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/plugins/util/env.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/pkg-loader.js @DataDog/lang-platform-js +/packages/dd-trace/test/profile.js @DataDog/lang-platform-js /packages/dd-trace/test/proxy.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/proxyquire.js @DataDog/lang-platform-js +/packages/dd-trace/test/rate_limiter.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/register.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/require-package-json.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/ritm-tests/ @DataDog/lang-platform-js /packages/dd-trace/test/ritm.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/scope.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/setup/ @DataDog/lang-platform-js +/packages/dd-trace/test/span_format.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/span_processor.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/span_stats.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/tagger.spec.js @DataDog/lang-platform-js +/packages/dd-trace/test/tracer.spec.js @DataDog/lang-platform-js /packages/dd-trace/test/util.spec.js @DataDog/lang-platform-js # Multiple teams +/devdocs/ @DataDog/apm-sdk-capabilities-js @DataDog/apm-idm-js +/docs/ @DataDog/apm-idm-js @DataDog/lang-platform-js +/docs/API.md @DataDog/apm-sdk-capabilities-js @DataDog/apm-idm-js +/docs/test.ts @DataDog/apm-sdk-capabilities-js @DataDog/apm-idm-js /.gitlab/benchmarks/ @DataDog/lang-platform-js @DataDog/ecosystems-performance /eslint-rules/* @DataDog/apm-sdk-capabilities-js @DataDog/lang-platform-js +/ext/exporters.* @DataDog/apm-idm-js @DataDog/ci-app-libraries @DataDog/lang-platform-js +/ext/formats.* @DataDog/apm-sdk-capabilities-js @DataDog/data-streams-monitoring +/ext/index.* @DataDog/apm-idm-js @DataDog/apm-sdk-capabilities-js @DataDog/apm-serverless @DataDog/ci-app-libraries @DataDog/data-streams-monitoring @DataDog/lang-platform-js +/ext/tags.* @DataDog/apm-idm-js @DataDog/apm-sdk-capabilities-js @DataDog/data-streams-monitoring +/ext/types.* @DataDog/apm-idm-js @DataDog/apm-serverless +/packages/dd-trace/src/plugin_manager.js @DataDog/lang-platform-js @DataDog/apm-idm-js diff --git a/.github/workflows/apm-integrations.yml b/.github/workflows/apm-integrations.yml index efaec6f2ba8..65fbfda0c5c 100644 --- a/.github/workflows/apm-integrations.yml +++ b/.github/workflows/apm-integrations.yml @@ -71,6 +71,7 @@ jobs: PLUGINS: aerospike SERVICES: aerospike PACKAGE_VERSION_RANGE: ${{ matrix.range }} + DD_INJECT_FORCE: "true" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/node diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bc4cd87e986..1d6f90af859 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,7 +45,7 @@ jobs: - name: Initialize CodeQL id: init-codeql - uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} config-file: .github/codeql_config.yml @@ -57,7 +57,7 @@ jobs: - name: Perform CodeQL Analysis id: analyze - uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: token: ${{ github.token }} wait-for-processing: false diff --git a/.github/workflows/instrumentation.yml b/.github/workflows/instrumentation.yml index 3077503a1cb..63179f945a5 100644 --- a/.github/workflows/instrumentation.yml +++ b/.github/workflows/instrumentation.yml @@ -61,6 +61,16 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/instrumentations/test + instrumentation-anthropic-lifecycle: + runs-on: ubuntu-latest + permissions: + id-token: write + env: + PLUGINS: anthropic|anthropic-lifecycle + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/instrumentations/test + instrumentation-aws-sdk: runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index bbb6cb42cf7..9a35d62ea41 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -2,7 +2,7 @@ name: Pull Request Title on: pull_request_target: - types: [opened, edited, reopened] + types: [opened, edited, reopened, synchronize] branches: - "master" @@ -14,6 +14,7 @@ jobs: conventional-commit: runs-on: ubuntu-latest permissions: + contents: read pull-requests: write env: # Shared between both steps. Must stay portable across bash ERE and JS @@ -21,6 +22,12 @@ jobs: # Revert PRs always get semver-patch regardless of the original change's type. PR_TITLE_PATTERN: '^(revert(!)?: .+|(feat|fix|docs|style|refactor|perf|test|bench|build|ci|chore)(\(([^)]+)\))?(!)?: .+)' steps: + - name: Checkout base revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + - name: Auto-rename GitHub revert title to Conventional Commit id: rename if: startsWith(github.event.pull_request.title, 'Revert "') @@ -39,22 +46,35 @@ jobs: }) core.setOutput('renamed', 'true') - - name: Validate PR title against Conventional Commits + - name: Validate PR title and release-note context if: >- steps.rename.outputs.renamed != 'true' && (github.event.action != 'edited' || github.event.changes.title != null) - env: - PR_TITLE: ${{ github.event.pull_request.title }} - run: | - if [[ ! "$PR_TITLE" =~ $PR_TITLE_PATTERN ]]; then - echo "::error::PR title does not follow Conventional Commits format." - echo "Got: $PR_TITLE" - echo "Expected: ()?(!)?: " - echo " revert(!)?: (for reverts, always semver-patch)" - echo "Allowed types: feat, fix, docs, style, refactor, perf, test, bench, build, ci, chore" - exit 1 - fi - echo "PR title OK: $PR_TITLE" + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const pullRequest = context.payload.pull_request + const title = pullRequest.title || '' + const match = title.match(new RegExp(process.env.PR_TITLE_PATTERN)) + if (!match) { + core.setFailed('PR title does not follow Conventional Commits format.') + return + } + core.info(`PR title OK: ${title}`) + + const type = match[3] + const changedFiles = await github.paginate(github.rest.pulls.listFiles, { + ...context.repo, + pull_number: pullRequest.number, + per_page: 100, + }) + const files = [] + const { appendChangedPaths, isInternalOnly } = require('./scripts/release/changelog') + appendChangedPaths(files, changedFiles) + if (/^(?:feat|fix|perf|docs)$/.test(type) && isInternalOnly(files)) { + core.setFailed(`PR title type "${type}" is public, but every changed file is internal. ` + + 'Use test, bench, ci, or chore.') + } - name: Sync labels with PR title if: steps.rename.outputs.renamed != 'true' diff --git a/.github/workflows/profiling.yml b/.github/workflows/profiling.yml index 6be7c32baa4..3c2e18bdcee 100644 --- a/.github/workflows/profiling.yml +++ b/.github/workflows/profiling.yml @@ -32,6 +32,7 @@ jobs: - uses: ./.github/actions/install - run: npm run test:profiler:ci - run: npm run test:integration:profiler:coverage + - run: npm run test:otel-thread-ctx - uses: ./.github/actions/coverage with: flags: profiling-macos @@ -60,6 +61,7 @@ jobs: - uses: ./.github/actions/node/latest - run: npm run test:profiler:ci - run: npm run test:integration:profiler:coverage + - run: npm run test:otel-thread-ctx - uses: ./.github/actions/coverage with: flags: profiling-ubuntu @@ -118,6 +120,7 @@ jobs: ) | ForEach-Object { "$_=" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 } - run: npm run test:profiler:ci - run: npm run test:integration:profiler:coverage + - run: npm run test:otel-thread-ctx - uses: ./.github/actions/node-crash-report if: failure() # Upload any WER minidumps that landed during this job, even on diff --git a/.gitlab/one-pipeline.locked.yml b/.gitlab/one-pipeline.locked.yml index 08c4ee68544..0d756c4f697 100644 --- a/.gitlab/one-pipeline.locked.yml +++ b/.gitlab/one-pipeline.locked.yml @@ -1,4 +1,4 @@ # DO NOT EDIT THIS FILE MANUALLY # This file is auto-generated by automation. include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.2.0/one-pipeline.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.3.1/one-pipeline.yml diff --git a/MIGRATING.md b/MIGRATING.md index 089c8db9fb7..03f967d7e8a 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -4,6 +4,16 @@ This guide describes the steps to upgrade dd-trace from a major version to the next. If you are having any issues related to migrating, please feel free to open an issue or contact our [support](https://www.datadoghq.com/support/) team. +## 6.0 to 7.0 (unreleased) + +### Electron plugin disabled by default + +The Electron integration has moved to the standalone +[Electron SDK](https://github.com/DataDog/electron-sdk). The `electron` plugin +and its instrumentation remain in dd-trace-js for backward compatibility, but +are now disabled by default. Set `DD_TRACE_ELECTRON_ENABLED=true` to keep +using the built-in plugin. + ## 5.0 to 6.0 ### Node 18 and 20 are no longer supported diff --git a/README.md b/README.md index 4b214a13bd8..04a8ee2b321 100644 --- a/README.md +++ b/README.md @@ -93,23 +93,6 @@ Regardless of where you open the issue, someone at Datadog will try to help. If you would like to trace your bundled application then please read this page on [bundling and dd-trace](https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/nodejs/#bundling). It includes information on how to use our ESBuild plugin and includes caveats for other bundlers. -When using the experimental OpenFeature provider, file-traced deployments can force the optional provider and -its dependencies into the output with a side-effect import before accessing `tracer.openfeature`: - -CommonJS: - -```js -require('dd-trace/openfeature') -``` - -ES modules: - -```js -import 'dd-trace/openfeature.js' -``` - -This is a fallback for build tools that do not recognize the provider's optional-require wrapper. - ## Security Vulnerabilities diff --git a/benchmark/sirun/appsec-iast/README.md b/benchmark/sirun/appsec-iast/README.md index 1a84854f08c..387f7bdf0b6 100644 --- a/benchmark/sirun/appsec-iast/README.md +++ b/benchmark/sirun/appsec-iast/README.md @@ -1,4 +1,3 @@ -Drives real Express request handling with the tracer loaded, measuring IAST's -per-request taint-tracking overhead -- IAST off vs on (default sampling, and -always-active), across a non-vulnerable endpoint and one with a command-injection -sink that triggers vulnerability reporting. +Drives real Express request handling with the tracer loaded, measuring default +IAST per-request taint-tracking across a non-vulnerable endpoint and one with a +command-injection sink. diff --git a/benchmark/sirun/appsec-iast/client.js b/benchmark/sirun/appsec-iast/client.js index 9068dc9fa63..0442abbdf69 100644 --- a/benchmark/sirun/appsec-iast/client.js +++ b/benchmark/sirun/appsec-iast/client.js @@ -1,30 +1,15 @@ 'use strict' -const http = require('http') -const { port, reqs } = require('./common') - -let connectionsMade = 0 -function request (opts) { - http.get(opts, (res) => { - res.on('data', () => {}) - res.on('end', () => { - if (++connectionsMade !== reqs) { - request(opts) - } - }) - }).on('error', (e) => { - setTimeout(() => { - request(opts) - }, 10) - }) -} +const runRequests = require('../http-client') +const { port, reqs, warmup } = require('./common') const path = '/?param=value' const opts = { headers: { accept: 'text/html', }, + host: '127.0.0.1', port, path, } -request(opts) +runRequests(opts, warmup, reqs, 1) diff --git a/benchmark/sirun/appsec-iast/common.js b/benchmark/sirun/appsec-iast/common.js index 57a6a1c6951..42940d47540 100644 --- a/benchmark/sirun/appsec-iast/common.js +++ b/benchmark/sirun/appsec-iast/common.js @@ -2,9 +2,6 @@ module.exports = { port: 3331 + parseInt(process.env.CPU_AFFINITY || '0', 10), - // Env-tunable like the other live benches. Local tuning (keep-alive, higher - // counts) did not reduce the run-to-run jitter -- it is express/IAST scheduling - // noise that CI core-pinning addresses, not connection churn -- so this is left - // at a modest default and gated on CI. - reqs: Number(process.env.REQS) || 100, + reqs: Number(process.env.OPERATIONS) || 100, + warmup: 100, } diff --git a/benchmark/sirun/appsec-iast/meta.json b/benchmark/sirun/appsec-iast/meta.json index 7cf881d4b35..7760ddeaaf8 100644 --- a/benchmark/sirun/appsec-iast/meta.json +++ b/benchmark/sirun/appsec-iast/meta.json @@ -4,71 +4,39 @@ "instructions": true, "iterations": 15, "variants": { - "no-vulnerability-control": { - "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node --require ../../../init.js server-without-vulnerability.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"", - "env": { - "DD_IAST_ENABLED": "0" - } - }, "no-vulnerability-iast-enabled-default-config": { + "cpus": 2, + "iterations": 5, "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node --require ../../../init.js server-without-vulnerability.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"", - "env": { - "DD_IAST_ENABLED": "1" - } - }, - "no-vulnerability-iast-enabled-always-active": { - "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node --require ../../../init.js server-without-vulnerability.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-without-vulnerability.js\"", + "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY_SECOND node client.js >/dev/null 2>&1 &\"", + "run": "node --require ../startup-guard.js --require ../noop-request.js --require ../../../init.js server-without-vulnerability.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../startup-guard.js --require ../noop-request.js --require ../../../init.js server-without-vulnerability.js\"", "env": { + "DD_INJECT_FORCE": "true", "DD_IAST_ENABLED": "1", - "DD_IAST_REQUEST_SAMPLING": "100", - "DD_IAST_MAX_CONCURRENT_REQUESTS": "1000", - "DD_IAST_MAX_CONTEXT_OPERATIONS": "100" - } - }, - "with-vulnerability-control": { - "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node --require ../../../init.js server-with-vulnerability.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"", - "env": { - "DD_IAST_ENABLED": "0" + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + "OPERATIONS": "30000" } }, "with-vulnerability-iast-enabled-default-config": { + "cpus": 2, + "iterations": 5, "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node --require ../../../init.js server-with-vulnerability.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"", - "env": { - "DD_IAST_ENABLED": "1" - } - }, - "with-vulnerability-iast-enabled-always-active": { - "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node --require ../../../init.js server-with-vulnerability.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js server-with-vulnerability.js\"", + "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY_SECOND node client.js >/dev/null 2>&1 &\"", + "run": "node --require ../startup-guard.js --require ../noop-request.js --require ../../../init.js server-with-vulnerability.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../startup-guard.js --require ../noop-request.js --require ../../../init.js server-with-vulnerability.js\"", "env": { + "DD_INJECT_FORCE": "true", "DD_IAST_ENABLED": "1", - "DD_IAST_REQUEST_SAMPLING": "100", - "DD_IAST_MAX_CONCURRENT_REQUESTS": "1000", - "DD_IAST_MAX_CONTEXT_OPERATIONS": "100" - } - }, - "startup-time-control": { - "run": "node --require ../../../init.js insecure-bank.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js insecure-bank.js\"", - "env": { - "DD_IAST_ENABLED": "0" + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + "OPERATIONS": "11500" } }, "startup-time-iast-enabled": { "run": "node --require ../../../init.js insecure-bank.js", "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js insecure-bank.js\"", "env": { + "DD_INJECT_FORCE": "true", "DD_IAST_ENABLED": "1" } } diff --git a/benchmark/sirun/appsec-iast/noop-child-process.js b/benchmark/sirun/appsec-iast/noop-child-process.js new file mode 100644 index 00000000000..7fdda6cb179 --- /dev/null +++ b/benchmark/sirun/appsec-iast/noop-child-process.js @@ -0,0 +1,27 @@ +'use strict' + +/** + * @param {typeof import('node:child_process').ChildProcess} ChildProcess + */ +module.exports = function installNoopSpawn (ChildProcess) { + /** + * @param {{ file: string, args?: string[] }} options + */ + ChildProcess.prototype.spawn = function noopSpawn (options) { + this.spawnfile = options.file + this.spawnargs = options.args ?? [] + this.pid = 1 + this.stdin = null + this.stdout = null + this.stderr = null + this.stdio = [null, null, null] + + process.nextTick(() => { + this.emit('spawn') + this.emit('exit', 0, null) + this.emit('close', 0, null) + }) + + return 0 + } +} diff --git a/benchmark/sirun/appsec-iast/server-with-vulnerability.js b/benchmark/sirun/appsec-iast/server-with-vulnerability.js index 772f99b9d2c..b77a6835ea8 100644 --- a/benchmark/sirun/appsec-iast/server-with-vulnerability.js +++ b/benchmark/sirun/appsec-iast/server-with-vulnerability.js @@ -1,28 +1,16 @@ 'use strict' -const assert = require('node:assert/strict') -const childProcess = require('child_process') -const express = require('../../../versions/express').get() -const cookieParser = require('../../../versions/cookie-parser').get() -const { port, reqs } = require('./common') +const childProcess = require('node:child_process') -const app = express() -app.use(cookieParser()) +require('./noop-child-process')(childProcess.ChildProcess) -let connectionsMade = 0 +/** + * @param {import('express').Request} req + */ +function runSink (req) { + childProcess.exec(req.query.param, noop) +} function noop () {} -app.get('/', (req, res) => { - childProcess.exec('echo #' + req.query.param, noop) - res.writeHead(200) - res.end('Hello, World!') - - if (++connectionsMade === reqs) { - server.close() - } -}) - -const server = app.listen(port, () => { - assert.ok(server.address(), 'appsec-iast server failed to bind') -}) +require('./server')(runSink, 'COMMAND_INJECTION') diff --git a/benchmark/sirun/appsec-iast/server-without-vulnerability.js b/benchmark/sirun/appsec-iast/server-without-vulnerability.js index 0de86819787..9d6020c6bbf 100644 --- a/benchmark/sirun/appsec-iast/server-without-vulnerability.js +++ b/benchmark/sirun/appsec-iast/server-without-vulnerability.js @@ -1,24 +1,3 @@ 'use strict' -const assert = require('node:assert/strict') -const express = require('../../../versions/express').get() -const cookieParser = require('../../../versions/cookie-parser').get() -const { port, reqs } = require('./common') - -const app = express() -app.use(cookieParser()) - -let connectionsMade = 0 - -app.get('/', (req, res) => { - res.writeHead(200) - res.end('Hello, World!') - - if (++connectionsMade === reqs) { - server.close() - } -}) - -const server = app.listen(port, () => { - assert.ok(server.address(), 'appsec-iast server failed to bind') -}) +require('./server')() diff --git a/benchmark/sirun/appsec-iast/server.js b/benchmark/sirun/appsec-iast/server.js new file mode 100644 index 00000000000..4b37b883d86 --- /dev/null +++ b/benchmark/sirun/appsec-iast/server.js @@ -0,0 +1,84 @@ +'use strict' + +const assert = require('node:assert/strict') + +const guard = require('../startup-guard') +const clearTimeoutGuard = require('../timeout-guard')('appsec-iast server') +const tracer = require('../../..') + +const express = require('../../../versions/express').get() +const cookieParser = require('../../../versions/cookie-parser').get() +const { port, reqs, warmup } = require('./common') + +/** + * @param {(request: import('express').Request) => void} [onRequest] + * @param {string} [expectedVulnerability] + */ +module.exports = function startServer (onRequest, expectedVulnerability) { + let featureValidated = false + let acquiredRequests = 0 + let hookCalls = 0 + let lastIastEnabled + let vulnerabilityReported = false + + tracer.use('express', { + hooks: { + /** @param {import('../../../index').Span} span */ + request (span) { + const context = span.context() + hookCalls++ + lastIastEnabled = context.getTag('_dd.iast.enabled') + if (lastIastEnabled !== 1) return + + acquiredRequests++ + if (expectedVulnerability) { + const iastJson = context.getTag('_dd.iast.json') + vulnerabilityReported = iastJson?.includes(`"type":"${expectedVulnerability}"`) === true + if (!vulnerabilityReported) return + } + + featureValidated = true + tracer.use('express', {}) + }, + }, + }) + + const app = express() + app.use(cookieParser()) + + let responsesFinished = 0 + + function onResponseFinish () { + responsesFinished++ + if (responsesFinished === warmup) { + assert.ok( + featureValidated, + `IAST preflight failed (hooks: ${hookCalls}, acquired: ${acquiredRequests}, ` + + `last enabled: ${lastIastEnabled}, vulnerability: ${vulnerabilityReported})` + ) + guard.loopStart() + } else if (responsesFinished === reqs + warmup) { + guard.done(0.1) + server.close() + } + } + + /** + * @param {import('express').Request} req + * @param {import('express').Response} res + */ + function handleRequest (req, res) { + res.once('finish', onResponseFinish) + onRequest?.(req) + + res.writeHead(200) + res.end('Hello, World!') + } + + app.get('/', handleRequest) + + const server = app.listen(port, () => { + assert.ok(server.address(), 'appsec-iast server failed to bind') + }) + server.once('close', clearTimeoutGuard) +} diff --git a/benchmark/sirun/appsec-waf/README.md b/benchmark/sirun/appsec-waf/README.md new file mode 100644 index 00000000000..8991664f21d --- /dev/null +++ b/benchmark/sirun/appsec-waf/README.md @@ -0,0 +1,5 @@ +Replays request payloads captured at the native WAF boundary. This isolates WAF +context creation and execution from HTTP client scheduling and tracer request +instrumentation. The matching `appsec` benchmark uses the same samples with a +replaying WAF implementation, so the two benchmarks cover both layers without +measuring them in the same timing window. diff --git a/benchmark/sirun/appsec-waf/index.js b/benchmark/sirun/appsec-waf/index.js new file mode 100644 index 00000000000..2464aeea65d --- /dev/null +++ b/benchmark/sirun/appsec-waf/index.js @@ -0,0 +1,70 @@ +'use strict' + +const assert = require('node:assert/strict') +const guard = require('../startup-guard') + +// eslint-disable-next-line import/order -- the startup guard must load before the native addon +const { DDWAF } = require('@datadog/native-appsec') + +const samples = require('../appsec/waf-samples.json') +const rules = require('../../../packages/dd-trace/src/appsec/recommended.json') + +const operations = Number(process.env.OPERATIONS) +const sample = samples[process.env.WAF_SAMPLE] + +assert.ok(operations > 0, 'OPERATIONS must be positive') +assert.ok(sample, `unknown WAF_SAMPLE: ${process.env.WAF_SAMPLE}`) + +// Keep these aligned with the production defaults without loading the configuration manifest into the benchmark. +const obfuscatorKeyRegex = '(?i)pass|pw(?:or)?d|secret|(?:api|private|public|access)[_-]?key|token|' + + 'consumer[_-]?(?:id|key|secret)|sign(?:ed|ature)|bearer|authorization|' + + 'jsessionid|phpsessid|asp\\.net[_-]sessionid|sid|jwt' +const obfuscatorValueRegex = + '(?i)(?:p(?:ass)?w(?:or)?d|pass(?:[_-]?phrase)?|secret(?:[_-]?key)?|' + + '(?:(?:api|private|public|access)[_-]?)key(?:[_-]?id)?|' + + '(?:(?:auth|access|id|refresh)[_-]?)?token|consumer[_-]?(?:id|key|secret)|' + + 'sign(?:ed|ature)?|auth(?:entication|orization)?|jsessionid|phpsessid|' + + 'asp\\.net(?:[_-]|-)sessionid|sid|jwt)(?:\\s*=([^;&]+)|"\\s*:\\s*("[^"]+"|\\d+))|' + + 'bearer\\s+([a-z0-9\\._\\-]+)|token\\s*:\\s*([a-z0-9]{13})|gh[opsu]_([0-9a-zA-Z]{36})|' + + 'ey[I-L][\\w=-]+\\.(ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?)|' + + '[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}([^\\-]+)[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|' + + 'ssh-rsa\\s*([a-z0-9\\/\\.+]{100,})' + +const waf = new DDWAF(rules, 'benchmark/sirun/appsec-waf', { + obfuscatorKeyRegex, + obfuscatorValueRegex, +}) + +/** + * @param {boolean} verify + * @returns {void} + */ +function runRequest (verify) { + const context = waf.createContext() + let matched = false + + for (const { payload } of sample) { + const result = context.run(payload, 5e3) + if (verify && result.events?.length) { + matched = true + } + } + + context.dispose() + + if (verify) { + assert.strictEqual(matched, process.env.WAF_SAMPLE === 'attack') + } +} + +for (let i = 0; i < 100; i++) { + runRequest(i === 0) +} + +guard.loopStart() +for (let i = 0; i < operations; i++) { + runRequest(false) +} +guard.done(0.1) + +waf.dispose() diff --git a/benchmark/sirun/appsec-waf/meta.json b/benchmark/sirun/appsec-waf/meta.json new file mode 100644 index 00000000000..b6d64197cbd --- /dev/null +++ b/benchmark/sirun/appsec-waf/meta.json @@ -0,0 +1,24 @@ +{ + "name": "appsec-waf", + "run": "node index.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"", + "cachegrind": false, + "iterations": 20, + "instructions": true, + "variants": { + "request": { + "iterations": 13, + "env": { + "WAF_SAMPLE": "request", + "OPERATIONS": "50000" + } + }, + "attack": { + "iterations": 14, + "env": { + "WAF_SAMPLE": "attack", + "OPERATIONS": "28000" + } + } + } +} diff --git a/benchmark/sirun/appsec/README.md b/benchmark/sirun/appsec/README.md index d0c29ef1704..d8eea5eb14f 100644 --- a/benchmark/sirun/appsec/README.md +++ b/benchmark/sirun/appsec/README.md @@ -1,6 +1,5 @@ -This benchmarks the cost Datadog AppSec adds to an HTTP server: per-request WAF -handling (a keep-alive client drives the tracer-instrumented server, with -`DD_APPSEC_ENABLED` toggled and an attack-payload variant) and process startup -(loading the tracer with AppSec on versus off). Variants, request counts, and -attack payloads are defined in `meta.json`. - +This benchmarks the HTTP instrumentation and result-processing cost Datadog +AppSec adds to a server. A keep-alive client drives the tracer-instrumented +server, while a replaying native-WAF implementation verifies the request data +against `waf-samples.json` and returns its captured result. Native WAF execution +is measured separately by `appsec-waf` using the same samples. diff --git a/benchmark/sirun/appsec/client.js b/benchmark/sirun/appsec/client.js index 9ed47d17615..ede5b9266f9 100644 --- a/benchmark/sirun/appsec/client.js +++ b/benchmark/sirun/appsec/client.js @@ -1,36 +1,17 @@ 'use strict' -const http = require('http') -const { port, reqs } = require('./common') +const runRequests = require('../http-client') +const { port, reqs, warmup } = require('./common') -// Reuse a single keep-alive connection so a high request count does not churn -// ephemeral ports on localhost (which collapses throughput). 127.0.0.1 avoids -// per-connection localhost -> ::1 lookups. -const agent = new http.Agent({ keepAlive: true, maxSockets: 1 }) - -let connectionsMade = 0 - -function request (opts) { - http.get(opts, (res) => { - res.on('data', () => {}) - res.on('end', () => { - if (++connectionsMade !== reqs) { - request(opts) - } - }) - }).on('error', () => { - setTimeout(() => { - request(opts) - }, 10) - }) -} +// Keep enough requests in flight that client scheduling does not leave the +// measured server idle between responses. +const concurrency = 4 const opts = { host: '127.0.0.1', headers: {}, port, path: '/', - agent, } if (Number(process.env.ATTACK_UA)) { @@ -47,4 +28,4 @@ if (Number(process.env.ATTACK_QS)) { opts.path += '?k=' } -request(opts) +runRequests(opts, warmup, reqs, concurrency) diff --git a/benchmark/sirun/appsec/common.js b/benchmark/sirun/appsec/common.js index a44e53333bf..06707cf528c 100644 --- a/benchmark/sirun/appsec/common.js +++ b/benchmark/sirun/appsec/common.js @@ -2,13 +2,6 @@ module.exports = { port: 3231 + parseInt(process.env.CPU_AFFINITY || '0', 10), - // Requests per iteration, sized per variant in meta.json (REQS). A higher count - // dilutes the fixed startup (node boot + tracer/AppSec init), which is otherwise - // a large, run-to-run-variable share of a short run and dominates stddev (control - // drifted 290-440 ms between runs at 1000). It can't just be maximized: with no - // draining agent, spans (control) and WAF events (AppSec on) accumulate, so each - // variant has a GC cliff above which stddev explodes (control ~8000). Each REQS - // sits in its variant's valley -- diluted startup, below the cliff, under ~45 s - // at 30 iterations. The keep-alive client avoids ephemeral-port churn. - reqs: Number(process.env.REQS) || 1000, + reqs: Number(process.env.OPERATIONS) || 1000, + warmup: 100, } diff --git a/benchmark/sirun/appsec/meta.json b/benchmark/sirun/appsec/meta.json index b72a4ae3c7e..6a9ab8dd72f 100644 --- a/benchmark/sirun/appsec/meta.json +++ b/benchmark/sirun/appsec/meta.json @@ -4,47 +4,42 @@ "instructions": true, "iterations": 30, "variants": { - "control": { - "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node server.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"", - "env": { - "DD_APPSEC_ENABLED": "0", - "REQS": "6000" - } - }, "appsec-enabled": { + "cpus": 2, + "iterations": 11, "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node server.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"", + "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY_SECOND node client.js >/dev/null 2>&1 &\"", + "run": "node --require ./mock-native-appsec.js server.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ./mock-native-appsec.js server.js\"", "env": { "DD_APPSEC_ENABLED": "1", - "REQS": "7000" + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + "WAF_SAMPLE": "request", + "OPERATIONS": "56000" } }, "appsec-enabled-with-attacks": { + "cpus": 2, + "iterations": 9, "setup": "bash -c \"nohup node client.js >/dev/null 2>&1 &\"", - "run": "node server.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node server.js\"", + "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY_SECOND node client.js >/dev/null 2>&1 &\"", + "run": "node --require ./mock-native-appsec.js server.js", + "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ./mock-native-appsec.js server.js\"", "env": { "DD_APPSEC_ENABLED": "1", + "DD_REMOTE_CONFIGURATION_ENABLED": "false", + "WAF_SAMPLE": "attack", "ATTACK_UA": "1", "ATTACK_404": "1", "ATTACK_QS": "1", - "REQS": "5000" - } - }, - "startup-time-control": { - "run": "node --require ../../../init.js insecure-bank.js", - "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js insecure-bank.js\"", - "env": { - "DD_APPSEC_ENABLED": "0" + "OPERATIONS": "60000" } }, "startup-time-appsec-enabled": { "run": "node --require ../../../init.js insecure-bank.js", "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --require ../../../init.js insecure-bank.js\"", "env": { + "DD_INJECT_FORCE": "true", "DD_APPSEC_ENABLED": "1" } } diff --git a/benchmark/sirun/appsec/mock-native-appsec.js b/benchmark/sirun/appsec/mock-native-appsec.js new file mode 100644 index 00000000000..9ebc158856b --- /dev/null +++ b/benchmark/sirun/appsec/mock-native-appsec.js @@ -0,0 +1,95 @@ +'use strict' + +const assert = require('node:assert/strict') + +const normalizePayload = require('./normalize-waf-sample') +const samples = require('./waf-samples.json') + +const nativeAppsecPath = require.resolve('@datadog/native-appsec') + +const sample = samples[process.env.WAF_SAMPLE] +assert.ok(sample, `unknown WAF_SAMPLE: ${process.env.WAF_SAMPLE}`) + +const expectedPayloads = [] +for (const { payload } of sample) { + expectedPayloads.push(payload) +} + +let validateNextContext = true +let replayValidated = false + +class ReplayingDDWAF { + /** + * @returns {string} + */ + static version () { + return 'replay' + } + + /** + * Create the metadata used by the production WAF wrapper. + */ + constructor () { + this.configPaths = ['datadog/00/ASM_DD/default/config'] + this.diagnostics = { rules: {}, ruleset_version: 'replay' } + this.knownAddresses = new Set(samples.knownAddresses) + } + + /** + * @returns {object} + */ + createContext () { + const validate = validateNextContext + validateNextContext = false + + const payloads = [] + let callIndex = 0 + + return { + disposed: false, + + /** + * @param {object} payload + * @returns {object} + */ + run (payload) { + if (validate) { + payloads.push(normalizePayload(payload)) + } + return sample[callIndex++].result + }, + + /** + * @returns {void} + */ + dispose () { + if (validate) { + assert.strictEqual(callIndex, sample.length) + assert.deepStrictEqual(payloads, expectedPayloads) + replayValidated = true + } + this.disposed = true + }, + } + } + + /** + * @returns {void} + */ + dispose () {} +} + +function assertReplayValidated () { + assert.ok(replayValidated, 'AppSec did not run the replayed WAF context during warmup') +} + +require.cache[nativeAppsecPath] = { + children: [], + exports: { DDWAF: ReplayingDDWAF }, + filename: nativeAppsecPath, + id: nativeAppsecPath, + loaded: true, + paths: [], +} + +module.exports = assertReplayValidated diff --git a/benchmark/sirun/appsec/normalize-waf-sample.js b/benchmark/sirun/appsec/normalize-waf-sample.js new file mode 100644 index 00000000000..c92213aad95 --- /dev/null +++ b/benchmark/sirun/appsec/normalize-waf-sample.js @@ -0,0 +1,20 @@ +'use strict' + +/** + * Remove request values that vary between benchmark processes. + * + * @param {object} payload + * @returns {object} + */ +module.exports = function normalizePayload (payload) { + const normalized = JSON.parse(JSON.stringify(payload)) + const persistent = normalized.persistent + const headers = persistent?.['server.request.headers.no_cookies'] + if (headers?.host) { + headers.host = '127.0.0.1:' + } + if (persistent?.['http.client_ip']) { + persistent['http.client_ip'] = '' + } + return normalized +} diff --git a/benchmark/sirun/appsec/server.js b/benchmark/sirun/appsec/server.js index c17f0f20636..2fd3a861551 100644 --- a/benchmark/sirun/appsec/server.js +++ b/benchmark/sirun/appsec/server.js @@ -2,7 +2,14 @@ const assert = require('node:assert/strict') +const guard = require('../startup-guard') +const clearTimeoutGuard = require('../timeout-guard')('appsec server') +const assertReplayValidated = require('./mock-native-appsec') + +require('../noop-request') + // AppSec is enabled from env config. +// eslint-disable-next-line import/order -- the request and WAF mocks must load before tracer initialization const tracer = require('../../..').init() // Fail loudly if the tracer did not load: a broken require would otherwise // measure a plain server and silently "pass". @@ -10,15 +17,31 @@ assert.equal(typeof tracer.startSpan, 'function', 'tracer did not initialize') // eslint-disable-next-line import/order -- the tracer must load before http to instrument it const http = require('http') -const { port, reqs } = require('./common') +const { port, reqs, warmup } = require('./common') -let connectionsMade = 0 +let responsesFinished = 0 -const server = http.createServer((req, res) => { - res.writeHead(404) - res.end('Hello, World!') - if (++connectionsMade === reqs) { +function onResponseFinish () { + responsesFinished++ + if (responsesFinished === warmup) { + assertReplayValidated() + guard.loopStart() + } else if (responsesFinished === reqs + warmup) { + guard.done(0.1) server.close() } -}) +} + +/** + * @param {import('node:http').IncomingMessage} req + * @param {import('node:http').ServerResponse} res + */ +function handleRequest (req, res) { + res.once('finish', onResponseFinish) + res.writeHead(404) + res.end('Hello, World!') +} + +const server = http.createServer(handleRequest) +server.once('close', clearTimeoutGuard) server.listen(port) diff --git a/benchmark/sirun/appsec/waf-samples.json b/benchmark/sirun/appsec/waf-samples.json new file mode 100644 index 00000000000..fed05eb6eae --- /dev/null +++ b/benchmark/sirun/appsec/waf-samples.json @@ -0,0 +1,228 @@ +{ + "knownAddresses": [ + "graphql.server.all_resolvers", + "graphql.server.resolver", + "grpc.server.request.message", + "grpc.server.request.metadata", + "http.client_ip", + "server.business_logic.llm.event", + "server.business_logic.payment.cancellation", + "server.business_logic.payment.creation", + "server.business_logic.payment.failure", + "server.business_logic.payment.success", + "server.db.statement", + "server.db.system", + "server.io.fs.file", + "server.io.fs.file_write", + "server.io.net.request.headers", + "server.io.net.response.headers", + "server.io.net.response.status", + "server.io.net.url", + "server.request.body", + "server.request.body.filenames", + "server.request.cookies", + "server.request.headers.no_cookies", + "server.request.jwt", + "server.request.method", + "server.request.path_params", + "server.request.query", + "server.request.uri.raw", + "server.response.body", + "server.response.headers.no_cookies", + "server.response.status", + "server.sys.exec.cmd", + "server.sys.shell.cmd", + "usr.id", + "usr.session_id", + "waf.context.processor" + ], + "request": [ + { + "payload": { + "persistent": { + "server.request.uri.raw": "/", + "server.request.headers.no_cookies": { + "host": "127.0.0.1:", + "connection": "keep-alive" + }, + "server.request.method": "GET", + "http.client_ip": "" + } + }, + "result": { + "metrics": {}, + "timeout": false, + "duration": 0, + "attributes": { + "_dd.appsec.fp.http.header": "hdr-0100000000--1-4740ae63", + "_dd.appsec.fp.http.network": "net-0-0000000000", + "_dd.appsec.fp.http.endpoint": "http-get-8a5edab2--" + }, + "keep": false + } + }, + { + "payload": { + "persistent": { + "server.response.status": "404", + "server.response.headers.no_cookies": {} + } + }, + "result": { + "metrics": {}, + "timeout": false, + "duration": 0, + "keep": false + } + } + ], + "attack": [ + { + "payload": { + "persistent": { + "server.request.uri.raw": "/../../../secret.txt?k=", + "server.request.headers.no_cookies": { + "user-agent": "Arachni/v1", + "host": "127.0.0.1:", + "connection": "keep-alive" + }, + "server.request.method": "GET", + "http.client_ip": "" + } + }, + "result": { + "metrics": {}, + "timeout": false, + "duration": 0, + "attributes": { + "_dd.appsec.fp.http.header": "hdr-0100000000-51fbd9d8-1-4740ae63", + "_dd.appsec.fp.http.network": "net-0-0000000000", + "_dd.appsec.fp.http.endpoint": "http-get-3f8012f9--" + }, + "status": "match", + "events": [ + { + "rule": { + "id": "ua0-600-12x", + "name": "Arachni", + "tags": { + "type": "attack_tool", + "category": "attack_attempt", + "confidence": "1", + "module": "waf", + "tool_name": "Arachni", + "cwe": "200", + "capec": "1000/118/169" + }, + "on_match": [] + }, + "rule_matches": [ + { + "operator": "match_regex", + "operator_value": "^Arachni\\/v", + "parameters": [ + { + "address": "server.request.headers.no_cookies", + "key_path": ["user-agent"], + "value": "Arachni/v1", + "highlight": ["Arachni/v"] + } + ] + } + ] + }, + { + "rule": { + "id": "crs-930-110", + "name": "Simple Path Traversal Attack (/../)", + "tags": { + "type": "lfi", + "category": "attack_attempt", + "crs_id": "930110", + "confidence": "1", + "cwe": "22", + "capec": "1000/255/153/126", + "module": "waf" + }, + "on_match": [] + }, + "rule_matches": [ + { + "operator": "match_regex", + "operator_value": "(?:(?:^|[\\x5c/])\\.{2,3}[\\x5c/]|[\\x5c/]\\.{2,3}(?:[\\x5c/]|$))", + "parameters": [ + { + "address": "server.request.uri.raw", + "key_path": [], + "value": "/../../../secret.txt?k=", + "highlight": ["/../"] + } + ] + } + ] + } + ], + "actions": {}, + "keep": true + } + }, + { + "payload": { + "persistent": { + "server.response.status": "404", + "server.response.headers.no_cookies": {} + } + }, + "result": { + "metrics": {}, + "timeout": false, + "duration": 0, + "status": "match", + "events": [ + { + "rule": { + "id": "nfd-000-001", + "name": "Detect common directory discovery scans", + "tags": { + "type": "security_scanner", + "category": "attack_attempt", + "cwe": "200", + "capec": "1000/118/169", + "confidence": "1" + }, + "on_match": [] + }, + "rule_matches": [ + { + "operator": "match_regex", + "operator_value": "^404$", + "parameters": [ + { + "address": "server.response.status", + "key_path": [], + "value": "404", + "highlight": ["404"] + } + ] + }, + { + "operator": "phrase_match", + "operator_value": "", + "parameters": [ + { + "address": "server.request.uri.raw", + "key_path": [], + "value": "/../../../secret.txt?k=", + "highlight": ["../../.."] + } + ] + } + ] + } + ], + "actions": {}, + "keep": true + } + } + ] +} diff --git a/benchmark/sirun/collect-overview.js b/benchmark/sirun/collect-overview.js index 9e21c8457b6..a1d36e9c71a 100644 --- a/benchmark/sirun/collect-overview.js +++ b/benchmark/sirun/collect-overview.js @@ -30,12 +30,14 @@ const SG_FILE = path.join(require('os').tmpdir(), 'sg-overview.txt') const HIGH_MEANING = new Set([ 'shimmer-runtime', 'shimmer-startup', 'scope', 'id', 'spans', 'encoding', 'exporting-pipeline', 'propagation', 'async_hooks', 'url', 'startup', 'fs', + 'appsec-waf', ]) const LOW_MEANING = new Set(['plugin-dns']) const CRITICAL_PATH = new Set([ 'shimmer-runtime', 'shimmer-startup', 'scope', 'id', 'spans', 'encoding', 'exporting-pipeline', 'propagation', 'async_hooks', 'startup', + 'appsec-waf', ]) const LIVE = new Set(['appsec', 'appsec-iast', 'plugin-http', 'plugin-net']) const BACKGROUND = new Set(['runtime-metrics', 'profiler', 'log', 'llmobs', 'debugger']) diff --git a/benchmark/sirun/debugger/README.md b/benchmark/sirun/debugger/README.md index 99d82104006..ec8d5c70738 100644 --- a/benchmark/sirun/debugger/README.md +++ b/benchmark/sirun/debugger/README.md @@ -1,3 +1,7 @@ # Dynamic Instrumentation Benchmarks -Benchmark the overhead on the instrumented application of different probe configurations. +Measures application overhead from installed line probes, including real stack and snapshot capture. Each active run +validates one probe payload before timing, resets its completion state, then waits for every production pause handler and +verifies the exact payload count and capture kind. HTTP export is stubbed because it runs after the application resumes +and is outside this benchmark's hot path. Line probes omit Sirun's main-thread-only instruction count because capture work +runs in the inspector worker. diff --git a/benchmark/sirun/debugger/agent.js b/benchmark/sirun/debugger/agent.js deleted file mode 100644 index 7d743581e9e..00000000000 --- a/benchmark/sirun/debugger/agent.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -// Minimal agent stand-in for the Dynamic Instrumentation benchmark. -// `debugger.start()` queries `/info` to choose the diagnostics upload route and -// blocks on it (~15s connection-retry budget) when no agent answers. Answer -// `/info` immediately and drain the diagnostics/input uploads so the benchmark -// measures probe overhead, not agent discovery. -// -// CI runs the variants in parallel, each pinned to its own core via -// `$CPU_AFFINITY`. Derive the port from the core (like `plugin-http`'s -// `3031 + CPU_AFFINITY`) so every variant gets a private agent; sharing one port -// leaves the losers of the bind race with no agent, and they then burn the ~15s -// retry budget per process. `app.js`'s agent URL derives the same port. Unset -// affinity (local, sequential runs) falls back to the conventional 8080. -const http = require('node:http') - -const port = 8080 + Number(process.env.CPU_AFFINITY || 0) -const info = JSON.stringify({ endpoints: ['/debugger/v1/diagnostics', '/debugger/v2/input'] }) - -http.createServer((req, res) => { - if (req.url === '/info') { - res.setHeader('content-type', 'application/json') - res.end(info) - return - } - // Drain and acknowledge diagnostics / snapshot uploads. - req.resume() - req.on('end', () => res.end()) -}).listen(port) diff --git a/benchmark/sirun/debugger/app.js b/benchmark/sirun/debugger/app.js index 83e211e6563..7f3ab00f7fe 100644 --- a/benchmark/sirun/debugger/app.js +++ b/benchmark/sirun/debugger/app.js @@ -5,34 +5,150 @@ // move the `data.n = n` line or the unreachable `return n` line. const guard = require('../startup-guard') +const { + CAPTURE_KIND_INDEX, + CAPTURE_KIND_NAMES, + CAPTURE_KINDS, + COMPLETED_PROBE_INDEX, + HANDLED_PROBE_INDEX, + MATCHED_CAPTURE_KIND_INDEX, +} = require('./benchmark-state') const OPERATIONS = Number(process.env.OPERATIONS) const STARTUP_GUARD_MAX_SHARE = Number(process.env.STARTUP_GUARD_MAX_SHARE) +const TRACK_PROBE_OUTPUT = process.env.TRACK_PROBE_OUTPUT === 'true' +const EXPECTED_CAPTURE_KIND = process.env.EXPECTED_CAPTURE_KIND +const OUTPUT_TIMEOUT = 15_000 +const dataFixture = createData() -if (process.env.DD_DYNAMIC_INSTRUMENTATION_ENABLED === 'true') { - // The devtools worker and its ports are unref'd, so nothing holds the event - // loop open while the breakpoint installs. Keep it alive until the install - // ack, then run the loop so the probe fires on every iteration instead of - // racing its installation. - const keepAlive = setInterval(() => {}, 2 ** 31 - 1) - require('./start-devtools-client')(() => { - clearInterval(keepAlive) - runWork() - }) -} else { - runWork() +// The devtools worker and its ports are unref'd, so nothing holds the event +// loop open while the breakpoint installs or while the app awaits final output. +const keepAlive = setInterval(() => {}, 2 ** 31 - 1) +const debuggerBenchmark = require('./start-devtools-client') +const { probeCounts, stop } = debuggerBenchmark +debuggerBenchmark.start(() => { + if (!TRACK_PROBE_OUTPUT) clearInterval(keepAlive) + setImmediate(TRACK_PROBE_OUTPUT ? preflightProbe : runLoop) +}) + +/** + * Run the passive-breakpoint control as a CPU loop. + * + * @returns {void} + */ +function runLoop () { + guard.loopStart() + for (let i = 0; i < OPERATIONS; i++) doSomeWork(i) + finish() } -function runWork () { +/** + * Run one continuous hot loop, then wait for every production pause handler. + * The final barrier makes completed work exact without adding work between hits. + * + * @returns {void} + */ +function runCapturedLoop () { guard.loopStart() - for (let i = 0; i < OPERATIONS; i++) { - doSomeWork(i) + for (let i = 0; i < OPERATIONS; i++) doSomeWork(i) + + if (!probeCounts) throw new Error('debugger completion counter was not initialized') + const expectedProbes = OPERATIONS + waitForProbeHandlers(probeCounts, expectedProbes, () => { + validateProbePayloads(probeCounts, expectedProbes) + validateMeasuredCaptureKinds(probeCounts, expectedProbes) + stop() + finish() + }) +} + +/** + * Wait without blocking the application thread until the debugger worker has + * finished each production pause handler. + * + * @param {Int32Array} counter + * @param {number} expected + * @param {() => void} done + * @returns {void} + */ +function waitForProbeHandlers (counter, expected, done) { + const actual = Atomics.load(counter, HANDLED_PROBE_INDEX) + if (actual === expected) return done() + if (actual > expected) throw new Error(`debugger handled ${actual} of ${expected} expected breakpoint events`) + + const waiter = Atomics.waitAsync(counter, HANDLED_PROBE_INDEX, actual, OUTPUT_TIMEOUT) + if (waiter.async) { + waiter.value.then(result => finishHandlerWait(counter, expected, done, result)) + } else { + finishHandlerWait(counter, expected, done, waiter.value) } +} + +/** + * Validate the worker handler count after an atomic wait. + * + * @param {Int32Array} counter + * @param {number} expected + * @param {() => void} done + * @param {'ok' | 'not-equal' | 'timed-out'} result + * @returns {void} + */ +function finishHandlerWait (counter, expected, done, result) { + const actual = Atomics.load(counter, HANDLED_PROBE_INDEX) + if (actual === expected) return done() + if (result === 'timed-out' || actual > expected) { + throw new Error(`debugger handled ${actual} of ${expected} expected breakpoint events`) + } + + waitForProbeHandlers(counter, expected, done) +} + +/** + * Ensure each handled breakpoint produces one payload. + * + * @param {Int32Array} counts + * @param {number} completed + * @returns {void} + */ +function validateProbePayloads (counts, completed) { + const payloads = Atomics.load(counts, COMPLETED_PROBE_INDEX) + if (payloads !== completed) { + throw new Error(`debugger completed ${payloads} of ${completed} expected probe payloads`) + } +} + +/** + * Ensure each measured payload uses the expected capture mode. + * + * @param {Int32Array} counts + * @param {number} completed + * @returns {void} + */ +function validateMeasuredCaptureKinds (counts, completed) { + const matched = Atomics.load(counts, MATCHED_CAPTURE_KIND_INDEX) + if (matched !== completed) { + throw new Error( + `debugger produced ${matched} of ${completed} expected ${EXPECTED_CAPTURE_KIND} capture payloads` + ) + } +} + +/** + * Report the measured loop and stop the benchmark after all worker output is complete. + * + * @returns {void} + */ +function finish () { + clearInterval(keepAlive) guard.done(STARTUP_GUARD_MAX_SHARE) } +/** + * @param {number} n + * @returns {number} + */ function doSomeWork (n) { - const data = getSomeData() + const data = dataFixture data.n = n // BREAKPOINT HERE! if (n < 0) { return n // BREAKPOINT HERE! @@ -40,11 +156,15 @@ function doSomeWork (n) { return data.n } -function getSomeData () { +/** + * @returns {Record} + */ +function createData () { const str = 'a'.repeat(1000) const arr = Array.from({ length: 1000 }, (_, i) => i) const data = { + n: 0, foo: 'bar', nil: null, undef: undefined, @@ -59,3 +179,50 @@ function getSomeData () { return data } + +/** + * Prove the selected capture path once before starting the measured loop. + * + * @returns {void} + */ +function preflightProbe () { + if (!probeCounts) throw new Error('debugger completion counter was not initialized') + + doSomeWork(0) + waitForProbeHandlers(probeCounts, 1, () => { + validateProbePayloads(probeCounts, 1) + validateCaptureKind(probeCounts) + resetProbeCounts(probeCounts) + setImmediate(runCapturedLoop) + }) +} + +/** + * Ensure the payload shape reflects the selected capture limits. + * + * @param {Int32Array} counts + * @returns {void} + */ +function validateCaptureKind (counts) { + const expected = CAPTURE_KINDS[EXPECTED_CAPTURE_KIND] + if (expected === undefined) { + throw new Error(`unknown expected debugger capture kind: ${EXPECTED_CAPTURE_KIND}`) + } + + const actual = Atomics.load(counts, CAPTURE_KIND_INDEX) + if (actual !== expected) { + throw new Error( + `debugger produced ${CAPTURE_KIND_NAMES[actual] ?? 'unknown'} capture output; expected ${EXPECTED_CAPTURE_KIND}` + ) + } +} + +/** + * Reset preflight state so only measured probe output contributes to completion. + * + * @param {Int32Array} counts + * @returns {void} + */ +function resetProbeCounts (counts) { + for (let i = 0; i < counts.length; i++) Atomics.store(counts, i, 0) +} diff --git a/benchmark/sirun/debugger/benchmark-state.js b/benchmark/sirun/debugger/benchmark-state.js new file mode 100644 index 00000000000..89fa44540d4 --- /dev/null +++ b/benchmark/sirun/debugger/benchmark-state.js @@ -0,0 +1,25 @@ +'use strict' + +const COMPLETED_PROBE_INDEX = 0 +const MATCHED_CAPTURE_KIND_INDEX = 1 +const CAPTURE_KIND_INDEX = 2 +const HANDLED_PROBE_INDEX = 3 +const PROBE_COUNT_LENGTH = 4 + +const CAPTURE_KINDS = { + none: 1, + default: 2, + minimal: 3, +} + +const CAPTURE_KIND_NAMES = [undefined, 'none', 'default', 'minimal'] + +module.exports = { + CAPTURE_KIND_INDEX, + CAPTURE_KIND_NAMES, + CAPTURE_KINDS, + COMPLETED_PROBE_INDEX, + HANDLED_PROBE_INDEX, + MATCHED_CAPTURE_KIND_INDEX, + PROBE_COUNT_LENGTH, +} diff --git a/benchmark/sirun/debugger/benchmark-worker.js b/benchmark/sirun/debugger/benchmark-worker.js new file mode 100644 index 00000000000..be7fa20f548 --- /dev/null +++ b/benchmark/sirun/debugger/benchmark-worker.js @@ -0,0 +1,126 @@ +'use strict' + +const { workerData } = require('node:worker_threads') + +const { + CAPTURE_KIND_INDEX, + CAPTURE_KINDS, + COMPLETED_PROBE_INDEX, + HANDLED_PROBE_INDEX, + MATCHED_CAPTURE_KIND_INDEX, +} = require('./benchmark-state') + +require('../noop-request') + +const EXPECTED_CAPTURE_KIND = CAPTURE_KINDS[process.env.EXPECTED_CAPTURE_KIND] + +/** @type {Int32Array | undefined} */ +let probeCounts +/** @type {typeof import('../../../packages/dd-trace/src/debugger/devtools_client/send') | undefined} */ +let send +let preflightPending = true + +if (workerData.probeCountBuffer !== undefined) { + probeCounts = new Int32Array(workerData.probeCountBuffer) + const sendPath = require.resolve('../../../packages/dd-trace/src/debugger/devtools_client/send') + send = require(sendPath) + require.cache[sendPath] = { exports: sendAndCount } +} + +loadDevtoolsClient() + +/** + * @typedef {object} CapturedValue + * @property {Record} [fields] + */ + +/** + * @typedef {object} DebuggerSnapshot + * @property {{ location: { lines: string[] } }} probe + * @property {{ lines: Record }> }} [captures] + */ + +/** + * Load the production client with its pause handler wrapped so the application + * can wait for post-resume formatting to finish. + * + * @returns {void} + */ +function loadDevtoolsClient () { + const session = require('../../../packages/dd-trace/src/debugger/devtools_client/session') + const originalOn = session.on + /** + * @param {string | symbol} eventName + * @param {(...args: unknown[]) => void} listener + * @returns {import('node:events').EventEmitter} + */ + session.on = function benchmarkOn (eventName, listener) { + if (eventName !== 'Debugger.paused' || probeCounts === undefined) { + return originalOn.call(this, eventName, listener) + } + + const paused = /** @type {(event: object) => Promise} */ (listener) + /** + * @param {object} event + * @returns {void} + */ + function benchmarkPaused (event) { + paused.call(this, event).then(markProbeHandled) + } + + return originalOn.call(this, eventName, benchmarkPaused) + } + + try { + require('../../../packages/dd-trace/src/debugger/devtools_client') + } finally { + session.on = originalOn + } +} + +function markProbeHandled () { + Atomics.add(probeCounts, HANDLED_PROBE_INDEX, 1) + Atomics.notify(probeCounts, HANDLED_PROBE_INDEX) +} + +/** + * Record completion after the production worker has captured and formatted the + * probe output. The one preflight payload is inspected but not buffered. + * + * @param {string} message + * @param {Record} logger + * @param {Record | undefined} dd + * @param {DebuggerSnapshot} snapshot + * @param {string | undefined} processTags + * @returns {void} + */ +function sendAndCount (message, logger, dd, snapshot, processTags) { + const captureKind = getCaptureKind(snapshot) + if (preflightPending) { + preflightPending = false + Atomics.store(probeCounts, CAPTURE_KIND_INDEX, captureKind) + } else { + send(message, logger, dd, snapshot, processTags) + if (captureKind === EXPECTED_CAPTURE_KIND) Atomics.add(probeCounts, MATCHED_CAPTURE_KIND_INDEX, 1) + } + + Atomics.add(probeCounts, COMPLETED_PROBE_INDEX, 1) + Atomics.notify(probeCounts, COMPLETED_PROBE_INDEX) +} + +/** + * Classify output by the production capture shape. + * + * @param {DebuggerSnapshot} snapshot + * @returns {number} + */ +function getCaptureKind (snapshot) { + if (snapshot.captures === undefined) return CAPTURE_KINDS.none + + const line = snapshot.probe.location.lines[0] + const locals = snapshot.captures.lines[line]?.locals + const data = locals?.data + if (locals !== undefined && data === undefined) return CAPTURE_KINDS.minimal + if (data?.fields !== undefined) return CAPTURE_KINDS.default + return 0 +} diff --git a/benchmark/sirun/debugger/meta.json b/benchmark/sirun/debugger/meta.json index ea0a0b02e22..b295d2e8820 100644 --- a/benchmark/sirun/debugger/meta.json +++ b/benchmark/sirun/debugger/meta.json @@ -2,66 +2,63 @@ "name": "debugger", "cachegrind": false, "iterations": 5, - "instructions": true, + "instructions": false, "variants": { "enabled-but-breakpoint-not-hit": { - "setup": "bash -c \"nohup node agent.js >/dev/null 2>&1 &\"", - "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node agent.js >/dev/null 2>&1 &\"", + "instructions": true, "run": "node app.js", "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node app.js\"", "env": { - "STARTUP_GUARD_MAX_SHARE": "0.1", - "OPERATIONS": "220000", - "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", + "STARTUP_GUARD_MAX_SHARE": "0.15", + "OPERATIONS": "250000000", "BREAKPOINT_FILE": "app.js", - "BREAKPOINT_LINE": "38" + "BREAKPOINT_LINE": "154" } }, "line-probe-without-snapshot": { - "setup": "bash -c \"nohup node agent.js >/dev/null 2>&1 &\"", - "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node agent.js >/dev/null 2>&1 &\"", "run": "node app.js", "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node app.js\"", "env": { - "STARTUP_GUARD_MAX_SHARE": "0.1", - "OPERATIONS": "32000", - "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", + "STARTUP_GUARD_MAX_SHARE": "0.15", + "OPERATIONS": "17000", "BREAKPOINT_FILE": "app.js", - "BREAKPOINT_LINE": "36" + "BREAKPOINT_LINE": "152", + "EXPECTED_CAPTURE_KIND": "none", + "TRACK_PROBE_OUTPUT": "true" } }, "line-probe-with-snapshot-default": { - "setup": "bash -c \"nohup node agent.js >/dev/null 2>&1 &\"", - "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node agent.js >/dev/null 2>&1 &\"", "run": "node app.js", "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node app.js\"", "env": { - "STARTUP_GUARD_MAX_SHARE": "0.1", - "OPERATIONS": "32000", - "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", + "STARTUP_GUARD_MAX_SHARE": "0.15", + "OPERATIONS": "700", + "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS": "1000", "BREAKPOINT_FILE": "app.js", - "BREAKPOINT_LINE": "36", + "BREAKPOINT_LINE": "152", "CAPTURE_SNAPSHOT": "true", - "MAX_SNAPSHOTS_PER_SECOND_GLOBALLY": "1000000" + "EXPECTED_CAPTURE_KIND": "default", + "MAX_SNAPSHOTS_PER_SECOND_GLOBALLY": "1000000", + "TRACK_PROBE_OUTPUT": "true" } }, "line-probe-with-snapshot-minimal": { - "setup": "bash -c \"nohup node agent.js >/dev/null 2>&1 &\"", - "setup_with_affinity": "bash -c \"nohup taskset -c $CPU_AFFINITY node agent.js >/dev/null 2>&1 &\"", "run": "node app.js", "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node app.js\"", "env": { - "STARTUP_GUARD_MAX_SHARE": "0.1", - "OPERATIONS": "32000", - "DD_DYNAMIC_INSTRUMENTATION_ENABLED": "true", + "STARTUP_GUARD_MAX_SHARE": "0.15", + "OPERATIONS": "12000", + "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS": "1000", "BREAKPOINT_FILE": "app.js", - "BREAKPOINT_LINE": "36", + "BREAKPOINT_LINE": "152", "CAPTURE_SNAPSHOT": "true", + "EXPECTED_CAPTURE_KIND": "minimal", "MAX_SNAPSHOTS_PER_SECOND_GLOBALLY": "1000000", "MAX_REFERENCE_DEPTH": "0", "MAX_COLLECTION_SIZE": "0", "MAX_FIELD_COUNT": "0", - "MAX_LENGTH": "9007199254740991" + "MAX_LENGTH": "9007199254740991", + "TRACK_PROBE_OUTPUT": "true" } } } diff --git a/benchmark/sirun/debugger/start-devtools-client.js b/benchmark/sirun/debugger/start-devtools-client.js index 4516c9b1dcc..0725f4274a6 100644 --- a/benchmark/sirun/debugger/start-devtools-client.js +++ b/benchmark/sirun/debugger/start-devtools-client.js @@ -2,100 +2,133 @@ const assert = require('node:assert/strict') const fs = require('node:fs') -const Module = require('node:module') - -// Point the tracer at this variant's per-core agent (port matches `agent.js`) -// before `getConfig()` reads the URL. Set it unconditionally so a globally -// inherited `DD_TRACE_AGENT_URL` can't redirect us back to a shared port that -// another parallel variant owns. -process.env.DD_TRACE_AGENT_URL = `http://127.0.0.1:${8080 + Number(process.env.CPU_AFFINITY || 0)}` - -// The trace-context expression the devtools client evaluates on the paused frame -// for every hit does `global.require('dd-trace')`. This bench loads the tracer by -// relative path, so the bare specifier would otherwise throw MODULE_NOT_FOUND on -// every hit and skew the measurement. Resolve it to this checkout's entry point. -const ddTraceEntry = require.resolve('../../..') -const originalResolveFilename = Module._resolveFilename -Module._resolveFilename = function (request, ...rest) { - return originalResolveFilename.call(this, request === 'dd-trace' ? ddTraceEntry : request, ...rest) +const workerThreads = require('node:worker_threads') + +const { DEBUGGER_INPUT_V2 } = require('../../../packages/dd-trace/src/debugger/constants') +const NoopTracer = require('../../../packages/dd-trace/src/noop/tracer') +const { generateProbeConfig } = require('../../../packages/dd-trace/test/debugger/devtools_client/utils') +const { PROBE_COUNT_LENGTH } = require('./benchmark-state') + +const TRACK_PROBE_OUTPUT = process.env.TRACK_PROBE_OUTPUT === 'true' +const probeCountBuffer = TRACK_PROBE_OUTPUT + ? new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT * PROBE_COUNT_LENGTH) + : undefined +const probeCounts = probeCountBuffer === undefined + ? undefined + : new Int32Array(probeCountBuffer) +const OriginalWorker = workerThreads.Worker + +class BenchmarkWorker extends OriginalWorker { + /** + * Start the debugger through the benchmark wrapper that disables exporter I/O + * and optionally records completed payloads. + * + * @param {string} filename + * @param {import('node:worker_threads').WorkerOptions} options + */ + constructor (filename, options) { + assert.equal(filename, require.resolve('../../../packages/dd-trace/src/debugger/devtools_client')) + super(require.resolve('./benchmark-worker'), { + ...options, + workerData: { ...options.workerData, probeCountBuffer }, + }) + } } -// The global snapshot cap (MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) is read in the -// devtools worker thread at module load, with no config or env path to override -// it. Rewrite the on-disk value before `start()` spawns the worker so the snapshot -// variants measure capture cost on every hit instead of the rate-limited path. -// No-op unless the variant opts in via the env var. -patchGlobalSnapshotCap(process.env.MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) +const agentInfoPath = require.resolve('../../../packages/dd-trace/src/agent/info') +require.cache[agentInfoPath] = { exports: { fetchAgentInfo } } + +// The benchmark has no active trace, so provide the production no-op tracer for +// the paused-frame expression without loading the debugger before its worker is replaced. +globalThis._ddtrace ??= new NoopTracer() // Entry point normally primes this; bench imports src directly. globalThis[Symbol.for('dd-trace')] ??= { beforeExitHandlers: new Set() } -const getConfig = require('../../../packages/dd-trace/src/config') -const { start } = require('../../../packages/dd-trace/src/debugger') -const { generateProbeConfig } = require('../../../packages/dd-trace/test/debugger/devtools_client/utils') +const globalSnapshotCap = process.env.MAX_SNAPSHOTS_PER_SECOND_GLOBALLY +if (globalSnapshotCap) { + const cap = Number(globalSnapshotCap) + assert(Number.isInteger(cap) && cap > 0, 'MAX_SNAPSHOTS_PER_SECOND_GLOBALLY must be a positive integer') + + // The application-thread sampler loads this value after the override, then passes accepted probe indexes to the + // debugger worker through shared memory. + require('../../../packages/dd-trace/src/debugger/devtools_client/defaults').MAX_SNAPSHOTS_PER_SECOND_GLOBALLY = cap +} + +const { start, stop } = loadDebugger() const sourceFile = process.env.BREAKPOINT_FILE const line = Number(process.env.BREAKPOINT_LINE) assert(sourceFile, 'BREAKPOINT_FILE environment variable must be set') assert(!Number.isNaN(line), 'BREAKPOINT_LINE environment variable must be a number') +const expectedBreakpoint = TRACK_PROBE_OUTPUT + ? 'data.n = n // BREAKPOINT HERE!' + : 'return n // BREAKPOINT HERE!' +assert.equal( + fs.readFileSync(sourceFile, 'utf8').split('\n')[line - 1]?.trim(), + expectedBreakpoint, + `BREAKPOINT_LINE must point at "${expectedBreakpoint}"` +) const breakpoint = { sourceFile, line } +// WARNING: Keep this fixture aligned with dd-trace's default config, apart from benchmark-specific overrides. +const config = { + DD_EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED: false, + DD_TRACE_GIT_METADATA_ENABLED: false, + debug: false, + dynamicInstrumentation: { + captureTimeoutMs: Number(process.env.DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS || '1000'), + enabled: true, + probeFile: undefined, + redactedIdentifiers: [], + redactionExcludedIdentifiers: [], + uploadIntervalSeconds: 1, + }, + env: undefined, + hostname: 'debugger-benchmark', + logLevel: 'error', + port: 8126, + service: 'debugger-benchmark', + tags: { 'runtime-id': 'debugger-benchmark' }, + url: new URL('http://127.0.0.1:8126'), + version: undefined, +} /** - * Parse an integer environment variable, returning undefined when it is unset so - * the probe config falls back to its defaults. + * Resolve the production debugger endpoint without querying an external agent. * - * @param {string} name - * @returns {number | undefined} + * @param {URL} url + * @param {(error: Error | null, info: { endpoints: string[] }) => void} callback + * @returns {void} */ -function intEnv (name) { - return process.env[name] ? parseInt(process.env[name], 10) : undefined +function fetchAgentInfo (url, callback) { + process.nextTick(callback, null, { endpoints: [DEBUGGER_INPUT_V2] }) } /** - * Replace `filePath` with `content` atomically, so the devtools worker's - * concurrent `require('./defaults')` read in another variant only ever sees - * complete file contents, never a half-written file. + * Load the production debugger with its worker constructor replaced only for + * this benchmark process. * - * @param {string} filePath - * @param {string} content + * @returns {typeof import('../../../packages/dd-trace/src/debugger')} */ -function writeFileAtomic (filePath, content) { - const tempPath = `${filePath}.${process.pid}.tmp` - fs.writeFileSync(tempPath, content) - fs.renameSync(tempPath, filePath) +function loadDebugger () { + workerThreads.Worker = BenchmarkWorker + try { + return require('../../../packages/dd-trace/src/debugger') + } finally { + workerThreads.Worker = OriginalWorker + } } /** - * Raise `MAX_SNAPSHOTS_PER_SECOND_GLOBALLY` in the devtools defaults file to `cap` - * and restore the committed default on exit. - * - * `runall.sh` pins one variant per core, so the two snapshot variants rewrite this - * shared file in parallel. Two properties keep that race-free without a lock (a - * lock would serialize startup and inflate the measured run): a sibling that has - * already raised the cap is left to restore it, so its value is never captured as - * the baseline; and the restore runs on exit, long after every worker has read the - * cap at load, so no worker can observe the default mid-run. + * Parse an integer environment variable, returning undefined when it is unset so + * the probe config falls back to its defaults. * - * @param {string | undefined} value Desired cap; skips the rewrite when unset. + * @param {string} name + * @returns {number | undefined} */ -function patchGlobalSnapshotCap (value) { - if (!value) return - - const cap = Number(value) - assert(Number.isInteger(cap) && cap > 0, 'MAX_SNAPSHOTS_PER_SECOND_GLOBALLY must be a positive integer') - - const defaultsPath = require.resolve('../../../packages/dd-trace/src/debugger/devtools_client/defaults') - const pattern = /(MAX_SNAPSHOTS_PER_SECOND_GLOBALLY:\s*)(\d+)/ - const committed = fs.readFileSync(defaultsPath, 'utf8') - const match = committed.match(pattern) - assert(match, 'MAX_SNAPSHOTS_PER_SECOND_GLOBALLY not found in defaults file') - - // Already raised by a sibling variant: leave its exit handler to restore it. - if (Number(match[2]) === cap) return - - writeFileAtomic(defaultsPath, committed.replace(pattern, `$1${cap}`)) - process.once('exit', () => writeFileAtomic(defaultsPath, committed)) +function intEnv (name) { + return process.env[name] ? parseInt(process.env[name], 10) : undefined } /** @@ -105,8 +138,7 @@ function patchGlobalSnapshotCap (value) { * * @param {() => void} onProbeInstalled */ -module.exports = function startDebugger (onProbeInstalled) { - const config = getConfig() +function startDebugger (onProbeInstalled) { const rc = { setProductHandler (product, cb) { const action = 'apply' @@ -118,15 +150,21 @@ module.exports = function startDebugger (onProbeInstalled) { maxFieldCount: intEnv('MAX_FIELD_COUNT'), maxLength: intEnv('MAX_LENGTH'), }, + // Accept every hit instead of letting the real-time per-probe limiter vary + // the amount of captured work. + sampling: { snapshotsPerSecond: 1e10 }, }) cb(action, conf, 'id', (error) => { if (error) throw error onProbeInstalled() }) }, + removeProductHandler () {}, } start(config, rc) assert.ok(globalThis[Symbol.for('dd-trace')].utilTypes, 'debugger.start did not populate utilTypes') } + +module.exports = { probeCounts, start: startDebugger, stop } diff --git a/benchmark/sirun/diff.html b/benchmark/sirun/diff.html index acad146d37a..8bf217d79fb 100644 --- a/benchmark/sirun/diff.html +++ b/benchmark/sirun/diff.html @@ -24,7 +24,10 @@

Sirun differences between REPLACE_ME_PREV and REPLACE_ME_CURR

Summary Statistics

-

These are the geometric mean (i.e. \(\sqrt[n]{\prod_{i=0}^n {x_i}}\)) of the measurements across all tests and variants, excluding control cases.

+

+ These are the geometric mean (i.e. \(\sqrt[n]{\prod_{i=0}^n {x_i}}\)) of the measurements across all tests + and variants, excluding control cases. Instruction means only include variants available in every result. +

@@ -65,14 +68,21 @@

Summary Statistics

const diffData = REPLACE_ME_DIFF_DATA const readmes = REPLACE_ME_READMES -const beforeStats = summaryStats(beforeSummary) -const afterStats = summaryStats(afterSummary) -const goalStats = summaryStats(goalSummary) +const comparedSummaries = [beforeSummary, afterSummary, goalSummary] +const beforeStats = summaryStats(beforeSummary, comparedSummaries) +const afterStats = summaryStats(afterSummary, comparedSummaries) +const goalStats = summaryStats(goalSummary, comparedSummaries) -document.getElementById('prev-instructions').innerHTML = Math.floor(beforeStats.instructions) -document.getElementById('curr-instructions').innerHTML = Math.floor(afterStats.instructions) +if (beforeStats.instructions !== undefined) { + document.getElementById('prev-instructions').innerHTML = Math.floor(beforeStats.instructions) +} +if (afterStats.instructions !== undefined) { + document.getElementById('curr-instructions').innerHTML = Math.floor(afterStats.instructions) +} diffPctElem('diff-instructions', beforeStats, afterStats, 'instructions') -document.getElementById('goal-instructions').innerHTML = Math.floor(goalStats.instructions) +if (goalStats.instructions !== undefined) { + document.getElementById('goal-instructions').innerHTML = Math.floor(goalStats.instructions) +} diffPctElem('diff-goal-instructions', goalStats, afterStats, 'instructions') document.getElementById('prev-mem').innerHTML = Math.floor(beforeStats.mem) @@ -82,6 +92,8 @@

Summary Statistics

diffPctElem('diff-goal-mem', goalStats, afterStats, 'mem') function diffPctElem(elemName, beforeStats, afterStats, prop) { + if (beforeStats[prop] === undefined || afterStats[prop] === undefined) return + const elem = document.getElementById(elemName) const val = (100*(afterStats[prop] - beforeStats[prop])/beforeStats[prop]).toFixed(2) elem.innerHTML = val @@ -94,8 +106,16 @@

Summary Statistics

} } -function summaryStats(summary) { +/** @typedef {{instructions?: number, summary: Record}} BenchmarkVariant */ +/** @typedef {Record>} BenchmarkSummary */ + +/** + * @param {BenchmarkSummary} summary + * @param {BenchmarkSummary[]} comparedSummaries + */ +function summaryStats(summary, comparedSummaries) { let count = 0 + let instructionCount = 0 let instructions = 1 let mem = 1 for (const testName in summary) { @@ -109,12 +129,18 @@

Summary Statistics

} count++ const variant = test[variantName] - instructions *= variant.instructions + const hasComparableInstructions = comparedSummaries.every((comparedSummary) => { + return comparedSummary[testName]?.[variantName]?.instructions !== undefined + }) + if (hasComparableInstructions) { + instructionCount++ + instructions *= variant.instructions + } mem *= variant.summary['max.res.size'].mean } } return { - instructions: Math.pow(instructions, 1/count), + instructions: instructionCount === 0 ? undefined : Math.pow(instructions, 1/instructionCount), mem: Math.pow(mem, 1/count) } } diff --git a/benchmark/sirun/diff.spec.js b/benchmark/sirun/diff.spec.js new file mode 100644 index 00000000000..fa9e2b791b4 --- /dev/null +++ b/benchmark/sirun/diff.spec.js @@ -0,0 +1,98 @@ +'use strict' + +const assert = require('node:assert/strict') +const vm = require('node:vm') + +const { describe, it } = require('mocha') + +describe('sirun diff summary', () => { + it('compares instruction means over variants available in every summary', () => { + const before = summary({ shared: 100, unavailable: 10_000 }) + const after = summary({ shared: 200 }) + const goal = summary({ shared: 50, unavailable: 5_000 }) + const elements = renderSummary(before, after, goal) + + assert.strictEqual(elements['prev-instructions'].innerHTML, 100) + assert.strictEqual(elements['curr-instructions'].innerHTML, 200) + assert.strictEqual(elements['diff-instructions'].innerHTML, '100.00') + assert.strictEqual(elements['goal-instructions'].innerHTML, 50) + assert.strictEqual(elements['diff-goal-instructions'].innerHTML, '300.00') + }) + + it('omits instruction means when no variant is available in every summary', () => { + const before = summary({ before: 100 }) + const after = summary({ after: 200 }) + const goal = summary({ goal: 50 }) + const elements = renderSummary(before, after, goal) + + assert.strictEqual(elements['prev-instructions'].innerHTML, '') + assert.strictEqual(elements['curr-instructions'].innerHTML, '') + assert.strictEqual(elements['diff-instructions'].innerHTML, '') + assert.strictEqual(elements['goal-instructions'].innerHTML, '') + assert.strictEqual(elements['diff-goal-instructions'].innerHTML, '') + }) +}) + +/** + * @param {Record} instructions + */ +function summary (instructions) { + const variants = {} + for (const [name, value] of Object.entries(instructions)) { + variants[name] = { + instructions: value, + summary: { + 'max.res.size': { mean: 1 }, + }, + } + } + return { benchmark: variants } +} + +/** + * @param {ReturnType} before + * @param {ReturnType} after + * @param {ReturnType} goal + */ +function renderSummary (before, after, goal) { + const goalPath = require.resolve('./goal.json') + const diffPath = require.resolve('./diff-recent') + const originalGoal = require.cache[goalPath] + let html + try { + require.cache[goalPath] = { exports: goal, loaded: true } + delete require.cache[diffPath] + const renderDiff = require('./diff-recent') + html = renderDiff(before, after).html + } finally { + if (originalGoal === undefined) { + delete require.cache[goalPath] + } else { + require.cache[goalPath] = originalGoal + } + delete require.cache[diffPath] + } + + const elements = new Proxy({}, { + get (target, name) { + target[name] ??= { + appendChild () {}, + getContext () {}, + innerHTML: '', + style: {}, + } + return target[name] + }, + }) + const script = html.match(/