feat(native-spans): native span pipeline via libdatadog wasm

.github/CODEOWNERS

@@ -115,10 +115,8 @@
 /packages/dd-trace/test/plugins/util/test.spec.js @DataDog/ci-app-libraries
 /packages/dd-trace/test/plugins/util/test-environment.spec.js @DataDog/ci-app-libraries
 /packages/dd-trace/src/encode/agentless-ci-visibility.js @DataDog/ci-app-libraries
-/packages/dd-trace/src/encode/agentless-json.js @DataDog/ci-app-libraries
 /packages/dd-trace/src/encode/coverage-ci-visibility.js @DataDog/ci-app-libraries
 /packages/dd-trace/src/encode/tags-processors.js @DataDog/ci-app-libraries
-/packages/dd-trace/src/exporters/agentless/ @DataDog/ci-app-libraries
 /packages/dd-trace/src/git_metadata.js @DataDog/ci-app-libraries
 /packages/dd-trace/src/git_metadata_tagger.js @DataDog/ci-app-libraries
 /packages/dd-trace/src/plugins/util/ci.js @DataDog/ci-app-libraries
@@ -340,9 +338,9 @@
 /benchmark/sirun/async_hooks/ @DataDog/lang-platform-js
 /benchmark/sirun/dogstatsd/ @DataDog/lang-platform-js
 /benchmark/sirun/encoding/ @DataDog/lang-platform-js
-/benchmark/sirun/exporting-pipeline/ @DataDog/lang-platform-js
 /benchmark/sirun/id/ @DataDog/lang-platform-js
 /benchmark/sirun/log/ @DataDog/lang-platform-js
+/benchmark/sirun/native-span-drain.js @DataDog/lang-platform-js
 /benchmark/sirun/runtime-metrics/ @DataDog/lang-platform-js
 /benchmark/sirun/scope/ @DataDog/lang-platform-js
 /benchmark/sirun/shimmer-runtime/ @DataDog/lang-platform-js
@@ -372,8 +370,13 @@
 /packages/datadog-core/ @DataDog/lang-platform-js
 /packages/datadog-shimmer/ @DataDog/lang-platform-js
 /packages/dd-trace/*/crashtracking/ @DataDog/lang-platform-js
+/packages/dd-trace/src/native/ @DataDog/lang-platform-js
+/packages/dd-trace/src/exporters/native/ @DataDog/lang-platform-js
+/packages/dd-trace/test/native/ @DataDog/lang-platform-js
 /packages/dd-trace/src/bootstrap.js @DataDog/lang-platform-js
 /packages/dd-trace/src/feature-registry.js @DataDog/lang-platform-js
+/packages/dd-trace/src/js_span_processor.js @DataDog/lang-platform-js
+/packages/dd-trace/test/js_span_processor.spec.js @DataDog/lang-platform-js
 /packages/dd-trace/src/exporters/common/ @DataDog/lang-platform-js
 /packages/dd-trace/src/exporters/common/client-library-headers.js @DataDog/lang-platform-js @DataDog/feature-flagging-and-experimentation-sdk
 /packages/dd-trace/src/proxy.js @DataDog/lang-platform-js
@@ -382,7 +385,6 @@
 /packages/dd-trace/test/dogstatsd.spec.js @DataDog/lang-platform-js
 /packages/dd-trace/test/encode/ @DataDog/lang-platform-js
 /packages/dd-trace/test/esm-named-exports.spec.js @DataDog/lang-platform-js
-/packages/dd-trace/test/exporter.spec.js @DataDog/lang-platform-js
 /packages/dd-trace/test/exporters/ @DataDog/lang-platform-js
 /packages/dd-trace/test/external-logger/ @DataDog/lang-platform-js
 /packages/dd-trace/test/flare.spec.js @DataDog/lang-platform-js

.github/workflows/system-tests.yml

@@ -39,6 +39,10 @@ jobs:
       packages: write
     with:
       library: nodejs
+      # TEMPORARY: pin system-tests to the branch that flushes native client stats
+      # on /trace/stats/flush (DataDog/system-tests#7293). Revert to the default
+      # (drop this `ref`) once #7293 merges to system-tests main.
+      ref: bengl/parametric-native-stats-flush
       binaries_artifact: system_tests_binaries
       desired_execution_time: 300 # 5 minutes
       scenarios_groups: tracer-release

.gitignore

@@ -33,6 +33,7 @@ Temporary Items
 logs
 *.log
 node-*-junit.xml
+.junit-tmp/
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*

.gitlab/benchmarks/gitlab-ci.yml

@@ -33,6 +33,8 @@ variables:
   needs: [ ]
   tags: ["runner:apm-k8s-m7i-metal"]
   image: $MICROBENCHMARKS_CI_IMAGE
+  variables:
+    KUBERNETES_POD_ANNOTATIONS_1: "beta.fabric.datadoghq.com/no-proxy-additions=registry.yarnpkg.com"
   rules:
     - if: '$CI_COMMIT_REF_NAME =~ /^graphite-base\/.*$/'
       when: never
@@ -177,4 +179,10 @@ benchmark-serverless-trigger:
     UPSTREAM_GITLAB_USER_EMAIL: $GITLAB_USER_EMAIL
     # only available on Merge Requests
     UPSTREAM_MERGE_TARGET_BRANCH: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
+    # The downstream serverless-tools hard cap can lag current main layer sizes;
+    # keep PRs gated by size increase while allowing the largest measured current-main layer.
+    MAX_LAYER_UNCOMPRESSED_SIZE_KB: "25280"
+    KUBERNETES_POD_ANNOTATIONS_1: "beta.fabric.datadoghq.com/no-proxy-additions=registry.yarnpkg.com,registry.npmjs.org"
+    YARN_REGISTRY: "https://registry.npmjs.org"
+    YARN_NETWORK_TIMEOUT: "600000"
     DD_TAGS: "SLS_CI_BRANCH:$SLS_CI_BRANCH"

benchmark/sirun/collect-overview.js

@@ -29,13 +29,13 @@ const SG_FILE = path.join(require('os').tmpdir(), 'sg-overview.txt')
 // Curated per-bench judgment the run cannot measure.
 const HIGH_MEANING = new Set([
   'shimmer-runtime', 'shimmer-startup', 'scope', 'id', 'spans', 'encoding',
-  'exporting-pipeline', 'propagation', 'async_hooks', 'url', 'startup', 'fs',
+  'native-spans', 'propagation', 'async_hooks', 'url', 'startup', 'fs',
 ])
 const LOW_MEANING = new Set(['plugin-dns'])
 
 const CRITICAL_PATH = new Set([
   'shimmer-runtime', 'shimmer-startup', 'scope', 'id', 'spans', 'encoding',
-  'exporting-pipeline', 'propagation', 'async_hooks', 'startup',
+  'native-spans', 'propagation', 'async_hooks', 'startup',
 ])
 const LIVE = new Set(['appsec', 'appsec-iast', 'plugin-http', 'plugin-net'])
 const BACKGROUND = new Set(['runtime-metrics', 'profiler', 'log', 'llmobs', 'debugger'])

benchmark/sirun/goal.json

@@ -251,90 +251,6 @@
       }
     }
   },
-  "exporting-pipeline": {
-    "0.4": {
-      "instructions": 30319059366,
-      "nodeVersion": "16.12.0",
-      "summary": {
-        "cpu.pct.wall.time": {
-          "mean": 105.89922768177193,
-          "stddev": 0.6434582243505186,
-          "stddev_pct": 0.6076137082737902,
-          "min": 105.2560612769908,
-          "max": 107.31576345467171
-        },
-        "max.res.size": {
-          "mean": 108270.4,
-          "stddev": 4134.782877008175,
-          "stddev_pct": 3.8189411667530324,
-          "min": 102932,
-          "max": 114208
-        },
-        "system.time": {
-          "mean": 143040.2,
-          "stddev": 25893.195657546792,
-          "stddev_pct": 18.102041004939025,
-          "min": 109613,
-          "max": 181938
-        },
-        "user.time": {
-          "mean": 3736568.4,
-          "stddev": 244340.5193095079,
-          "stddev_pct": 6.5391689152407295,
-          "min": 3418318,
-          "max": 4132215
-        },
-        "wall.time": {
-          "mean": 3664722.9,
-          "stddev": 264119.1155397314,
-          "stddev_pct": 7.207069204051727,
-          "min": 3313910,
-          "max": 4067171
-        }
-      }
-    },
-    "0.5": {
-      "instructions": 14527746495,
-      "nodeVersion": "16.12.0",
-      "summary": {
-        "cpu.pct.wall.time": {
-          "mean": 104.65799849318564,
-          "stddev": 2.245230539716932,
-          "stddev_pct": 2.1453023868625962,
-          "min": 99.96941158237492,
-          "max": 106.81915703535817
-        },
-        "max.res.size": {
-          "mean": 96533.6,
-          "stddev": 3349.0842688711195,
-          "stddev_pct": 3.469345667074593,
-          "min": 92404,
-          "max": 100888
-        },
-        "system.time": {
-          "mean": 131690,
-          "stddev": 22527.920401137784,
-          "stddev_pct": 17.106781381378834,
-          "min": 93039,
-          "max": 168131
-        },
-        "user.time": {
-          "mean": 2837721.2,
-          "stddev": 152809.6321360666,
-          "stddev_pct": 5.384941696741265,
-          "min": 2628362,
-          "max": 3084636
-        },
-        "wall.time": {
-          "mean": 2840945.6,
-          "stddev": 202824.1615070552,
-          "stddev_pct": 7.139318736235399,
-          "min": 2570523,
-          "max": 3187481
-        }
-      }
-    }
-  },
   "log": {
     "without-log": {
       "instructions": 631069048,

benchmark/sirun/native-span-drain.js

@@ -0,0 +1,47 @@
+'use strict'
+
+const DEFAULT_DRAIN_THRESHOLD = 5000
+
+function createNativeSpanDrain (tracer, threshold = DEFAULT_DRAIN_THRESHOLD) {
+  const nativeSpans = tracer._tracer._nativeSpans
+  const pendingSpanIds = nativeSpans ? [] : null
+
+  function add (span) {
+    if (pendingSpanIds) {
+      pendingSpanIds.push(span.context()._nativeSpanId)
+    }
+  }
+
+  function addAll (spans) {
+    if (!pendingSpanIds) return
+
+    for (const span of spans) {
+      pendingSpanIds.push(span.context()._nativeSpanId)
+    }
+  }
+
+  async function drain () {
+    if (!pendingSpanIds || pendingSpanIds.length === 0) return
+
+    nativeSpans.flushChangeQueue()
+
+    const spanIds = Buffer.allocUnsafe(pendingSpanIds.length * 8)
+    let offset = 0
+    for (const spanId of pendingSpanIds) {
+      spanIds.set(spanId, offset)
+      offset += 8
+    }
+
+    nativeSpans._state.prepareChunk(pendingSpanIds.length, false, spanIds)
+    await nativeSpans._state.sendPreparedChunk().catch(() => {})
+    pendingSpanIds.length = 0
+  }
+
+  function needsDrain () {
+    return pendingSpanIds && pendingSpanIds.length >= threshold
+  }
+
+  return { add, addAll, drain, needsDrain }
+}
+
+module.exports = { createNativeSpanDrain }