chore: update iitm and fix initialize (#7387)

The initialize hook was expecting that it could import a cjs
module. That expectation was wrong due to only working due to
iitm formerly instrumenting the top level module. Since that is
not the case anymore it broke our instrumentation, since that
would not be possible by Node.js itself.

As drive by, this removes code from not supported versions as well
as making some parts faster and cleaning up code.

Author: BridgeAR

.github/workflows/platform.yml

@@ -443,7 +443,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [14.0.0, 14, 16.0.0, 18.0.0, 20.0.0, 22.0.0, 24.0.0]
+        version: [16.0.0, 18.0.0, 20.0.0, 22.0.0, 24.0.0, 25]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -464,7 +464,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ['0.8', '0.10', '0.12', '4', '6', '8', '10', '12']
+        version: ['0.8', '0.10', '0.12', '4', '6', '8', '10', '12', '14.0.0', '14']
     runs-on: ubuntu-latest
     env:
       DD_TRACE_DEBUG: 'true' # This exercises more of the guardrails code

eslint.config.mjs

@@ -475,6 +475,7 @@ export default [
       'unicorn/explicit-length-check': 'off', // 68 errors
       'unicorn/filename-case': ['off', { case: 'kebabCase' }], // 59 errors
       'unicorn/prefer-at': 'off', // 17 errors | Difficult to fix
+      'unicorn/prefer-export-from': ['error', { ignoreUsedVariables: true }],
       'unicorn/prevent-abbreviations': 'off', // too strict
 
       // These rules require a newer Node.js version than we support

initialize.mjs

@@ -2,8 +2,7 @@
  * This file serves one of two purposes, depending on how it's used.
  *
  * If used with --import, it will import init.js and register the loader hook.
- * If used with --loader, it will act as the loader hook, except that it will
- * also import init.js inside the source code of the entrypoint file.
+ * If used with --loader, it will act as the loader hook.
  *
  * The result is that no matter how this file is used, so long as it's with
  * one of the two flags, the tracer will always be initialized, and the loader
@@ -12,49 +11,68 @@
 
 /* eslint n/no-unsupported-features/node-builtins: ['error', { ignores: ['module.register'] }] */
 
+import { Buffer } from 'buffer'
+import * as Module from 'module'
+import { types } from 'util'
 import { isMainThread } from 'worker_threads'
 
-import * as Module from 'node:module'
-import { fileURLToPath } from 'node:url'
+// This file must support Node.js 12.0.0 syntax
+
 import {
-  load as origLoad,
-  resolve as origResolve,
-  getSource as origGetSource,
-} from 'import-in-the-middle/hook.mjs'
+  iitmExclusions,
+  load as hookLoad,
+  resolve as hookResolve,
+} from './loader-hook.mjs'
 
 let hasInsertedInit = false
-function insertInit (result) {
-  if (!hasInsertedInit) {
-    hasInsertedInit = true
-    result.source = `
-import '${fileURLToPath(new URL('init.js', import.meta.url))}';
-${result.source}`
+const initJsUrl = new URL('init.js', import.meta.url).href
+// `--loader` only reliably influences ESM entrypoints; for CJS apps use `--import`/`--require`.
+
+/**
+ * @param {{ source?: string|Buffer|Uint8Array, format?: string }} result
+ * @param {unknown} _url_
+ * @param {{ format?: string, isMain?: boolean }} context
+ * @returns {{ source?: string|Buffer|Uint8Array, format?: string }}
+ */
+function insertInit (result, _url_, context) {
+  if (hasInsertedInit) return result
+  // If Node provides `isMain`, only inject into the entrypoint module.
+  if (context && context.isMain === false) return result
+
+  let { source } = result
+  if (typeof source !== 'string') {
+    // Fast decode: handle bytes sources without extra copies when possible.
+    if (Buffer.isBuffer(source)) {
+      source = source.toString('utf8')
+    } else if (types.isUint8Array(source)) {
+      // Create a Buffer view over the same ArrayBuffer segment (no copy).
+      source = Buffer.from(source.buffer, source.byteOffset, source.byteLength).toString('utf8')
+    } else {
+      return result
+    }
   }
+
+  const format = result.format || (context && context.format)
+  if (format !== 'module') return result
+
+  hasInsertedInit = true
+
+  result.source = `import ${JSON.stringify(initJsUrl)};\n${source}`
+
   return result
 }
 
 const [NODE_MAJOR, NODE_MINOR] = process.versions.node.split('.').map(Number)
 
 const brokenLoaders = NODE_MAJOR === 18 && NODE_MINOR === 0
-const iitmExclusions = [
-  /langsmith/,
-  /openai\/_shims/,
-  /openai\/resources\/chat\/completions\/messages/,
-  /openai\/agents-core\/dist\/shims/,
-  /@anthropic-ai\/sdk\/_shims/,
-]
 
 export async function load (url, context, nextLoad) {
   const iitmExclusionsMatch = iitmExclusions.some((exclusion) => exclusion.test(url))
-  const loadHook = (brokenLoaders || iitmExclusionsMatch) ? nextLoad : origLoad
-  return insertInit(await loadHook(url, context, nextLoad))
+  const loadHook = (brokenLoaders || iitmExclusionsMatch) ? nextLoad : hookLoad
+  return insertInit(await loadHook(url, context, nextLoad), url, context)
 }
 
-export const resolve = brokenLoaders ? undefined : origResolve
-
-export async function getSource (...args) {
-  return insertInit(await origGetSource(...args))
-}
+export const resolve = brokenLoaders ? undefined : hookResolve
 
 if (isMainThread) {
   const require = Module.createRequire(import.meta.url)
@@ -65,5 +83,3 @@ if (isMainThread) {
     })
   }
 }
-
-export { getFormat } from 'import-in-the-middle/hook.mjs'

integration-tests/helpers/index.js

@@ -60,7 +60,7 @@ async function runAndCheckOutput (filename, cwd, expectedOut, expectedSource) {
       // Debug adds this, which we don't care about in these tests
       out = out.replace('Flushing 0 metrics via HTTP\n', '')
     }
-    assert.match(out, new RegExp(expectedOut), `output "${out} does not contain expected output "${expectedOut}"`)
+    assert.match(out, new RegExp(expectedOut), `output "${out}" does not contain expected output "${expectedOut}"`)
   }
 
   if (expectedSource) {

integration-tests/init.spec.js

@@ -31,7 +31,12 @@ delete process.env.DD_INJECT_FORCE
 
 function testInjectionScenarios (arg, filename, esmWorks = false) {
   if (!currentVersionIsSupported) return
-  const doTest = (file, ...args) => testFile(file, ...args)
+
+  // For `--loader`, we generally want ESM fixtures to ensure the loader hook actually applies.
+  // However, Node 18.0.0 is a known outlier where ESM via custom loaders is not supportable.
+  const isNode1800 = process.versions.node === '18.0.0'
+  const tracerFile = arg === 'loader' && !isNode1800 ? 'init/trace.mjs' : 'init/trace.js'
+  const instrFile = arg === 'loader' && !isNode1800 ? 'init/instrument.mjs' : 'init/instrument.js'
 
   context('preferring app-dir dd-trace', () => {
     context('when dd-trace is not in the app dir', () => {
@@ -40,23 +45,23 @@ function testInjectionScenarios (arg, filename, esmWorks = false) {
 
       if (currentVersionIsSupported) {
         context('without DD_INJECTION_ENABLED', () => {
-          it('should initialize the tracer', () => doTest('init/trace.js', 'true\n', [], 'manual'))
+          it('should initialize the tracer', () => testFile(tracerFile, 'true\n', [], 'manual'))
 
-          it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n', [], 'manual'))
+          it('should initialize instrumentation', () => testFile(instrFile, 'true\n', [], 'manual'))
 
           it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () =>
-            doTest('init/instrument.mjs', `${esmWorks}\n`, [], 'manual'))
+            testFile('init/instrument.mjs', `${esmWorks}\n`, [], 'manual'))
         })
       }
 
       context('with DD_INJECTION_ENABLED', () => {
         useEnv({ DD_INJECTION_ENABLED })
 
-        it('should not initialize the tracer', () => doTest('init/trace.js', 'false\n', []))
+        it('should not initialize the tracer', () => testFile(tracerFile, 'false\n', [], ''))
 
-        it('should not initialize instrumentation', () => doTest('init/instrument.js', 'false\n', []))
+        it('should not initialize instrumentation', () => testFile(instrFile, 'false\n', [], ''))
 
-        it('should not initialize ESM instrumentation', () => doTest('init/instrument.mjs', 'false\n', []))
+        it('should not initialize ESM instrumentation', () => testFile('init/instrument.mjs', 'false\n', [], ''))
       })
     })
 
@@ -65,23 +70,23 @@ function testInjectionScenarios (arg, filename, esmWorks = false) {
       useEnv({ NODE_OPTIONS })
 
       context('without DD_INJECTION_ENABLED', () => {
-        it('should initialize the tracer', () => doTest('init/trace.js', 'true\n', [], 'manual'))
+        it('should initialize the tracer', () => testFile(tracerFile, 'true\n', [], 'manual'))
 
-        it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n', [], 'manual'))
+        it('should initialize instrumentation', () => testFile(instrFile, 'true\n', [], 'manual'))
 
         it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () =>
-          doTest('init/instrument.mjs', `${esmWorks}\n`, [], 'manual'))
+          testFile('init/instrument.mjs', `${esmWorks}\n`, [], 'manual'))
       })
 
       context('with DD_INJECTION_ENABLED', () => {
         useEnv({ DD_INJECTION_ENABLED, DD_TRACE_DEBUG })
 
-        it('should initialize the tracer', () => doTest('init/trace.js', 'true\n', telemetryGood, 'ssi'))
+        it('should initialize the tracer', () => testFile(tracerFile, 'true\n', telemetryGood, 'ssi'))
 
-        it('should initialize instrumentation', () => doTest('init/instrument.js', 'true\n', telemetryGood, 'ssi'))
+        it('should initialize instrumentation', () => testFile(instrFile, 'true\n', telemetryGood, 'ssi'))
 
         it(`should ${esmWorks ? '' : 'not '}initialize ESM instrumentation`, () =>
-          doTest('init/instrument.mjs', `${esmWorks}\n`, telemetryGood, 'ssi'))
+          testFile('init/instrument.mjs', `${esmWorks}\n`, telemetryGood, 'ssi'))
       })
     })
   })
@@ -90,11 +95,13 @@ function testInjectionScenarios (arg, filename, esmWorks = false) {
 function testRuntimeVersionChecks (arg, filename) {
   context('runtime version check', () => {
     const NODE_OPTIONS = `--${arg} dd-trace/${filename}`
-    const doTest = (...args) => testFile('init/trace.js', ...args)
-    const doTestForced = async (...args) => {
+    const entryFile = arg === 'loader' ? 'init/trace.mjs' : 'init/trace.js'
+    const doTest = (expectedOut, expectedTelemetryPoints, expectedSource) =>
+      testFile(entryFile, expectedOut, expectedTelemetryPoints, expectedSource)
+    const doTestForced = async (expectedOut, expectedTelemetryPoints, expectedSource) => {
       Object.assign(process.env, { DD_INJECT_FORCE })
       try {
-        await testFile('init/trace.js', ...args)
+        await testFile(entryFile, expectedOut, expectedTelemetryPoints, expectedSource)
       } finally {
         delete process.env.DD_INJECT_FORCE
       }
@@ -279,7 +286,7 @@ if (semver.satisfies(process.versions.node, '>=14.13.1')) {
     if (semver.satisfies(process.versions.node, '>=20.6.0')) {
       context('as --import', () => {
         testInjectionScenarios('import', 'initialize.mjs', true)
-        testRuntimeVersionChecks('loader', 'initialize.mjs')
+        testRuntimeVersionChecks('import', 'initialize.mjs')
       })
     }
   })

integration-tests/init/trace.mjs

@@ -0,0 +1,5 @@
+// eslint-disable-next-line no-console
+console.log(!!global._ddtrace)
+// eslint-disable-next-line no-console
+console.log('instrumentation source:', global._ddtrace._tracer._config.instrumentationSource)
+process.exit()

integration-tests/package-guardrails.spec.js

@@ -14,6 +14,8 @@ const NODE_OPTIONS = '--require dd-trace/init.js'
 const DD_TRACE_DEBUG = 'true'
 const DD_INJECTION_ENABLED = 'tracing'
 const DD_LOG_LEVEL = 'error'
+const NODE_MAJOR = Number(process.versions.node.split('.')[0])
+const FASTIFY_DEP = NODE_MAJOR < 20 ? 'fastify@4' : 'fastify'
 
 // These are on by default in release tests, so we'll turn them off for
 // more fine-grained control of these variables in these tests.
@@ -22,8 +24,8 @@ delete process.env.DD_INJECT_FORCE
 
 describe('package guardrails', () => {
   useEnv({ NODE_OPTIONS })
-  const runTest = (...args) =>
-    testFile('package-guardrails/index.js', ...args)
+  const runTest = (expectedOut, expectedTelemetryPoints, expectedSource = '') =>
+    testFile('package-guardrails/index.js', expectedOut, expectedTelemetryPoints, expectedSource)
 
   context('when package is out of range', () => {
     useSandbox(['bluebird@1.0.0'])
@@ -70,13 +72,13 @@ false
 
   context('when package is in range (fastify)', () => {
     context('when fastify is latest', () => {
-      useSandbox(['fastify'])
+      useSandbox([FASTIFY_DEP])
 
       it('should instrument the package', () => runTest('true\n', [], 'manual'))
     })
 
     context('when fastify is latest and logging enabled', () => {
-      useSandbox(['fastify'])
+      useSandbox([FASTIFY_DEP])
       useEnv({ DD_TRACE_DEBUG })
 
       it('should instrument the package', () =>

integration-tests/startup.spec.js

@@ -23,6 +23,10 @@ const execArgvs = [
     execArgv: ['--import', 'dd-trace/register.js'],
     skip: semver.satisfies(process.versions.node, '<20.6'),
   },
+  {
+    execArgv: ['--import', 'dd-trace/loader-hook.mjs'],
+    skip: semver.satisfies(process.versions.node, '<20.6'),
+  },
   {
     execArgv: ['--loader', 'dd-trace/loader-hook.mjs'],
     skip: semver.satisfies(process.versions.node, '>=20.6'),

loader-hook.mjs

@@ -1,28 +1,30 @@
-import * as iitm from 'import-in-the-middle/hook.mjs'
+import { initialize as origInitialize, load as origLoad, resolve } from 'import-in-the-middle/hook.mjs'
 import regexpEscapeModule from './vendor/dist/escape-string-regexp/index.js'
 import hooks from './packages/datadog-instrumentations/src/helpers/hooks.js'
 import configHelper from './packages/dd-trace/src/config/helper.js'
 import * as rewriterLoader from './packages/datadog-instrumentations/src/helpers/rewriter/loader.mjs'
 import { isRelativeRequire } from './packages/datadog-instrumentations/src/helpers/shared-utils.js'
 
+// This file must support Node.js 12.0.0 syntax
+
 const regexpEscape = regexpEscapeModule.default
 
 // For some reason `getEnvironmentVariable` is not otherwise available to ESM.
 const env = configHelper.getEnvironmentVariable
 
 function initialize (data = {}) {
-  data.include ??= []
-  data.exclude ??= []
+  if (data.include == null) data.include = []
+  if (data.exclude == null) data.exclude = []
 
   addInstrumentations(data)
   addSecurityControls(data)
   addExclusions(data)
 
-  return iitm.initialize(data)
+  return origInitialize(data)
 }
 
 function load (url, context, nextLoad) {
-  return rewriterLoader.load(url, context, (url, context) => iitm.load(url, context, nextLoad))
+  return rewriterLoader.load(url, context, (url, context) => origLoad(url, context, nextLoad))
 }
 
 function addInstrumentations (data) {
@@ -38,27 +40,38 @@ function addInstrumentations (data) {
 }
 
 function addSecurityControls (data) {
-  const securityControls = (env('DD_IAST_SECURITY_CONTROLS_CONFIGURATION') || '')
-    .split(';')
-    .map(sc => sc.trim().split(':')[2])
-    .filter(Boolean)
-    .map(sc => sc.trim())
+  const raw = env('DD_IAST_SECURITY_CONTROLS_CONFIGURATION')
+  if (!raw) return
+  // Parse `;`-separated entries and take the 3rd `:`-separated segment.
+  // Expected form (per entry): `<...>:<...>:<subpath>:<...>`
+  const entries = raw.split(';')
+  for (const entry of entries) {
+    if (entry) {
+      const first = entry.indexOf(':')
+      if (first === -1) continue
+      const second = entry.indexOf(':', first + 1)
+      if (second === -1) continue
+      const third = entry.indexOf(':', second + 1)
 
-  for (const subpath of securityControls) {
-    data.include.push(new RegExp(regexpEscape(subpath)))
+      const subpath = entry.slice(second + 1, third === -1 ? undefined : third).trim()
+      if (subpath) {
+        data.include.push(new RegExp(regexpEscape(subpath)))
+      }
+    }
   }
 }
 
 function addExclusions (data) {
-  data.exclude.push(
-    /middle/,
-    /langsmith/,
-    /openai\/_shims/,
-    /openai\/resources\/chat\/completions\/messages/,
-    /openai\/agents-core\/dist\/shims/,
-    /@anthropic-ai\/sdk\/_shims/
-  )
+  data.exclude.push(...iitmExclusions)
 }
 
-export { initialize, load }
-export { getFormat, resolve, getSource } from 'import-in-the-middle/hook.mjs'
+export const iitmExclusions = [
+  /middle/,
+  /langsmith/,
+  /openai\/_shims/,
+  /openai\/resources\/chat\/completions\/messages/,
+  /openai\/agents-core\/dist\/shims/,
+  /@anthropic-ai\/sdk\/_shims/,
+]
+
+export { initialize, load, resolve }

package.json

@@ -131,7 +131,7 @@
   ],
   "dependencies": {
     "dc-polyfill": "^0.1.10",
-    "import-in-the-middle": "2.0.3"
+    "import-in-the-middle": "^2.0.6"
   },
   "optionalDependencies": {
     "@datadog/libdatadog": "0.7.0",