feat(evp_proxy): add direct intake route (#9739)

* feat(evp_proxy): add direct intake route

* fix(evp_proxy): validate direct intake URL

Author: leoromanovsky

LICENSE-3rdparty.csv

@@ -44,18 +44,21 @@
 "@oxc-project/types","https://github.com/oxc-project/oxc","['MIT']","['Boshen and oxc contributors']"
 "@tybys/wasm-util","https://github.com/toyobayashi/wasm-util","['MIT']","['toyobayashi']"
 "@types/estree","https://github.com/DefinitelyTyped/DefinitelyTyped","['MIT']","['DefinitelyTyped']"
+"agent-base","https://github.com/TooTallNate/proxy-agents","['MIT']","['Nathan Rajlich']"
 "argparse","https://github.com/nodeca/argparse","['Python-2.0']","['nodeca']"
 "astring","https://github.com/davidbonnet/astring","['MIT']","['David Bonnet']"
 "cjs-module-lexer","https://github.com/nodejs/cjs-module-lexer","['MIT']","['Guy Bedford']"
 "crypto-randomuuid","npm:crypto-randomuuid","['MIT']","['Stephen Belanger']"
 "dc-polyfill","https://github.com/DataDog/dc-polyfill","['MIT']","['Thomas Hunter II']"
 "dd-trace","https://github.com/DataDog/dd-trace-js","['(Apache-2.0 OR BSD-3-Clause)']","['Datadog Inc. <info@datadoghq.com>']"
+"debug","https://github.com/debug-js/debug","['MIT']","['Josh Junon']"
 "detect-newline","https://github.com/sindresorhus/detect-newline","['MIT']","['Sindre Sorhus']"
 "es-module-lexer","https://github.com/guybedford/es-module-lexer","['MIT']","['Guy Bedford']"
 "escape-string-regexp","https://github.com/sindresorhus/escape-string-regexp","['MIT']","['Sindre Sorhus']"
 "esquery","https://github.com/estools/esquery","['BSD-3-Clause']","['Joel Feenstra']"
 "estraverse","https://github.com/estools/estraverse","['BSD-2-Clause']","['estools']"
 "fast-fifo","https://github.com/mafintosh/fast-fifo","['MIT']","['Mathias Buus']"
+"https-proxy-agent","https://github.com/TooTallNate/proxy-agents","['MIT']","['Nathan Rajlich']"
 "import-in-the-middle","https://github.com/nodejs/import-in-the-middle","['Apache-2.0']","['Bryan English']"
 "istanbul-lib-coverage","https://github.com/istanbuljs/istanbuljs","['BSD-3-Clause']","['Krishnan Anantheswaran']"
 "jest-docblock","https://github.com/jestjs/jest","['MIT']","['jestjs']"
@@ -68,13 +71,15 @@
 "lru-cache","https://github.com/isaacs/node-lru-cache","['ISC']","['Isaac Z. Schlueter']"
 "meriyah","https://github.com/meriyah/meriyah","['ISC']","['Kenny F.']"
 "module-details-from-path","https://github.com/watson/module-details-from-path","['MIT']","['Thomas Watson']"
+"ms","https://github.com/vercel/ms","['MIT']","['vercel']"
 "mutexify","https://github.com/mafintosh/mutexify","['MIT']","['Mathias Buus']"
 "node-addon-api","https://github.com/nodejs/node-addon-api","['MIT']","['nodejs']"
 "node-gyp-build","https://github.com/prebuild/node-gyp-build","['MIT']","['Mathias Buus']"
 "opentracing","https://github.com/opentracing/opentracing-javascript","['Apache-2.0']","['opentracing']"
 "oxc-parser","https://github.com/oxc-project/oxc","['MIT']","['Boshen and oxc contributors']"
 "pprof-format","https://github.com/DataDog/pprof-format","['MIT']","['Datadog Inc.']"
 "protobufjs","https://github.com/protobufjs/protobuf.js","['BSD-3-Clause']","['Daniel Wirtz']"
+"proxy-from-env","https://github.com/Rob--W/proxy-from-env","['MIT']","['Rob Wu']"
 "queue-tick","https://github.com/mafintosh/queue-tick","['MIT']","['Mathias Buus']"
 "retry","https://github.com/tim-kos/node-retry","['MIT']","['Tim Koschützki']"
 "rfdc","https://github.com/davidmarkclements/rfdc","['MIT']","['David Mark Clements']"

package.json

@@ -179,8 +179,10 @@
   ],
   "dependencies": {
     "dc-polyfill": "^0.1.11",
+    "https-proxy-agent": "^7.0.6",
     "import-in-the-middle": "^3.3.2",
-    "opentracing": ">=0.14.7"
+    "opentracing": ">=0.14.7",
+    "proxy-from-env": "^2.1.0"
   },
   "optionalDependencies": {
     "@datadog/libdatadog": "0.12.1",

packages/dd-trace/src/evp_proxy/direct.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const { format } = require('node:url')
+
+const { HttpsProxyAgent } = require('https-proxy-agent')
+const { getProxyForUrl } = require('proxy-from-env')
+const log = require('../log')
+
+/**
+ * @typedef {object} DirectEVPRoute
+ * @property {URL} url - Direct intake URL
+ * @property {string} basePath - Direct intake base path
+ * @property {object} headers - Direct intake authentication headers
+ * @property {import('node:https').Agent} [agent] - Optional HTTPS proxy agent
+ */
+
+/**
+ * Creates an authenticated direct EVP intake route.
+ *
+ * This helper does not perform local receiver discovery.
+ *
+ * @param {import('../config/config-base')} config - Tracer configuration
+ * @param {string} intake - EVP intake subdomain
+ * @returns {DirectEVPRoute|undefined} Direct route when credentials and site are available
+ */
+function createDirectEVPRoute (config, intake) {
+  const apiKey = config.DD_API_KEY
+  if (!apiKey || !config.site) return
+
+  try {
+    const hostname = `${intake}.${config.site}`.toLowerCase()
+    const url = new URL(format({
+      protocol: 'https:',
+      hostname,
+    }))
+    if (
+      url.hostname !== hostname ||
+      url.username ||
+      url.password ||
+      url.port ||
+      url.pathname !== '/' ||
+      url.search ||
+      url.hash
+    ) {
+      throw new Error('Invalid direct EVP intake URL')
+    }
+
+    const proxyUrl = getProxyForUrl(url.href)
+    const agent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined
+
+    return {
+      url,
+      basePath: '',
+      headers: {
+        'DD-API-KEY': apiKey,
+      },
+      ...(agent && { agent }),
+    }
+  } catch (error) {
+    log.debug('Unable to configure direct EVP intake: %s', error.message)
+  }
+}
+
+module.exports = { createDirectEVPRoute }

packages/dd-trace/src/exporters/common/request.js

@@ -93,7 +93,10 @@ function request (data, options, callback) {
 
   docker.inject(options.headers)
 
-  options.agent = isSecure ? httpsAgent : httpAgent
+  const connectionOptions = {
+    ...options,
+    agent: options.agent ?? (isSecure ? httpsAgent : httpAgent),
+  }
 
   /**
    * @param {import('node:http').IncomingMessage} res
@@ -218,7 +221,7 @@ function request (data, options, callback) {
         }
       }
 
-      const req = client.request(options, (res) => onResponse(res, complete, handleError))
+      const req = client.request(connectionOptions, (res) => onResponse(res, complete, handleError))
 
       req.once('close', finalize)
       req.once('timeout', finalize)

packages/dd-trace/test/evp_proxy/direct.spec.js

@@ -0,0 +1,120 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+
+const { describe, it, beforeEach } = require('mocha')
+const proxyquire = require('proxyquire').noPreserveCache()
+const sinon = require('sinon')
+
+describe('direct EVP route', () => {
+  let createDirectEVPRoute
+  let getProxyForUrl
+  let HttpsProxyAgent
+  let log
+
+  beforeEach(() => {
+    getProxyForUrl = sinon.stub().returns('')
+    HttpsProxyAgent = sinon.stub().callsFake(proxyUrl => ({ proxyUrl }))
+    log = { debug: sinon.spy() }
+
+    ;({ createDirectEVPRoute } = proxyquire('../../src/evp_proxy/direct', {
+      'https-proxy-agent': { HttpsProxyAgent },
+      'proxy-from-env': { getProxyForUrl },
+      '../log': log,
+    }))
+  })
+
+  it('creates an authenticated route from API key and site', () => {
+    const route = createDirectEVPRoute({
+      DD_API_KEY: 'test-api-key',
+      site: 'datadoghq.com',
+    }, 'event-platform-intake')
+
+    assert.deepStrictEqual(route, {
+      url: new URL('https://event-platform-intake.datadoghq.com'),
+      basePath: '',
+      headers: {
+        'DD-API-KEY': 'test-api-key',
+      },
+    })
+  })
+
+  it('normalizes site casing', () => {
+    const route = createDirectEVPRoute({
+      DD_API_KEY: 'test-api-key',
+      site: 'DATADOGHQ.EU',
+    }, 'event-platform-intake')
+
+    assert.deepStrictEqual(route, {
+      url: new URL('https://event-platform-intake.datadoghq.eu'),
+      basePath: '',
+      headers: {
+        'DD-API-KEY': 'test-api-key',
+      },
+    })
+  })
+
+  it('uses the standard HTTPS proxy for direct intake', () => {
+    const proxyUrl = 'http://proxy:8202'
+    getProxyForUrl.returns(proxyUrl)
+
+    const route = createDirectEVPRoute({
+      DD_API_KEY: 'test-api-key',
+      site: 'datadoghq.com',
+    }, 'event-platform-intake')
+
+    assert.deepStrictEqual(route.agent, { proxyUrl })
+    sinon.assert.calledOnceWithExactly(
+      getProxyForUrl,
+      'https://event-platform-intake.datadoghq.com/'
+    )
+    sinon.assert.calledOnceWithExactly(HttpsProxyAgent, proxyUrl)
+  })
+
+  it('does not create a route without an API key', () => {
+    assert.strictEqual(createDirectEVPRoute({
+      site: 'datadoghq.com',
+    }, 'event-platform-intake'), undefined)
+  })
+
+  it('does not create a route without a site', () => {
+    assert.strictEqual(createDirectEVPRoute({
+      DD_API_KEY: 'test-api-key',
+    }, 'event-platform-intake'), undefined)
+  })
+
+  it('does not create a route for an invalid site', () => {
+    assert.strictEqual(createDirectEVPRoute({
+      DD_API_KEY: 'test-api-key',
+      site: 'not a host',
+    }, 'event-platform-intake'), undefined)
+
+    sinon.assert.calledOnceWithExactly(
+      log.debug,
+      'Unable to configure direct EVP intake: %s',
+      sinon.match.string
+    )
+  })
+
+  for (const site of [
+    'datadoghq.com@evil.example',
+    'datadoghq.com:password@evil.example',
+    'datadoghq.com:443',
+    'datadoghq.com/path',
+    'datadoghq.com?query',
+    'datadoghq.com#fragment',
+  ]) {
+    it(`does not create a route for a site with URL components: ${site}`, () => {
+      assert.strictEqual(createDirectEVPRoute({
+        DD_API_KEY: 'test-api-key',
+        site,
+      }, 'event-platform-intake'), undefined)
+
+      sinon.assert.calledOnceWithExactly(
+        log.debug,
+        'Unable to configure direct EVP intake: %s',
+        sinon.match.string
+      )
+    })
+  }
+})

packages/dd-trace/test/exporters/common/request.spec.js

@@ -111,6 +111,50 @@ describe('request', function () {
       })
   })
 
+  it('preserves a caller-supplied connection agent', (done) => {
+    const customAgent = new http.Agent()
+    const sandbox = sinon.createSandbox()
+    sandbox.spy(http, 'request')
+    nock('http://test:123').get('/path').reply(200, 'OK')
+
+    request(Buffer.from(''), {
+      agent: customAgent,
+      protocol: 'http:',
+      hostname: 'test',
+      port: 123,
+      path: '/path',
+      method: 'GET',
+    }, (error) => {
+      const callOptions = http.request.getCall(0).args[0]
+      sandbox.restore()
+      customAgent.destroy()
+      assert.strictEqual(callOptions.agent, customAgent)
+      done(error)
+    })
+  })
+
+  it('selects a new default agent when callers reuse options with another protocol', (done) => {
+    const options = {
+      url: new URL('http://test:123'),
+      path: '/path',
+      method: 'GET',
+    }
+    nock('http://test:123').get('/path').reply(200, 'OK')
+
+    request(Buffer.from(''), options, (httpError) => {
+      if (httpError) return done(httpError)
+
+      assert.strictEqual(options.agent, undefined)
+      options.url = new URL('https://test:443')
+      nock('https://test:443').get('/path').reply(200, 'OK')
+
+      request(Buffer.from(''), options, (httpsError) => {
+        assert.strictEqual(options.agent, undefined)
+        done(httpsError)
+      })
+    })
+  })
+
   it('does not retry when retries are disabled', (done) => {
     maxAttempts = 5
     const error = Object.assign(new Error('ECONNRESET'), { code: 'ECONNRESET' })

yarn.lock

@@ -2588,7 +2588,7 @@ https-proxy-agent@^5.0.1:
     agent-base "6"
     debug "4"
 
-https-proxy-agent@^7.0.5:
+https-proxy-agent@^7.0.5, https-proxy-agent@^7.0.6:
   version "7.0.6"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9"
   integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==