fix(openfeature): retry exposure fallback failures

Author: leoromanovsky

packages/dd-trace/src/evp_proxy/discovery.js

@@ -29,9 +29,10 @@ const TRAILING_SLASHES = /\/+$/
  * `/info` endpoint returns an error through the shared request timeout and
  * retry policy. A valid response without a compatible path returns no route.
  * Discovery sends no events, so the caller can safely select direct intake
- * after either result. The caller also owns later delivery failures. It can
- * switch future batches after an ambiguous timeout or reset, but it must not
- * replay the current batch because the first receiver might have accepted it.
+ * after either result. The caller also owns later delivery failures. Exposure
+ * delivery uses retries and can therefore produce duplicates. After local
+ * retries fail, the caller can retry through direct intake and use that route
+ * for future batches.
  *
  * Reference implementations:
  *

packages/dd-trace/src/openfeature/writers/base.js

@@ -40,18 +40,19 @@ const log = require('../../log')
  * @returns {boolean} Whether direct retry is safe
  */
 function isDefinitiveRejection (error, statusCode) {
-  return error?.code === 'ECONNREFUSED' || error?.code === 'ENOENT' ||
+  return error?.code === 'EAI_AGAIN' || error?.code === 'ECONNREFUSED' ||
+    error?.code === 'ENOENT' || error?.code === 'ENOTFOUND' ||
     statusCode === 403 || statusCode === 404 || statusCode === 405
 }
 
 /**
  * Tests whether a local route can have accepted an event batch before failing.
  *
  * @param {Error | null} error - Request error
- * @returns {boolean} Whether the current batch must not be replayed
+ * @returns {boolean} Whether the delivery result is ambiguous
  */
 function isAmbiguousNetworkFailure (error) {
-  return error?.code === 'ECONNRESET' || error?.code === 'ETIMEDOUT'
+  return error?.code === 'ECONNRESET' || error?.code === 'EPIPE' || error?.code === 'ETIMEDOUT'
 }
 
 /**
@@ -257,7 +258,7 @@ class BaseFFEWriter {
   }
 
   /**
-   * Sends an encoded batch and retries it directly only after definitive rejection.
+   * Sends an encoded batch and retries it through direct intake after a local route failure.
    *
    * @param {string} payload - Encoded event batch
    * @param {number} eventCount - Event count
@@ -282,13 +283,15 @@ class BaseFFEWriter {
 
       if (fallbackRoute && isAmbiguousNetworkFailure(error)) {
         log.debug(
-          '%s switching future batches from %s%s to direct intake after ambiguous failure',
+          '%s retrying through direct intake and switching future batches from %s%s after ambiguous failure',
           this.constructor.name,
           route.url.href,
           route.endpoint
         )
         this.#activateRoute(fallbackRoute)
         this._fallbackRoute = undefined
+        this.#sendRequest(payload, eventCount, fallbackRoute)
+        return
       }
 
       if (error) {

packages/dd-trace/test/openfeature/writers/exposures.spec.js

@@ -424,11 +424,13 @@ describe('OpenFeature Exposures Writer', () => {
     })
 
     for (const [name, error, statusCode] of [
+      ['temporary DNS failure', Object.assign(new Error('getaddrinfo EAI_AGAIN'), { code: 'EAI_AGAIN' })],
       ['connection refusal', Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })],
       ['missing Unix socket', Object.assign(
         new Error('connect ENOENT /var/run/datadog/apm.socket'),
         { code: 'ENOENT' }
       )],
+      ['unresolvable hostname', Object.assign(new Error('getaddrinfo ENOTFOUND'), { code: 'ENOTFOUND' })],
       ['HTTP 403', Object.assign(new Error('Forbidden'), { status: 403 }), 403],
       ['HTTP 404', Object.assign(new Error('Not Found'), { status: 404 }), 404],
       ['HTTP 405', Object.assign(new Error('Method Not Allowed'), { status: 405 }), 405],
@@ -476,9 +478,10 @@ describe('OpenFeature Exposures Writer', () => {
 
     for (const [name, error, statusCode] of [
       ['connection reset', Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' })],
+      ['broken pipe', Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })],
       ['timeout', Object.assign(new Error('request timed out'), { code: 'ETIMEDOUT' })],
     ]) {
-      it(`should switch future batches after ambiguous local ${name} without replaying the current batch`, async () => {
+      it(`should retry ambiguous local ${name} through direct intake and switch future batches`, async () => {
         const localUrl = new URL('http://serverless-init:8126')
         const directUrl = new URL('https://event-platform-intake.datadoghq.com')
         request.onFirstCall().yieldsAsync(error, null, statusCode)
@@ -501,14 +504,15 @@ describe('OpenFeature Exposures Writer', () => {
         writer.flush()
         await clock.tickAsync(0)
 
-        sinon.assert.calledOnce(request)
+        sinon.assert.calledTwice(request)
         assert.strictEqual(request.firstCall.args[1].url, localUrl)
+        assert.strictEqual(request.secondCall.args[1].url, directUrl)
 
         writer.append(exposureEvent)
         writer.flush()
 
-        sinon.assert.calledTwice(request)
-        assert.strictEqual(request.secondCall.args[1].url, directUrl)
+        sinon.assert.calledThrice(request)
+        assert.strictEqual(request.thirdCall.args[1].url, directUrl)
       })
     }