feat: Evaluate the accuracy of cross-feature attribution (#1573)

Author: metal-messiah

docs/supportability-metrics.md

@@ -390,4 +390,32 @@ EventBuffer/soft_navigations/Dropped/Bytes
 <!--- Soft nav harvest was sent before the interval elapsed (bytes captured) --->
 * soft_navigations/Harvest/Early/Seen
 <!--- Logging harvest was sent before the interval elapsed (bytes captured) --->
-* spa/Harvest/Early/Seen
+* spa/Harvest/Early/Seen
+
+### Audit
+<!--- Page view event had hasReplay true but no session replay harvest (false positive) --->
+* audit/page_view/hasReplay/false/positive
+<!--- Page view event had hasReplay false but a session replay harvest occurred (false negative) --->
+* audit/page_view/hasReplay/false/negative
+<!--- Page view event had hasReplay true and a session replay harvest occurred (true positive) --->
+* audit/page_view/hasReplay/true/positive
+<!--- Page view event had hasReplay false and no session replay harvest occurred (true negative) --->
+* audit/page_view/hasReplay/true/negative
+
+<!--- Page view event had hasTrace true but no session trace harvest (false positive) --->
+* audit/page_view/hasTrace/false/positive
+<!--- Page view event had hasTrace false but a session trace harvest occurred (false negative) --->
+* audit/page_view/hasTrace/false/negative
+<!--- Page view event had hasTrace true and a session trace harvest occurred (true positive) --->
+* audit/page_view/hasTrace/true/positive
+<!--- Page view event had hasTrace false and no session trace harvest occurred (true negative) --->
+* audit/page_view/hasTrace/true/negative
+
+<!--- Session replay had hasError true but no js error harvest occurred (false positive) --->
+* audit/session_replay/hasError/false/positive
+<!--- Session replay had hasError false but a js error harvest occurred (false negative) --->
+* audit/session_replay/hasError/false/negative
+<!--- Session replay had hasError true and a js error harvest occurred (true positive) --->
+* audit/session_replay/hasError/true/positive
+<!--- Session replay had hasError false and no js error harvest occurred (true negative) --->
+* audit/session_replay/hasError/true/negative

src/common/harvest/harvester.js

@@ -160,15 +160,34 @@ export function send (agentRef, { endpoint, targetApp, payload, localOpts = {},
         const cbResult = { sent: this.status !== 0, status: this.status, retry: shouldRetry(this.status), fullUrl, xhr: this, targetApp }
         if (localOpts.needResponse) cbResult.responseText = this.responseText
         cbFinished(cbResult)
+
+        /** temporary audit of consistency of harvest metadata flags */
+        if (!shouldRetry(this.status)) trackHarvestMetadata()
       }, eventListenerOpts(false))
     } else if (submitMethod === fetchMethod) {
       result.then(async function (response) {
         const status = response.status
         const cbResult = { sent: true, status, retry: shouldRetry(status), fullUrl, fetchResponse: response, targetApp }
         if (localOpts.needResponse) cbResult.responseText = await response.text()
         cbFinished(cbResult)
+        /** temporary audit of consistency of harvest metadata flags */
+        if (!shouldRetry(status)) trackHarvestMetadata()
       })
     }
+
+    function trackHarvestMetadata () {
+      const hasReplay = baseParams.includes('hr=1')
+      const hasTrace = baseParams.includes('ht=1')
+      const hasError = qs?.attributes?.includes('hasError=true')
+
+      handle('harvest-metadata', [{
+        [featureName]: {
+          ...(hasReplay && { hasReplay }),
+          ...(hasTrace && { hasTrace }),
+          ...(hasError && { hasError })
+        }
+      }], undefined, FEATURE_NAMES.metrics, agentRef.ee)
+    }
   }
 
   dispatchGlobalEvent({

src/features/metrics/aggregate/harvest-metadata.js

@@ -0,0 +1,42 @@
+/**
+ * Copyright 2020-2025 New Relic, Inc. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export function evaluateHarvestMetadata (pageMetadata) {
+  try {
+    const supportabilityTags = []
+
+    // Report SM like... audit/<feature_name>/<hasReplay|hasTrace|hasError>/<true|false>/<negative|positive>
+    const formTag = (...strings) => strings.join('/')
+
+    // Track if replay/trace/error harvests actually occurred (key only exists when harvested)
+    function evaluateTag (feature, flag, hasFlag, hasHarvest) {
+      const AUDIT = 'audit'
+      if (hasFlag) {
+      // False positive: flag true, but no harvest
+        if (!hasHarvest) supportabilityTags.push(formTag(AUDIT, feature, flag, 'false', 'positive'))
+        // True positive (correct)
+        else supportabilityTags.push(formTag(AUDIT, feature, flag, 'true', 'positive'))
+      } else {
+      // False negative: flag false, but harvest occurred
+        if (hasHarvest) supportabilityTags.push(formTag(AUDIT, feature, flag, 'false', 'negative'))
+        // True negative (correct)
+        else supportabilityTags.push(formTag(AUDIT, feature, flag, 'true', 'negative'))
+      }
+    }
+
+    if (pageMetadata.page_view_event) {
+      evaluateTag('page_view', 'hasReplay', pageMetadata.page_view_event.hasReplay, !!pageMetadata.session_replay)
+      evaluateTag('page_view', 'hasTrace', pageMetadata.page_view_event.hasTrace, !!pageMetadata.session_trace)
+    }
+
+    if (pageMetadata.session_replay) {
+      evaluateTag('session_replay', 'hasError', pageMetadata.session_replay.hasError, !!pageMetadata.jserrors)
+    }
+
+    return supportabilityTags
+  } catch (err) {
+    return []
+  }
+}

src/features/metrics/aggregate/index.js

@@ -11,6 +11,7 @@ import { windowAddEventListener } from '../../../common/event-listener/event-lis
 import { isBrowserScope, isWorkerScope } from '../../../common/constants/runtime'
 import { AggregateBase } from '../../utils/aggregate-base'
 import { isIFrameWindow } from '../../../common/dom/iframe'
+import { evaluateHarvestMetadata } from './harvest-metadata'
 // import { WEBSOCKET_TAG } from '../../../common/wrap/wrap-websocket'
 // import { handleWebsocketEvents } from './websocket-detection'
 
@@ -19,6 +20,15 @@ export class Aggregate extends AggregateBase {
   constructor (agentRef) {
     super(agentRef, FEATURE_NAME)
     this.harvestOpts.aggregatorTypes = ['cm', 'sm'] // the types in EventAggregator this feature cares about
+
+    /** all the harvest metadata metrics need to be evaluated simulataneously at unload time so just temporarily buffer them and dont make SMs immediately from the data */
+    this.harvestMetadata = {}
+    this.harvestOpts.beforeUnload = () => {
+      evaluateHarvestMetadata(this.harvestMetadata).forEach(smTag => {
+        this.storeSupportabilityMetrics(smTag)
+      })
+    }
+
     // This feature only harvests once per potential EoL of the page, which is handled by the central harvester.
 
     // this must be read/stored synchronously, as the currentScript is removed from the DOM after this script is executed and this lookup will be void
@@ -126,6 +136,17 @@ export class Aggregate extends AggregateBase {
     //     handleWebsocketEvents(this.storeSupportabilityMetrics.bind(this), tag, ...args)
     //   }, this.featureName, this.ee)
     // })
+
+    /** all the harvest metadata metrics need to be evaluated simulataneously at unload time so just temporarily buffer them and dont make SMs immediately from the data */
+    registerHandler('harvest-metadata', (harvestMetadataObject = {}) => {
+      try {
+        Object.keys(harvestMetadataObject).forEach(key => {
+          Object.assign(this.harvestMetadata[key] ??= {}, harvestMetadataObject[key])
+        })
+      } catch (e) {
+      // failed to merge harvest metadata... ignore
+      }
+    }, this.featureName, this.ee)
   }
 
   eachSessionChecks () {

tests/unit/harvest-metadata.test.js

@@ -0,0 +1,69 @@
+import { evaluateHarvestMetadata } from '../../src/features/metrics/aggregate/harvest-metadata'
+
+describe('evaluateHarvestMetadata', () => {
+  it('returns SR false positive when hasReplay true but no replay harvest', () => {
+    const meta = { page_view_event: { hasReplay: true } }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining(['audit/page_view/hasReplay/false/positive']))
+  })
+
+  it('returns SR false negative when hasReplay false but replay harvest occurred', () => {
+    const meta = { page_view_event: { hasReplay: false }, session_replay: {} }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining(['audit/page_view/hasReplay/false/negative']))
+  })
+
+  it('returns ST false positive when hasTrace true but no trace harvest', () => {
+    const meta = { page_view_event: { hasTrace: true } }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining(['audit/page_view/hasTrace/false/positive']))
+  })
+
+  it('returns ST false negative when hasTrace false but trace harvest occurred', () => {
+    const meta = { page_view_event: { hasTrace: false }, session_trace: {} }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining(['audit/page_view/hasTrace/false/negative']))
+  })
+
+  it('returns SR event error false positive when SR harvest hasError true but no error harvest', () => {
+    const meta = { session_replay: { hasError: true } }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining(['audit/session_replay/hasError/false/positive']))
+  })
+
+  it('returns SR event error false negative when error harvested during active SR but no SR harvest with hasError true', () => {
+    const meta = { session_replay: { hasError: false }, jserrors: { } }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining(['audit/session_replay/hasError/false/negative']))
+  })
+
+  it('returns positive tags when all conditions are normal', () => {
+    const meta = { page_view_event: { hasReplay: true, hasTrace: true }, session_replay: { }, session_trace: { } }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining([
+      'audit/page_view/hasReplay/true/positive',
+      'audit/page_view/hasTrace/true/positive',
+      'audit/session_replay/hasError/true/negative'
+    ]))
+  })
+
+  it('returns true negative tags when all features are correctly absent', () => {
+    const meta = { page_view_event: { hasReplay: false, hasTrace: false } }
+    expect(evaluateHarvestMetadata(meta)).toEqual(expect.arrayContaining([
+      'audit/page_view/hasReplay/true/negative',
+      'audit/page_view/hasTrace/true/negative'
+    ]))
+  })
+
+  it('returns multiple tags if multiple conditions are met', () => {
+    const meta = { page_view_event: { hasReplay: true, hasTrace: false }, session_trace: { } }
+    const tags = evaluateHarvestMetadata(meta)
+    expect(tags).toEqual(expect.arrayContaining([
+      'audit/page_view/hasReplay/false/positive',
+      'audit/page_view/hasTrace/false/negative'
+    ]))
+  })
+
+  it('handles errors', () => {
+    const meta = {}
+    Object.defineProperty(meta, 'page_view_event', {
+      get () {
+        throw new Error('Test error accessing page_view_event')
+      }
+    })
+    expect(evaluateHarvestMetadata(meta)).toEqual([])
+  })
+})