Make Live Debugger per-second budgets configurable (#4509)

Co-authored-by: thomas.watson <thomas.watson@datadoghq.com>

Author: watson

packages/debugger/src/domain/api.spec.ts

@@ -7,11 +7,21 @@ import type { Probe } from './probes'
 describe('api', () => {
   let mockBatchAdd: jasmine.Spy
 
+  function initTransport(overrides: Record<string, unknown> = {}) {
+    resetDebuggerTransport()
+    initDebuggerTransport(
+      { service: 'test-service', env: 'test-env', ...overrides } as any,
+      {
+        add: mockBatchAdd,
+      } as any
+    )
+  }
+
   beforeEach(() => {
     clearProbes()
 
     mockBatchAdd = jasmine.createSpy('batchAdd')
-    initDebuggerTransport({ service: 'test-service', env: 'test-env' } as any, { add: mockBatchAdd } as any)
+    initTransport()
     ;(window as any).DD_DEBUGGER = {
       version: '0.0.1',
     }
@@ -476,6 +486,85 @@ describe('api', () => {
       // Should only get 25 calls (global limit)
       expect(mockBatchAdd).toHaveBeenCalledTimes(25)
     })
+
+    it('should respect configured global snapshot rate limit', () => {
+      initTransport({ maxSnapshotsPerSecondGlobally: 2 })
+
+      for (let i = 0; i < 3; i++) {
+        const probe: Probe = {
+          id: `configured-global-probe-${i}`,
+          version: 0,
+          type: 'LOG_PROBE',
+          where: { typeName: 'TestClass', methodName: `configuredGlobal${i}` },
+          template: 'Test',
+          captureSnapshot: true,
+          capture: {},
+          sampling: { snapshotsPerSecond: 5000 },
+          evaluateAt: 'ENTRY',
+        }
+        addProbe(probe)
+      }
+
+      for (let i = 0; i < 3; i++) {
+        const probes = getProbes(`TestClass;configuredGlobal${i}`)!
+        onEntry(probes, {}, {})
+        onReturn(probes, null, {}, {}, {})
+      }
+
+      expect(mockBatchAdd).toHaveBeenCalledTimes(2)
+    })
+  })
+
+  describe('configured per-second budgets', () => {
+    it('should respect configured default snapshot per-probe rate limit', () => {
+      initTransport({ maxSnapshotsPerSecondPerProbe: 0.5 })
+
+      const probe: Probe = {
+        id: 'configured-snapshot-rate-probe',
+        version: 0,
+        type: 'LOG_PROBE',
+        where: { typeName: 'TestClass', methodName: 'configuredSnapshotRate' },
+        template: 'Test',
+        captureSnapshot: true,
+        capture: { maxReferenceDepth: 1 },
+        sampling: {},
+        evaluateAt: 'ENTRY',
+      }
+      addProbe(probe)
+
+      const probes = getProbes('TestClass;configuredSnapshotRate')!
+      onEntry(probes, {}, {})
+      onReturn(probes, null, {}, {}, {})
+      onEntry(probes, {}, {})
+      onReturn(probes, null, {}, {}, {})
+
+      expect(mockBatchAdd).toHaveBeenCalledTimes(1)
+    })
+
+    it('should respect configured default non-snapshot per-probe rate limit', () => {
+      initTransport({ maxNonSnapshotsPerSecondPerProbe: 1 })
+
+      const probe: Probe = {
+        id: 'configured-non-snapshot-rate-probe',
+        version: 0,
+        type: 'LOG_PROBE',
+        where: { typeName: 'TestClass', methodName: 'configuredNonSnapshotRate' },
+        template: 'Test',
+        captureSnapshot: false,
+        capture: {},
+        sampling: {},
+        evaluateAt: 'ENTRY',
+      }
+      addProbe(probe)
+
+      const probes = getProbes('TestClass;configuredNonSnapshotRate')!
+      onEntry(probes, {}, {})
+      onReturn(probes, null, {}, {}, {})
+      onEntry(probes, {}, {})
+      onReturn(probes, null, {}, {}, {})
+
+      expect(mockBatchAdd).toHaveBeenCalledTimes(1)
+    })
   })
 
   describe('active entries cleanup', () => {

packages/debugger/src/domain/api.ts

@@ -4,7 +4,7 @@ import { timeStampNow, display, buildTag, generateUUID, getGlobalObject } from '
 import type { BrowserWindow, DebuggerInitConfiguration } from '../entries/main'
 import { capture, captureFields } from './capture'
 import type { InitializedProbe } from './probes'
-import { checkGlobalSnapshotBudget } from './probes'
+import { checkGlobalSnapshotBudget, resetProbeBudgetConfiguration, setProbeBudgetConfiguration } from './probes'
 import type { ActiveEntry } from './activeEntries'
 import { active } from './activeEntries'
 import { captureStackTrace, parseStackTrace } from './stacktrace'
@@ -23,13 +23,15 @@ export function initDebuggerTransport(config: DebuggerInitConfiguration, batch:
   debuggerConfig = config
   debuggerBatch = batch
   cachedDDtags = undefined
+  setProbeBudgetConfiguration(config)
 }
 
 export function resetDebuggerTransport(): void {
   debuggerBatch = undefined
   debuggerConfig = undefined
   cachedDDtags = undefined
   active.clear()
+  resetProbeBudgetConfiguration()
 }
 
 /**

packages/debugger/src/domain/probes.spec.ts

@@ -1,6 +1,14 @@
 import { display } from '@datadog/browser-core'
 import { registerCleanupTask } from '@datadog/browser-core/test'
-import { initializeProbe, getProbes, addProbe, removeProbe, checkGlobalSnapshotBudget, clearProbes } from './probes'
+import {
+  initializeProbe,
+  getProbes,
+  addProbe,
+  removeProbe,
+  checkGlobalSnapshotBudget,
+  clearProbes,
+  resetProbeBudgetConfiguration,
+} from './probes'
 import type { Probe } from './probes'
 
 interface TemplateWithCache {
@@ -11,6 +19,7 @@ interface TemplateWithCache {
 describe('probes', () => {
   beforeEach(() => {
     clearProbes()
+    resetProbeBudgetConfiguration()
 
     registerCleanupTask(() => clearProbes())
   })

packages/debugger/src/domain/probes.ts

@@ -8,9 +8,9 @@ import type { TemplateSegment, CompiledTemplate } from './template'
 import type { CaptureOptions } from './capture'
 
 // Sampling rate limits
-const MAX_SNAPSHOTS_PER_SECOND_GLOBALLY = 25
-const MAX_SNAPSHOTS_PER_SECOND_PER_PROBE = 1
-const MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE = 5000
+const DEFAULT_MAX_SNAPSHOTS_PER_SECOND_GLOBALLY = 25
+const DEFAULT_MAX_SNAPSHOTS_PER_SECOND_PER_PROBE = 1
+const DEFAULT_MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE = 5000
 
 // Global snapshot rate limiting
 let globalSnapshotSamplingRateWindowStart = 0
@@ -32,6 +32,12 @@ export interface ProbeSampling {
   snapshotsPerSecond?: number
 }
 
+export interface ProbeBudgetConfiguration {
+  maxSnapshotsPerSecondGlobally?: number
+  maxSnapshotsPerSecondPerProbe?: number
+  maxNonSnapshotsPerSecondPerProbe?: number
+}
+
 export interface Probe {
   id: string
   version: number
@@ -70,6 +76,32 @@ const probeIdToFunctionId: Record<string, string> = {
   // @ts-expect-error - Pre-populate with a placeholder key to help V8 optimize property lookups.
   __placeholder__: undefined,
 }
+let currentProbeBudgetConfiguration: Required<ProbeBudgetConfiguration> = {
+  maxSnapshotsPerSecondGlobally: DEFAULT_MAX_SNAPSHOTS_PER_SECOND_GLOBALLY,
+  maxSnapshotsPerSecondPerProbe: DEFAULT_MAX_SNAPSHOTS_PER_SECOND_PER_PROBE,
+  maxNonSnapshotsPerSecondPerProbe: DEFAULT_MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE,
+}
+
+export function setProbeBudgetConfiguration(configuration: ProbeBudgetConfiguration = {}): void {
+  currentProbeBudgetConfiguration = {
+    maxSnapshotsPerSecondGlobally: normalizeProbeBudgetRate(
+      configuration.maxSnapshotsPerSecondGlobally,
+      DEFAULT_MAX_SNAPSHOTS_PER_SECOND_GLOBALLY
+    ),
+    maxSnapshotsPerSecondPerProbe: normalizeProbeBudgetRate(
+      configuration.maxSnapshotsPerSecondPerProbe,
+      DEFAULT_MAX_SNAPSHOTS_PER_SECOND_PER_PROBE
+    ),
+    maxNonSnapshotsPerSecondPerProbe: normalizeProbeBudgetRate(
+      configuration.maxNonSnapshotsPerSecondPerProbe,
+      DEFAULT_MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE
+    ),
+  }
+}
+
+export function resetProbeBudgetConfiguration(): void {
+  setProbeBudgetConfiguration()
+}
 
 /**
  * Add a probe to the registry
@@ -136,6 +168,9 @@ export function removeProbe(id: string): void {
         probe.condition.clearCache()
       }
       probes.splice(i, 1)
+      // TODO: Gracefully drain in-flight entries instead of clearing them immediately.
+      // Deleting a probe can currently race with return/throw handling, whether removal
+      // comes from delivery updates or budget-based auto-unregistering.
       clearActiveEntries(id)
       break
     }
@@ -199,7 +234,7 @@ export function checkGlobalSnapshotBudget(now: number, captureSnapshot: boolean)
   }
 
   // Check if we've exceeded the global limit
-  if (snapshotsSampledWithinTheLastSecond >= MAX_SNAPSHOTS_PER_SECOND_GLOBALLY) {
+  if (snapshotsSampledWithinTheLastSecond >= currentProbeBudgetConfiguration.maxSnapshotsPerSecondGlobally) {
     return false
   }
 
@@ -266,7 +301,13 @@ export function initializeProbe(probe: Probe): asserts probe is InitializedProbe
   // Optimize for fast calculations when probe is hit - calculate sampling budget
   const snapshotsPerSecond =
     probe.sampling?.snapshotsPerSecond ??
-    (probe.captureSnapshot ? MAX_SNAPSHOTS_PER_SECOND_PER_PROBE : MAX_NON_SNAPSHOTS_PER_SECOND_PER_PROBE)
+    (probe.captureSnapshot
+      ? currentProbeBudgetConfiguration.maxSnapshotsPerSecondPerProbe
+      : currentProbeBudgetConfiguration.maxNonSnapshotsPerSecondPerProbe)
   ;(probe as InitializedProbe).msBetweenSampling = (1 / snapshotsPerSecond) * 1000 // Convert to milliseconds
   ;(probe as InitializedProbe).lastCaptureMs = -Infinity // Initialize to -Infinity to allow first call
 }
+
+function normalizeProbeBudgetRate(rate: number | undefined, defaultRate: number): number {
+  return typeof rate === 'number' && Number.isFinite(rate) && rate > 0 ? rate : defaultRate
+}

packages/debugger/src/entries/main.ts

@@ -67,6 +67,30 @@ export interface DebuggerInitConfiguration {
    * @defaultValue 60000
    */
   pollInterval?: number
+
+  /**
+   * Maximum number of snapshot events allowed globally per second
+   *
+   * @category Data Collection
+   * @defaultValue 25
+   */
+  maxSnapshotsPerSecondGlobally?: number
+
+  /**
+   * Default maximum number of snapshot events allowed per probe per second
+   *
+   * @category Data Collection
+   * @defaultValue 1
+   */
+  maxSnapshotsPerSecondPerProbe?: number
+
+  /**
+   * Default maximum number of non-snapshot events allowed per probe per second
+   *
+   * @category Data Collection
+   * @defaultValue 5000
+   */
+  maxNonSnapshotsPerSecondPerProbe?: number
 }
 
 /**