Add configuration option for controlling the snapshot profiling sampling rate.

Author: tduncan

profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java

@@ -77,6 +77,9 @@ public class Configuration implements AutoConfigurationCustomizerProvider {
   private static final String CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH =
       "splunk.snapshot.profiler.max.stack.depth";
   private static final int DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH = 1024;
+  private static final String CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL =
+      "splunk.snapshot.profiler.sampling.interval";
+  public static final Duration DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL = Duration.ofMillis(20);
 
   @Override
   public void customize(AutoConfigurationCustomizer autoConfiguration) {
@@ -219,4 +222,12 @@ public static int getSnapshotProfilerStackDepth(ConfigProperties properties) {
     return properties.getInt(
         CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH, DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH);
   }
+
+  public static Duration getSnapshotProfilerSamplingInterval(ConfigProperties properties) {
+    int millis = properties.getInt(CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL, 0);
+    if (millis > 0) {
+      return Duration.ofMillis(millis);
+    }
+    return DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL;
+  }
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java

@@ -16,13 +16,11 @@
 
 package com.splunk.opentelemetry.profiler.snapshot;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.splunk.opentelemetry.profiler.InstrumentationSource;
 import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter;
 import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter;
 import io.opentelemetry.api.logs.Logger;
-import java.time.Clock;
-import java.time.Instant;
+import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -34,18 +32,13 @@ class AsyncStackTraceExporter implements StackTraceExporter {
 
   private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor();
   private final Logger otelLogger;
+  private final Duration samplingPeriod;
   private final int maxDepth;
-  private final Clock clock;
 
-  AsyncStackTraceExporter(Logger logger, int maxDepth) {
-    this(logger, maxDepth, Clock.systemUTC());
-  }
-
-  @VisibleForTesting
-  AsyncStackTraceExporter(Logger logger, int maxDepth, Clock clock) {
+  AsyncStackTraceExporter(Logger logger, Duration samplingPeriod, int maxDepth) {
     this.otelLogger = logger;
+    this.samplingPeriod = samplingPeriod;
     this.maxDepth = maxDepth;
-    this.clock = clock;
   }
 
   @Override
@@ -60,18 +53,17 @@ private Runnable pprofExporter(Logger otelLogger, List<StackTrace> stackTraces)
             PprofCpuEventExporter.builder()
                 .otelLogger(otelLogger)
                 .stackDepth(maxDepth)
-                .period(ScheduledExecutorStackTraceSampler.SCHEDULER_PERIOD)
+                .period(samplingPeriod)
                 .instrumentationSource(InstrumentationSource.SNAPSHOT)
                 .build();
 
-        Instant now = Instant.now(clock);
         for (StackTrace stackTrace : stackTraces) {
           cpuEventExporter.export(
               stackTrace.getThreadId(),
               stackTrace.getThreadName(),
               stackTrace.getThreadState(),
               stackTrace.getStackFrames(),
-              now,
+              stackTrace.getTimestamp(),
               stackTrace.getTraceId(),
               null);
         }

profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java

@@ -16,7 +16,6 @@
 
 package com.splunk.opentelemetry.profiler.snapshot;
 
-import com.google.common.annotations.VisibleForTesting;
 import io.opentelemetry.api.trace.SpanContext;
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
@@ -36,19 +35,13 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler {
   private static final Logger logger =
       Logger.getLogger(ScheduledExecutorStackTraceSampler.class.getName());
   private static final int SCHEDULER_INITIAL_DELAY = 0;
-  static final Duration SCHEDULER_PERIOD = Duration.ofMillis(20);
 
   private final ConcurrentMap<String, ScheduledExecutorService> samplers =
       new ConcurrentHashMap<>();
   private final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
   private final StagingArea stagingArea;
   private final Duration samplingPeriod;
 
-  ScheduledExecutorStackTraceSampler(StagingArea stagingArea) {
-    this(stagingArea, SCHEDULER_PERIOD);
-  }
-
-  @VisibleForTesting
   ScheduledExecutorStackTraceSampler(StagingArea stagingArea, Duration samplingPeriod) {
     this.stagingArea = stagingArea;
     this.samplingPeriod = samplingPeriod;

profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java

@@ -23,6 +23,7 @@
 import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
 import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
 import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
+import java.time.Duration;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.Map;
@@ -33,19 +34,29 @@
 @AutoService(AutoConfigurationCustomizerProvider.class)
 public class SnapshotProfilingSdkCustomizer implements AutoConfigurationCustomizerProvider {
   private final TraceRegistry registry;
-  private final StackTraceSampler sampler;
+  private final Function<ConfigProperties, StackTraceSampler> samplerProvider;
 
   public SnapshotProfilingSdkCustomizer() {
-    this(
-        new TraceRegistry(),
-        new ScheduledExecutorStackTraceSampler(
-            new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE)));
+    this(new TraceRegistry(), stackTraceSamplerProvider());
+  }
+
+  private static Function<ConfigProperties, StackTraceSampler> stackTraceSamplerProvider() {
+    return properties -> {
+      Duration samplingPeriod = Configuration.getSnapshotProfilerSamplingInterval(properties);
+      return new ScheduledExecutorStackTraceSampler(
+          new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE), samplingPeriod);
+    };
   }
 
   @VisibleForTesting
-  SnapshotProfilingSdkCustomizer(TraceRegistry registry, StackTraceSampler sampler) {
+  SnapshotProfilingSdkCustomizer(TraceRegistry registry, StackTraceSampler samplerProvider) {
+    this(registry, properties -> samplerProvider);
+  }
+
+  private SnapshotProfilingSdkCustomizer(
+      TraceRegistry registry, Function<ConfigProperties, StackTraceSampler> samplerProvider) {
     this.registry = registry;
-    this.sampler = sampler;
+    this.samplerProvider = samplerProvider;
   }
 
   @Override
@@ -59,6 +70,7 @@ public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) {
       snapshotProfilingSpanProcessor(TraceRegistry registry) {
     return (builder, properties) -> {
       if (snapshotProfilingEnabled(properties)) {
+        StackTraceSampler sampler = samplerProvider.apply(properties);
         return builder.addSpanProcessor(new SnapshotProfilingSpanProcessor(registry, sampler));
       }
       return builder;

profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java

@@ -26,6 +26,7 @@
 import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
 import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
 import io.opentelemetry.sdk.resources.Resource;
+import java.time.Duration;
 
 @AutoService(AgentListener.class)
 public class StackTraceExporterActivator implements AgentListener {
@@ -45,14 +46,20 @@ public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetr
     ConfigProperties properties = AutoConfigureUtil.getConfig(autoConfiguredOpenTelemetrySdk);
     if (snapshotProfilingEnabled(properties)) {
       int maxDepth = Configuration.getSnapshotProfilerStackDepth(properties);
-      Resource resource = AutoConfigureUtil.getResource(autoConfiguredOpenTelemetrySdk);
-      Logger logger = otelLoggerFactory.build(properties, resource);
-      AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, maxDepth);
+      Duration samplingPeriod = Configuration.getSnapshotProfilerSamplingInterval(properties);
+      Logger logger = buildLogger(autoConfiguredOpenTelemetrySdk, properties);
+      AsyncStackTraceExporter exporter =
+          new AsyncStackTraceExporter(logger, samplingPeriod, maxDepth);
       StackTraceExporterProvider.INSTANCE.configure(exporter);
     }
   }
 
   private boolean snapshotProfilingEnabled(ConfigProperties properties) {
     return Configuration.isSnapshotProfilingEnabled(properties);
   }
+
+  private Logger buildLogger(AutoConfiguredOpenTelemetrySdk sdk, ConfigProperties properties) {
+    Resource resource = AutoConfigureUtil.getResource(sdk);
+    return otelLoggerFactory.build(properties, resource);
+  }
 }

profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java

@@ -24,6 +24,7 @@
 
 import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
 import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties;
+import java.time.Duration;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -185,4 +186,22 @@ void getDefaultSnapshotProfilerStackDepthWhenNotSpecified() {
     var properties = DefaultConfigProperties.create(Collections.emptyMap());
     assertEquals(1024, Configuration.getSnapshotProfilerStackDepth(properties));
   }
+
+  @ParameterizedTest
+  @ValueSource(ints = {128, 512, 2056})
+  void getConfiguredSnapshotProfilerSamplingInterval(int milliseconds) {
+    var properties =
+        DefaultConfigProperties.create(
+            Map.of("splunk.snapshot.profiler.sampling.interval", String.valueOf(milliseconds)));
+    assertEquals(
+        Duration.ofMillis(milliseconds),
+        Configuration.getSnapshotProfilerSamplingInterval(properties));
+  }
+
+  @Test
+  void getDefaultSnapshotProfilerSamplingInterval() {
+    var properties = DefaultConfigProperties.create(Collections.emptyMap());
+    assertEquals(
+        Duration.ofMillis(20), Configuration.getSnapshotProfilerSamplingInterval(properties));
+  }
 }

profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java

@@ -20,6 +20,7 @@
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.FRAME_COUNT;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE;
+import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_TIME;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_TYPE;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.awaitility.Awaitility.await;
@@ -30,13 +31,15 @@
 import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger;
 import com.splunk.opentelemetry.profiler.pprof.PprofUtils;
 import java.io.ByteArrayInputStream;
+import java.time.Duration;
 import java.util.List;
 import java.util.zip.GZIPInputStream;
 import org.junit.jupiter.api.Test;
 
 class AsyncStackTraceExporterTest {
   private final InMemoryOtelLogger logger = new InMemoryOtelLogger();
-  private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, 200);
+  private final AsyncStackTraceExporter exporter =
+      new AsyncStackTraceExporter(logger, Duration.ofMillis(20), 200);
 
   @Test
   void exportStackTraceAsOpenTelemetryLog() {
@@ -153,4 +156,19 @@ void includeFrameCountOpenTelemetryAttributeInLogMessage() {
     assertThat(attributes.asMap())
         .containsEntry(FRAME_COUNT, (long) stackTrace.getStackFrames().length);
   }
+
+  @Test
+  void includeStackTraceDurationInSamples() throws Exception {
+    var stackTrace = Snapshotting.stackTrace().build();
+
+    exporter.export(List.of(stackTrace));
+    await().until(() -> !logger.records().isEmpty());
+
+    var profile = Profile.parseFrom(PprofUtils.deserialize(logger.records().get(0)));
+    var sample = profile.getSample(0);
+
+    var labels = PprofUtils.toLabelString(sample, profile);
+    assertThat(labels)
+        .containsEntry(SOURCE_EVENT_TIME.getKey(), stackTrace.getTimestamp().toEpochMilli());
+  }
 }

profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java

@@ -16,6 +16,8 @@
 
 package com.splunk.opentelemetry.profiler.snapshot;
 
+import java.time.Duration;
+
 class SnapshotProfilingSdkCustomizerBuilder {
   private TraceRegistry registry = new TraceRegistry();
   private StackTraceSampler sampler = new ObservableStackTraceSampler();
@@ -28,7 +30,8 @@ SnapshotProfilingSdkCustomizerBuilder with(TraceRegistry registry) {
   SnapshotProfilingSdkCustomizerBuilder withRealStackTraceSampler() {
     return with(
         new ScheduledExecutorStackTraceSampler(
-            new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE)));
+            new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE),
+            Duration.ofMillis(20)));
   }
 
   SnapshotProfilingSdkCustomizerBuilder with(StackTraceSampler sampler) {