Export Snapshot Profiling Stack Traces As OpenTelemetry Log Messages

profiler/build.gradle.kts

@@ -37,6 +37,7 @@ dependencies {
   annotationProcessor("com.google.auto.service:auto-service")
   compileOnly("com.google.auto.service:auto-service")
 
+  testImplementation(project(":custom"))
   testImplementation("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api")
   testImplementation("io.grpc:grpc-netty")
   testImplementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api")

profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java

@@ -74,6 +74,12 @@ public class Configuration implements AutoConfigurationCustomizerProvider {
   public static final String CONFIG_KEY_SNAPSHOT_SELECTION_RATE = "splunk.snapshot.selection.rate";
   private static final double DEFAULT_SNAPSHOT_SELECTION_RATE = 0.01;
   private static final double MAX_SNAPSHOT_SELECTION_RATE = 0.10;
+  private static final String CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH =
+      "splunk.snapshot.profiler.max.stack.depth";
+  private static final int DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH = 1024;
+  private static final String CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL =
+      "splunk.snapshot.profiler.sampling.interval";
+  public static final Duration DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL = Duration.ofMillis(20);
 
   @Override
   public void customize(AutoConfigurationCustomizer autoConfiguration) {
@@ -181,6 +187,10 @@ private static int getJavaVersion() {
     return Integer.parseInt(javaSpecVersion);
   }
 
+  public static boolean isSnapshotProfilingEnabled(ConfigProperties properties) {
+    return properties.getBoolean(CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, false);
+  }
+
   public static double getSnapshotSelectionRate(ConfigProperties properties) {
     String selectionRatePropertyValue =
         properties.getString(
@@ -207,4 +217,17 @@ public static double getSnapshotSelectionRate(ConfigProperties properties) {
       return DEFAULT_SNAPSHOT_SELECTION_RATE;
     }
   }
+
+  public static int getSnapshotProfilerStackDepth(ConfigProperties properties) {
+    return properties.getInt(
+        CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH, DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH);
+  }
+
+  public static Duration getSnapshotProfilerSamplingInterval(ConfigProperties properties) {
+    int millis = properties.getInt(CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL, 0);
+    if (millis > 0) {
+      return Duration.ofMillis(millis);
+    }
+    return DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL;
+  }
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java

@@ -29,7 +29,6 @@
 import com.splunk.opentelemetry.profiler.allocation.exporter.AllocationEventExporter;
 import com.splunk.opentelemetry.profiler.allocation.exporter.PprofAllocationEventExporter;
 import com.splunk.opentelemetry.profiler.context.SpanContextualizer;
-import com.splunk.opentelemetry.profiler.events.EventPeriods;
 import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter;
 import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter;
 import com.splunk.opentelemetry.profiler.util.HelpfulExecutors;
@@ -130,13 +129,12 @@ private void activateJfrAndRunForever(ConfigProperties config, Resource resource
 
     EventReader eventReader = new EventReader();
     SpanContextualizer spanContextualizer = new SpanContextualizer(eventReader);
-    EventPeriods periods = new EventPeriods(jfrSettings::get);
     LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config);
 
     CpuEventExporter cpuEventExporter =
         PprofCpuEventExporter.builder()
             .otelLogger(buildOtelLogger(SimpleLogRecordProcessor.create(logsExporter), resource))
-            .eventPeriods(periods)
+            .period(Configuration.getCallStackInterval(config))
             .stackDepth(stackDepth)
             .build();
 

profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Splunk Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.splunk.opentelemetry.profiler;
+
+import com.google.common.annotations.VisibleForTesting;
+import io.opentelemetry.api.logs.Logger;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+import io.opentelemetry.sdk.logs.LogRecordProcessor;
+import io.opentelemetry.sdk.logs.SdkLoggerProvider;
+import io.opentelemetry.sdk.logs.export.LogRecordExporter;
+import io.opentelemetry.sdk.logs.export.SimpleLogRecordProcessor;
+import io.opentelemetry.sdk.resources.Resource;
+import java.util.function.Function;
+
+public class OtelLoggerFactory {
+  private final Function<ConfigProperties, LogRecordExporter> logRecordExporter;
+
+  public OtelLoggerFactory() {
+    this(LogExporterBuilder::fromConfig);
+  }
+
+  @VisibleForTesting
+  public OtelLoggerFactory(Function<ConfigProperties, LogRecordExporter> logRecordExporter) {
+    this.logRecordExporter = logRecordExporter;
+  }
+
+  public Logger build(ConfigProperties properties, Resource resource) {
+    LogRecordExporter exporter = createLogRecordExporter(properties);
+    LogRecordProcessor processor = SimpleLogRecordProcessor.create(exporter);
+    return buildOtelLogger(processor, resource);
+  }
+
+  private LogRecordExporter createLogRecordExporter(ConfigProperties properties) {
+    return logRecordExporter.apply(properties);
+  }
+
+  private Logger buildOtelLogger(LogRecordProcessor logProcessor, Resource resource) {
+    return SdkLoggerProvider.builder()
+        .addLogRecordProcessor(logProcessor)
+        .setResource(resource)
+        .build()
+        .loggerBuilder(ProfilingSemanticAttributes.OTEL_INSTRUMENTATION_NAME)
+        .setInstrumentationVersion(ProfilingSemanticAttributes.OTEL_INSTRUMENTATION_VERSION)
+        .build();
+  }
+}

profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java

@@ -17,10 +17,20 @@
 package com.splunk.opentelemetry.profiler.exporter;
 
 import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage;
+import java.time.Instant;
 
 public interface CpuEventExporter {
 
   void export(StackToSpanLinkage stackToSpanLinkage);
 
+  default void export(
+      long threadId,
+      String threadName,
+      Thread.State threadState,
+      StackTraceElement[] stackTrace,
+      Instant eventTime,
+      String traceId,
+      String spanId) {}
+
   default void flush() {}
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java

@@ -30,26 +30,27 @@
 import com.splunk.opentelemetry.profiler.InstrumentationSource;
 import com.splunk.opentelemetry.profiler.ProfilingDataType;
 import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage;
-import com.splunk.opentelemetry.profiler.events.EventPeriods;
 import com.splunk.opentelemetry.profiler.exporter.StackTraceParser.StackTrace;
 import com.splunk.opentelemetry.profiler.pprof.Pprof;
 import io.opentelemetry.api.logs.Logger;
 import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.SpanId;
+import io.opentelemetry.api.trace.TraceId;
 import java.time.Duration;
 import java.time.Instant;
 
 public class PprofCpuEventExporter implements CpuEventExporter {
-  private final EventPeriods eventPeriods;
+  private final Duration period;
   private final int stackDepth;
   private final PprofLogDataExporter pprofLogDataExporter;
   private Pprof pprof = createPprof();
 
   private PprofCpuEventExporter(Builder builder) {
-    this.eventPeriods = builder.eventPeriods;
+    this.period = builder.period;
     this.stackDepth = builder.stackDepth;
     this.pprofLogDataExporter =
         new PprofLogDataExporter(
-            builder.otelLogger, ProfilingDataType.CPU, InstrumentationSource.CONTINUOUS);
+            builder.otelLogger, ProfilingDataType.CPU, builder.instrumentationSource);
   }
 
   @Override
@@ -80,10 +81,7 @@ public void export(StackToSpanLinkage stackToSpanLinkage) {
 
     String eventName = stackToSpanLinkage.getSourceEventName();
     pprof.addLabel(sample, SOURCE_EVENT_NAME, eventName);
-    Duration eventPeriod = eventPeriods.getDuration(eventName);
-    if (!EventPeriods.UNKNOWN.equals(eventPeriod)) {
-      pprof.addLabel(sample, SOURCE_EVENT_PERIOD, eventPeriod.toMillis());
-    }
+    pprof.addLabel(sample, SOURCE_EVENT_PERIOD, period.toMillis());
     Instant time = stackToSpanLinkage.getTime();
     pprof.addLabel(sample, SOURCE_EVENT_TIME, time.toEpochMilli());
 
@@ -96,6 +94,52 @@ public void export(StackToSpanLinkage stackToSpanLinkage) {
     pprof.getProfileBuilder().addSample(sample);
   }
 
+  @Override
+  public void export(
+      long threadId,
+      String threadName,
+      Thread.State threadState,
+      StackTraceElement[] stackTrace,
+      Instant eventTime,
+      String traceId,
+      String spanId) {
+    Sample.Builder sample = Sample.newBuilder();
+
+    pprof.addLabel(sample, THREAD_ID, threadId);
+    pprof.addLabel(sample, THREAD_NAME, threadName);
+    pprof.addLabel(sample, THREAD_STATE, threadState.name());
+
+    if (stackTrace.length > stackDepth) {
+      pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true);
+    }
+
+    for (int i = 0; i < Math.min(stackDepth, stackTrace.length); i++) {
+      StackTraceElement ste = stackTrace[i];
+
+      String fileName = ste.getFileName();
+      if (fileName == null) {
+        fileName = "unknown";
+      }
+      String className = ste.getClassName();
+      String methodName = ste.getMethodName();
+      int lineNumber = Math.max(ste.getLineNumber(), 0);
+      sample.addLocationId(pprof.getLocationId(fileName, className, methodName, lineNumber));
+      pprof.incFrameCount();
+    }
+
+    pprof.addLabel(sample, SOURCE_EVENT_PERIOD, period.toMillis());
+    pprof.addLabel(sample, SOURCE_EVENT_TIME, eventTime.toEpochMilli());
+
+    if (TraceId.isValid(traceId)) {
+      pprof.addLabel(sample, TRACE_ID, traceId);
+    }
+    if (SpanId.isValid(spanId)) {
+      pprof.addLabel(sample, SPAN_ID, spanId);
+    }
+
+    pprof.getProfileBuilder().addSample(sample);
+  }
+
   private static Pprof createPprof() {
     return new Pprof();
   }
@@ -123,8 +167,9 @@ public static Builder builder() {
 
   public static class Builder {
     private Logger otelLogger;
-    private EventPeriods eventPeriods;
+    private Duration period;
     private int stackDepth;
+    private InstrumentationSource instrumentationSource = InstrumentationSource.CONTINUOUS;
 
     public PprofCpuEventExporter build() {
       return new PprofCpuEventExporter(this);
@@ -135,14 +180,19 @@ public Builder otelLogger(Logger otelLogger) {
       return this;
     }
 
-    public Builder eventPeriods(EventPeriods eventPeriods) {
-      this.eventPeriods = eventPeriods;
+    public Builder period(Duration period) {
+      this.period = period;
       return this;
     }
 
     public Builder stackDepth(int stackDepth) {
       this.stackDepth = stackDepth;
       return this;
     }
+
+    public Builder instrumentationSource(InstrumentationSource instrumentationSource) {
+      this.instrumentationSource = instrumentationSource;
+      return this;
+    }
   }
 }