From 72d1404015e91410c27a672bc81ee284a7ef6622 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Thu, 6 Mar 2025 15:28:52 -0800 Subject: [PATCH 01/28] Add an accumulating staging area. --- .../snapshot/AccumulatingStagingArea.java | 52 +++++++++ .../profiler/snapshot/StackTrace.java | 5 +- .../profiler/snapshot/StackTraceExporter.java | 24 +++++ .../snapshot/AccumulatingStagingAreaTest.java | 102 ++++++++++++++++++ .../snapshot/InMemoryStackTraceExporter.java | 38 +++++++ .../profiler/snapshot/Snapshotting.java | 11 ++ .../profiler/snapshot/StackTraceBuilder.java | 50 +++++++++ 7 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStackTraceExporter.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java new file mode 100644 index 000000000..585b9e4b6 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java @@ -0,0 +1,52 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +class AccumulatingStagingArea implements StagingArea { + private final ConcurrentMap> stackTraces = new ConcurrentHashMap<>(); + private final StackTraceExporter exporter; + + AccumulatingStagingArea(StackTraceExporter exporter) { + this.exporter = exporter; + } + + @Override + public void stage(String traceId, StackTrace stackTrace) { + stackTraces.compute( + traceId, + (id, stackTraces) -> { + if (stackTraces == null) { + stackTraces = new ArrayList<>(); + } + stackTraces.add(stackTrace); + return stackTraces; + }); + } + + @Override + public void empty(String traceId) { + List stackTraces = this.stackTraces.remove(traceId); + if (stackTraces != null) { + exporter.export(stackTraces); + } + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java index 32a868266..34ae768b7 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java @@ -16,6 +16,7 @@ package com.splunk.opentelemetry.profiler.snapshot; +import com.google.common.annotations.VisibleForTesting; import java.lang.management.ThreadInfo; import java.time.Instant; @@ -30,8 +31,8 @@ static StackTrace from(Instant timestamp, ThreadInfo thread) { private final String threadName; private final StackTraceElement[] stackFrames; - private StackTrace( - Instant timestamp, long threadId, String threadName, StackTraceElement[] stackFrames) { + @VisibleForTesting + StackTrace(Instant timestamp, long threadId, String threadName, StackTraceElement[] stackFrames) { this.timestamp = timestamp; this.threadId = threadId; this.threadName = threadName; diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java new file mode 100644 index 000000000..af27874b5 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java @@ -0,0 +1,24 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import java.util.List; + +/** Works in concert with the {@link StagingArea} to export a batch of {@link StackTrace}s */ +interface StackTraceExporter { + void export(List stackTraces); +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java new file mode 100644 index 000000000..bf59c1e73 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java @@ -0,0 +1,102 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.opentelemetry.sdk.trace.IdGenerator; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +class AccumulatingStagingAreaTest { + private final IdGenerator idGenerator = IdGenerator.random(); + private final InMemoryStackTraceExporter exporter = new InMemoryStackTraceExporter(); + private final AccumulatingStagingArea stagingArea = new AccumulatingStagingArea(exporter); + + @Test + void exportStackTracesToLogExporter() { + var traceId = idGenerator.generateTraceId(); + var stackTrace = Snapshotting.stackTrace().build(); + + stagingArea.stage(traceId, stackTrace); + stagingArea.empty(traceId); + + assertEquals(List.of(stackTrace), exporter.stackTraces()); + } + + @Test + void onlyExportStackTracesWhenAtLeastOneHasBeenStaged() { + var traceId = idGenerator.generateTraceId(); + stagingArea.empty(traceId); + assertEquals(Collections.emptyList(), exporter.stackTraces()); + } + + @Test + void exportMultipleStackTracesToLogExporter() { + var traceId = idGenerator.generateTraceId(); + var stackTrace1 = Snapshotting.stackTrace().withId(1).withName("one").build(); + var stackTrace2 = Snapshotting.stackTrace().withId(1).withName("two").build(); + + stagingArea.stage(traceId, stackTrace1); + stagingArea.stage(traceId, stackTrace2); + stagingArea.empty(traceId); + + assertEquals(List.of(stackTrace1, stackTrace2), exporter.stackTraces()); + } + + @Test + void exportStackTracesForOnlySpecifiedThread() { + var traceId1 = idGenerator.generateTraceId(); + var traceId2 = idGenerator.generateTraceId(); + var stackTrace1 = Snapshotting.stackTrace().withId(1).withName("one").build(); + var stackTrace2 = Snapshotting.stackTrace().withId(1).withName("two").build(); + + stagingArea.stage(traceId1, stackTrace1); + stagingArea.stage(traceId2, stackTrace2); + stagingArea.empty(traceId1); + + assertEquals(List.of(stackTrace1), exporter.stackTraces()); + } + + @Test + void exportStackTracesForMultipleThreads() { + var traceId1 = idGenerator.generateTraceId(); + var traceId2 = idGenerator.generateTraceId(); + var stackTrace1 = Snapshotting.stackTrace().withId(1).withName("one").build(); + var stackTrace2 = Snapshotting.stackTrace().withId(1).withName("two").build(); + + stagingArea.stage(traceId1, stackTrace1); + stagingArea.stage(traceId2, stackTrace2); + stagingArea.empty(traceId1); + stagingArea.empty(traceId2); + + assertEquals(List.of(stackTrace1, stackTrace2), exporter.stackTraces()); + } + + @Test + void stackTracesAreNotExportedMultipleTimes() { + var traceId = idGenerator.generateTraceId(); + var stackTrace = Snapshotting.stackTrace().build(); + + stagingArea.stage(traceId, stackTrace); + stagingArea.empty(traceId); + stagingArea.empty(traceId); + + assertEquals(List.of(stackTrace), exporter.stackTraces()); + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStackTraceExporter.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStackTraceExporter.java new file mode 100644 index 000000000..f05434697 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/InMemoryStackTraceExporter.java @@ -0,0 +1,38 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * In memory implementation of the {@link StackTraceExporter} interface that allows for direct + * access to the exported {@link StackTrace}s. Intended for testing use only. + */ +class InMemoryStackTraceExporter implements StackTraceExporter { + private final List stackTraces = new ArrayList<>(); + + @Override + public void export(List stackTraces) { + this.stackTraces.addAll(stackTraces); + } + + List stackTraces() { + return Collections.unmodifiableList(stackTraces); + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java index 6688199fc..0a6154dd0 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java @@ -16,10 +16,21 @@ package com.splunk.opentelemetry.profiler.snapshot; +import java.time.Instant; + class Snapshotting { static SnapshotProfilingSdkCustomizerBuilder customizer() { return new SnapshotProfilingSdkCustomizerBuilder(); } + static StackTraceBuilder stackTrace() { + var threadId = 1; + return new StackTraceBuilder() + .with(Instant.now()) + .withId(threadId) + .withName("thread-" + threadId) + .with(new RuntimeException()); + } + private Snapshotting() {} } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java new file mode 100644 index 000000000..30cda7fea --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java @@ -0,0 +1,50 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import java.time.Instant; + +class StackTraceBuilder { + private Instant timestamp; + private long threadId; + private String threadName; + private Exception exception; + + public StackTraceBuilder with(Instant timestamp) { + this.timestamp = timestamp; + return this; + } + + public StackTraceBuilder withId(long threadId) { + this.threadId = threadId; + return this; + } + + public StackTraceBuilder withName(String threadName) { + this.threadName = threadName; + return this; + } + + public StackTraceBuilder with(Exception exception) { + this.exception = exception; + return this; + } + + StackTrace build() { + return new StackTrace(timestamp, threadId, threadName, exception.getStackTrace()); + } +} From 28772e7080cd3ece8719f5f62962937c6f3ece41 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Thu, 6 Mar 2025 16:14:03 -0800 Subject: [PATCH 02/28] Add the AsyncStackTraceExporter. --- .../snapshot/AsyncStackTraceExporter.java | 40 +++++++++ .../profiler/exporter/InMemoryOtelLogger.java | 2 +- .../snapshot/AsyncStackTraceExporterTest.java | 84 +++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java new file mode 100644 index 000000000..8cd937e67 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -0,0 +1,40 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import io.opentelemetry.api.logs.Logger; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.BiFunction; + +class AsyncStackTraceExporter implements StackTraceExporter { + private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + private final Logger logger; + private final BiFunction, Runnable> workerFactory; + + AsyncStackTraceExporter( + Logger logger, BiFunction, Runnable> workerFactory) { + this.logger = logger; + this.workerFactory = workerFactory; + } + + @Override + public void export(List stackTraces) { + executor.submit(workerFactory.apply(logger, stackTraces)); + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java index c17f8d3a7..41f729d38 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java @@ -43,7 +43,7 @@ * access to the collected logs. Intended for testing use only. */ @SuppressWarnings("deprecation") // uses deprecated io.opentelemetry.sdk.logs.data.Body -class InMemoryOtelLogger implements Logger, AfterEachCallback { +public class InMemoryOtelLogger implements Logger, AfterEachCallback { private final List records = new ArrayList<>(); @Override diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java new file mode 100644 index 000000000..7219f8df6 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -0,0 +1,84 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; +import io.opentelemetry.api.logs.Logger; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; + +class AsyncStackTraceExporterTest { + private final InMemoryOtelLogger logger = new InMemoryOtelLogger(); + private final ObservableRunnable worker = new ObservableRunnable(); + private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, worker); + + @Test + void exportStackTrace() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().atMost(Duration.ofSeconds(5)).until(() -> !worker.stackTraces().isEmpty()); + + assertEquals(List.of(stackTrace), worker.stackTraces()); + assertEquals(1, worker.invocations()); + } + + @Test + void exportStackTraceMultipleTimes() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + exporter.export(List.of(stackTrace)); + exporter.export(List.of(stackTrace)); + await().atMost(Duration.ofSeconds(5)).until(() -> !worker.stackTraces().isEmpty()); + + assertEquals(3, worker.invocations()); + } + + static final class ObservableRunnable + implements Runnable, BiFunction, Runnable> { + private final List stackTraces = new CopyOnWriteArrayList<>(); + private final AtomicInteger invocations = new AtomicInteger(); + + @Override + public void run() { + invocations.incrementAndGet(); + } + + @Override + public Runnable apply(Logger logger, List stackTraces) { + this.stackTraces.addAll(stackTraces); + return this; + } + + List stackTraces() { + return Collections.unmodifiableList(stackTraces); + } + + int invocations() { + return invocations.get(); + } + } +} From 2eaf7e6108fac09d74cda4cc20eee4e1ead26a8c Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 08:35:43 -0700 Subject: [PATCH 03/28] Add StackTraceExporter implementation that emits stack traces translated into the pprof data format as otel logs. --- .../snapshot/AsyncStackTraceExporter.java | 78 +++++- .../profiler/snapshot/Pprof.java | 226 ++++++++++++++++++ .../profiler/snapshot/PprofTranslator.java | 58 +++++ .../ScheduledExecutorStackTraceSampler.java | 19 +- .../profiler/snapshot/StackTrace.java | 38 ++- .../profiler/exporter/InMemoryOtelLogger.java | 2 +- .../snapshot/AsyncStackTraceExporterTest.java | 175 +++++++++++--- .../snapshot/PprofTranslatorTest.java | 205 ++++++++++++++++ ...cheduledExecutorStackTraceSamplerTest.java | 100 +++++++- .../profiler/snapshot/Snapshotting.java | 15 +- .../profiler/snapshot/StackTraceBuilder.java | 22 +- 11 files changed, 879 insertions(+), 59 deletions(-) create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index 8cd937e67..c435dbb58 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -16,25 +16,87 @@ package com.splunk.opentelemetry.profiler.snapshot; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.PPROF_GZIP_BASE64; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.PROFILING_SOURCE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_TYPE; + +import com.google.common.annotations.VisibleForTesting; +import com.google.perftools.profiles.ProfileProto.Profile; +import com.splunk.opentelemetry.profiler.InstrumentationSource; +import com.splunk.opentelemetry.profiler.ProfilingDataType; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.logs.Severity; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.time.Clock; +import java.time.Instant; +import java.util.Base64; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.function.BiFunction; +import java.util.logging.Level; +import java.util.zip.GZIPOutputStream; class AsyncStackTraceExporter implements StackTraceExporter { + private static final java.util.logging.Logger logger = + java.util.logging.Logger.getLogger(AsyncStackTraceExporter.class.getName()); + private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - private final Logger logger; - private final BiFunction, Runnable> workerFactory; + private final PprofTranslator translator = new PprofTranslator(); + private final Logger otelLogger; + private final Clock clock; - AsyncStackTraceExporter( - Logger logger, BiFunction, Runnable> workerFactory) { - this.logger = logger; - this.workerFactory = workerFactory; + AsyncStackTraceExporter(Logger logger) { + this(logger, Clock.systemUTC()); + } + + @VisibleForTesting + AsyncStackTraceExporter(Logger logger, Clock clock) { + this.otelLogger = logger; + this.clock = clock; } @Override public void export(List stackTraces) { - executor.submit(workerFactory.apply(logger, stackTraces)); + executor.submit(pprofExporter(otelLogger, stackTraces)); + } + + private Runnable pprofExporter(Logger otelLogger, List stackTraces) { + return () -> { + try { + Profile profile = translator.translateToPprof(stackTraces); + otelLogger + .logRecordBuilder() + .setTimestamp(Instant.now(clock)) + .setSeverity(Severity.INFO) + .setAllAttributes(profilingAttributes()) + .setBody(serialize(profile)) + .emit(); + } catch (Exception e) { + logger.log(Level.SEVERE, "an exception was thrown", e); + } + }; + } + + private Attributes profilingAttributes() { + return Attributes.builder() + .put(SOURCE_TYPE, PROFILING_SOURCE) + .put(DATA_TYPE, ProfilingDataType.CPU.value()) + .put(DATA_FORMAT, PPROF_GZIP_BASE64) + .put(INSTRUMENTATION_SOURCE, InstrumentationSource.SNAPSHOT.value()) + .build(); + } + + private String serialize(Profile profile) throws IOException { + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (OutputStream outputStream = new GZIPOutputStream(Base64.getEncoder().wrap(byteStream))) { + profile.writeTo(outputStream); + } + return byteStream.toString(); } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java new file mode 100644 index 000000000..1822660a4 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java @@ -0,0 +1,226 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static com.google.perftools.profiles.ProfileProto.Function; +import static com.google.perftools.profiles.ProfileProto.Label; +import static com.google.perftools.profiles.ProfileProto.Line; +import static com.google.perftools.profiles.ProfileProto.Location; +import static com.google.perftools.profiles.ProfileProto.Profile; +import static com.google.perftools.profiles.ProfileProto.Sample; + +import io.opentelemetry.api.common.AttributeKey; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Adapted from the Splunk OpenTelemetry profiler here, + * which is itself adapted from Google's Bazel build system here. + */ +class Pprof { + private final Profile.Builder profileBuilder = Profile.newBuilder(); + private final StringTable stringTable = new StringTable(profileBuilder); + private final FunctionTable functionTable = new FunctionTable(profileBuilder, stringTable); + private final LocationTable locationTable = new LocationTable(profileBuilder, functionTable); + private int frameCount; + + Profile build() { + return profileBuilder.build(); + } + + void add(Sample sample) { + profileBuilder.addSample(sample); + } + + long getLocationId(StackTraceElement stackFrame) { + return locationTable.get(stackFrame); + } + + Label newLabel(AttributeKey key, String value) { + return newLabel(key.getKey(), label -> label.setStr(stringTable.get(value))); + } + + Label newLabel(AttributeKey key, long value) { + return newLabel(key.getKey(), value); + } + + Label newLabel(String key, long value) { + return newLabel(key, label -> label.setNum(value)); + } + + private Label newLabel(String name, Consumer valueSetter) { + Label.Builder label = Label.newBuilder(); + label.setKey(stringTable.get(name)); + valueSetter.accept(label); + return label.build(); + } + + public void incFrameCount() { + frameCount++; + } + + /** + * @return non unique stack frames in this pprof batch + */ + public int frameCount() { + return frameCount; + } + + // copied from + // https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/profiler/memory/AllocationTracker.java + private static class StringTable { + final Profile.Builder profile; + final Map table = new HashMap<>(); + long index = 0; + + StringTable(Profile.Builder profile) { + this.profile = profile; + get(""); // 0 is reserved for the empty string + } + + long get(String str) { + return table.computeIfAbsent( + str, + key -> { + profile.addStringTable(key); + return index++; + }); + } + } + + private static class LocationTable { + final Profile.Builder profile; + final FunctionTable functionTable; + final Map table = new HashMap<>(); + long index = 1; // 0 is reserved + + LocationTable(Profile.Builder profile, FunctionTable functionTable) { + this.profile = profile; + this.functionTable = functionTable; + } + + long get(StackTraceElement stackFrame) { + LocationKey locationKey = LocationKey.from(stackFrame); + Location location = + Location.newBuilder() + .setId(index) + .addLine( + Line.newBuilder() + .setFunctionId(functionTable.get(locationKey.functionKey)) + .setLine(locationKey.line)) + .build(); + return table.computeIfAbsent( + locationKey, + key -> { + profile.addLocation(location); + return index++; + }); + } + } + + private static class LocationKey { + private final FunctionKey functionKey; + private final long line; + + static LocationKey from(StackTraceElement stackFrame) { + return new LocationKey(FunctionKey.from(stackFrame), stackFrame.getLineNumber()); + } + + private LocationKey(FunctionKey functionKey, long line) { + this.functionKey = functionKey; + this.line = line; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LocationKey that = (LocationKey) o; + return line == that.line && Objects.equals(functionKey, that.functionKey); + } + + @Override + public int hashCode() { + return Objects.hash(functionKey, line); + } + } + + private static class FunctionTable { + final Profile.Builder profile; + final StringTable stringTable; + final Map table = new HashMap<>(); + long index = 1; // 0 is reserved + + FunctionTable(Profile.Builder profile, StringTable stringTable) { + this.profile = profile; + this.stringTable = stringTable; + } + + long get(FunctionKey functionKey) { + Function fn = + Function.newBuilder() + .setId(index) + .setFilename(stringTable.get(functionKey.file)) + .setName(stringTable.get(functionKey.className + "." + functionKey.function)) + .build(); + return table.computeIfAbsent( + functionKey, + key -> { + profile.addFunction(fn); + return index++; + }); + } + } + + private static class FunctionKey { + private final String file; + private final String className; + private final String function; + + static FunctionKey from(StackTraceElement stackFrame) { + return new FunctionKey( + stackFrame.getFileName() == null ? "Unknown Source" : stackFrame.getFileName(), + stackFrame.getClassName(), + stackFrame.getMethodName()); + } + + private FunctionKey(String file, String className, String function) { + this.file = file; + this.className = className; + this.function = function; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FunctionKey that = (FunctionKey) o; + return Objects.equals(file, that.file) + && Objects.equals(className, that.className) + && Objects.equals(function, that.function); + } + + @Override + public int hashCode() { + return Objects.hash(file, className, function); + } + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java new file mode 100644 index 000000000..437b65092 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java @@ -0,0 +1,58 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_NAME; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_PERIOD; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_TIME; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_ID; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_NAME; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID; + +import com.google.perftools.profiles.ProfileProto.Profile; +import com.google.perftools.profiles.ProfileProto.Sample; +import java.util.List; + +class PprofTranslator { + public Profile translateToPprof(List stackTraces) { + Pprof pprof = new Pprof(); + for (StackTrace stackTrace : stackTraces) { + pprof.add(translateToPprofSample(stackTrace, pprof)); + } + return pprof.build(); + } + + private Sample translateToPprofSample(StackTrace stackTrace, Pprof pprof) { + Sample.Builder sample = Sample.newBuilder(); + sample.addLabel(pprof.newLabel(THREAD_ID, stackTrace.getThreadId())); + sample.addLabel(pprof.newLabel(THREAD_NAME, stackTrace.getThreadName())); + sample.addLabel( + pprof.newLabel(THREAD_STATE, stackTrace.getThreadState().toString().toLowerCase())); + sample.addLabel(pprof.newLabel(SOURCE_EVENT_NAME, "snapshot-profiling")); + sample.addLabel(pprof.newLabel(SOURCE_EVENT_TIME, stackTrace.getTimestamp().toEpochMilli())); + sample.addLabel(pprof.newLabel(SOURCE_EVENT_PERIOD, stackTrace.getDuration().toMillis())); + + for (StackTraceElement stackFrame : stackTrace.getStackFrames()) { + sample.addLocationId(pprof.getLocationId(stackFrame)); + // pprof.incFrameCount(); + } + sample.addLabel(pprof.newLabel(TRACE_ID, stackTrace.getTraceId())); + + return sample.build(); + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java index 9175b0043..1884af7ff 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java @@ -64,7 +64,8 @@ public void start(SpanContext spanContext) { ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); samplers.put(spanContext.getTraceId(), scheduler); scheduler.scheduleAtFixedRate( - new StackTraceGatherer(spanContext.getTraceId(), Thread.currentThread().getId()), + new StackTraceGatherer( + samplingPeriod, spanContext.getTraceId(), Thread.currentThread().getId()), SCHEDULER_INITIAL_DELAY, samplingPeriod.toMillis(), TimeUnit.MILLISECONDS); @@ -80,10 +81,13 @@ public void stop(SpanContext spanContext) { } class StackTraceGatherer implements Runnable { + private final Duration samplingPeriod; private final String traceId; private final long threadId; + private Instant lastExecution; - StackTraceGatherer(String traceId, long threadId) { + StackTraceGatherer(Duration samplingPeroid, String traceId, long threadId) { + this.samplingPeriod = samplingPeroid; this.traceId = traceId; this.threadId = threadId; } @@ -93,10 +97,12 @@ public void run() { Instant now = Instant.now(); try { ThreadInfo threadInfo = threadMXBean.getThreadInfo(threadId, MAX_ENTRY_DEPTH); - StackTrace stackTrace = StackTrace.from(now, threadInfo); + StackTrace stackTrace = StackTrace.from(now, sampleDuration(now), traceId, threadInfo); stagingArea.stage(traceId, stackTrace); } catch (Exception e) { logger.log(Level.SEVERE, e, samplerErrorMessage(traceId, threadId)); + } finally { + lastExecution = now; } } @@ -107,5 +113,12 @@ private Supplier samplerErrorMessage(String traceId, long threadId) { + "' on profiled thread " + threadId; } + + private Duration sampleDuration(Instant now) { + if (lastExecution == null) { + return samplingPeriod; + } + return Duration.between(lastExecution, now); + } } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java index 34ae768b7..eab765901 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTrace.java @@ -18,24 +18,44 @@ import com.google.common.annotations.VisibleForTesting; import java.lang.management.ThreadInfo; +import java.time.Duration; import java.time.Instant; class StackTrace { - static StackTrace from(Instant timestamp, ThreadInfo thread) { + static StackTrace from(Instant timestamp, Duration duration, String traceId, ThreadInfo thread) { return new StackTrace( - timestamp, thread.getThreadId(), thread.getThreadName(), thread.getStackTrace()); + timestamp, + duration, + traceId, + thread.getThreadId(), + thread.getThreadName(), + thread.getThreadState(), + thread.getStackTrace()); } private final Instant timestamp; + private final Duration duration; + private final String traceId; private final long threadId; private final String threadName; + private final Thread.State threadState; private final StackTraceElement[] stackFrames; @VisibleForTesting - StackTrace(Instant timestamp, long threadId, String threadName, StackTraceElement[] stackFrames) { + StackTrace( + Instant timestamp, + Duration duration, + String traceId, + long threadId, + String threadName, + Thread.State threadState, + StackTraceElement[] stackFrames) { this.timestamp = timestamp; + this.duration = duration; + this.traceId = traceId; this.threadId = threadId; this.threadName = threadName; + this.threadState = threadState; this.stackFrames = stackFrames; } @@ -43,6 +63,14 @@ Instant getTimestamp() { return timestamp; } + Duration getDuration() { + return duration; + } + + String getTraceId() { + return traceId; + } + long getThreadId() { return threadId; } @@ -51,6 +79,10 @@ String getThreadName() { return threadName; } + Thread.State getThreadState() { + return threadState; + } + StackTraceElement[] getStackFrames() { return stackFrames; } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java index 41f729d38..2ad68aeb0 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/InMemoryOtelLogger.java @@ -56,7 +56,7 @@ public void afterEach(ExtensionContext extensionContext) { records.clear(); } - List records() { + public List records() { return Collections.unmodifiableList(records); } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index 7219f8df6..1605d9642 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -16,69 +16,168 @@ package com.splunk.opentelemetry.profiler.snapshot; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_TYPE; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; -import io.opentelemetry.api.logs.Logger; -import java.time.Duration; -import java.util.Collections; +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Base64; import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; +import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.Test; class AsyncStackTraceExporterTest { private final InMemoryOtelLogger logger = new InMemoryOtelLogger(); - private final ObservableRunnable worker = new ObservableRunnable(); - private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, worker); + private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger); @Test - void exportStackTrace() { + void exportStackTraceAsOpenTelemetryLog() { var stackTrace = Snapshotting.stackTrace().build(); exporter.export(List.of(stackTrace)); - await().atMost(Duration.ofSeconds(5)).until(() -> !worker.stackTraces().isEmpty()); + await().until(() -> !logger.records().isEmpty()); - assertEquals(List.of(stackTrace), worker.stackTraces()); - assertEquals(1, worker.invocations()); + assertEquals(1, logger.records().size()); } @Test - void exportStackTraceMultipleTimes() { + void exportMultipleStackTraceAsSingleOpenTelemetryLog() { + var one = Snapshotting.stackTrace().with(new RuntimeException()).build(); + var two = Snapshotting.stackTrace().with(new IllegalArgumentException()).build(); + var three = Snapshotting.stackTrace().with(new NullPointerException()).build(); + + exporter.export(List.of(one, two, three)); + await().until(() -> !logger.records().isEmpty()); + + assertEquals(1, logger.records().size()); + } + + @Test + void setTimestampOnLogMessage() { var stackTrace = Snapshotting.stackTrace().build(); + var timestamp = Instant.ofEpochMilli(System.currentTimeMillis()); + var clock = Clock.fixed(timestamp, ZoneId.systemDefault()); + var exporter = new AsyncStackTraceExporter(logger, clock); exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var logRecord = logger.records().get(0); + assertEquals(timestamp, Instant.ofEpochSecond(0L, logRecord.getTimestampEpochNanos())); + } + + @Test + void setSeverityToInfoOnLogMessage() { + var stackTrace = Snapshotting.stackTrace().build(); + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + assertEquals(Severity.INFO, logger.records().get(0).getSeverity()); + } + + @Test + void encodedLogBodyIsPprofProtobufMessage() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var logRecord = logger.records().get(0); + assertDoesNotThrow(() -> Profile.parseFrom(deserialize(logRecord))); + } + + @Test + void encodeLogBodyUsingBase64() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var logRecord = logger.records().get(0); + assertDoesNotThrow(() -> decode(logRecord)); + } + + @Test + void logBodyIsGZipped() { + var stackTrace = Snapshotting.stackTrace().build(); + exporter.export(List.of(stackTrace)); - await().atMost(Duration.ofSeconds(5)).until(() -> !worker.stackTraces().isEmpty()); + await().until(() -> !logger.records().isEmpty()); - assertEquals(3, worker.invocations()); + var logRecord = logger.records().get(0); + assertDoesNotThrow( + () -> { + var bytes = new ByteArrayInputStream(decode(logRecord)); + var inputStream = new GZIPInputStream(bytes); + inputStream.readAllBytes(); + }); } - static final class ObservableRunnable - implements Runnable, BiFunction, Runnable> { - private final List stackTraces = new CopyOnWriteArrayList<>(); - private final AtomicInteger invocations = new AtomicInteger(); - - @Override - public void run() { - invocations.incrementAndGet(); - } - - @Override - public Runnable apply(Logger logger, List stackTraces) { - this.stackTraces.addAll(stackTraces); - return this; - } - - List stackTraces() { - return Collections.unmodifiableList(stackTraces); - } - - int invocations() { - return invocations.get(); - } + private byte[] deserialize(LogRecordData logRecord) throws IOException { + var bytes = new ByteArrayInputStream(decode(logRecord)); + var inputStream = new GZIPInputStream(bytes); + return inputStream.readAllBytes(); + } + + private byte[] decode(LogRecordData logRecord) { + return Base64.getDecoder().decode(logRecord.getBody().asString()); + } + + @Test + void includeSourceTypeOpenTelemetryAttribute() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var attributes = logger.records().get(0).getAttributes(); + assertThat(attributes.asMap()).containsEntry(SOURCE_TYPE, "otel.profiling"); + } + + @Test + void includeDataTypeOpenTelemetryAttributeWithValueOfCpu() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var attributes = logger.records().get(0).getAttributes(); + assertThat(attributes.asMap()).containsEntry(DATA_TYPE, "cpu"); + } + + @Test + void includeDataFormatOpenTelemetryAttributeWithValueOfGzipBase64() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var attributes = logger.records().get(0).getAttributes(); + assertThat(attributes.asMap()).containsEntry(DATA_FORMAT, "pprof-gzip-base64"); + } + + @Test + void includeInstrumentationSourceOpenTelemetryAttributeWithValueOfSnapshot() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var attributes = logger.records().get(0).getAttributes(); + assertThat(attributes.asMap()).containsEntry(INSTRUMENTATION_SOURCE, "snapshot"); } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java new file mode 100644 index 000000000..da7191081 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java @@ -0,0 +1,205 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.perftools.profiles.ProfileProto.Location; +import com.google.perftools.profiles.ProfileProto.Profile; +import com.google.perftools.profiles.ProfileProto.Sample; +import com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +class PprofTranslatorTest { + private final PprofTranslator translator = new PprofTranslator(); + + @Test + void allStackFramesAreInPprofStringTable() throws Exception { + var exception = new RuntimeException(); + var stackTrace = Snapshotting.stackTrace().with(exception).build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + var table = profile.getStringTableList(); + + assertThat(table).containsAll(fullyQualifiedMethodNames(exception.getStackTrace())); + } + + private List fullyQualifiedMethodNames(StackTraceElement[] stackTrace) { + return Arrays.stream(stackTrace) + .map(stackFrame -> stackFrame.getClassName() + "." + stackFrame.getMethodName()) + .collect(Collectors.toList()); + } + + @Test + void allStackFramesIncludedInSample() { + var exception = new RuntimeException(); + var stackTrace = Snapshotting.stackTrace().with(exception).build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + + var expectedStackTrace = removeModuleInfo(exception.getStackTrace()); + var reportedStackTrace = toStackTrace(profile.getSample(0), profile); + assertEquals(expectedStackTrace.size(), reportedStackTrace.size()); + for (int i = 0; i < expectedStackTrace.size(); i++) { + var expectedStackFrame = expectedStackTrace.get(0); + var actualStackFrame = reportedStackTrace.get(0); + assertAll( + () -> + assertEquals( + expectedStackFrame.getClassLoaderName(), actualStackFrame.getClassLoaderName()), + () -> assertEquals(expectedStackFrame.getModuleName(), actualStackFrame.getModuleName()), + () -> + assertEquals( + expectedStackFrame.getModuleVersion(), actualStackFrame.getModuleVersion()), + () -> assertEquals(expectedStackFrame.getClassName(), actualStackFrame.getClassName()), + () -> assertEquals(expectedStackFrame.getLineNumber(), actualStackFrame.getLineNumber()), + () -> assertEquals(expectedStackFrame.getMethodName(), actualStackFrame.getMethodName()), + () -> assertEquals(expectedStackFrame.getFileName(), actualStackFrame.getFileName())); + } + } + + private List toStackTrace(Sample sample, Profile profile) { + List stackTrace = new ArrayList<>(); + for (var locationId : sample.getLocationIdList()) { + var location = profile.getLocation(locationId.intValue() - 1); + stackTrace.add(toStackTrace(location, profile)); + } + return stackTrace; + } + + private StackTraceElement toStackTrace(Location location, Profile profile) { + var line = location.getLine(0); + var functionId = line.getFunctionId(); + var function = profile.getFunction((int) functionId - 1); + var fileName = profile.getStringTable((int) function.getFilename()); + var functionName = profile.getStringTable((int) function.getName()); + + var declaringClass = functionName.substring(0, functionName.lastIndexOf('.')); + var methodName = functionName.substring(functionName.lastIndexOf('.') + 1); + return new StackTraceElement(declaringClass, methodName, fileName, (int) line.getLine()); + } + + private List removeModuleInfo(StackTraceElement[] originalStackTrace) { + List stackTrace = new ArrayList<>(); + for (var stackFrame : originalStackTrace) { + stackTrace.add(removeModuleInfo(stackFrame)); + } + return stackTrace; + } + + private StackTraceElement removeModuleInfo(StackTraceElement stackFrame) { + return new StackTraceElement( + stackFrame.getClassName(), + stackFrame.getMethodName(), + stackFrame.getFileName(), + stackFrame.getLineNumber()); + } + + @Test + void includeThreadInformationInSamples() { + var stackTrace = Snapshotting.stackTrace().build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + var sample = profile.getSample(0); + + var labels = toLabelString(sample, profile); + assertThat(labels) + .containsEntry(ProfilingSemanticAttributes.THREAD_ID.getKey(), stackTrace.getThreadId()); + assertThat(labels) + .containsEntry( + ProfilingSemanticAttributes.THREAD_NAME.getKey(), stackTrace.getThreadName()); + assertThat(labels) + .containsEntry( + ProfilingSemanticAttributes.THREAD_STATE.getKey(), + stackTrace.getThreadState().toString().toLowerCase()); + } + + @Test + void includeTraceIdInformationInSamples() { + var stackTrace = Snapshotting.stackTrace().build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + var sample = profile.getSample(0); + + var labels = toLabelString(sample, profile); + assertThat(labels) + .containsEntry(ProfilingSemanticAttributes.TRACE_ID.getKey(), stackTrace.getTraceId()); + } + + @Test + void includeSourceEventNameAsSnapshotProfilingInSamples() { + var stackTrace = Snapshotting.stackTrace().build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + var sample = profile.getSample(0); + + var labels = toLabelString(sample, profile); + assertThat(labels) + .containsEntry( + ProfilingSemanticAttributes.SOURCE_EVENT_NAME.getKey(), "snapshot-profiling"); + } + + @Test + void includeStackTraceTimestampInSamples() { + var stackTrace = Snapshotting.stackTrace().build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + var sample = profile.getSample(0); + + var labels = toLabelString(sample, profile); + assertThat(labels) + .containsEntry( + ProfilingSemanticAttributes.SOURCE_EVENT_TIME.getKey(), + stackTrace.getTimestamp().toEpochMilli()); + } + + @Test + void includeStackTraceDurationInSamples() { + var stackTrace = Snapshotting.stackTrace().build(); + + var profile = translator.translateToPprof(List.of(stackTrace)); + var sample = profile.getSample(0); + + var labels = toLabelString(sample, profile); + assertThat(labels) + .containsEntry( + ProfilingSemanticAttributes.SOURCE_EVENT_PERIOD.getKey(), + stackTrace.getDuration().toMillis()); + } + + private Map toLabelString(Sample sample, Profile profile) { + var labels = new HashMap(); + for (var label : sample.getLabelList()) { + var stringTableIndex = label.getKey(); + var key = profile.getStringTable((int) stringTableIndex); + if (label.getStr() > 0) { + labels.put(key, profile.getStringTable((int) label.getStr())); + } else { + labels.put(key, label.getNum()); + } + } + return labels; + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java index 75c4bf61f..c994c9cf1 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java @@ -18,26 +18,30 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.TraceFlags; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.sdk.trace.IdGenerator; import java.time.Duration; +import java.time.Instant; import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; class ScheduledExecutorStackTraceSamplerTest { private static final Duration HALF_SECOND = Duration.ofMillis(500); - private static final Duration PERIOD = Duration.ofMillis(20); + private static final Duration SAMPLING_PERIOD = Duration.ofMillis(20); private final IdGenerator idGenerator = IdGenerator.random(); private final InMemoryStagingArea staging = new InMemoryStagingArea(); private final ScheduledExecutorStackTraceSampler sampler = - new ScheduledExecutorStackTraceSampler(staging, PERIOD); + new ScheduledExecutorStackTraceSampler(staging, SAMPLING_PERIOD); @Test void takeStackTraceSampleForGivenThread() { @@ -54,7 +58,7 @@ void takeStackTraceSampleForGivenThread() { @Test void continuallySampleThreadForStackTraces() { var spanContext = randomSpanContext(); - int expectedSamples = (int) HALF_SECOND.dividedBy(PERIOD.multipliedBy(2)); + int expectedSamples = (int) HALF_SECOND.dividedBy(SAMPLING_PERIOD.multipliedBy(2)); try { sampler.start(spanContext); @@ -67,7 +71,7 @@ void continuallySampleThreadForStackTraces() { @Test void emptyStagingAreaAfterSamplingStops() { var spanContext = randomSpanContext(); - int expectedSamples = (int) HALF_SECOND.dividedBy(PERIOD.multipliedBy(2)); + int expectedSamples = (int) HALF_SECOND.dividedBy(SAMPLING_PERIOD.multipliedBy(2)); try { sampler.start(spanContext); @@ -80,6 +84,7 @@ void emptyStagingAreaAfterSamplingStops() { } @Test + @Disabled void onlyTakeStackTraceSamplesForOneThreadPerTrace() { var latch = new CountDownLatch(1); var traceId = idGenerator.generateTraceId(); @@ -109,6 +114,93 @@ private Thread startAndStopSampler(SpanContext spanContext, CountDownLatch latch }); } + @Test + void includeTimestampOnStackTraces() { + var now = Instant.now(); + var spanContext = randomSpanContext(); + + try { + sampler.start(spanContext); + await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + + var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); + assertThat(stackTrace.getTimestamp()).isNotNull().isAfter(now); + } finally { + sampler.stop(spanContext); + } + } + + @Test + void includeDefaultSamplingPeriodOnFirstRecordedStackTraces() { + var spanContext = randomSpanContext(); + + try { + sampler.start(spanContext); + await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + + var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); + assertThat(stackTrace.getDuration()).isNotNull().isEqualTo(SAMPLING_PERIOD); + } finally { + sampler.stop(spanContext); + } + } + + @Test + void calculateSamplingPeriodAfterFirstRecordedStackTraces() { + var spanContext = randomSpanContext(); + + try { + sampler.start(spanContext); + await().until(() -> staging.allStackTraces().size() > 1); + + var stackTrace = staging.allStackTraces().stream().skip(1).findFirst().orElseThrow(); + assertThat(stackTrace.getDuration()) + .isNotNull() + .isCloseTo(SAMPLING_PERIOD, Duration.ofMillis(5)); + } finally { + sampler.stop(spanContext); + } + } + + @Test + void includeTraceIdOnStackTraces() { + var spanContext = randomSpanContext(); + + try { + sampler.start(spanContext); + await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + + var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); + assertEquals(spanContext.getTraceId(), stackTrace.getTraceId()); + } finally { + sampler.stop(spanContext); + } + } + + @Test + void includeThreadDetailsOnStackTraces() { + var traceId = idGenerator.generateTraceId(); + var spanContext = randomSpanContext(traceId); + var latch = new CountDownLatch(1); + try { + var thread = startAndStopSampler(spanContext, latch); + + thread.start(); + await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + + var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); + assertAll( + () -> assertEquals(thread.getId(), stackTrace.getThreadId()), + () -> assertEquals(thread.getName(), stackTrace.getThreadName()), + () -> assertNotNull(stackTrace.getThreadState()), + () -> assertThat(stackTrace.getStackFrames()).isNotEmpty()); + + latch.countDown(); + } finally { + sampler.stop(spanContext); + } + } + private SpanContext randomSpanContext() { return randomSpanContext(idGenerator.generateTraceId()); } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java index 0a6154dd0..9dfdc4d09 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/Snapshotting.java @@ -16,21 +16,34 @@ package com.splunk.opentelemetry.profiler.snapshot; +import io.opentelemetry.sdk.trace.IdGenerator; +import java.time.Duration; import java.time.Instant; +import java.util.Random; class Snapshotting { + private static final Random RANDOM = new Random(); + private static final IdGenerator ID_GENERATOR = IdGenerator.random(); + static SnapshotProfilingSdkCustomizerBuilder customizer() { return new SnapshotProfilingSdkCustomizerBuilder(); } static StackTraceBuilder stackTrace() { - var threadId = 1; + var threadId = RANDOM.nextLong(10_000); return new StackTraceBuilder() .with(Instant.now()) + .with(Duration.ofMillis(20)) + .withTraceId(randomTraceId()) .withId(threadId) .withName("thread-" + threadId) + .with(Thread.State.WAITING) .with(new RuntimeException()); } + static String randomTraceId() { + return ID_GENERATOR.generateTraceId(); + } + private Snapshotting() {} } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java index 30cda7fea..5143b3b27 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceBuilder.java @@ -16,12 +16,16 @@ package com.splunk.opentelemetry.profiler.snapshot; +import java.time.Duration; import java.time.Instant; class StackTraceBuilder { private Instant timestamp; + private Duration duration; + private String traceId; private long threadId; private String threadName; + private Thread.State state; private Exception exception; public StackTraceBuilder with(Instant timestamp) { @@ -29,6 +33,16 @@ public StackTraceBuilder with(Instant timestamp) { return this; } + public StackTraceBuilder with(Duration duration) { + this.duration = duration; + return this; + } + + public StackTraceBuilder withTraceId(String traceId) { + this.traceId = traceId; + return this; + } + public StackTraceBuilder withId(long threadId) { this.threadId = threadId; return this; @@ -39,12 +53,18 @@ public StackTraceBuilder withName(String threadName) { return this; } + public StackTraceBuilder with(Thread.State state) { + this.state = state; + return this; + } + public StackTraceBuilder with(Exception exception) { this.exception = exception; return this; } StackTrace build() { - return new StackTrace(timestamp, threadId, threadName, exception.getStackTrace()); + return new StackTrace( + timestamp, duration, traceId, threadId, threadName, state, exception.getStackTrace()); } } From b78aa4b1b2079b15592629553a404d0cc7d2eb49 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 13:26:40 -0700 Subject: [PATCH 04/28] Add the StackTraceExporterProvider. --- .../profiler/snapshot/StackTraceExporter.java | 2 ++ .../snapshot/StackTraceExporterProvider.java | 22 +++++++++++++++++++ .../StackTraceExporterProviderTest.java | 21 ++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java index af27874b5..e583ba3ad 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporter.java @@ -20,5 +20,7 @@ /** Works in concert with the {@link StagingArea} to export a batch of {@link StackTrace}s */ interface StackTraceExporter { + StackTraceExporter NOOP = stackTraces -> {}; + void export(List stackTraces); } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java new file mode 100644 index 000000000..84f4f67db --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java @@ -0,0 +1,22 @@ +package com.splunk.opentelemetry.profiler.snapshot; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +class StackTraceExporterProvider implements Supplier { + private StackTraceExporter exporter; + + @Override + public StackTraceExporter get() { + if (exporter == null) { + return StackTraceExporter.NOOP; + } + return exporter; + } + + void configure(StackTraceExporter exporter) { + AtomicReference exporterRef = new AtomicReference<>(); + exporterRef.set(exporter); + this.exporter = exporter; + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java new file mode 100644 index 000000000..d8fda8f37 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java @@ -0,0 +1,21 @@ +package com.splunk.opentelemetry.profiler.snapshot; + +import static org.junit.jupiter.api.Assertions.assertSame; + +import org.junit.jupiter.api.Test; + +class StackTraceExporterProviderTest { + private final StackTraceExporterProvider provider = new StackTraceExporterProvider(); + + @Test + void provideNoopExporterWhenNotConfigured() { + assertSame(StackTraceExporter.NOOP, provider.get()); + } + + @Test + void providedConfiguredExporter() { + var exporter = new InMemoryStackTraceExporter(); + provider.configure(exporter); + assertSame(exporter, provider.get()); + } +} From cff68ecefd10e74cfd8b3953982d5dbf6ab806b9 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 13:39:32 -0700 Subject: [PATCH 05/28] Update AccumulatingStagingArea to accept a Supplier of StackTraceExporters. --- .../snapshot/AccumulatingStagingArea.java | 7 ++++--- .../snapshot/StackTraceExporterProvider.java | 16 ++++++++++++++++ .../snapshot/AccumulatingStagingAreaTest.java | 2 +- .../snapshot/StackTraceExporterProviderTest.java | 16 ++++++++++++++++ 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java index 585b9e4b6..3af6f4778 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingArea.java @@ -20,12 +20,13 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.Supplier; class AccumulatingStagingArea implements StagingArea { private final ConcurrentMap> stackTraces = new ConcurrentHashMap<>(); - private final StackTraceExporter exporter; + private final Supplier exporter; - AccumulatingStagingArea(StackTraceExporter exporter) { + AccumulatingStagingArea(Supplier exporter) { this.exporter = exporter; } @@ -46,7 +47,7 @@ public void stage(String traceId, StackTrace stackTrace) { public void empty(String traceId) { List stackTraces = this.stackTraces.remove(traceId); if (stackTraces != null) { - exporter.export(stackTraces); + exporter.get().export(stackTraces); } } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java index 84f4f67db..d6ecd7732 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java @@ -1,3 +1,19 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.splunk.opentelemetry.profiler.snapshot; import java.util.concurrent.atomic.AtomicReference; diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java index bf59c1e73..7177abda6 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AccumulatingStagingAreaTest.java @@ -26,7 +26,7 @@ class AccumulatingStagingAreaTest { private final IdGenerator idGenerator = IdGenerator.random(); private final InMemoryStackTraceExporter exporter = new InMemoryStackTraceExporter(); - private final AccumulatingStagingArea stagingArea = new AccumulatingStagingArea(exporter); + private final AccumulatingStagingArea stagingArea = new AccumulatingStagingArea(() -> exporter); @Test void exportStackTracesToLogExporter() { diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java index d8fda8f37..a1925af68 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java @@ -1,3 +1,19 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.splunk.opentelemetry.profiler.snapshot; import static org.junit.jupiter.api.Assertions.assertSame; From 4ec661d7692d8c803d20c12f065476cb04e29f38 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 15:52:23 -0700 Subject: [PATCH 06/28] Add an AgentListener to configure the Otel logger for for the snapshot profiler. --- profiler/build.gradle.kts | 1 + .../profiler/OtelLoggerFactory.java | 60 ++++ .../snapshot/StackTraceExporterActivator.java | 48 +++ .../snapshot/StackTraceExporterProvider.java | 13 +- .../profiler/OtelLoggerFactoryTest.java | 68 +++++ .../StackTraceExporterActivatorTest.java | 66 +++++ .../StackTraceExporterProviderTest.java | 15 +- .../OpenTelemetrySdkExtension.java | 278 ++++++++++++++++++ 8 files changed, 545 insertions(+), 4 deletions(-) create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java create mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/OtelLoggerFactoryTest.java create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java create mode 100644 profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java diff --git a/profiler/build.gradle.kts b/profiler/build.gradle.kts index 8db13e10e..e6ea278fd 100644 --- a/profiler/build.gradle.kts +++ b/profiler/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { annotationProcessor("com.google.auto.service:auto-service") compileOnly("com.google.auto.service:auto-service") + testImplementation(project(":custom")) testImplementation("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api") testImplementation("io.grpc:grpc-netty") testImplementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api") diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java new file mode 100644 index 000000000..146da43a3 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java @@ -0,0 +1,60 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler; + +import com.google.common.annotations.VisibleForTesting; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.sdk.logs.LogRecordProcessor; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.LogRecordExporter; +import io.opentelemetry.sdk.logs.export.SimpleLogRecordProcessor; +import io.opentelemetry.sdk.resources.Resource; +import java.util.function.Function; + +public class OtelLoggerFactory { + private final Function logRecordExporter; + + public OtelLoggerFactory() { + this(LogExporterBuilder::fromConfig); + } + + @VisibleForTesting + OtelLoggerFactory(Function logRecordExporter) { + this.logRecordExporter = logRecordExporter; + } + + public Logger build(ConfigProperties properties, Resource resource) { + LogRecordExporter exporter = createLogRecordExporter(properties); + LogRecordProcessor processor = SimpleLogRecordProcessor.create(exporter); + return buildOtelLogger(processor, resource); + } + + private LogRecordExporter createLogRecordExporter(ConfigProperties properties) { + return logRecordExporter.apply(properties); + } + + private Logger buildOtelLogger(LogRecordProcessor logProcessor, Resource resource) { + return SdkLoggerProvider.builder() + .addLogRecordProcessor(logProcessor) + .setResource(resource) + .build() + .loggerBuilder(ProfilingSemanticAttributes.OTEL_INSTRUMENTATION_NAME) + .setInstrumentationVersion(ProfilingSemanticAttributes.OTEL_INSTRUMENTATION_VERSION) + .build(); + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java new file mode 100644 index 000000000..b68d46b42 --- /dev/null +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java @@ -0,0 +1,48 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER; + +import com.google.auto.service.AutoService; +import com.splunk.opentelemetry.profiler.OtelLoggerFactory; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.javaagent.extension.AgentListener; +import io.opentelemetry.sdk.autoconfigure.AutoConfigureUtil; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.sdk.resources.Resource; + +@AutoService(AgentListener.class) +public class StackTraceExporterActivator implements AgentListener { + private final OtelLoggerFactory otelLoggerFactory = new OtelLoggerFactory(); + + @Override + public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) { + ConfigProperties properties = AutoConfigureUtil.getConfig(autoConfiguredOpenTelemetrySdk); + if (snapshotProfilingEnabled(properties)) { + Resource resource = AutoConfigureUtil.getResource(autoConfiguredOpenTelemetrySdk); + Logger logger = otelLoggerFactory.build(properties, resource); + AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger); + StackTraceExporterProvider.INSTANCE.configure(exporter); + } + } + + private boolean snapshotProfilingEnabled(ConfigProperties config) { + return config.getBoolean(CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, false); + } +} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java index d6ecd7732..72c0320a3 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProvider.java @@ -16,10 +16,12 @@ package com.splunk.opentelemetry.profiler.snapshot; -import java.util.concurrent.atomic.AtomicReference; +import com.google.common.annotations.VisibleForTesting; import java.util.function.Supplier; class StackTraceExporterProvider implements Supplier { + public static final StackTraceExporterProvider INSTANCE = new StackTraceExporterProvider(); + private StackTraceExporter exporter; @Override @@ -31,8 +33,13 @@ public StackTraceExporter get() { } void configure(StackTraceExporter exporter) { - AtomicReference exporterRef = new AtomicReference<>(); - exporterRef.set(exporter); this.exporter = exporter; } + + @VisibleForTesting + void reset() { + exporter = null; + } + + private StackTraceExporterProvider() {} } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/OtelLoggerFactoryTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/OtelLoggerFactoryTest.java new file mode 100644 index 000000000..309d006f4 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/OtelLoggerFactoryTest.java @@ -0,0 +1,68 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +class OtelLoggerFactoryTest { + private final InMemoryLogRecordExporter exporter = InMemoryLogRecordExporter.create(); + private final OtelLoggerFactory factory = new OtelLoggerFactory(properties -> exporter); + + @Test + void configureLoggerWithProfilingInstrumentationScopeName() { + var properties = DefaultConfigProperties.create(Collections.emptyMap()); + var resource = Resource.getDefault(); + + var logger = factory.build(properties, resource); + logger.logRecordBuilder().setBody("test").emit(); + + var logRecord = exporter.getFinishedLogRecordItems().get(0); + assertEquals("otel.profiling", logRecord.getInstrumentationScopeInfo().getName()); + } + + @Test + void configureLoggerWithProfilingInstrumentationVersion() { + var properties = DefaultConfigProperties.create(Collections.emptyMap()); + var resource = Resource.getDefault(); + + var logger = factory.build(properties, resource); + logger.logRecordBuilder().setBody("test").emit(); + + var logRecord = exporter.getFinishedLogRecordItems().get(0); + assertEquals("0.1.0", logRecord.getInstrumentationScopeInfo().getVersion()); + } + + @Test + void configureLoggerWithOpenTelemetryResource() { + var properties = DefaultConfigProperties.create(Collections.emptyMap()); + var resource = Resource.create(Attributes.of(AttributeKey.stringKey("test"), "value")); + + var logger = factory.build(properties, resource); + logger.logRecordBuilder().setBody("test").emit(); + + var logRecord = exporter.getFinishedLogRecordItems().get(0); + assertEquals(resource, logRecord.getResource()); + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java new file mode 100644 index 000000000..a0ec49d01 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java @@ -0,0 +1,66 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +class StackTraceExporterActivatorTest { + @AfterEach + void tearDown() { + StackTraceExporterProvider.INSTANCE.reset(); + } + + @Nested + class SnapshotProfilingEnabled { + @RegisterExtension + public final io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension s = + io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension.configure() + .withProperty("splunk.snapshot.profiler.enabled", "true") + .with(new StackTraceExporterActivator()) + .build(); + + @Test + void configureStackTraceExporterProvider() { + var exporter = StackTraceExporterProvider.INSTANCE.get(); + assertNotSame(StackTraceExporter.NOOP, exporter); + assertInstanceOf(AsyncStackTraceExporter.class, exporter); + } + } + + @Nested + class SnapshotProfilingDisabled { + @RegisterExtension + public final io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension s = + io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension.configure() + .withProperty("splunk.snapshot.profiler.enabled", "false") + .with(new StackTraceExporterActivator()) + .build(); + + @Test + void doNotConfigureStackTraceExporterProvider() { + var exporter = StackTraceExporterProvider.INSTANCE.get(); + assertSame(StackTraceExporter.NOOP, exporter); + } + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java index a1925af68..a6f5ebd24 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterProviderTest.java @@ -18,10 +18,16 @@ import static org.junit.jupiter.api.Assertions.assertSame; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; class StackTraceExporterProviderTest { - private final StackTraceExporterProvider provider = new StackTraceExporterProvider(); + private final StackTraceExporterProvider provider = StackTraceExporterProvider.INSTANCE; + + @AfterEach + void tearDown() { + provider.reset(); + } @Test void provideNoopExporterWhenNotConfigured() { @@ -34,4 +40,11 @@ void providedConfiguredExporter() { provider.configure(exporter); assertSame(exporter, provider.get()); } + + @Test + void canResetConfiguredExporter() { + provider.configure(new InMemoryStackTraceExporter()); + provider.reset(); + assertSame(StackTraceExporter.NOOP, provider.get()); + } } diff --git a/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java b/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java new file mode 100644 index 000000000..37ef72207 --- /dev/null +++ b/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java @@ -0,0 +1,278 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.sdk.autoconfigure; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.logs.LoggerProvider; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.javaagent.extension.AgentListener; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; +import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; +import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.annotation.Nullable; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolver; + +public class OpenTelemetrySdkExtension extends AutoConfiguredOpenTelemetrySdk + implements OpenTelemetry, + BeforeEachCallback, + AfterEachCallback, + ParameterResolver, + AutoCloseable { + public static Builder configure() { + return new Builder(); + } + + private final OpenTelemetrySdk sdk; + private final ConfigProperties properties; + private final List agentListeners; + + private OpenTelemetrySdkExtension( + OpenTelemetrySdk sdk, ConfigProperties properties, List agentListeners) { + this.sdk = sdk; + this.properties = properties; + this.agentListeners = agentListeners; + } + + @Override + public TracerProvider getTracerProvider() { + return sdk.getTracerProvider(); + } + + @Override + public MeterProvider getMeterProvider() { + return sdk.getMeterProvider(); + } + + @Override + public LoggerProvider getLogsBridge() { + return sdk.getLogsBridge(); + } + + @Override + public ContextPropagators getPropagators() { + return sdk.getPropagators(); + } + + @Override + public void beforeEach(ExtensionContext context) { + agentListeners.forEach(listener -> listener.afterAgent(this)); + } + + @Override + public void afterEach(ExtensionContext extensionContext) { + sdk.close(); + } + + @Override + public void close() { + sdk.close(); + } + + @Override + public OpenTelemetrySdk getOpenTelemetrySdk() { + return sdk; + } + + @Override + Resource getResource() { + return Resource.getDefault(); + } + + @Nullable + @Override + ConfigProperties getConfig() { + return properties; + } + + @Nullable + @Override + Object getConfigProvider() { + return null; + } + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) { + return parameterContext.getParameter().getType() == Tracer.class; + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) { + return sdk.getTracer(extensionContext.getRequiredTestClass().getName(), "test"); + } + + /** + * An extremely simplified adaptation of the OpenTelemetry class + * AutoConfiguredOpenTelemetrySdkBuilder, designed explicitly to facilitate easier component-like + * testing of custom OpenTelemetry Java Agent extensions. + */ + public static class Builder { + private final SdkCustomizer customizer = new SdkCustomizer(); + private final Map properties = new HashMap<>(); + private final List propagators = new ArrayList<>(); + private final List agentListeners = new ArrayList<>(); + private Sampler sampler = Sampler.alwaysOn(); + + public Builder withProperty(String name, String value) { + properties.put(name, value); + return this; + } + + public Builder with(AutoConfigurationCustomizerProvider provider) { + provider.customize(customizer); + return this; + } + + public Builder withSampler(Sampler sampler) { + this.sampler = sampler; + return this; + } + + public Builder with(TextMapPropagator propagator) { + this.propagators.add(propagator); + return this; + } + + public Builder with(AgentListener agentListener) { + this.agentListeners.add(agentListener); + return this; + } + + /** + * Simplified re-implementation of AutoConfiguredOpenTelemetrySdkBuilder's build method. The + * OpenTelemetry SDK is only configured with features necessary to pass existing test use cases. + */ + public OpenTelemetrySdkExtension build() { + ConfigProperties configProperties = customizeProperties(); + SdkTracerProvider tracerProvider = customizeTracerProvider(configProperties); + + TextMapPropagator propagator = TextMapPropagator.composite(configuredPropagators()); + ContextPropagators contextPropagators = ContextPropagators.create(propagator); + + OpenTelemetrySdk sdk = + OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(contextPropagators) + .build(); + return new OpenTelemetrySdkExtension(sdk, configProperties, agentListeners); + } + + private ConfigProperties customizeProperties() { + var properties = DefaultConfigProperties.createFromMap(this.properties); + for (var customizer : customizer.propertyCustomizers) { + var overrides = customizer.apply(properties); + properties = properties.withOverrides(overrides); + } + return properties; + } + + private SdkTracerProvider customizeTracerProvider(ConfigProperties properties) { + var builder = SdkTracerProvider.builder().setSampler(sampler); + customizer.tracerProviderCustomizers.forEach( + customizer -> customizer.apply(builder, properties)); + return builder.build(); + } + + private List configuredPropagators() { + List propagators = new ArrayList<>(); + propagators.add(W3CBaggagePropagator.getInstance()); + propagators.add(W3CTraceContextPropagator.getInstance()); + propagators.addAll(this.propagators); + return propagators; + } + } + + private static class SdkCustomizer implements AutoConfigurationCustomizer { + private final List>> propertyCustomizers = + new ArrayList<>(); + private final List< + BiFunction> + tracerProviderCustomizers = new ArrayList<>(); + + @Override + public AutoConfigurationCustomizer addTracerProviderCustomizer( + BiFunction + tracerProviderCustomizer) { + tracerProviderCustomizers.add(Objects.requireNonNull(tracerProviderCustomizer)); + return this; + } + + @Override + public AutoConfigurationCustomizer addPropagatorCustomizer( + BiFunction + textMapPropagator) { + return this; + } + + @Override + public AutoConfigurationCustomizer addPropertiesCustomizer( + Function> propertiesCustomizer) { + this.propertyCustomizers.add(propertiesCustomizer); + return this; + } + + @Override + public AutoConfigurationCustomizer addResourceCustomizer( + BiFunction biFunction) { + return this; + } + + @Override + public AutoConfigurationCustomizer addSamplerCustomizer( + BiFunction biFunction) { + return null; + } + + @Override + public AutoConfigurationCustomizer addSpanExporterCustomizer( + BiFunction biFunction) { + return this; + } + + @Override + public AutoConfigurationCustomizer addPropertiesSupplier( + Supplier> supplier) { + return this; + } + } +} From 62cbcfb62babb839d7f7d64b097787e4f046ec7a Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 16:00:09 -0700 Subject: [PATCH 07/28] Add function to Configuration for the snapshot profiling feature flag. --- .../opentelemetry/profiler/Configuration.java | 5 +++++ .../opentelemetry/profiler/ConfigurationTest.java | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index 9276fe91d..ae2b986de 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -24,6 +24,7 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; +import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties; import java.time.Duration; import java.util.HashMap; import java.util.Map; @@ -181,6 +182,10 @@ private static int getJavaVersion() { return Integer.parseInt(javaSpecVersion); } + public static boolean isSnapshotProfilingEnabled(ConfigProperties properties) { + return properties.getBoolean(CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, false); + } + public static double getSnapshotSelectionRate(ConfigProperties properties) { String selectionRatePropertyValue = properties.getString( diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java index 672d9d1c4..bb55bdb8c 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java @@ -17,6 +17,7 @@ package com.splunk.opentelemetry.profiler; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -106,6 +107,20 @@ void snapshotProfilingDisabledByDefault() { assertEquals("false", properties.get("splunk.snapshot.profiler.enabled")); } + @Test + void isSnapshotProfilingEnabledIsFalseByDefault() { + var properties = DefaultConfigProperties.create(Collections.emptyMap()); + assertFalse(Configuration.isSnapshotProfilingEnabled(properties)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void isSnapshotProfilingEnabled(boolean enabled) { + var properties = DefaultConfigProperties.create(Map.of( + "splunk.snapshot.profiler.enabled", String.valueOf(enabled))); + assertEquals(enabled, Configuration.isSnapshotProfilingEnabled(properties)); + } + @Test void snapshotSelectionDefaultRate() { Configuration configuration = new Configuration(); From be77245d85ad834e84c22be3d093ca6de6777c5f Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 16:09:51 -0700 Subject: [PATCH 08/28] Apply spotless code formatting. --- .../com/splunk/opentelemetry/profiler/Configuration.java | 1 - .../profiler/snapshot/SnapshotProfilingSdkCustomizer.java | 7 +++---- .../profiler/snapshot/StackTraceExporterActivator.java | 7 +++---- .../splunk/opentelemetry/profiler/ConfigurationTest.java | 5 +++-- .../snapshot/ScheduledExecutorStackTraceSamplerTest.java | 2 -- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index ae2b986de..5b986e36e 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -24,7 +24,6 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; -import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties; import java.time.Duration; import java.util.HashMap; import java.util.Map; diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java index f77434a10..871ef4dd0 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java @@ -16,10 +16,9 @@ package com.splunk.opentelemetry.profiler.snapshot; -import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER; - import com.google.auto.service.AutoService; import com.google.common.annotations.VisibleForTesting; +import com.splunk.opentelemetry.profiler.Configuration; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; @@ -97,7 +96,7 @@ private boolean includeTraceContextPropagator(Set configuredPropagators) return configuredPropagators.isEmpty(); } - private boolean snapshotProfilingEnabled(ConfigProperties config) { - return config.getBoolean(CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, false); + private boolean snapshotProfilingEnabled(ConfigProperties properties) { + return Configuration.isSnapshotProfilingEnabled(properties); } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java index b68d46b42..7475f8a84 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java @@ -16,9 +16,8 @@ package com.splunk.opentelemetry.profiler.snapshot; -import static com.splunk.opentelemetry.profiler.Configuration.CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER; - import com.google.auto.service.AutoService; +import com.splunk.opentelemetry.profiler.Configuration; import com.splunk.opentelemetry.profiler.OtelLoggerFactory; import io.opentelemetry.api.logs.Logger; import io.opentelemetry.javaagent.extension.AgentListener; @@ -42,7 +41,7 @@ public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetr } } - private boolean snapshotProfilingEnabled(ConfigProperties config) { - return config.getBoolean(CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, false); + private boolean snapshotProfilingEnabled(ConfigProperties properties) { + return Configuration.isSnapshotProfilingEnabled(properties); } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java index bb55bdb8c..16db64d83 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java @@ -116,8 +116,9 @@ void isSnapshotProfilingEnabledIsFalseByDefault() { @ParameterizedTest @ValueSource(booleans = {true, false}) void isSnapshotProfilingEnabled(boolean enabled) { - var properties = DefaultConfigProperties.create(Map.of( - "splunk.snapshot.profiler.enabled", String.valueOf(enabled))); + var properties = + DefaultConfigProperties.create( + Map.of("splunk.snapshot.profiler.enabled", String.valueOf(enabled))); assertEquals(enabled, Configuration.isSnapshotProfilingEnabled(properties)); } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java index c994c9cf1..92a580254 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java @@ -31,7 +31,6 @@ import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; class ScheduledExecutorStackTraceSamplerTest { @@ -84,7 +83,6 @@ void emptyStagingAreaAfterSamplingStops() { } @Test - @Disabled void onlyTakeStackTraceSamplesForOneThreadPerTrace() { var latch = new CountDownLatch(1); var traceId = idGenerator.generateTraceId(); From a9a98addb3380742c0d1bc40a847a434ca5b82ab Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 16:13:43 -0700 Subject: [PATCH 09/28] Replace noop staging area with a real implememtation. --- .../profiler/snapshot/NoopStagingArea.java | 25 ------------------- .../SnapshotProfilingSdkCustomizer.java | 5 +++- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/NoopStagingArea.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/NoopStagingArea.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/NoopStagingArea.java deleted file mode 100644 index d9455d9ac..000000000 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/NoopStagingArea.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.snapshot; - -class NoopStagingArea implements StagingArea { - @Override - public void stage(String traceId, StackTrace stackTrace) {} - - @Override - public void empty(String traceId) {} -} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java index 871ef4dd0..64f726e10 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java @@ -36,7 +36,10 @@ public class SnapshotProfilingSdkCustomizer implements AutoConfigurationCustomiz private final StackTraceSampler sampler; public SnapshotProfilingSdkCustomizer() { - this(new TraceRegistry(), new ScheduledExecutorStackTraceSampler(new NoopStagingArea())); + this( + new TraceRegistry(), + new ScheduledExecutorStackTraceSampler( + new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE))); } @VisibleForTesting From 7f708408445830ac20b224290b368b484a645a2a Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 16:34:42 -0700 Subject: [PATCH 10/28] Update all usages of OpenTelemetrySdkExtension to the version in the io.opentelemetry.sdk.autoconfigure package. --- ...ConfigureSnapshotVolumePropagatorTest.java | 24 +- .../DistributedProfilingSignalTest.java | 9 +- .../snapshot/OpenTelemetrySdkExtension.java | 239 ------------------ .../SnapshotProfilingFeatureFlagTest.java | 12 +- .../snapshot/SnapshotSpanAttributeTest.java | 4 +- .../SnapshotVolumePropagatorTest.java | 3 +- .../profiler/snapshot/SpanSamplingTest.java | 7 +- .../StackTraceExporterActivatorTest.java | 7 +- .../profiler/snapshot/TraceProfilingTest.java | 4 +- .../snapshot/TraceRegistrationTest.java | 4 +- .../OpenTelemetrySdkExtension.java | 5 +- 11 files changed, 38 insertions(+), 280 deletions(-) delete mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/OpenTelemetrySdkExtension.java diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AutoConfigureSnapshotVolumePropagatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AutoConfigureSnapshotVolumePropagatorTest.java index 86a82aa5d..f1cb29ff9 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AutoConfigureSnapshotVolumePropagatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AutoConfigureSnapshotVolumePropagatorTest.java @@ -19,6 +19,8 @@ import static org.assertj.core.api.Assertions.assertThat; import com.splunk.opentelemetry.profiler.Configuration; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension.Builder; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -29,7 +31,7 @@ class AutoConfigureSnapshotVolumePropagatorTest { @Test void autoConfigureSnapshotVolumePropagator() { try (var sdk = newSdk().build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .contains(SnapshotVolumePropagatorProvider.NAME); } @@ -38,7 +40,7 @@ void autoConfigureSnapshotVolumePropagator() { @Test void snapshotVolumePropagatorMustBeAfterTraceContextAndBaggage() { try (var sdk = newSdk().build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .containsExactly("tracecontext", "baggage", SnapshotVolumePropagatorProvider.NAME); } @@ -50,7 +52,7 @@ void appendSnapshotPropagatorToEndOfAlreadyConfiguredPropagators() { newSdk() .withProperty(OTEL_PROPAGATORS, "tracecontext,baggage,some-other-propagator") .build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .containsExactly( "tracecontext", @@ -64,7 +66,7 @@ void appendSnapshotPropagatorToEndOfAlreadyConfiguredPropagators() { @ValueSource(strings = {"baggage", "tracecontext"}) void doNotDoubleCountDefaultOpenTelemetryPropagators(String propagatorName) { try (var sdk = newSdk().withProperty(OTEL_PROPAGATORS, propagatorName).build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)).containsOnlyOnce(propagatorName); } } @@ -73,7 +75,7 @@ void doNotDoubleCountDefaultOpenTelemetryPropagators(String propagatorName) { void doNotDoubleCountSnapshotVolumePropagator() { try (var sdk = newSdk().withProperty(OTEL_PROPAGATORS, SnapshotVolumePropagatorProvider.NAME).build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .containsOnlyOnce(SnapshotVolumePropagatorProvider.NAME); } @@ -83,7 +85,7 @@ void doNotDoubleCountSnapshotVolumePropagator() { void doNotAddSnapshotVolumePropagatorWhenTraceSnapshottingIsDisabled() { try (var sdk = newSdk().withProperty(Configuration.CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, "false").build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .doesNotContain(SnapshotVolumePropagatorProvider.NAME); } @@ -92,7 +94,7 @@ void doNotAddSnapshotVolumePropagatorWhenTraceSnapshottingIsDisabled() { @Test void doNotAddSnapshotVolumePropagatorsConfiguredAsNone() { try (var sdk = newSdk().withProperty(OTEL_PROPAGATORS, "none").build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .doesNotContain(SnapshotVolumePropagatorProvider.NAME); } @@ -102,7 +104,7 @@ void doNotAddSnapshotVolumePropagatorsConfiguredAsNone() { void doNotAddTraceContextPropagatorWhenOtherPropagatorsAreExplicitlyConfigured() { try (var sdk = newSdk().withProperty(OTEL_PROPAGATORS, "some-other-propagator,baggage").build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .containsExactly( "some-other-propagator", "baggage", SnapshotVolumePropagatorProvider.NAME); @@ -112,15 +114,15 @@ void doNotAddTraceContextPropagatorWhenOtherPropagatorsAreExplicitlyConfigured() @Test void addBaggagePropagatorWhenOtherPropagatorsAreExplicitlyConfiguredButBaggageIsMissing() { try (var sdk = newSdk().withProperty(OTEL_PROPAGATORS, "some-other-propagator").build()) { - var properties = sdk.getProperties(); + var properties = sdk.getConfig(); assertThat(properties.getList(OTEL_PROPAGATORS)) .containsExactly( "some-other-propagator", "baggage", SnapshotVolumePropagatorProvider.NAME); } } - private OpenTelemetrySdkExtension.Builder newSdk() { - return OpenTelemetrySdkExtension.builder() + private Builder newSdk() { + return OpenTelemetrySdkExtension.configure() .withProperty(Configuration.CONFIG_KEY_ENABLE_SNAPSHOT_PROFILER, "true") .with(new SnapshotProfilingSdkCustomizer()); } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java index 41be44c01..2e5243f02 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java @@ -25,6 +25,7 @@ import com.splunk.opentelemetry.profiler.snapshot.simulation.Server; import java.time.Duration; import java.util.function.UnaryOperator; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -34,8 +35,7 @@ class DistributedProfilingSignalTest { Snapshotting.customizer().with(downstreamRegistry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension downstreamSdk = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension downstreamSdk = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(downstreamCustomizer) .with(new SnapshotVolumePropagator(() -> true)) @@ -49,7 +49,7 @@ class DistributedProfilingSignalTest { .build(); @RegisterExtension - public final OpenTelemetrySdkExtension middleSdk = OpenTelemetrySdkExtension.builder().build(); + public final OpenTelemetrySdkExtension middleSdk = OpenTelemetrySdkExtension.configure().build(); @RegisterExtension public final Server middle = @@ -60,8 +60,7 @@ class DistributedProfilingSignalTest { Snapshotting.customizer().with(upstreamRegistry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension upstreamSdk = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension upstreamSdk = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(upstreamCustomizer) .with(new SnapshotVolumePropagator(() -> true)) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/OpenTelemetrySdkExtension.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/OpenTelemetrySdkExtension.java deleted file mode 100644 index 519124ff3..000000000 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/OpenTelemetrySdkExtension.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.snapshot; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; -import io.opentelemetry.api.logs.LoggerProvider; -import io.opentelemetry.api.metrics.MeterProvider; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.api.trace.TracerProvider; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.context.propagation.TextMapPropagator; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; -import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; -import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; -import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import io.opentelemetry.sdk.trace.samplers.Sampler; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.Supplier; -import org.junit.jupiter.api.extension.AfterEachCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolver; - -public class OpenTelemetrySdkExtension - implements OpenTelemetry, AfterEachCallback, ParameterResolver, AutoCloseable { - public static Builder builder() { - return new Builder(); - } - - private final OpenTelemetrySdk sdk; - private final ConfigProperties properties; - - private OpenTelemetrySdkExtension(OpenTelemetrySdk sdk, ConfigProperties properties) { - this.sdk = sdk; - this.properties = properties; - } - - @Override - public TracerProvider getTracerProvider() { - return sdk.getTracerProvider(); - } - - @Override - public MeterProvider getMeterProvider() { - return sdk.getMeterProvider(); - } - - @Override - public LoggerProvider getLogsBridge() { - return sdk.getLogsBridge(); - } - - @Override - public ContextPropagators getPropagators() { - return sdk.getPropagators(); - } - - @Override - public void afterEach(ExtensionContext extensionContext) { - sdk.close(); - } - - @Override - public void close() { - sdk.close(); - } - - public ConfigProperties getProperties() { - return properties; - } - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) { - return parameterContext.getParameter().getType() == Tracer.class; - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) { - return sdk.getTracer(extensionContext.getRequiredTestClass().getName(), "test"); - } - - /** - * An extremely simplified adaptation of the OpenTelemetry class - * AutoConfiguredOpenTelemetrySdkBuilder, designed explicitly to facilitate easier component-like - * testing of custom OpenTelemetry Java Agent extensions. - */ - public static class Builder { - private final SdkCustomizer customizer = new SdkCustomizer(); - private final Map properties = new HashMap<>(); - private final List propagators = new ArrayList<>(); - private Sampler sampler = Sampler.alwaysOn(); - - public Builder withProperty(String name, String value) { - properties.put(name, value); - return this; - } - - public Builder with(AutoConfigurationCustomizerProvider provider) { - provider.customize(customizer); - return this; - } - - public Builder withSampler(Sampler sampler) { - this.sampler = sampler; - return this; - } - - public Builder with(TextMapPropagator propagator) { - this.propagators.add(propagator); - return this; - } - - /** - * Simplified re-implementation of AutoConfiguredOpenTelemetrySdkBuilder's build method. The - * OpenTelemetry SDK is only configured with features necessary to pass existing test use cases. - */ - public OpenTelemetrySdkExtension build() { - ConfigProperties configProperties = customizeProperties(); - SdkTracerProvider tracerProvider = customizeTracerProvider(configProperties); - - TextMapPropagator propagator = TextMapPropagator.composite(configuredPropagators()); - ContextPropagators contextPropagators = ContextPropagators.create(propagator); - - OpenTelemetrySdk sdk = - OpenTelemetrySdk.builder() - .setTracerProvider(tracerProvider) - .setPropagators(contextPropagators) - .build(); - return new OpenTelemetrySdkExtension(sdk, configProperties); - } - - private ConfigProperties customizeProperties() { - var properties = DefaultConfigProperties.createFromMap(this.properties); - for (var customizer : customizer.propertyCustomizers) { - var overrides = customizer.apply(properties); - properties = properties.withOverrides(overrides); - } - return properties; - } - - private SdkTracerProvider customizeTracerProvider(ConfigProperties properties) { - var builder = SdkTracerProvider.builder().setSampler(sampler); - customizer.tracerProviderCustomizers.forEach( - customizer -> customizer.apply(builder, properties)); - return builder.build(); - } - - private List configuredPropagators() { - List propagators = new ArrayList<>(); - propagators.add(W3CBaggagePropagator.getInstance()); - propagators.add(W3CTraceContextPropagator.getInstance()); - propagators.addAll(this.propagators); - return propagators; - } - } - - private static class SdkCustomizer implements AutoConfigurationCustomizer { - private final List>> propertyCustomizers = - new ArrayList<>(); - private final List< - BiFunction> - tracerProviderCustomizers = new ArrayList<>(); - - @Override - public AutoConfigurationCustomizer addTracerProviderCustomizer( - BiFunction - tracerProviderCustomizer) { - tracerProviderCustomizers.add(Objects.requireNonNull(tracerProviderCustomizer)); - return this; - } - - @Override - public AutoConfigurationCustomizer addPropagatorCustomizer( - BiFunction - textMapPropagator) { - return this; - } - - @Override - public AutoConfigurationCustomizer addPropertiesCustomizer( - Function> propertiesCustomizer) { - this.propertyCustomizers.add(propertiesCustomizer); - return this; - } - - @Override - public AutoConfigurationCustomizer addResourceCustomizer( - BiFunction biFunction) { - return this; - } - - @Override - public AutoConfigurationCustomizer addSamplerCustomizer( - BiFunction biFunction) { - return null; - } - - @Override - public AutoConfigurationCustomizer addSpanExporterCustomizer( - BiFunction biFunction) { - return this; - } - - @Override - public AutoConfigurationCustomizer addPropertiesSupplier( - Supplier> supplier) { - return this; - } - } -} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java index 8ea45ceef..70ca46fb9 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java @@ -21,6 +21,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; @@ -33,8 +34,9 @@ class SnapshotProfilingFeatureFlagTest { @Nested class SnapshotProfilingDisabledByDefaultTest { @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder().with(customizer).build(); + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + .with(customizer) + .build(); @ParameterizedTest @SpanKinds.Entry @@ -49,8 +51,7 @@ void snapshotProfilingIsDisabledByDefault(SpanKind kind, Tracer tracer) { @Nested class SnapshotProfilingEnabledTest { @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .with(customizer) .withProperty("splunk.snapshot.profiler.enabled", "true") .build(); @@ -68,8 +69,7 @@ void snapshotProfilingIsExplicitlyEnabled(SpanKind kind, Tracer tracer) { @Nested class SnapshotProfilingDisabledTest { @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .with(customizer) .withProperty("splunk.snapshot.profiler.enabled", "false") .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java index 57ec1314a..57bc30858 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java @@ -23,6 +23,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import io.opentelemetry.sdk.trace.ReadWriteSpan; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; @@ -33,8 +34,7 @@ class SnapshotSpanAttributeTest { Snapshotting.customizer().with(registry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .with(customizer) .withProperty("splunk.snapshot.profiler.enabled", "true") .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java index 70e980105..9fc6d609e 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java @@ -23,12 +23,13 @@ import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import java.util.Collections; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; class SnapshotVolumePropagatorTest { @RegisterExtension - public final OpenTelemetrySdkExtension sdk = OpenTelemetrySdkExtension.builder().build(); + public final OpenTelemetrySdkExtension sdk = OpenTelemetrySdkExtension.configure().build(); @RegisterExtension public final ObservableCarrier carrier = new ObservableCarrier(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java index 2d514e214..04e3340a1 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java @@ -21,6 +21,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import io.opentelemetry.sdk.trace.samplers.Sampler; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.extension.RegisterExtension; @@ -34,8 +35,7 @@ class SpanSamplingTest { @Nested class SpanSamplingDisabled { @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .withSampler(Sampler.alwaysOff()) .with(customizer) @@ -54,8 +54,7 @@ void doNotRegisterTraceForProfilingWhenSpanSamplingIsOff(SpanKind kind, Tracer t @Nested class SpanSamplingEnabled { @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .withSampler(Sampler.alwaysOn()) .with(customizer) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java index a0ec49d01..76c122643 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -34,8 +35,7 @@ void tearDown() { @Nested class SnapshotProfilingEnabled { @RegisterExtension - public final io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension s = - io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(new StackTraceExporterActivator()) .build(); @@ -51,8 +51,7 @@ void configureStackTraceExporterProvider() { @Nested class SnapshotProfilingDisabled { @RegisterExtension - public final io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension s = - io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "false") .with(new StackTraceExporterActivator()) .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java index e8d824c8a..882973d94 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java @@ -22,6 +22,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; @@ -32,8 +33,7 @@ class TraceProfilingTest { Snapshotting.customizer().with(registry).with(sampler).build(); @RegisterExtension - public final OpenTelemetrySdkExtension sdk = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension sdk = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(customizer) .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java index c3c677f8c..e03ec95b7 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java @@ -21,6 +21,7 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; @@ -30,8 +31,7 @@ class TraceRegistrationTest { Snapshotting.customizer().with(registry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension s = - OpenTelemetrySdkExtension.builder() + public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(customizer) .build(); diff --git a/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java b/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java index 37ef72207..7ed4194c0 100644 --- a/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java +++ b/profiler/src/test/java/io/opentelemetry/sdk/autoconfigure/OpenTelemetrySdkExtension.java @@ -44,7 +44,6 @@ import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; -import javax.annotation.Nullable; import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; @@ -117,13 +116,11 @@ Resource getResource() { return Resource.getDefault(); } - @Nullable @Override - ConfigProperties getConfig() { + public ConfigProperties getConfig() { return properties; } - @Nullable @Override Object getConfigProvider() { return null; From 1190956b77f96ee22d5c735137b5f16a47dbe56e Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 16:36:11 -0700 Subject: [PATCH 11/28] Apply spotless code formatting. --- .../snapshot/DistributedProfilingSignalTest.java | 8 +++++--- .../snapshot/SnapshotProfilingFeatureFlagTest.java | 11 ++++++----- .../profiler/snapshot/SnapshotSpanAttributeTest.java | 3 ++- .../snapshot/SnapshotVolumePropagatorTest.java | 2 +- .../profiler/snapshot/SpanSamplingTest.java | 6 ++++-- .../snapshot/StackTraceExporterActivatorTest.java | 6 ++++-- .../profiler/snapshot/TraceProfilingTest.java | 3 ++- .../profiler/snapshot/TraceRegistrationTest.java | 3 ++- 8 files changed, 26 insertions(+), 16 deletions(-) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java index 2e5243f02..a21b951be 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/DistributedProfilingSignalTest.java @@ -23,9 +23,9 @@ import com.splunk.opentelemetry.profiler.snapshot.simulation.ExitCall; import com.splunk.opentelemetry.profiler.snapshot.simulation.Message; import com.splunk.opentelemetry.profiler.snapshot.simulation.Server; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import java.time.Duration; import java.util.function.UnaryOperator; -import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -35,7 +35,8 @@ class DistributedProfilingSignalTest { Snapshotting.customizer().with(downstreamRegistry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension downstreamSdk = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension downstreamSdk = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(downstreamCustomizer) .with(new SnapshotVolumePropagator(() -> true)) @@ -60,7 +61,8 @@ class DistributedProfilingSignalTest { Snapshotting.customizer().with(upstreamRegistry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension upstreamSdk = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension upstreamSdk = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(upstreamCustomizer) .with(new SnapshotVolumePropagator(() -> true)) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java index 70ca46fb9..05386c643 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingFeatureFlagTest.java @@ -34,9 +34,8 @@ class SnapshotProfilingFeatureFlagTest { @Nested class SnapshotProfilingDisabledByDefaultTest { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() - .with(customizer) - .build(); + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure().with(customizer).build(); @ParameterizedTest @SpanKinds.Entry @@ -51,7 +50,8 @@ void snapshotProfilingIsDisabledByDefault(SpanKind kind, Tracer tracer) { @Nested class SnapshotProfilingEnabledTest { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .with(customizer) .withProperty("splunk.snapshot.profiler.enabled", "true") .build(); @@ -69,7 +69,8 @@ void snapshotProfilingIsExplicitlyEnabled(SpanKind kind, Tracer tracer) { @Nested class SnapshotProfilingDisabledTest { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .with(customizer) .withProperty("splunk.snapshot.profiler.enabled", "false") .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java index 57bc30858..d8828c1c8 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotSpanAttributeTest.java @@ -34,7 +34,8 @@ class SnapshotSpanAttributeTest { Snapshotting.customizer().with(registry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .with(customizer) .withProperty("splunk.snapshot.profiler.enabled", "true") .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java index 9fc6d609e..6fcea4d99 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotVolumePropagatorTest.java @@ -22,8 +22,8 @@ import io.opentelemetry.api.baggage.Baggage; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; -import java.util.Collections; import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; +import java.util.Collections; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java index 04e3340a1..42b3fa8dc 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SpanSamplingTest.java @@ -35,7 +35,8 @@ class SpanSamplingTest { @Nested class SpanSamplingDisabled { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .withSampler(Sampler.alwaysOff()) .with(customizer) @@ -54,7 +55,8 @@ void doNotRegisterTraceForProfilingWhenSpanSamplingIsOff(SpanKind kind, Tracer t @Nested class SpanSamplingEnabled { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .withSampler(Sampler.alwaysOn()) .with(customizer) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java index 76c122643..4b8f1c950 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java @@ -35,7 +35,8 @@ void tearDown() { @Nested class SnapshotProfilingEnabled { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(new StackTraceExporterActivator()) .build(); @@ -51,7 +52,8 @@ void configureStackTraceExporterProvider() { @Nested class SnapshotProfilingDisabled { @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "false") .with(new StackTraceExporterActivator()) .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java index 882973d94..66d13a12a 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceProfilingTest.java @@ -33,7 +33,8 @@ class TraceProfilingTest { Snapshotting.customizer().with(registry).with(sampler).build(); @RegisterExtension - public final OpenTelemetrySdkExtension sdk = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension sdk = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(customizer) .build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java index e03ec95b7..4bec19cdf 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/TraceRegistrationTest.java @@ -31,7 +31,8 @@ class TraceRegistrationTest { Snapshotting.customizer().with(registry).build(); @RegisterExtension - public final OpenTelemetrySdkExtension s = OpenTelemetrySdkExtension.configure() + public final OpenTelemetrySdkExtension s = + OpenTelemetrySdkExtension.configure() .withProperty("splunk.snapshot.profiler.enabled", "true") .with(customizer) .build(); From 02ed77a5896f64217ed009e5a1fd635478ece158 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 12 Mar 2025 17:08:16 -0700 Subject: [PATCH 12/28] Add test verifying that trace snapshot profiling log message exporting is properly configured and that logs are exported to the OpenTelemetry log exporter. --- .../profiler/OtelLoggerFactory.java | 2 +- .../snapshot/AsyncStackTraceExporter.java | 27 ++++++++ .../snapshot/StackTraceExporterActivator.java | 12 +++- .../snapshot/AsyncStackTraceExporterTest.java | 10 +++ .../SnapshotProfilingLogExportingTest.java | 64 +++++++++++++++++++ ...SnapshotProfilingSdkCustomizerBuilder.java | 6 ++ 6 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java index 146da43a3..8deb5178b 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/OtelLoggerFactory.java @@ -34,7 +34,7 @@ public OtelLoggerFactory() { } @VisibleForTesting - OtelLoggerFactory(Function logRecordExporter) { + public OtelLoggerFactory(Function logRecordExporter) { this.logRecordExporter = logRecordExporter; } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index c435dbb58..540d2575b 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -28,8 +28,15 @@ import com.splunk.opentelemetry.profiler.InstrumentationSource; import com.splunk.opentelemetry.profiler.ProfilingDataType; import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.internal.ImmutableSpanContext; import io.opentelemetry.api.logs.Logger; import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanId; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -69,9 +76,11 @@ public void export(List stackTraces) { private Runnable pprofExporter(Logger otelLogger, List stackTraces) { return () -> { try { + Context context = createProfilingContext(stackTraces); Profile profile = translator.translateToPprof(stackTraces); otelLogger .logRecordBuilder() + .setContext(context) .setTimestamp(Instant.now(clock)) .setSeverity(Severity.INFO) .setAllAttributes(profilingAttributes()) @@ -83,6 +92,24 @@ private Runnable pprofExporter(Logger otelLogger, List stackTraces) }; } + private Context createProfilingContext(List stackTraces) { + String traceId = extractTraceId(stackTraces); + SpanContext spanContext = + ImmutableSpanContext.create( + traceId, + SpanId.getInvalid(), + TraceFlags.getDefault(), + TraceState.getDefault(), + false, + true); + Span span = Span.wrap(spanContext); + return span.storeInContext(Context.root()); + } + + private String extractTraceId(List stackTraces) { + return stackTraces.stream().findFirst().map(StackTrace::getTraceId).orElse(null); + } + private Attributes profilingAttributes() { return Attributes.builder() .put(SOURCE_TYPE, PROFILING_SOURCE) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java index 7475f8a84..d2e7b68a8 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java @@ -17,6 +17,7 @@ package com.splunk.opentelemetry.profiler.snapshot; import com.google.auto.service.AutoService; +import com.google.common.annotations.VisibleForTesting; import com.splunk.opentelemetry.profiler.Configuration; import com.splunk.opentelemetry.profiler.OtelLoggerFactory; import io.opentelemetry.api.logs.Logger; @@ -28,7 +29,16 @@ @AutoService(AgentListener.class) public class StackTraceExporterActivator implements AgentListener { - private final OtelLoggerFactory otelLoggerFactory = new OtelLoggerFactory(); + private final OtelLoggerFactory otelLoggerFactory; + + public StackTraceExporterActivator() { + this(new OtelLoggerFactory()); + } + + @VisibleForTesting + StackTraceExporterActivator(OtelLoggerFactory otelLoggerFactory) { + this.otelLoggerFactory = otelLoggerFactory; + } @Override public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) { diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index 1605d9642..d8cfa52aa 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -89,6 +89,16 @@ void setSeverityToInfoOnLogMessage() { assertEquals(Severity.INFO, logger.records().get(0).getSeverity()); } + @Test + void includeTraceIdInLogMessageContext() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + assertEquals(stackTrace.getTraceId(), logger.records().get(0).getSpanContext().getTraceId()); + } + @Test void encodedLogBodyIsPprofProtobufMessage() { var stackTrace = Snapshotting.stackTrace().build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java new file mode 100644 index 000000000..48de15c85 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java @@ -0,0 +1,64 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.splunk.opentelemetry.profiler.OtelLoggerFactory; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; + +class SnapshotProfilingLogExportingTest { + private final InMemoryLogRecordExporter logExporter = InMemoryLogRecordExporter.create(); + + @AfterEach + void tearDown() { + logExporter.reset(); + } + + @RegisterExtension + public final OpenTelemetrySdkExtension sdk = + OpenTelemetrySdkExtension.configure() + .withProperty("splunk.snapshot.profiler.enabled", "true") + .with(Snapshotting.customizer().withRealStackTraceSampler().build()) + .with(new StackTraceExporterActivator(new OtelLoggerFactory(properties -> logExporter))) + .build(); + + @ParameterizedTest + @SpanKinds.Entry + void exportStackTracesForProfiledTraces(SpanKind kind, Tracer tracer) throws Exception { + String traceId; + try (var ignored = Context.root().with(Volume.HIGHEST).makeCurrent()) { + var span = tracer.spanBuilder("root").setSpanKind(kind).startSpan(); + traceId = span.getSpanContext().getTraceId(); + Thread.sleep(250); + span.end(); + } + + await().until(() -> !logExporter.getFinishedLogRecordItems().isEmpty()); + + var logRecord = logExporter.getFinishedLogRecordItems().get(0); + assertEquals(traceId, logRecord.getSpanContext().getTraceId()); + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java index e9096490c..babe8dc12 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java @@ -25,6 +25,12 @@ SnapshotProfilingSdkCustomizerBuilder with(TraceRegistry registry) { return this; } + SnapshotProfilingSdkCustomizerBuilder withRealStackTraceSampler() { + return with( + new ScheduledExecutorStackTraceSampler( + new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE))); + } + SnapshotProfilingSdkCustomizerBuilder with(StackTraceSampler sampler) { this.sampler = sampler; return this; From 30551e48b1c5df2130247709756137eb86ee1473 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Thu, 13 Mar 2025 10:36:13 -0700 Subject: [PATCH 13/28] Include the overall stack frame count in the snapshot profiling log message attributes. --- .../snapshot/AsyncStackTraceExporter.java | 26 +++++++----- .../profiler/snapshot/PprofTranslator.java | 10 ++++- .../snapshot/AsyncStackTraceExporterTest.java | 20 ++++++++- .../snapshot/PprofTranslatorTest.java | 41 +++++++++++++++---- .../StackTraceExporterActivatorTest.java | 2 + 5 files changed, 79 insertions(+), 20 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index 540d2575b..80ba61e9d 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -18,6 +18,7 @@ import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.FRAME_COUNT; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.PPROF_GZIP_BASE64; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.PROFILING_SOURCE; @@ -40,6 +41,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.time.Clock; import java.time.Instant; import java.util.Base64; @@ -53,6 +55,14 @@ class AsyncStackTraceExporter implements StackTraceExporter { private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(AsyncStackTraceExporter.class.getName()); + private static final Attributes COMMON_ATTRIBUTES = + Attributes.builder() + .put(SOURCE_TYPE, PROFILING_SOURCE) + .put(DATA_TYPE, ProfilingDataType.CPU.value()) + .put(DATA_FORMAT, PPROF_GZIP_BASE64) + .put(INSTRUMENTATION_SOURCE, InstrumentationSource.SNAPSHOT.value()) + .build(); + private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final PprofTranslator translator = new PprofTranslator(); private final Logger otelLogger; @@ -77,13 +87,14 @@ private Runnable pprofExporter(Logger otelLogger, List stackTraces) return () -> { try { Context context = createProfilingContext(stackTraces); - Profile profile = translator.translateToPprof(stackTraces); + Pprof pprof = translator.toPprof(stackTraces); + Profile profile = pprof.build(); otelLogger .logRecordBuilder() .setContext(context) .setTimestamp(Instant.now(clock)) .setSeverity(Severity.INFO) - .setAllAttributes(profilingAttributes()) + .setAllAttributes(profilingAttributes(pprof)) .setBody(serialize(profile)) .emit(); } catch (Exception e) { @@ -110,13 +121,8 @@ private String extractTraceId(List stackTraces) { return stackTraces.stream().findFirst().map(StackTrace::getTraceId).orElse(null); } - private Attributes profilingAttributes() { - return Attributes.builder() - .put(SOURCE_TYPE, PROFILING_SOURCE) - .put(DATA_TYPE, ProfilingDataType.CPU.value()) - .put(DATA_FORMAT, PPROF_GZIP_BASE64) - .put(INSTRUMENTATION_SOURCE, InstrumentationSource.SNAPSHOT.value()) - .build(); + private Attributes profilingAttributes(Pprof pprof) { + return COMMON_ATTRIBUTES.toBuilder().put(FRAME_COUNT, pprof.frameCount()).build(); } private String serialize(Profile profile) throws IOException { @@ -124,6 +130,6 @@ private String serialize(Profile profile) throws IOException { try (OutputStream outputStream = new GZIPOutputStream(Base64.getEncoder().wrap(byteStream))) { profile.writeTo(outputStream); } - return byteStream.toString(); + return byteStream.toString(StandardCharsets.ISO_8859_1.name()); } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java index 437b65092..a2ae5c8ea 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java @@ -37,6 +37,14 @@ public Profile translateToPprof(List stackTraces) { return pprof.build(); } + public Pprof toPprof(List stackTraces) { + Pprof pprof = new Pprof(); + for (StackTrace stackTrace : stackTraces) { + pprof.add(translateToPprofSample(stackTrace, pprof)); + } + return pprof; + } + private Sample translateToPprofSample(StackTrace stackTrace, Pprof pprof) { Sample.Builder sample = Sample.newBuilder(); sample.addLabel(pprof.newLabel(THREAD_ID, stackTrace.getThreadId())); @@ -49,7 +57,7 @@ private Sample translateToPprofSample(StackTrace stackTrace, Pprof pprof) { for (StackTraceElement stackFrame : stackTrace.getStackFrames()) { sample.addLocationId(pprof.getLocationId(stackFrame)); - // pprof.incFrameCount(); + pprof.incFrameCount(); } sample.addLabel(pprof.newLabel(TRACE_ID, stackTrace.getTraceId())); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index d8cfa52aa..4d9115c02 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -18,6 +18,7 @@ import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.FRAME_COUNT; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_TYPE; import static org.assertj.core.api.Assertions.assertThat; @@ -27,6 +28,7 @@ import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; +import io.opentelemetry.api.common.Value; import io.opentelemetry.api.logs.Severity; import io.opentelemetry.sdk.logs.data.LogRecordData; import java.io.ByteArrayInputStream; @@ -144,7 +146,11 @@ private byte[] deserialize(LogRecordData logRecord) throws IOException { } private byte[] decode(LogRecordData logRecord) { - return Base64.getDecoder().decode(logRecord.getBody().asString()); + Value body = logRecord.getBodyValue(); + if (body == null) { + throw new RuntimeException("Log record body is null"); + } + return Base64.getDecoder().decode(body.asString()); } @Test @@ -190,4 +196,16 @@ void includeInstrumentationSourceOpenTelemetryAttributeWithValueOfSnapshot() { var attributes = logger.records().get(0).getAttributes(); assertThat(attributes.asMap()).containsEntry(INSTRUMENTATION_SOURCE, "snapshot"); } + + @Test + void includeFrameCountOpenTelemetryAttributeInLogMessage() { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var attributes = logger.records().get(0).getAttributes(); + assertThat(attributes.asMap()) + .containsEntry(FRAME_COUNT, (long) stackTrace.getStackFrames().length); + } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java index da7191081..07aae21b0 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java @@ -24,6 +24,7 @@ import com.google.perftools.profiles.ProfileProto.Profile; import com.google.perftools.profiles.ProfileProto.Sample; import com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -36,11 +37,11 @@ class PprofTranslatorTest { private final PprofTranslator translator = new PprofTranslator(); @Test - void allStackFramesAreInPprofStringTable() throws Exception { + void allStackFramesAreInPprofStringTable() { var exception = new RuntimeException(); var stackTrace = Snapshotting.stackTrace().with(exception).build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var table = profile.getStringTableList(); assertThat(table).containsAll(fullyQualifiedMethodNames(exception.getStackTrace())); @@ -57,7 +58,7 @@ void allStackFramesIncludedInSample() { var exception = new RuntimeException(); var stackTrace = Snapshotting.stackTrace().with(exception).build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var expectedStackTrace = removeModuleInfo(exception.getStackTrace()); var reportedStackTrace = toStackTrace(profile.getSample(0), profile); @@ -117,11 +118,35 @@ private StackTraceElement removeModuleInfo(StackTraceElement stackFrame) { stackFrame.getLineNumber()); } + @Test + void maintainStackFrameCount() { + var stackTrace = Snapshotting.stackTrace().with(new RuntimeException()).build(); + + var pprof = translator.toPprof(List.of(stackTrace)); + + assertEquals(stackTrace.getStackFrames().length, pprof.frameCount()); + } + + @Test + void maintainStackFrameCountAcrossMultipleStackTraces() { + var stackTrace1 = Snapshotting.stackTrace().with(new RuntimeException()).build(); + var stackTrace2 = Snapshotting.stackTrace().with(new IllegalArgumentException()).build(); + var stackTrace3 = Snapshotting.stackTrace().with(new IOException()).build(); + + var pprof = translator.toPprof(List.of(stackTrace1, stackTrace2, stackTrace3)); + + var expectedFrameCount = + stackTrace1.getStackFrames().length + + stackTrace2.getStackFrames().length + + stackTrace3.getStackFrames().length; + assertEquals(expectedFrameCount, pprof.frameCount()); + } + @Test void includeThreadInformationInSamples() { var stackTrace = Snapshotting.stackTrace().build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); var labels = toLabelString(sample, profile); @@ -140,7 +165,7 @@ void includeThreadInformationInSamples() { void includeTraceIdInformationInSamples() { var stackTrace = Snapshotting.stackTrace().build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); var labels = toLabelString(sample, profile); @@ -152,7 +177,7 @@ void includeTraceIdInformationInSamples() { void includeSourceEventNameAsSnapshotProfilingInSamples() { var stackTrace = Snapshotting.stackTrace().build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); var labels = toLabelString(sample, profile); @@ -165,7 +190,7 @@ void includeSourceEventNameAsSnapshotProfilingInSamples() { void includeStackTraceTimestampInSamples() { var stackTrace = Snapshotting.stackTrace().build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); var labels = toLabelString(sample, profile); @@ -179,7 +204,7 @@ void includeStackTraceTimestampInSamples() { void includeStackTraceDurationInSamples() { var stackTrace = Snapshotting.stackTrace().build(); - var profile = translator.translateToPprof(List.of(stackTrace)); + var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); var labels = toLabelString(sample, profile); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java index 4b8f1c950..b2bf6fdda 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivatorTest.java @@ -43,6 +43,7 @@ class SnapshotProfilingEnabled { @Test void configureStackTraceExporterProvider() { + System.out.println("one"); var exporter = StackTraceExporterProvider.INSTANCE.get(); assertNotSame(StackTraceExporter.NOOP, exporter); assertInstanceOf(AsyncStackTraceExporter.class, exporter); @@ -60,6 +61,7 @@ class SnapshotProfilingDisabled { @Test void doNotConfigureStackTraceExporterProvider() { + System.out.println("two"); var exporter = StackTraceExporterProvider.INSTANCE.get(); assertSame(StackTraceExporter.NOOP, exporter); } From aab9b3d81b8268efc84a7489d9669cc3a59b7495 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Thu, 13 Mar 2025 10:51:41 -0700 Subject: [PATCH 14/28] Properly reset the StackTraceExporterProvider after SnapshotProfilingLogExportingTest. --- .../snapshot/SnapshotProfilingLogExportingTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java index 48de15c85..3812667f3 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java @@ -32,11 +32,6 @@ class SnapshotProfilingLogExportingTest { private final InMemoryLogRecordExporter logExporter = InMemoryLogRecordExporter.create(); - @AfterEach - void tearDown() { - logExporter.reset(); - } - @RegisterExtension public final OpenTelemetrySdkExtension sdk = OpenTelemetrySdkExtension.configure() @@ -45,6 +40,11 @@ void tearDown() { .with(new StackTraceExporterActivator(new OtelLoggerFactory(properties -> logExporter))) .build(); + @AfterEach + void tearDown() { + StackTraceExporterProvider.INSTANCE.reset(); + } + @ParameterizedTest @SpanKinds.Entry void exportStackTracesForProfiledTraces(SpanKind kind, Tracer tracer) throws Exception { From 98d7943ec7efcc7ec92a0feb8798c287cdbadb1e Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Thu, 13 Mar 2025 10:56:20 -0700 Subject: [PATCH 15/28] Remove unused method from PprofTranslator. --- .../opentelemetry/profiler/snapshot/PprofTranslator.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java index a2ae5c8ea..03f9158de 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java @@ -24,19 +24,10 @@ import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID; -import com.google.perftools.profiles.ProfileProto.Profile; import com.google.perftools.profiles.ProfileProto.Sample; import java.util.List; class PprofTranslator { - public Profile translateToPprof(List stackTraces) { - Pprof pprof = new Pprof(); - for (StackTrace stackTrace : stackTraces) { - pprof.add(translateToPprofSample(stackTrace, pprof)); - } - return pprof.build(); - } - public Pprof toPprof(List stackTraces) { Pprof pprof = new Pprof(); for (StackTrace stackTrace : stackTraces) { From fad3539e0faaf47ffc1811d815870fac42ac29ea Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Mon, 17 Mar 2025 14:37:35 -0700 Subject: [PATCH 16/28] Apply spotless code formatting. --- .../ScheduledExecutorStackTraceSampler.java | 19 ++++++++++++------- ...cheduledExecutorStackTraceSamplerTest.java | 3 ++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java index d7336f993..dbebaccf9 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java @@ -57,13 +57,18 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler { @Override public void start(SpanContext spanContext) { - samplers.computeIfAbsent(spanContext.getTraceId(), traceId -> { - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); - scheduler.scheduleAtFixedRate(new StackTraceGatherer(samplingPeriod, spanContext.getTraceId(), - Thread.currentThread().getId()), SCHEDULER_INITIAL_DELAY, samplingPeriod.toMillis(), - TimeUnit.MILLISECONDS); - return scheduler; - }); + samplers.computeIfAbsent( + spanContext.getTraceId(), + traceId -> { + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(); + scheduler.scheduleAtFixedRate( + new StackTraceGatherer( + samplingPeriod, spanContext.getTraceId(), Thread.currentThread().getId()), + SCHEDULER_INITIAL_DELAY, + samplingPeriod.toMillis(), + TimeUnit.MILLISECONDS); + return scheduler; + }); } @Override diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java index 1df90f473..8f51a0ab9 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java @@ -203,7 +203,8 @@ void includeThreadDetailsOnStackTraces() throws Exception { } } - private Callable startSampling(SpanContext spanContext, CountDownLatch startSpanLatch, CountDownLatch shutdownLatch) { + private Callable startSampling( + SpanContext spanContext, CountDownLatch startSpanLatch, CountDownLatch shutdownLatch) { return (() -> { try { startSpanLatch.await(); From b50f54c38405ab0c126cf42d2ae479c406addb61 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 19 Mar 2025 02:42:24 -0700 Subject: [PATCH 17/28] Remove the trace ID from the snapshot profiling log message context. --- .../snapshot/AsyncStackTraceExporter.java | 27 -------- .../snapshot/AsyncStackTraceExporterTest.java | 34 +---------- .../snapshot/PprofTranslatorTest.java | 26 ++------ .../profiler/snapshot/PprofUtils.java | 61 +++++++++++++++++++ .../SnapshotProfilingLogExportingTest.java | 26 +++++++- 5 files changed, 94 insertions(+), 80 deletions(-) create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index 80ba61e9d..6e37e90b8 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -29,15 +29,8 @@ import com.splunk.opentelemetry.profiler.InstrumentationSource; import com.splunk.opentelemetry.profiler.ProfilingDataType; import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.internal.ImmutableSpanContext; import io.opentelemetry.api.logs.Logger; import io.opentelemetry.api.logs.Severity; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.SpanId; -import io.opentelemetry.api.trace.TraceFlags; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.context.Context; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -86,12 +79,10 @@ public void export(List stackTraces) { private Runnable pprofExporter(Logger otelLogger, List stackTraces) { return () -> { try { - Context context = createProfilingContext(stackTraces); Pprof pprof = translator.toPprof(stackTraces); Profile profile = pprof.build(); otelLogger .logRecordBuilder() - .setContext(context) .setTimestamp(Instant.now(clock)) .setSeverity(Severity.INFO) .setAllAttributes(profilingAttributes(pprof)) @@ -103,24 +94,6 @@ private Runnable pprofExporter(Logger otelLogger, List stackTraces) }; } - private Context createProfilingContext(List stackTraces) { - String traceId = extractTraceId(stackTraces); - SpanContext spanContext = - ImmutableSpanContext.create( - traceId, - SpanId.getInvalid(), - TraceFlags.getDefault(), - TraceState.getDefault(), - false, - true); - Span span = Span.wrap(spanContext); - return span.storeInContext(Context.root()); - } - - private String extractTraceId(List stackTraces) { - return stackTraces.stream().findFirst().map(StackTrace::getTraceId).orElse(null); - } - private Attributes profilingAttributes(Pprof pprof) { return COMMON_ATTRIBUTES.toBuilder().put(FRAME_COUNT, pprof.frameCount()).build(); } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index 4d9115c02..7bc1009bd 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -28,15 +28,11 @@ import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; -import io.opentelemetry.api.common.Value; import io.opentelemetry.api.logs.Severity; -import io.opentelemetry.sdk.logs.data.LogRecordData; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.time.Clock; import java.time.Instant; import java.time.ZoneId; -import java.util.Base64; import java.util.List; import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.Test; @@ -91,16 +87,6 @@ void setSeverityToInfoOnLogMessage() { assertEquals(Severity.INFO, logger.records().get(0).getSeverity()); } - @Test - void includeTraceIdInLogMessageContext() { - var stackTrace = Snapshotting.stackTrace().build(); - - exporter.export(List.of(stackTrace)); - await().until(() -> !logger.records().isEmpty()); - - assertEquals(stackTrace.getTraceId(), logger.records().get(0).getSpanContext().getTraceId()); - } - @Test void encodedLogBodyIsPprofProtobufMessage() { var stackTrace = Snapshotting.stackTrace().build(); @@ -109,7 +95,7 @@ void encodedLogBodyIsPprofProtobufMessage() { await().until(() -> !logger.records().isEmpty()); var logRecord = logger.records().get(0); - assertDoesNotThrow(() -> Profile.parseFrom(deserialize(logRecord))); + assertDoesNotThrow(() -> Profile.parseFrom(PprofUtils.deserialize(logRecord))); } @Test @@ -120,7 +106,7 @@ void encodeLogBodyUsingBase64() { await().until(() -> !logger.records().isEmpty()); var logRecord = logger.records().get(0); - assertDoesNotThrow(() -> decode(logRecord)); + assertDoesNotThrow(() -> PprofUtils.decode(logRecord)); } @Test @@ -133,26 +119,12 @@ void logBodyIsGZipped() { var logRecord = logger.records().get(0); assertDoesNotThrow( () -> { - var bytes = new ByteArrayInputStream(decode(logRecord)); + var bytes = new ByteArrayInputStream(PprofUtils.decode(logRecord)); var inputStream = new GZIPInputStream(bytes); inputStream.readAllBytes(); }); } - private byte[] deserialize(LogRecordData logRecord) throws IOException { - var bytes = new ByteArrayInputStream(decode(logRecord)); - var inputStream = new GZIPInputStream(bytes); - return inputStream.readAllBytes(); - } - - private byte[] decode(LogRecordData logRecord) { - Value body = logRecord.getBodyValue(); - if (body == null) { - throw new RuntimeException("Log record body is null"); - } - return Base64.getDecoder().decode(body.asString()); - } - @Test void includeSourceTypeOpenTelemetryAttribute() { var stackTrace = Snapshotting.stackTrace().build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java index 07aae21b0..4288e003d 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java @@ -27,9 +27,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; @@ -149,7 +147,7 @@ void includeThreadInformationInSamples() { var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); - var labels = toLabelString(sample, profile); + var labels = PprofUtils.toLabelString(sample, profile); assertThat(labels) .containsEntry(ProfilingSemanticAttributes.THREAD_ID.getKey(), stackTrace.getThreadId()); assertThat(labels) @@ -168,7 +166,7 @@ void includeTraceIdInformationInSamples() { var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); - var labels = toLabelString(sample, profile); + var labels = PprofUtils.toLabelString(sample, profile); assertThat(labels) .containsEntry(ProfilingSemanticAttributes.TRACE_ID.getKey(), stackTrace.getTraceId()); } @@ -180,7 +178,7 @@ void includeSourceEventNameAsSnapshotProfilingInSamples() { var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); - var labels = toLabelString(sample, profile); + var labels = PprofUtils.toLabelString(sample, profile); assertThat(labels) .containsEntry( ProfilingSemanticAttributes.SOURCE_EVENT_NAME.getKey(), "snapshot-profiling"); @@ -193,7 +191,7 @@ void includeStackTraceTimestampInSamples() { var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); - var labels = toLabelString(sample, profile); + var labels = PprofUtils.toLabelString(sample, profile); assertThat(labels) .containsEntry( ProfilingSemanticAttributes.SOURCE_EVENT_TIME.getKey(), @@ -207,24 +205,10 @@ void includeStackTraceDurationInSamples() { var profile = translator.toPprof(List.of(stackTrace)).build(); var sample = profile.getSample(0); - var labels = toLabelString(sample, profile); + var labels = PprofUtils.toLabelString(sample, profile); assertThat(labels) .containsEntry( ProfilingSemanticAttributes.SOURCE_EVENT_PERIOD.getKey(), stackTrace.getDuration().toMillis()); } - - private Map toLabelString(Sample sample, Profile profile) { - var labels = new HashMap(); - for (var label : sample.getLabelList()) { - var stringTableIndex = label.getKey(); - var key = profile.getStringTable((int) stringTableIndex); - if (label.getStr() > 0) { - labels.put(key, profile.getStringTable((int) label.getStr())); - } else { - labels.put(key, label.getNum()); - } - } - return labels; - } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java new file mode 100644 index 000000000..32b33e7d0 --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java @@ -0,0 +1,61 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.snapshot; + +import com.google.perftools.profiles.ProfileProto.Profile; +import com.google.perftools.profiles.ProfileProto.Sample; +import io.opentelemetry.api.common.Value; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.zip.GZIPInputStream; + +class PprofUtils { + + static byte[] deserialize(LogRecordData logRecord) throws IOException { + var bytes = new ByteArrayInputStream(decode(logRecord)); + var inputStream = new GZIPInputStream(bytes); + return inputStream.readAllBytes(); + } + + static byte[] decode(LogRecordData logRecord) { + Value body = logRecord.getBodyValue(); + if (body == null) { + throw new RuntimeException("Log record body is null"); + } + return Base64.getDecoder().decode(body.asString()); + } + + static Map toLabelString(Sample sample, Profile profile) { + var labels = new HashMap(); + for (var label : sample.getLabelList()) { + var stringTableIndex = label.getKey(); + var key = profile.getStringTable((int) stringTableIndex); + if (label.getStr() > 0) { + labels.put(key, profile.getStringTable((int) label.getStr())); + } else { + labels.put(key, label.getNum()); + } + } + return labels; + } + + private PprofUtils() {} +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java index 3812667f3..d579c1113 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java @@ -16,20 +16,32 @@ package com.splunk.opentelemetry.profiler.snapshot; +import static com.google.perftools.profiles.ProfileProto.Sample; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; +import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.OtelLoggerFactory; +import com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import io.opentelemetry.sdk.autoconfigure.OpenTelemetrySdkExtension; import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; class SnapshotProfilingLogExportingTest { + private static final Predicate> TRACE_ID_LABEL = + kv -> ProfilingSemanticAttributes.TRACE_ID.getKey().equals(kv.getKey()); + private final InMemoryLogRecordExporter logExporter = InMemoryLogRecordExporter.create(); @RegisterExtension @@ -59,6 +71,18 @@ void exportStackTracesForProfiledTraces(SpanKind kind, Tracer tracer) throws Exc await().until(() -> !logExporter.getFinishedLogRecordItems().isEmpty()); var logRecord = logExporter.getFinishedLogRecordItems().get(0); - assertEquals(traceId, logRecord.getSpanContext().getTraceId()); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + + var traceIds = + profile.getSampleList().stream() + .flatMap(sampleLabels(profile)) + .filter(TRACE_ID_LABEL) + .map(Map.Entry::getValue) + .collect(Collectors.toSet()); + assertEquals(Set.of(traceId), traceIds); + } + + private Function>> sampleLabels(Profile profile) { + return sample -> PprofUtils.toLabelString(sample, profile).entrySet().stream(); } } From 072faa0bf0c0781465466b33bcb4ac81f816ca6a Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 19 Mar 2025 02:47:24 -0700 Subject: [PATCH 18/28] Assume a consistent sampling duration rather than compute it. --- .../ScheduledExecutorStackTraceSampler.java | 14 ++------------ .../ScheduledExecutorStackTraceSamplerTest.java | 2 +- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java index dbebaccf9..4c3559144 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java @@ -84,7 +84,6 @@ class StackTraceGatherer implements Runnable { private final Duration samplingPeriod; private final String traceId; private final long threadId; - private Instant lastExecution; StackTraceGatherer(Duration samplingPeroid, String traceId, long threadId) { this.samplingPeriod = samplingPeroid; @@ -94,15 +93,13 @@ class StackTraceGatherer implements Runnable { @Override public void run() { - Instant now = Instant.now(); try { + Instant now = Instant.now(); ThreadInfo threadInfo = threadMXBean.getThreadInfo(threadId, MAX_ENTRY_DEPTH); - StackTrace stackTrace = StackTrace.from(now, sampleDuration(now), traceId, threadInfo); + StackTrace stackTrace = StackTrace.from(now, samplingPeriod, traceId, threadInfo); stagingArea.stage(traceId, stackTrace); } catch (Exception e) { logger.log(Level.SEVERE, e, samplerErrorMessage(traceId, threadId)); - } finally { - lastExecution = now; } } @@ -113,12 +110,5 @@ private Supplier samplerErrorMessage(String traceId, long threadId) { + "' on profiled thread " + threadId; } - - private Duration sampleDuration(Instant now) { - if (lastExecution == null) { - return samplingPeriod; - } - return Duration.between(lastExecution, now); - } } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java index 8f51a0ab9..2eb3e1011 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java @@ -155,7 +155,7 @@ void calculateSamplingPeriodAfterFirstRecordedStackTraces() { var stackTrace = staging.allStackTraces().stream().skip(1).findFirst().orElseThrow(); assertThat(stackTrace.getDuration()) .isNotNull() - .isCloseTo(SAMPLING_PERIOD, Duration.ofMillis(5)); + .isEqualTo(SAMPLING_PERIOD); } finally { sampler.stop(spanContext); } From 826cfae39a9576daf2ec9424a09051aa1f00783e Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Wed, 19 Mar 2025 03:39:32 -0700 Subject: [PATCH 19/28] Apply spotless code formatting. --- .../snapshot/ScheduledExecutorStackTraceSamplerTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java index 2eb3e1011..548fdb590 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java @@ -153,9 +153,7 @@ void calculateSamplingPeriodAfterFirstRecordedStackTraces() { await().until(() -> staging.allStackTraces().size() > 1); var stackTrace = staging.allStackTraces().stream().skip(1).findFirst().orElseThrow(); - assertThat(stackTrace.getDuration()) - .isNotNull() - .isEqualTo(SAMPLING_PERIOD); + assertThat(stackTrace.getDuration()).isNotNull().isEqualTo(SAMPLING_PERIOD); } finally { sampler.stop(spanContext); } From 9b10d29a15343e8b3f5f0d2757bcd0a5ae5d4012 Mon Sep 17 00:00:00 2001 From: Lauri Tulmin Date: Wed, 19 Mar 2025 17:30:40 +0200 Subject: [PATCH 20/28] Share pprof code --- .../opentelemetry/profiler/JfrActivator.java | 4 +- .../profiler/events/EventPeriods.java | 62 ----- .../profiler/exporter/CpuEventExporter.java | 10 + .../exporter/PprofCpuEventExporter.java | 75 +++++- .../snapshot/AsyncStackTraceExporter.java | 70 ++---- .../profiler/snapshot/Pprof.java | 226 ------------------ .../profiler/snapshot/PprofTranslator.java | 57 ----- .../ScheduledExecutorStackTraceSampler.java | 2 +- .../profiler/events/EventPeriodsTest.java | 96 -------- .../snapshot/AsyncStackTraceExporterTest.java | 7 +- .../snapshot/PprofTranslatorTest.java | 214 ----------------- 11 files changed, 101 insertions(+), 722 deletions(-) delete mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java delete mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java delete mode 100644 profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java delete mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java delete mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java index 906eedeb4..8a93d9702 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java @@ -29,7 +29,6 @@ import com.splunk.opentelemetry.profiler.allocation.exporter.AllocationEventExporter; import com.splunk.opentelemetry.profiler.allocation.exporter.PprofAllocationEventExporter; import com.splunk.opentelemetry.profiler.context.SpanContextualizer; -import com.splunk.opentelemetry.profiler.events.EventPeriods; import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter; import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter; import com.splunk.opentelemetry.profiler.util.HelpfulExecutors; @@ -130,13 +129,12 @@ private void activateJfrAndRunForever(ConfigProperties config, Resource resource EventReader eventReader = new EventReader(); SpanContextualizer spanContextualizer = new SpanContextualizer(eventReader); - EventPeriods periods = new EventPeriods(jfrSettings::get); LogRecordExporter logsExporter = LogExporterBuilder.fromConfig(config); CpuEventExporter cpuEventExporter = PprofCpuEventExporter.builder() .otelLogger(buildOtelLogger(SimpleLogRecordProcessor.create(logsExporter), resource)) - .eventPeriods(periods) + .period(Configuration.getCallStackInterval(config)) .stackDepth(stackDepth) .build(); diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java deleted file mode 100644 index ded9b170a..000000000 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/events/EventPeriods.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.events; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -public class EventPeriods { - - public static final Duration UNKNOWN = Duration.ZERO; - private final Map cache = new HashMap<>(); - private final Function configFinder; - - public EventPeriods(Function configFinder) { - this.configFinder = configFinder; - } - - public Duration getDuration(String eventName) { - return cache.computeIfAbsent( - eventName, - event -> { - String value = configFinder.apply(event + "#period"); - return parseToDuration(value); - }); - } - - private Duration parseToDuration(String value) { - if (value == null) { - return UNKNOWN; - } - // format is "TTT UUU" where TTT is some numbers and UUU is some units suffix (ms or s) - try { - String[] parts = value.split(" "); - if (parts.length < 2) { - return UNKNOWN; - } - long multiplier = 1; - if ("s".equals(parts[1])) { - multiplier = 1000; - } - return Duration.ofMillis(multiplier * Integer.parseInt(parts[0])); - } catch (NumberFormatException e) { - return UNKNOWN; - } - } -} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java index 2c239b692..ac6499c97 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/CpuEventExporter.java @@ -17,10 +17,20 @@ package com.splunk.opentelemetry.profiler.exporter; import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage; +import java.time.Instant; public interface CpuEventExporter { void export(StackToSpanLinkage stackToSpanLinkage); + default void export( + long threadId, + String threadName, + Thread.State threadState, + StackTraceElement[] stackTrace, + Instant eventTime, + String traceId, + String spanId) {} + default void flush() {} } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java index 4996c5b15..816f620ca 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java @@ -30,26 +30,27 @@ import com.splunk.opentelemetry.profiler.InstrumentationSource; import com.splunk.opentelemetry.profiler.ProfilingDataType; import com.splunk.opentelemetry.profiler.context.StackToSpanLinkage; -import com.splunk.opentelemetry.profiler.events.EventPeriods; import com.splunk.opentelemetry.profiler.exporter.StackTraceParser.StackTrace; import com.splunk.opentelemetry.profiler.pprof.Pprof; import io.opentelemetry.api.logs.Logger; import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanId; +import io.opentelemetry.api.trace.TraceId; import java.time.Duration; import java.time.Instant; public class PprofCpuEventExporter implements CpuEventExporter { - private final EventPeriods eventPeriods; + private final Duration period; private final int stackDepth; private final PprofLogDataExporter pprofLogDataExporter; private Pprof pprof = createPprof(); private PprofCpuEventExporter(Builder builder) { - this.eventPeriods = builder.eventPeriods; + this.period = builder.period; this.stackDepth = builder.stackDepth; this.pprofLogDataExporter = new PprofLogDataExporter( - builder.otelLogger, ProfilingDataType.CPU, InstrumentationSource.CONTINUOUS); + builder.otelLogger, ProfilingDataType.CPU, builder.instrumentationSource); } @Override @@ -80,10 +81,7 @@ public void export(StackToSpanLinkage stackToSpanLinkage) { String eventName = stackToSpanLinkage.getSourceEventName(); pprof.addLabel(sample, SOURCE_EVENT_NAME, eventName); - Duration eventPeriod = eventPeriods.getDuration(eventName); - if (!EventPeriods.UNKNOWN.equals(eventPeriod)) { - pprof.addLabel(sample, SOURCE_EVENT_PERIOD, eventPeriod.toMillis()); - } + pprof.addLabel(sample, SOURCE_EVENT_PERIOD, period.toMillis()); Instant time = stackToSpanLinkage.getTime(); pprof.addLabel(sample, SOURCE_EVENT_TIME, time.toEpochMilli()); @@ -96,6 +94,55 @@ public void export(StackToSpanLinkage stackToSpanLinkage) { pprof.getProfileBuilder().addSample(sample); } + @Override + public void export( + long threadId, + String threadName, + Thread.State threadState, + StackTraceElement[] stackTrace, + Instant eventTime, + String traceId, + String spanId) { + Sample.Builder sample = Sample.newBuilder(); + + pprof.addLabel(sample, THREAD_ID, threadId); + pprof.addLabel(sample, THREAD_NAME, threadName); + pprof.addLabel(sample, THREAD_STATE, threadState.name()); + + if (stackTrace.length > stackDepth) { + pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true); + } + + for (int i = 0; i < Math.min(stackDepth, stackTrace.length); i++) { + StackTraceElement ste = stackTrace[i]; + + String fileName = ste.getFileName(); + if (fileName == null) { + fileName = "unknown"; + } + String className = ste.getClassName(); + String methodName = ste.getMethodName(); + int lineNumber = ste.getLineNumber(); + if (lineNumber < 0) { + lineNumber = 0; + } + sample.addLocationId(pprof.getLocationId(fileName, className, methodName, lineNumber)); + pprof.incFrameCount(); + } + + pprof.addLabel(sample, SOURCE_EVENT_PERIOD, period.toMillis()); + pprof.addLabel(sample, SOURCE_EVENT_TIME, eventTime.toEpochMilli()); + + if (TraceId.isValid(traceId)) { + pprof.addLabel(sample, TRACE_ID, traceId); + } + if (SpanId.isValid(spanId)) { + pprof.addLabel(sample, SPAN_ID, spanId); + } + + pprof.getProfileBuilder().addSample(sample); + } + private static Pprof createPprof() { return new Pprof(); } @@ -123,8 +170,9 @@ public static Builder builder() { public static class Builder { private Logger otelLogger; - private EventPeriods eventPeriods; + private Duration period; private int stackDepth; + private InstrumentationSource instrumentationSource = InstrumentationSource.CONTINUOUS; public PprofCpuEventExporter build() { return new PprofCpuEventExporter(this); @@ -135,8 +183,8 @@ public Builder otelLogger(Logger otelLogger) { return this; } - public Builder eventPeriods(EventPeriods eventPeriods) { - this.eventPeriods = eventPeriods; + public Builder period(Duration period) { + this.period = period; return this; } @@ -144,5 +192,10 @@ public Builder stackDepth(int stackDepth) { this.stackDepth = stackDepth; return this; } + + public Builder instrumentationSource(InstrumentationSource instrumentationSource) { + this.instrumentationSource = instrumentationSource; + return this; + } } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index 6e37e90b8..f12f44bec 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -16,48 +16,23 @@ package com.splunk.opentelemetry.profiler.snapshot; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.FRAME_COUNT; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.PPROF_GZIP_BASE64; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.PROFILING_SOURCE; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_TYPE; - import com.google.common.annotations.VisibleForTesting; -import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.InstrumentationSource; -import com.splunk.opentelemetry.profiler.ProfilingDataType; -import io.opentelemetry.api.common.Attributes; +import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter; +import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter; import io.opentelemetry.api.logs.Logger; -import io.opentelemetry.api.logs.Severity; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; import java.time.Clock; import java.time.Instant; -import java.util.Base64; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.logging.Level; -import java.util.zip.GZIPOutputStream; class AsyncStackTraceExporter implements StackTraceExporter { private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(AsyncStackTraceExporter.class.getName()); - private static final Attributes COMMON_ATTRIBUTES = - Attributes.builder() - .put(SOURCE_TYPE, PROFILING_SOURCE) - .put(DATA_TYPE, ProfilingDataType.CPU.value()) - .put(DATA_FORMAT, PPROF_GZIP_BASE64) - .put(INSTRUMENTATION_SOURCE, InstrumentationSource.SNAPSHOT.value()) - .build(); - private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - private final PprofTranslator translator = new PprofTranslator(); private final Logger otelLogger; private final Clock clock; @@ -79,30 +54,29 @@ public void export(List stackTraces) { private Runnable pprofExporter(Logger otelLogger, List stackTraces) { return () -> { try { - Pprof pprof = translator.toPprof(stackTraces); - Profile profile = pprof.build(); - otelLogger - .logRecordBuilder() - .setTimestamp(Instant.now(clock)) - .setSeverity(Severity.INFO) - .setAllAttributes(profilingAttributes(pprof)) - .setBody(serialize(profile)) - .emit(); + CpuEventExporter cpuEventExporter = + PprofCpuEventExporter.builder() + .otelLogger(otelLogger) + .stackDepth(200) + .period(ScheduledExecutorStackTraceSampler.SCHEDULER_PERIOD) + .instrumentationSource(InstrumentationSource.SNAPSHOT) + .build(); + + Instant now = Instant.now(clock); + for (StackTrace stackTrace : stackTraces) { + cpuEventExporter.export( + stackTrace.getThreadId(), + stackTrace.getThreadName(), + stackTrace.getThreadState(), + stackTrace.getStackFrames(), + now, + stackTrace.getTraceId(), + null); + } + cpuEventExporter.flush(); } catch (Exception e) { logger.log(Level.SEVERE, "an exception was thrown", e); } }; } - - private Attributes profilingAttributes(Pprof pprof) { - return COMMON_ATTRIBUTES.toBuilder().put(FRAME_COUNT, pprof.frameCount()).build(); - } - - private String serialize(Profile profile) throws IOException { - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - try (OutputStream outputStream = new GZIPOutputStream(Base64.getEncoder().wrap(byteStream))) { - profile.writeTo(outputStream); - } - return byteStream.toString(StandardCharsets.ISO_8859_1.name()); - } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java deleted file mode 100644 index 1822660a4..000000000 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/Pprof.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.snapshot; - -import static com.google.perftools.profiles.ProfileProto.Function; -import static com.google.perftools.profiles.ProfileProto.Label; -import static com.google.perftools.profiles.ProfileProto.Line; -import static com.google.perftools.profiles.ProfileProto.Location; -import static com.google.perftools.profiles.ProfileProto.Profile; -import static com.google.perftools.profiles.ProfileProto.Sample; - -import io.opentelemetry.api.common.AttributeKey; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.function.Consumer; - -/** - * Adapted from the Splunk OpenTelemetry profiler here, - * which is itself adapted from Google's Bazel build system here. - */ -class Pprof { - private final Profile.Builder profileBuilder = Profile.newBuilder(); - private final StringTable stringTable = new StringTable(profileBuilder); - private final FunctionTable functionTable = new FunctionTable(profileBuilder, stringTable); - private final LocationTable locationTable = new LocationTable(profileBuilder, functionTable); - private int frameCount; - - Profile build() { - return profileBuilder.build(); - } - - void add(Sample sample) { - profileBuilder.addSample(sample); - } - - long getLocationId(StackTraceElement stackFrame) { - return locationTable.get(stackFrame); - } - - Label newLabel(AttributeKey key, String value) { - return newLabel(key.getKey(), label -> label.setStr(stringTable.get(value))); - } - - Label newLabel(AttributeKey key, long value) { - return newLabel(key.getKey(), value); - } - - Label newLabel(String key, long value) { - return newLabel(key, label -> label.setNum(value)); - } - - private Label newLabel(String name, Consumer valueSetter) { - Label.Builder label = Label.newBuilder(); - label.setKey(stringTable.get(name)); - valueSetter.accept(label); - return label.build(); - } - - public void incFrameCount() { - frameCount++; - } - - /** - * @return non unique stack frames in this pprof batch - */ - public int frameCount() { - return frameCount; - } - - // copied from - // https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/profiler/memory/AllocationTracker.java - private static class StringTable { - final Profile.Builder profile; - final Map table = new HashMap<>(); - long index = 0; - - StringTable(Profile.Builder profile) { - this.profile = profile; - get(""); // 0 is reserved for the empty string - } - - long get(String str) { - return table.computeIfAbsent( - str, - key -> { - profile.addStringTable(key); - return index++; - }); - } - } - - private static class LocationTable { - final Profile.Builder profile; - final FunctionTable functionTable; - final Map table = new HashMap<>(); - long index = 1; // 0 is reserved - - LocationTable(Profile.Builder profile, FunctionTable functionTable) { - this.profile = profile; - this.functionTable = functionTable; - } - - long get(StackTraceElement stackFrame) { - LocationKey locationKey = LocationKey.from(stackFrame); - Location location = - Location.newBuilder() - .setId(index) - .addLine( - Line.newBuilder() - .setFunctionId(functionTable.get(locationKey.functionKey)) - .setLine(locationKey.line)) - .build(); - return table.computeIfAbsent( - locationKey, - key -> { - profile.addLocation(location); - return index++; - }); - } - } - - private static class LocationKey { - private final FunctionKey functionKey; - private final long line; - - static LocationKey from(StackTraceElement stackFrame) { - return new LocationKey(FunctionKey.from(stackFrame), stackFrame.getLineNumber()); - } - - private LocationKey(FunctionKey functionKey, long line) { - this.functionKey = functionKey; - this.line = line; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - LocationKey that = (LocationKey) o; - return line == that.line && Objects.equals(functionKey, that.functionKey); - } - - @Override - public int hashCode() { - return Objects.hash(functionKey, line); - } - } - - private static class FunctionTable { - final Profile.Builder profile; - final StringTable stringTable; - final Map table = new HashMap<>(); - long index = 1; // 0 is reserved - - FunctionTable(Profile.Builder profile, StringTable stringTable) { - this.profile = profile; - this.stringTable = stringTable; - } - - long get(FunctionKey functionKey) { - Function fn = - Function.newBuilder() - .setId(index) - .setFilename(stringTable.get(functionKey.file)) - .setName(stringTable.get(functionKey.className + "." + functionKey.function)) - .build(); - return table.computeIfAbsent( - functionKey, - key -> { - profile.addFunction(fn); - return index++; - }); - } - } - - private static class FunctionKey { - private final String file; - private final String className; - private final String function; - - static FunctionKey from(StackTraceElement stackFrame) { - return new FunctionKey( - stackFrame.getFileName() == null ? "Unknown Source" : stackFrame.getFileName(), - stackFrame.getClassName(), - stackFrame.getMethodName()); - } - - private FunctionKey(String file, String className, String function) { - this.file = file; - this.className = className; - this.function = function; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FunctionKey that = (FunctionKey) o; - return Objects.equals(file, that.file) - && Objects.equals(className, that.className) - && Objects.equals(function, that.function); - } - - @Override - public int hashCode() { - return Objects.hash(file, className, function); - } - } -} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java deleted file mode 100644 index 03f9158de..000000000 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslator.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.snapshot; - -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_NAME; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_PERIOD; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_TIME; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_ID; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_NAME; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE; -import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID; - -import com.google.perftools.profiles.ProfileProto.Sample; -import java.util.List; - -class PprofTranslator { - public Pprof toPprof(List stackTraces) { - Pprof pprof = new Pprof(); - for (StackTrace stackTrace : stackTraces) { - pprof.add(translateToPprofSample(stackTrace, pprof)); - } - return pprof; - } - - private Sample translateToPprofSample(StackTrace stackTrace, Pprof pprof) { - Sample.Builder sample = Sample.newBuilder(); - sample.addLabel(pprof.newLabel(THREAD_ID, stackTrace.getThreadId())); - sample.addLabel(pprof.newLabel(THREAD_NAME, stackTrace.getThreadName())); - sample.addLabel( - pprof.newLabel(THREAD_STATE, stackTrace.getThreadState().toString().toLowerCase())); - sample.addLabel(pprof.newLabel(SOURCE_EVENT_NAME, "snapshot-profiling")); - sample.addLabel(pprof.newLabel(SOURCE_EVENT_TIME, stackTrace.getTimestamp().toEpochMilli())); - sample.addLabel(pprof.newLabel(SOURCE_EVENT_PERIOD, stackTrace.getDuration().toMillis())); - - for (StackTraceElement stackFrame : stackTrace.getStackFrames()) { - sample.addLocationId(pprof.getLocationId(stackFrame)); - pprof.incFrameCount(); - } - sample.addLabel(pprof.newLabel(TRACE_ID, stackTrace.getTraceId())); - - return sample.build(); - } -} diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java index 4c3559144..23687ff28 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java @@ -36,7 +36,7 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler { private static final Logger logger = Logger.getLogger(ScheduledExecutorStackTraceSampler.class.getName()); private static final int SCHEDULER_INITIAL_DELAY = 0; - private static final Duration SCHEDULER_PERIOD = Duration.ofMillis(20); + static final Duration SCHEDULER_PERIOD = Duration.ofMillis(20); private static final int MAX_ENTRY_DEPTH = 200; private final ConcurrentMap samplers = diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java deleted file mode 100644 index fa1284405..000000000 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/events/EventPeriodsTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.events; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; - -import java.time.Duration; -import java.util.function.Function; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class EventPeriodsTest { - - @Mock Function configFinder; - - @Test - void testNotCachedFirstParse() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")).thenReturn("250 ms"); - Duration result = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(Duration.ofMillis(250), result); - } - - @Test - void testCached() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")) - .thenReturn("26 s") - .thenThrow(new IllegalStateException()); - Duration result1 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result2 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result3 = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(Duration.ofSeconds(26), result1); - assertEquals(Duration.ofSeconds(26), result2); - assertEquals(Duration.ofSeconds(26), result3); - } - - @Test - void testNotFoundAlsoCached() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")) - .thenReturn(null) - .thenThrow(new IllegalStateException()); - Duration result1 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result2 = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result1); - assertEquals(EventPeriods.UNKNOWN, result2); - } - - @Test - void testNotParsedAlsoCached() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")) - .thenReturn("BLEAK BLOOP") - .thenThrow(new IllegalStateException()); - Duration result1 = eventPeriods.getDuration("jdk.SomeEvent"); - Duration result2 = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result1); - assertEquals(EventPeriods.UNKNOWN, result2); - } - - @Test - void testConfigNotFound() { - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")).thenReturn(null); - Duration result = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result); - } - - @Test - void testEveryChunk() { - // Sometimes the JFR config might have the word "everyChunk" instead of an actual value - EventPeriods eventPeriods = new EventPeriods(configFinder); - when(configFinder.apply("jdk.SomeEvent#period")).thenReturn("everyChunk"); - Duration result = eventPeriods.getDuration("jdk.SomeEvent"); - assertEquals(EventPeriods.UNKNOWN, result); - } -} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index 7bc1009bd..de7b7f528 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -28,11 +28,7 @@ import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; -import io.opentelemetry.api.logs.Severity; import java.io.ByteArrayInputStream; -import java.time.Clock; -import java.time.Instant; -import java.time.ZoneId; import java.util.List; import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.Test; @@ -63,6 +59,7 @@ void exportMultipleStackTraceAsSingleOpenTelemetryLog() { assertEquals(1, logger.records().size()); } + /* @Test void setTimestampOnLogMessage() { var stackTrace = Snapshotting.stackTrace().build(); @@ -87,6 +84,8 @@ void setSeverityToInfoOnLogMessage() { assertEquals(Severity.INFO, logger.records().get(0).getSeverity()); } + */ + @Test void encodedLogBodyIsPprofProtobufMessage() { var stackTrace = Snapshotting.stackTrace().build(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java deleted file mode 100644 index 4288e003d..000000000 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofTranslatorTest.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright Splunk Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.splunk.opentelemetry.profiler.snapshot; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.perftools.profiles.ProfileProto.Location; -import com.google.perftools.profiles.ProfileProto.Profile; -import com.google.perftools.profiles.ProfileProto.Sample; -import com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.junit.jupiter.api.Test; - -class PprofTranslatorTest { - private final PprofTranslator translator = new PprofTranslator(); - - @Test - void allStackFramesAreInPprofStringTable() { - var exception = new RuntimeException(); - var stackTrace = Snapshotting.stackTrace().with(exception).build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - var table = profile.getStringTableList(); - - assertThat(table).containsAll(fullyQualifiedMethodNames(exception.getStackTrace())); - } - - private List fullyQualifiedMethodNames(StackTraceElement[] stackTrace) { - return Arrays.stream(stackTrace) - .map(stackFrame -> stackFrame.getClassName() + "." + stackFrame.getMethodName()) - .collect(Collectors.toList()); - } - - @Test - void allStackFramesIncludedInSample() { - var exception = new RuntimeException(); - var stackTrace = Snapshotting.stackTrace().with(exception).build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - - var expectedStackTrace = removeModuleInfo(exception.getStackTrace()); - var reportedStackTrace = toStackTrace(profile.getSample(0), profile); - assertEquals(expectedStackTrace.size(), reportedStackTrace.size()); - for (int i = 0; i < expectedStackTrace.size(); i++) { - var expectedStackFrame = expectedStackTrace.get(0); - var actualStackFrame = reportedStackTrace.get(0); - assertAll( - () -> - assertEquals( - expectedStackFrame.getClassLoaderName(), actualStackFrame.getClassLoaderName()), - () -> assertEquals(expectedStackFrame.getModuleName(), actualStackFrame.getModuleName()), - () -> - assertEquals( - expectedStackFrame.getModuleVersion(), actualStackFrame.getModuleVersion()), - () -> assertEquals(expectedStackFrame.getClassName(), actualStackFrame.getClassName()), - () -> assertEquals(expectedStackFrame.getLineNumber(), actualStackFrame.getLineNumber()), - () -> assertEquals(expectedStackFrame.getMethodName(), actualStackFrame.getMethodName()), - () -> assertEquals(expectedStackFrame.getFileName(), actualStackFrame.getFileName())); - } - } - - private List toStackTrace(Sample sample, Profile profile) { - List stackTrace = new ArrayList<>(); - for (var locationId : sample.getLocationIdList()) { - var location = profile.getLocation(locationId.intValue() - 1); - stackTrace.add(toStackTrace(location, profile)); - } - return stackTrace; - } - - private StackTraceElement toStackTrace(Location location, Profile profile) { - var line = location.getLine(0); - var functionId = line.getFunctionId(); - var function = profile.getFunction((int) functionId - 1); - var fileName = profile.getStringTable((int) function.getFilename()); - var functionName = profile.getStringTable((int) function.getName()); - - var declaringClass = functionName.substring(0, functionName.lastIndexOf('.')); - var methodName = functionName.substring(functionName.lastIndexOf('.') + 1); - return new StackTraceElement(declaringClass, methodName, fileName, (int) line.getLine()); - } - - private List removeModuleInfo(StackTraceElement[] originalStackTrace) { - List stackTrace = new ArrayList<>(); - for (var stackFrame : originalStackTrace) { - stackTrace.add(removeModuleInfo(stackFrame)); - } - return stackTrace; - } - - private StackTraceElement removeModuleInfo(StackTraceElement stackFrame) { - return new StackTraceElement( - stackFrame.getClassName(), - stackFrame.getMethodName(), - stackFrame.getFileName(), - stackFrame.getLineNumber()); - } - - @Test - void maintainStackFrameCount() { - var stackTrace = Snapshotting.stackTrace().with(new RuntimeException()).build(); - - var pprof = translator.toPprof(List.of(stackTrace)); - - assertEquals(stackTrace.getStackFrames().length, pprof.frameCount()); - } - - @Test - void maintainStackFrameCountAcrossMultipleStackTraces() { - var stackTrace1 = Snapshotting.stackTrace().with(new RuntimeException()).build(); - var stackTrace2 = Snapshotting.stackTrace().with(new IllegalArgumentException()).build(); - var stackTrace3 = Snapshotting.stackTrace().with(new IOException()).build(); - - var pprof = translator.toPprof(List.of(stackTrace1, stackTrace2, stackTrace3)); - - var expectedFrameCount = - stackTrace1.getStackFrames().length - + stackTrace2.getStackFrames().length - + stackTrace3.getStackFrames().length; - assertEquals(expectedFrameCount, pprof.frameCount()); - } - - @Test - void includeThreadInformationInSamples() { - var stackTrace = Snapshotting.stackTrace().build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - var sample = profile.getSample(0); - - var labels = PprofUtils.toLabelString(sample, profile); - assertThat(labels) - .containsEntry(ProfilingSemanticAttributes.THREAD_ID.getKey(), stackTrace.getThreadId()); - assertThat(labels) - .containsEntry( - ProfilingSemanticAttributes.THREAD_NAME.getKey(), stackTrace.getThreadName()); - assertThat(labels) - .containsEntry( - ProfilingSemanticAttributes.THREAD_STATE.getKey(), - stackTrace.getThreadState().toString().toLowerCase()); - } - - @Test - void includeTraceIdInformationInSamples() { - var stackTrace = Snapshotting.stackTrace().build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - var sample = profile.getSample(0); - - var labels = PprofUtils.toLabelString(sample, profile); - assertThat(labels) - .containsEntry(ProfilingSemanticAttributes.TRACE_ID.getKey(), stackTrace.getTraceId()); - } - - @Test - void includeSourceEventNameAsSnapshotProfilingInSamples() { - var stackTrace = Snapshotting.stackTrace().build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - var sample = profile.getSample(0); - - var labels = PprofUtils.toLabelString(sample, profile); - assertThat(labels) - .containsEntry( - ProfilingSemanticAttributes.SOURCE_EVENT_NAME.getKey(), "snapshot-profiling"); - } - - @Test - void includeStackTraceTimestampInSamples() { - var stackTrace = Snapshotting.stackTrace().build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - var sample = profile.getSample(0); - - var labels = PprofUtils.toLabelString(sample, profile); - assertThat(labels) - .containsEntry( - ProfilingSemanticAttributes.SOURCE_EVENT_TIME.getKey(), - stackTrace.getTimestamp().toEpochMilli()); - } - - @Test - void includeStackTraceDurationInSamples() { - var stackTrace = Snapshotting.stackTrace().build(); - - var profile = translator.toPprof(List.of(stackTrace)).build(); - var sample = profile.getSample(0); - - var labels = PprofUtils.toLabelString(sample, profile); - assertThat(labels) - .containsEntry( - ProfilingSemanticAttributes.SOURCE_EVENT_PERIOD.getKey(), - stackTrace.getDuration().toMillis()); - } -} From cc4c27d5b9c09540d4931d222b032c76746efcbb Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Fri, 21 Mar 2025 10:46:55 -0700 Subject: [PATCH 21/28] Remove commented out tests in AsyncStackTraceExporterTest. --- .../snapshot/AsyncStackTraceExporterTest.java | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index de7b7f528..88d863abf 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -59,33 +59,6 @@ void exportMultipleStackTraceAsSingleOpenTelemetryLog() { assertEquals(1, logger.records().size()); } - /* - @Test - void setTimestampOnLogMessage() { - var stackTrace = Snapshotting.stackTrace().build(); - var timestamp = Instant.ofEpochMilli(System.currentTimeMillis()); - var clock = Clock.fixed(timestamp, ZoneId.systemDefault()); - - var exporter = new AsyncStackTraceExporter(logger, clock); - exporter.export(List.of(stackTrace)); - await().until(() -> !logger.records().isEmpty()); - - var logRecord = logger.records().get(0); - assertEquals(timestamp, Instant.ofEpochSecond(0L, logRecord.getTimestampEpochNanos())); - } - - @Test - void setSeverityToInfoOnLogMessage() { - var stackTrace = Snapshotting.stackTrace().build(); - - exporter.export(List.of(stackTrace)); - await().until(() -> !logger.records().isEmpty()); - - assertEquals(Severity.INFO, logger.records().get(0).getSeverity()); - } - - */ - @Test void encodedLogBodyIsPprofProtobufMessage() { var stackTrace = Snapshotting.stackTrace().build(); From 475b5d9fd5782f4ca4316db8395d4504858d04d0 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Fri, 21 Mar 2025 10:56:04 -0700 Subject: [PATCH 22/28] Add configuration option for controlling the max depth of stack traces collected by the snaphot profiler. --- .../opentelemetry/profiler/Configuration.java | 6 ++++++ .../opentelemetry/profiler/ConfigurationTest.java | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index 5b986e36e..752294561 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -74,6 +74,8 @@ public class Configuration implements AutoConfigurationCustomizerProvider { public static final String CONFIG_KEY_SNAPSHOT_SELECTION_RATE = "splunk.snapshot.selection.rate"; private static final double DEFAULT_SNAPSHOT_SELECTION_RATE = 0.01; private static final double MAX_SNAPSHOT_SELECTION_RATE = 0.10; + private static final String CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH = "splunk.snapshot.profiler.max.stack.depth"; + private static final int DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH = 1024; @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { @@ -211,4 +213,8 @@ public static double getSnapshotSelectionRate(ConfigProperties properties) { return DEFAULT_SNAPSHOT_SELECTION_RATE; } } + + public static int getSnapshotProfilerStackDepth(ConfigProperties properties) { + return properties.getInt(CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH, DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH); + } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java index 16db64d83..9cbc3ee8d 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java @@ -170,4 +170,19 @@ void getSnapshotSelectionRateUsesMaxSelectionRateWhenConfiguredRateIsHigher( double actualSelectionRate = Configuration.getSnapshotSelectionRate(properties); assertEquals(0.10, actualSelectionRate); } + + @ParameterizedTest + @ValueSource(ints = {128, 512, 2056}) + void getConfiguredSnapshotProfilerStackDepth(int depth) { + var properties = DefaultConfigProperties.create(Map.of( + "splunk.snapshot.profiler.max.stack.depth", String.valueOf(depth) + )); + assertEquals(depth, Configuration.getSnapshotProfilerStackDepth(properties)); + } + + @Test + void getDefaultSnapshotProfilerStackDepthWhenNotSpecified() { + var properties = DefaultConfigProperties.create(Collections.emptyMap()); + assertEquals(1024, Configuration.getSnapshotProfilerStackDepth(properties)); + } } From a5bfcbcf03c2800dea10b358f3bd353cef210063 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Fri, 21 Mar 2025 13:51:04 -0700 Subject: [PATCH 23/28] Add tests for new stack trace exporting and configuration option for max stack trace depth for the snapshot profiler. --- .../opentelemetry/profiler/Configuration.java | 6 +- .../exporter/PprofCpuEventExporter.java | 5 +- .../snapshot/AsyncStackTraceExporter.java | 10 +- .../ScheduledExecutorStackTraceSampler.java | 7 +- .../snapshot/StackTraceExporterActivator.java | 3 +- .../profiler/ConfigurationTest.java | 8 +- .../exporter/PprofCpuEventExporterTest.java | 406 ++++++++++++++++++ .../{snapshot => pprof}/PprofUtils.java | 11 +- .../snapshot/AsyncStackTraceExporterTest.java | 3 +- .../SnapshotProfilingLogExportingTest.java | 1 + 10 files changed, 434 insertions(+), 26 deletions(-) create mode 100644 profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporterTest.java rename profiler/src/test/java/com/splunk/opentelemetry/profiler/{snapshot => pprof}/PprofUtils.java (85%) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index 752294561..89aa86121 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -74,7 +74,8 @@ public class Configuration implements AutoConfigurationCustomizerProvider { public static final String CONFIG_KEY_SNAPSHOT_SELECTION_RATE = "splunk.snapshot.selection.rate"; private static final double DEFAULT_SNAPSHOT_SELECTION_RATE = 0.01; private static final double MAX_SNAPSHOT_SELECTION_RATE = 0.10; - private static final String CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH = "splunk.snapshot.profiler.max.stack.depth"; + private static final String CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH = + "splunk.snapshot.profiler.max.stack.depth"; private static final int DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH = 1024; @Override @@ -215,6 +216,7 @@ public static double getSnapshotSelectionRate(ConfigProperties properties) { } public static int getSnapshotProfilerStackDepth(ConfigProperties properties) { - return properties.getInt(CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH, DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH); + return properties.getInt( + CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH, DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH); } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java index 816f620ca..0af068360 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java @@ -122,10 +122,7 @@ public void export( } String className = ste.getClassName(); String methodName = ste.getMethodName(); - int lineNumber = ste.getLineNumber(); - if (lineNumber < 0) { - lineNumber = 0; - } + int lineNumber = Math.max(ste.getLineNumber(), 0); sample.addLocationId(pprof.getLocationId(fileName, className, methodName, lineNumber)); pprof.incFrameCount(); } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index f12f44bec..ecd163208 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -34,15 +34,17 @@ class AsyncStackTraceExporter implements StackTraceExporter { private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final Logger otelLogger; + private final int maxDepth; private final Clock clock; - AsyncStackTraceExporter(Logger logger) { - this(logger, Clock.systemUTC()); + AsyncStackTraceExporter(Logger logger, int maxDepth) { + this(logger, maxDepth, Clock.systemUTC()); } @VisibleForTesting - AsyncStackTraceExporter(Logger logger, Clock clock) { + AsyncStackTraceExporter(Logger logger, int maxDepth, Clock clock) { this.otelLogger = logger; + this.maxDepth = maxDepth; this.clock = clock; } @@ -57,7 +59,7 @@ private Runnable pprofExporter(Logger otelLogger, List stackTraces) CpuEventExporter cpuEventExporter = PprofCpuEventExporter.builder() .otelLogger(otelLogger) - .stackDepth(200) + .stackDepth(maxDepth) .period(ScheduledExecutorStackTraceSampler.SCHEDULER_PERIOD) .instrumentationSource(InstrumentationSource.SNAPSHOT) .build(); diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java index 23687ff28..37b096f4d 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java @@ -37,7 +37,6 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler { Logger.getLogger(ScheduledExecutorStackTraceSampler.class.getName()); private static final int SCHEDULER_INITIAL_DELAY = 0; static final Duration SCHEDULER_PERIOD = Duration.ofMillis(20); - private static final int MAX_ENTRY_DEPTH = 200; private final ConcurrentMap samplers = new ConcurrentHashMap<>(); @@ -85,8 +84,8 @@ class StackTraceGatherer implements Runnable { private final String traceId; private final long threadId; - StackTraceGatherer(Duration samplingPeroid, String traceId, long threadId) { - this.samplingPeriod = samplingPeroid; + StackTraceGatherer(Duration samplingPeriod, String traceId, long threadId) { + this.samplingPeriod = samplingPeriod; this.traceId = traceId; this.threadId = threadId; } @@ -95,7 +94,7 @@ class StackTraceGatherer implements Runnable { public void run() { try { Instant now = Instant.now(); - ThreadInfo threadInfo = threadMXBean.getThreadInfo(threadId, MAX_ENTRY_DEPTH); + ThreadInfo threadInfo = threadMXBean.getThreadInfo(threadId, Integer.MAX_VALUE); StackTrace stackTrace = StackTrace.from(now, samplingPeriod, traceId, threadInfo); stagingArea.stage(traceId, stackTrace); } catch (Exception e) { diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java index d2e7b68a8..fad807813 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java @@ -44,9 +44,10 @@ public StackTraceExporterActivator() { public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) { ConfigProperties properties = AutoConfigureUtil.getConfig(autoConfiguredOpenTelemetrySdk); if (snapshotProfilingEnabled(properties)) { + int maxDepth = Configuration.getSnapshotProfilerStackDepth(properties); Resource resource = AutoConfigureUtil.getResource(autoConfiguredOpenTelemetrySdk); Logger logger = otelLoggerFactory.build(properties, resource); - AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger); + AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, maxDepth); StackTraceExporterProvider.INSTANCE.configure(exporter); } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java index 9cbc3ee8d..7072c773c 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java @@ -172,11 +172,11 @@ void getSnapshotSelectionRateUsesMaxSelectionRateWhenConfiguredRateIsHigher( } @ParameterizedTest - @ValueSource(ints = {128, 512, 2056}) + @ValueSource(ints = {128, 512, 2056}) void getConfiguredSnapshotProfilerStackDepth(int depth) { - var properties = DefaultConfigProperties.create(Map.of( - "splunk.snapshot.profiler.max.stack.depth", String.valueOf(depth) - )); + var properties = + DefaultConfigProperties.create( + Map.of("splunk.snapshot.profiler.max.stack.depth", String.valueOf(depth))); assertEquals(depth, Configuration.getSnapshotProfilerStackDepth(properties)); } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporterTest.java new file mode 100644 index 000000000..e7caae19f --- /dev/null +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporterTest.java @@ -0,0 +1,406 @@ +/* + * Copyright Splunk Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.splunk.opentelemetry.profiler.exporter; + +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.FRAME_COUNT; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.perftools.profiles.ProfileProto.Location; +import com.google.perftools.profiles.ProfileProto.Profile; +import com.google.perftools.profiles.ProfileProto.Sample; +import com.splunk.opentelemetry.profiler.InstrumentationSource; +import com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes; +import com.splunk.opentelemetry.profiler.pprof.PprofUtils; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.IdGenerator; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +class PprofCpuEventExporterTest { + private final InMemoryOtelLogger logger = new InMemoryOtelLogger(); + private final PprofCpuEventExporter exporter = + new PprofCpuEventExporter.Builder() + .otelLogger(logger) + .period(Duration.ofMillis(20)) + .stackDepth(1024) + .instrumentationSource(InstrumentationSource.SNAPSHOT) + .build(); + + @Test + void noLogRecordWhenNothingToExport() { + exporter.flush(); + assertThat(logger.records()).isEmpty(); + } + + @Test + void allStackFramesAreInPprofStringTable() throws Exception { + var exception = new RuntimeException(); + + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception.getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var table = profile.getStringTableList(); + + assertThat(table).containsAll(fullyQualifiedMethodNames(exception.getStackTrace())); + } + + private List fullyQualifiedMethodNames(StackTraceElement[] stackTrace) { + return Arrays.stream(stackTrace) + .map(stackFrame -> stackFrame.getClassName() + "." + stackFrame.getMethodName()) + .collect(Collectors.toList()); + } + + @Test + void reportStackTraceWasTruncated() throws Exception { + var exception = new RuntimeException(); + var exporter = + new PprofCpuEventExporter.Builder() + .otelLogger(logger) + .period(Duration.ofMillis(20)) + .stackDepth(exception.getStackTrace().length - 1) + .instrumentationSource(InstrumentationSource.SNAPSHOT) + .build(); + + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception.getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels) + .containsEntry(ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED.getKey(), "true"); + } + + @Test + void includeOnlyTruncatedStackFrames() throws Exception { + var exception = new RuntimeException(); + var depth = exception.getStackTrace().length - 1; + var exporter = + new PprofCpuEventExporter.Builder() + .otelLogger(logger) + .period(Duration.ofMillis(20)) + .stackDepth(depth) + .instrumentationSource(InstrumentationSource.SNAPSHOT) + .build(); + + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception.getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + + var expectedStackTrace = removeModuleInfo(exception.getStackTrace()).subList(0, depth); + var reportedStackTrace = toStackTrace(profile.getSample(0), profile); + for (int i = 0; i < depth; i++) { + var expectedStackFrame = expectedStackTrace.get(i); + var actualStackFrame = reportedStackTrace.get(i); + assertAll( + () -> + assertEquals( + expectedStackFrame.getClassLoaderName(), actualStackFrame.getClassLoaderName()), + () -> assertEquals(expectedStackFrame.getModuleName(), actualStackFrame.getModuleName()), + () -> + assertEquals( + expectedStackFrame.getModuleVersion(), actualStackFrame.getModuleVersion()), + () -> assertEquals(expectedStackFrame.getClassName(), actualStackFrame.getClassName()), + () -> assertEquals(expectedStackFrame.getLineNumber(), actualStackFrame.getLineNumber()), + () -> assertEquals(expectedStackFrame.getMethodName(), actualStackFrame.getMethodName()), + () -> assertEquals(expectedStackFrame.getFileName(), actualStackFrame.getFileName())); + } + } + + @Test + void allStackFramesIncludedInSample() throws Exception { + var exception = new RuntimeException(); + + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception.getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + + var expectedStackTrace = removeModuleInfo(exception.getStackTrace()); + var reportedStackTrace = toStackTrace(profile.getSample(0), profile); + assertEquals(expectedStackTrace.size(), reportedStackTrace.size()); + for (int i = 0; i < expectedStackTrace.size(); i++) { + var expectedStackFrame = expectedStackTrace.get(i); + var actualStackFrame = reportedStackTrace.get(i); + assertAll( + () -> + assertEquals( + expectedStackFrame.getClassLoaderName(), actualStackFrame.getClassLoaderName()), + () -> assertEquals(expectedStackFrame.getModuleName(), actualStackFrame.getModuleName()), + () -> + assertEquals( + expectedStackFrame.getModuleVersion(), actualStackFrame.getModuleVersion()), + () -> assertEquals(expectedStackFrame.getClassName(), actualStackFrame.getClassName()), + () -> assertEquals(expectedStackFrame.getLineNumber(), actualStackFrame.getLineNumber()), + () -> assertEquals(expectedStackFrame.getMethodName(), actualStackFrame.getMethodName()), + () -> assertEquals(expectedStackFrame.getFileName(), actualStackFrame.getFileName())); + } + } + + private List toStackTrace(Sample sample, Profile profile) { + List stackTrace = new ArrayList<>(); + for (var locationId : sample.getLocationIdList()) { + var location = profile.getLocation(locationId.intValue() - 1); + stackTrace.add(toStackTrace(location, profile)); + } + return stackTrace; + } + + private StackTraceElement toStackTrace(Location location, Profile profile) { + var line = location.getLine(0); + var functionId = line.getFunctionId(); + var function = profile.getFunction((int) functionId - 1); + var fileName = profile.getStringTable((int) function.getFilename()); + var functionName = profile.getStringTable((int) function.getName()); + + var declaringClass = functionName.substring(0, functionName.lastIndexOf('.')); + var methodName = functionName.substring(functionName.lastIndexOf('.') + 1); + return new StackTraceElement(declaringClass, methodName, fileName, (int) line.getLine()); + } + + private List removeModuleInfo(StackTraceElement[] originalStackTrace) { + List stackTrace = new ArrayList<>(); + for (var stackFrame : originalStackTrace) { + stackTrace.add(removeModuleInfo(stackFrame)); + } + return stackTrace; + } + + private StackTraceElement removeModuleInfo(StackTraceElement stackFrame) { + return new StackTraceElement( + stackFrame.getClassName(), + stackFrame.getMethodName(), + valueOrUnknown(stackFrame.getFileName()), + Math.max(stackFrame.getLineNumber(), 0)); + } + + private String valueOrUnknown(String value) { + return value == null ? "unknown" : value; + } + + @Test + void maintainStackFrameCount() { + var exception = new RuntimeException(); + + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception.getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + assertEquals(exception.getStackTrace().length, logRecord.getAttributes().get(FRAME_COUNT)); + } + + @Test + void maintainStackFrameCountAcrossMultipleStackTraces() { + var exception1 = new RuntimeException(); + var exception2 = new IllegalArgumentException(); + var exception3 = new IOException(); + + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception1.getStackTrace(), Instant.now(), "", ""); + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception2.getStackTrace(), Instant.now(), "", ""); + exporter.export( + 1, "thread-name", Thread.State.RUNNABLE, exception3.getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var expectedFrameCount = + exception1.getStackTrace().length + + exception2.getStackTrace().length + + exception3.getStackTrace().length; + var logRecord = logger.records().get(0); + assertEquals(expectedFrameCount, logRecord.getAttributes().get(FRAME_COUNT)); + } + + @ParameterizedTest + @EnumSource(Thread.State.class) + void includeThreadInformationInSamples(Thread.State state) throws Exception { + var random = new Random(); + var threadId = new Random().nextLong(10_000); + var threadName = "thread-name-" + random.nextInt(1000); + + exporter.export( + threadId, threadName, state, new RuntimeException().getStackTrace(), Instant.now(), "", ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels).contains(entry(ProfilingSemanticAttributes.THREAD_ID, threadId)); + assertThat(labels).contains(entry(ProfilingSemanticAttributes.THREAD_NAME, threadName)); + assertThat(labels).contains(entry(ProfilingSemanticAttributes.THREAD_STATE, state.toString())); + } + + @Test + void includeTraceIdInformationInSamples() throws Exception { + var traceId = IdGenerator.random().generateTraceId(); + + exporter.export( + 1, + "thread-name", + Thread.State.RUNNABLE, + new RuntimeException().getStackTrace(), + Instant.now(), + traceId, + ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels).contains(entry(ProfilingSemanticAttributes.TRACE_ID, traceId)); + } + + @Test + void doNotIncludeInvalidTraceIdsInformationInSamples() throws Exception { + exporter.export( + 1, + "thread-name", + Thread.State.RUNNABLE, + new RuntimeException().getStackTrace(), + Instant.now(), + "", + ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels).doesNotContainKey(ProfilingSemanticAttributes.TRACE_ID.getKey()); + } + + @Test + void includeSpanIdInformationInSamples() throws Exception { + var spanId = IdGenerator.random().generateSpanId(); + + exporter.export( + 1, + "thread-name", + Thread.State.RUNNABLE, + new RuntimeException().getStackTrace(), + Instant.now(), + "", + spanId); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels).contains(entry(ProfilingSemanticAttributes.SPAN_ID, spanId)); + } + + @Test + void doNotIncludeInvalidSpanIdsInformationInSamples() throws Exception { + exporter.export( + 1, + "thread-name", + Thread.State.RUNNABLE, + new RuntimeException().getStackTrace(), + Instant.now(), + "", + ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels).doesNotContainKey(ProfilingSemanticAttributes.SPAN_ID.getKey()); + } + + @Test + void includeStackTraceTimestampInSamples() throws Exception { + var time = Instant.now(); + + exporter.export( + 1, + "thread-name", + Thread.State.RUNNABLE, + new RuntimeException().getStackTrace(), + time, + "", + ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels) + .contains(entry(ProfilingSemanticAttributes.SOURCE_EVENT_TIME, time.toEpochMilli())); + } + + @Test + void includeStackTraceDurationInSamples() throws Exception { + exporter.export( + 1, + "thread-name", + Thread.State.RUNNABLE, + new RuntimeException().getStackTrace(), + Instant.now(), + "", + ""); + exporter.flush(); + + var logRecord = logger.records().get(0); + var profile = Profile.parseFrom(PprofUtils.deserialize(logRecord)); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels) + .contains( + entry( + ProfilingSemanticAttributes.SOURCE_EVENT_PERIOD, Duration.ofMillis(20).toMillis())); + } + + private Map.Entry entry(AttributeKey attribute, T value) { + return Map.entry(attribute.getKey(), value); + } +} diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/pprof/PprofUtils.java similarity index 85% rename from profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java rename to profiler/src/test/java/com/splunk/opentelemetry/profiler/pprof/PprofUtils.java index 32b33e7d0..d0758272e 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/PprofUtils.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/pprof/PprofUtils.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.splunk.opentelemetry.profiler.snapshot; +package com.splunk.opentelemetry.profiler.pprof; import com.google.perftools.profiles.ProfileProto.Profile; import com.google.perftools.profiles.ProfileProto.Sample; @@ -27,15 +27,14 @@ import java.util.Map; import java.util.zip.GZIPInputStream; -class PprofUtils { - - static byte[] deserialize(LogRecordData logRecord) throws IOException { +public class PprofUtils { + public static byte[] deserialize(LogRecordData logRecord) throws IOException { var bytes = new ByteArrayInputStream(decode(logRecord)); var inputStream = new GZIPInputStream(bytes); return inputStream.readAllBytes(); } - static byte[] decode(LogRecordData logRecord) { + public static byte[] decode(LogRecordData logRecord) { Value body = logRecord.getBodyValue(); if (body == null) { throw new RuntimeException("Log record body is null"); @@ -43,7 +42,7 @@ static byte[] decode(LogRecordData logRecord) { return Base64.getDecoder().decode(body.asString()); } - static Map toLabelString(Sample sample, Profile profile) { + public static Map toLabelString(Sample sample, Profile profile) { var labels = new HashMap(); for (var label : sample.getLabelList()) { var stringTableIndex = label.getKey(); diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index 88d863abf..e4191c27e 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -28,6 +28,7 @@ import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; +import com.splunk.opentelemetry.profiler.pprof.PprofUtils; import java.io.ByteArrayInputStream; import java.util.List; import java.util.zip.GZIPInputStream; @@ -35,7 +36,7 @@ class AsyncStackTraceExporterTest { private final InMemoryOtelLogger logger = new InMemoryOtelLogger(); - private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger); + private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, 200); @Test void exportStackTraceAsOpenTelemetryLog() { diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java index d579c1113..6dadb86b8 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingLogExportingTest.java @@ -23,6 +23,7 @@ import com.google.perftools.profiles.ProfileProto.Profile; import com.splunk.opentelemetry.profiler.OtelLoggerFactory; import com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes; +import com.splunk.opentelemetry.profiler.pprof.PprofUtils; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; From bd562fd3ed6a21be371176826272bbd8198d6476 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Fri, 21 Mar 2025 14:56:26 -0700 Subject: [PATCH 24/28] Add configuration option for controlling the snapshot profiling sampling rate. --- .../opentelemetry/profiler/Configuration.java | 11 ++++++++ .../snapshot/AsyncStackTraceExporter.java | 20 +++++--------- .../ScheduledExecutorStackTraceSampler.java | 7 ----- .../SnapshotProfilingSdkCustomizer.java | 26 ++++++++++++++----- .../snapshot/StackTraceExporterActivator.java | 13 +++++++--- .../profiler/ConfigurationTest.java | 19 ++++++++++++++ .../snapshot/AsyncStackTraceExporterTest.java | 20 +++++++++++++- ...SnapshotProfilingSdkCustomizerBuilder.java | 5 +++- 8 files changed, 88 insertions(+), 33 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index 89aa86121..5f2ee40dd 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -77,6 +77,9 @@ public class Configuration implements AutoConfigurationCustomizerProvider { private static final String CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH = "splunk.snapshot.profiler.max.stack.depth"; private static final int DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH = 1024; + private static final String CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL = + "splunk.snapshot.profiler.sampling.interval"; + public static final Duration DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL = Duration.ofMillis(20); @Override public void customize(AutoConfigurationCustomizer autoConfiguration) { @@ -219,4 +222,12 @@ public static int getSnapshotProfilerStackDepth(ConfigProperties properties) { return properties.getInt( CONFIG_KEY_SNAPSHOT_PROFILER_STACK_DEPTH, DEFAULT_SNAPSHOT_PROFILER_STACK_DEPTH); } + + public static Duration getSnapshotProfilerSamplingInterval(ConfigProperties properties) { + int millis = properties.getInt(CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL, 0); + if (millis > 0) { + return Duration.ofMillis(millis); + } + return DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL; + } } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index ecd163208..26fda6a3d 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -16,13 +16,11 @@ package com.splunk.opentelemetry.profiler.snapshot; -import com.google.common.annotations.VisibleForTesting; import com.splunk.opentelemetry.profiler.InstrumentationSource; import com.splunk.opentelemetry.profiler.exporter.CpuEventExporter; import com.splunk.opentelemetry.profiler.exporter.PprofCpuEventExporter; import io.opentelemetry.api.logs.Logger; -import java.time.Clock; -import java.time.Instant; +import java.time.Duration; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -34,18 +32,13 @@ class AsyncStackTraceExporter implements StackTraceExporter { private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final Logger otelLogger; + private final Duration samplingPeriod; private final int maxDepth; - private final Clock clock; - AsyncStackTraceExporter(Logger logger, int maxDepth) { - this(logger, maxDepth, Clock.systemUTC()); - } - - @VisibleForTesting - AsyncStackTraceExporter(Logger logger, int maxDepth, Clock clock) { + AsyncStackTraceExporter(Logger logger, Duration samplingPeriod, int maxDepth) { this.otelLogger = logger; + this.samplingPeriod = samplingPeriod; this.maxDepth = maxDepth; - this.clock = clock; } @Override @@ -60,18 +53,17 @@ private Runnable pprofExporter(Logger otelLogger, List stackTraces) PprofCpuEventExporter.builder() .otelLogger(otelLogger) .stackDepth(maxDepth) - .period(ScheduledExecutorStackTraceSampler.SCHEDULER_PERIOD) + .period(samplingPeriod) .instrumentationSource(InstrumentationSource.SNAPSHOT) .build(); - Instant now = Instant.now(clock); for (StackTrace stackTrace : stackTraces) { cpuEventExporter.export( stackTrace.getThreadId(), stackTrace.getThreadName(), stackTrace.getThreadState(), stackTrace.getStackFrames(), - now, + stackTrace.getTimestamp(), stackTrace.getTraceId(), null); } diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java index 37b096f4d..1d8a4669d 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSampler.java @@ -16,7 +16,6 @@ package com.splunk.opentelemetry.profiler.snapshot; -import com.google.common.annotations.VisibleForTesting; import io.opentelemetry.api.trace.SpanContext; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; @@ -36,7 +35,6 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler { private static final Logger logger = Logger.getLogger(ScheduledExecutorStackTraceSampler.class.getName()); private static final int SCHEDULER_INITIAL_DELAY = 0; - static final Duration SCHEDULER_PERIOD = Duration.ofMillis(20); private final ConcurrentMap samplers = new ConcurrentHashMap<>(); @@ -44,11 +42,6 @@ class ScheduledExecutorStackTraceSampler implements StackTraceSampler { private final StagingArea stagingArea; private final Duration samplingPeriod; - ScheduledExecutorStackTraceSampler(StagingArea stagingArea) { - this(stagingArea, SCHEDULER_PERIOD); - } - - @VisibleForTesting ScheduledExecutorStackTraceSampler(StagingArea stagingArea, Duration samplingPeriod) { this.stagingArea = stagingArea; this.samplingPeriod = samplingPeriod; diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java index 3717e3822..fc035b4c5 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizer.java @@ -23,6 +23,7 @@ import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import java.time.Duration; import java.util.Collections; import java.util.LinkedHashSet; import java.util.Map; @@ -33,19 +34,29 @@ @AutoService(AutoConfigurationCustomizerProvider.class) public class SnapshotProfilingSdkCustomizer implements AutoConfigurationCustomizerProvider { private final TraceRegistry registry; - private final StackTraceSampler sampler; + private final Function samplerProvider; public SnapshotProfilingSdkCustomizer() { - this( - new TraceRegistry(), - new ScheduledExecutorStackTraceSampler( - new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE))); + this(new TraceRegistry(), stackTraceSamplerProvider()); + } + + private static Function stackTraceSamplerProvider() { + return properties -> { + Duration samplingPeriod = Configuration.getSnapshotProfilerSamplingInterval(properties); + return new ScheduledExecutorStackTraceSampler( + new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE), samplingPeriod); + }; } @VisibleForTesting - SnapshotProfilingSdkCustomizer(TraceRegistry registry, StackTraceSampler sampler) { + SnapshotProfilingSdkCustomizer(TraceRegistry registry, StackTraceSampler samplerProvider) { + this(registry, properties -> samplerProvider); + } + + private SnapshotProfilingSdkCustomizer( + TraceRegistry registry, Function samplerProvider) { this.registry = registry; - this.sampler = sampler; + this.samplerProvider = samplerProvider; } @Override @@ -59,6 +70,7 @@ public void customize(AutoConfigurationCustomizer autoConfigurationCustomizer) { snapshotProfilingSpanProcessor(TraceRegistry registry) { return (builder, properties) -> { if (snapshotProfilingEnabled(properties)) { + StackTraceSampler sampler = samplerProvider.apply(properties); return builder.addSpanProcessor(new SnapshotProfilingSpanProcessor(registry, sampler)); } return builder; diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java index fad807813..c6d2d90e7 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/StackTraceExporterActivator.java @@ -26,6 +26,7 @@ import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.resources.Resource; +import java.time.Duration; @AutoService(AgentListener.class) public class StackTraceExporterActivator implements AgentListener { @@ -45,9 +46,10 @@ public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetr ConfigProperties properties = AutoConfigureUtil.getConfig(autoConfiguredOpenTelemetrySdk); if (snapshotProfilingEnabled(properties)) { int maxDepth = Configuration.getSnapshotProfilerStackDepth(properties); - Resource resource = AutoConfigureUtil.getResource(autoConfiguredOpenTelemetrySdk); - Logger logger = otelLoggerFactory.build(properties, resource); - AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, maxDepth); + Duration samplingPeriod = Configuration.getSnapshotProfilerSamplingInterval(properties); + Logger logger = buildLogger(autoConfiguredOpenTelemetrySdk, properties); + AsyncStackTraceExporter exporter = + new AsyncStackTraceExporter(logger, samplingPeriod, maxDepth); StackTraceExporterProvider.INSTANCE.configure(exporter); } } @@ -55,4 +57,9 @@ public void afterAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetr private boolean snapshotProfilingEnabled(ConfigProperties properties) { return Configuration.isSnapshotProfilingEnabled(properties); } + + private Logger buildLogger(AutoConfiguredOpenTelemetrySdk sdk, ConfigProperties properties) { + Resource resource = AutoConfigureUtil.getResource(sdk); + return otelLoggerFactory.build(properties, resource); + } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java index 7072c773c..5ecc3e112 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/ConfigurationTest.java @@ -24,6 +24,7 @@ import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties; import io.opentelemetry.sdk.autoconfigure.spi.internal.DefaultConfigProperties; +import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -185,4 +186,22 @@ void getDefaultSnapshotProfilerStackDepthWhenNotSpecified() { var properties = DefaultConfigProperties.create(Collections.emptyMap()); assertEquals(1024, Configuration.getSnapshotProfilerStackDepth(properties)); } + + @ParameterizedTest + @ValueSource(ints = {128, 512, 2056}) + void getConfiguredSnapshotProfilerSamplingInterval(int milliseconds) { + var properties = + DefaultConfigProperties.create( + Map.of("splunk.snapshot.profiler.sampling.interval", String.valueOf(milliseconds))); + assertEquals( + Duration.ofMillis(milliseconds), + Configuration.getSnapshotProfilerSamplingInterval(properties)); + } + + @Test + void getDefaultSnapshotProfilerSamplingInterval() { + var properties = DefaultConfigProperties.create(Collections.emptyMap()); + assertEquals( + Duration.ofMillis(20), Configuration.getSnapshotProfilerSamplingInterval(properties)); + } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java index e4191c27e..14ec0b56b 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporterTest.java @@ -20,6 +20,7 @@ import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.FRAME_COUNT; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SOURCE; +import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_EVENT_TIME; import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.SOURCE_TYPE; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @@ -30,13 +31,15 @@ import com.splunk.opentelemetry.profiler.exporter.InMemoryOtelLogger; import com.splunk.opentelemetry.profiler.pprof.PprofUtils; import java.io.ByteArrayInputStream; +import java.time.Duration; import java.util.List; import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.Test; class AsyncStackTraceExporterTest { private final InMemoryOtelLogger logger = new InMemoryOtelLogger(); - private final AsyncStackTraceExporter exporter = new AsyncStackTraceExporter(logger, 200); + private final AsyncStackTraceExporter exporter = + new AsyncStackTraceExporter(logger, Duration.ofMillis(20), 200); @Test void exportStackTraceAsOpenTelemetryLog() { @@ -153,4 +156,19 @@ void includeFrameCountOpenTelemetryAttributeInLogMessage() { assertThat(attributes.asMap()) .containsEntry(FRAME_COUNT, (long) stackTrace.getStackFrames().length); } + + @Test + void includeStackTraceDurationInSamples() throws Exception { + var stackTrace = Snapshotting.stackTrace().build(); + + exporter.export(List.of(stackTrace)); + await().until(() -> !logger.records().isEmpty()); + + var profile = Profile.parseFrom(PprofUtils.deserialize(logger.records().get(0))); + var sample = profile.getSample(0); + + var labels = PprofUtils.toLabelString(sample, profile); + assertThat(labels) + .containsEntry(SOURCE_EVENT_TIME.getKey(), stackTrace.getTimestamp().toEpochMilli()); + } } diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java index babe8dc12..a639bdbdc 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/SnapshotProfilingSdkCustomizerBuilder.java @@ -16,6 +16,8 @@ package com.splunk.opentelemetry.profiler.snapshot; +import java.time.Duration; + class SnapshotProfilingSdkCustomizerBuilder { private TraceRegistry registry = new TraceRegistry(); private StackTraceSampler sampler = new ObservableStackTraceSampler(); @@ -28,7 +30,8 @@ SnapshotProfilingSdkCustomizerBuilder with(TraceRegistry registry) { SnapshotProfilingSdkCustomizerBuilder withRealStackTraceSampler() { return with( new ScheduledExecutorStackTraceSampler( - new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE))); + new AccumulatingStagingArea(StackTraceExporterProvider.INSTANCE), + Duration.ofMillis(20))); } SnapshotProfilingSdkCustomizerBuilder with(StackTraceSampler sampler) { From c89be8a809ae80e6ce3797bb82c8a7cae2d44aa9 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Mon, 24 Mar 2025 08:21:25 -0700 Subject: [PATCH 25/28] Use a single threaded Executor in AsyncStackTraceExporter, not a ScheduledExecutor. --- .../profiler/snapshot/AsyncStackTraceExporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index 26fda6a3d..26815f6df 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -30,7 +30,7 @@ class AsyncStackTraceExporter implements StackTraceExporter { private static final java.util.logging.Logger logger = java.util.logging.Logger.getLogger(AsyncStackTraceExporter.class.getName()); - private final ExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + private final ExecutorService executor = Executors.newSingleThreadExecutor(); private final Logger otelLogger; private final Duration samplingPeriod; private final int maxDepth; From f8ef40e33109367a7b2d1cd060fd820c2e83366d Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Mon, 24 Mar 2025 08:22:42 -0700 Subject: [PATCH 26/28] Improve exception error message in AsyncStackTraceExporter. --- .../profiler/snapshot/AsyncStackTraceExporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java index 26815f6df..3fb36c8ae 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/snapshot/AsyncStackTraceExporter.java @@ -69,7 +69,7 @@ private Runnable pprofExporter(Logger otelLogger, List stackTraces) } cpuEventExporter.flush(); } catch (Exception e) { - logger.log(Level.SEVERE, "an exception was thrown", e); + logger.log(Level.SEVERE, "An exception was thrown while exporting profiling snapshots.", e); } }; } From b6f60559a27b6fca2e05930d2567f7dcb66bfb8b Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Mon, 24 Mar 2025 08:25:15 -0700 Subject: [PATCH 27/28] Remove configured wait timeout for tests in ScheduledExecutorStackTraceSamplerTest. --- ...cheduledExecutorStackTraceSamplerTest.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java index 548fdb590..6ec211b6a 100644 --- a/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java +++ b/profiler/src/test/java/com/splunk/opentelemetry/profiler/snapshot/ScheduledExecutorStackTraceSamplerTest.java @@ -36,7 +36,6 @@ import org.junit.jupiter.api.Test; class ScheduledExecutorStackTraceSamplerTest { - private static final Duration HALF_SECOND = Duration.ofMillis(500); private static final Duration SAMPLING_PERIOD = Duration.ofMillis(20); private final IdGenerator idGenerator = IdGenerator.random(); @@ -50,7 +49,7 @@ void takeStackTraceSampleForGivenThread() { try { sampler.start(spanContext); - await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + await().until(() -> !staging.allStackTraces().isEmpty()); } finally { sampler.stop(spanContext); } @@ -58,12 +57,13 @@ void takeStackTraceSampleForGivenThread() { @Test void continuallySampleThreadForStackTraces() { + var halfSecond = Duration.ofMillis(500); var spanContext = randomSpanContext(); - int expectedSamples = (int) HALF_SECOND.dividedBy(SAMPLING_PERIOD.multipliedBy(2)); + int expectedSamples = (int) halfSecond.dividedBy(SAMPLING_PERIOD.multipliedBy(2)); try { sampler.start(spanContext); - await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() >= expectedSamples); + await().until(() -> staging.allStackTraces().size() >= expectedSamples); } finally { sampler.stop(spanContext); } @@ -71,12 +71,13 @@ void continuallySampleThreadForStackTraces() { @Test void emptyStagingAreaAfterSamplingStops() { + var halfSecond = Duration.ofMillis(500); var spanContext = randomSpanContext(); - int expectedSamples = (int) HALF_SECOND.dividedBy(SAMPLING_PERIOD.multipliedBy(2)); + int expectedSamples = (int) halfSecond.dividedBy(SAMPLING_PERIOD.multipliedBy(2)); try { sampler.start(spanContext); - await().atMost(HALF_SECOND).until(() -> staging.allStackTraces().size() >= expectedSamples); + await().until(() -> staging.allStackTraces().size() >= expectedSamples); } finally { sampler.stop(spanContext); } @@ -120,7 +121,7 @@ void includeTimestampOnStackTraces() { try { sampler.start(spanContext); - await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + await().until(() -> !staging.allStackTraces().isEmpty()); var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); assertThat(stackTrace.getTimestamp()).isNotNull().isAfter(now); @@ -135,7 +136,7 @@ void includeDefaultSamplingPeriodOnFirstRecordedStackTraces() { try { sampler.start(spanContext); - await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + await().until(() -> !staging.allStackTraces().isEmpty()); var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); assertThat(stackTrace.getDuration()).isNotNull().isEqualTo(SAMPLING_PERIOD); @@ -165,7 +166,7 @@ void includeTraceIdOnStackTraces() { try { sampler.start(spanContext); - await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + await().until(() -> !staging.allStackTraces().isEmpty()); var stackTrace = staging.allStackTraces().stream().findFirst().orElseThrow(); assertEquals(spanContext.getTraceId(), stackTrace.getTraceId()); @@ -185,7 +186,7 @@ void includeThreadDetailsOnStackTraces() throws Exception { var future = executor.submit(startSampling(spanContext, startLatch, stopLatch)); startLatch.countDown(); - await().atMost(HALF_SECOND).until(() -> !staging.allStackTraces().isEmpty()); + await().until(() -> !staging.allStackTraces().isEmpty()); stopLatch.countDown(); var thread = future.get(); From 9358d523d36fb78c201758fdc8ddc1ed194d44e4 Mon Sep 17 00:00:00 2001 From: thomasduncan Date: Mon, 24 Mar 2025 10:48:44 -0700 Subject: [PATCH 28/28] Use ConfigProperties method 'getDuration' when looking for the snapshot profiling sampling interval. --- .../com/splunk/opentelemetry/profiler/Configuration.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java index 5f2ee40dd..6780b3a30 100644 --- a/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java +++ b/profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java @@ -224,10 +224,8 @@ public static int getSnapshotProfilerStackDepth(ConfigProperties properties) { } public static Duration getSnapshotProfilerSamplingInterval(ConfigProperties properties) { - int millis = properties.getInt(CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL, 0); - if (millis > 0) { - return Duration.ofMillis(millis); - } - return DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL; + return properties.getDuration( + CONFIG_KEY_SNAPSHOT_PROFILER_SAMPLING_INTERVAL, + DEFAULT_SNAPSHOT_PROFILER_SAMPLING_INTERVAL); } }