Skip to content

feat(diagnostics): Add thread dumps endpoint and storage #894

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat(diagnostics): Add thread dumps endpoint and storage #894

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
325df77
Initial implementation of thread dumps
Josh-Matsuoka May 2, 2025
e74a9d5
Support other thread dump format, sanity check format query parameter
Josh-Matsuoka May 2, 2025
caa7d2a
Merge remote-tracking branch 'upstream/main' into thread-dumps
Josh-Matsuoka May 16, 2025
368fa46
refactor, long-running api handling
Josh-Matsuoka May 30, 2025
4fe54df
Fix error notification class
Josh-Matsuoka Jun 2, 2025
fb193d5
Fix downloads, filter by target, temp debug logging
Josh-Matsuoka Jun 2, 2025
f4d9a40
Merge remote-tracking branch 'upstream/main' into thread-dumps
Josh-Matsuoka Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion smoketest.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ PULL_IMAGES=${PULL_IMAGES:-true}
KEEP_VOLUMES=${KEEP_VOLUMES:-false}
OPEN_TABS=${OPEN_TABS:-false}

PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes}
PRECREATE_BUCKETS=${PRECREATE_BUCKETS:-archivedrecordings,archivedreports,eventtemplates,probes,threaddumps}

LOG_LEVEL=0
CRYOSTAT_HTTP_HOST=${CRYOSTAT_HTTP_HOST:-cryostat}
Expand Down
1 change: 1 addition & 0 deletions src/main/java/io/cryostat/ConfigProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ public class ConfigProperties {
"storage.buckets.event-templates.name";
public static final String AWS_BUCKET_NAME_PROBE_TEMPLATES =
"storage.buckets.probe-templates.name";
public static final String AWS_BUCKET_NAME_THREAD_DUMPS = "storage.buckets.thread-dumps.name";

public static final String CONTAINERS_POLL_PERIOD = "cryostat.discovery.containers.poll-period";
public static final String CONTAINERS_REQUEST_TIMEOUT =
Expand Down
177 changes: 175 additions & 2 deletions src/main/java/io/cryostat/diagnostic/Diagnostics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,185 @@
*/
package io.cryostat.diagnostic;

import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;

import io.cryostat.ConfigProperties;
import io.cryostat.Producers;
import io.cryostat.recordings.LongRunningRequestGenerator;
import io.cryostat.recordings.LongRunningRequestGenerator.ThreadDumpRequest;
import io.cryostat.targets.Target;
import io.cryostat.targets.TargetConnectionManager;
import io.cryostat.util.HttpMimeType;

import io.smallrye.common.annotation.Blocking;
import io.vertx.core.http.HttpServerResponse;
import io.vertx.mutiny.core.eventbus.EventBus;
import jakarta.annotation.security.RolesAllowed;
import jakarta.inject.Inject;
import jakarta.inject.Named;
import jakarta.ws.rs.DELETE;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.NotFoundException;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.core.HttpHeaders;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import org.jboss.resteasy.reactive.RestPath;
import org.jboss.resteasy.reactive.RestQuery;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest;

@Path("/api/beta/diagnostics/targets/{targetId}")
@Path("/api/beta/diagnostics/")
public class Diagnostics {

@Inject TargetConnectionManager targetConnectionManager;
@Inject S3Client storage;
@Inject S3Presigner presigner;
@Inject Logger log;
@Inject LongRunningRequestGenerator generator;

@Inject
@Named(Producers.BASE64_URL)
Base64 base64Url;

@ConfigProperty(name = ConfigProperties.AWS_BUCKET_NAME_THREAD_DUMPS)
String bucket;

@ConfigProperty(name = ConfigProperties.STORAGE_PRESIGNED_DOWNLOADS_ENABLED)
boolean presignedDownloadsEnabled;

@ConfigProperty(name = ConfigProperties.STORAGE_EXT_URL)
Optional<String> externalStorageUrl;

@Inject EventBus bus;
@Inject DiagnosticsHelper helper;

@Path("targets/{targetId}/threaddump")
@RolesAllowed("write")
@Blocking
@POST
public String threadDump(
HttpServerResponse response, @RestPath long targetId, @RestQuery String format) {
log.trace("Creating new thread dump request for target: " + targetId);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Best to use the log.tracev("message {0} {1}", obj1, obj2) format specifier style rather than direct string concatenation. Using concatenation will result in an eager .toString() conversion being called on the operands, even if the message is logged at a lower level than currently configured.

ThreadDumpRequest request =
new ThreadDumpRequest(
UUID.randomUUID().toString(), Long.toString(targetId), format);
response.endHandler(
(e) -> bus.publish(LongRunningRequestGenerator.THREAD_DUMP_ADDRESS, request));
return request.id();
}

@Path("targets/{targetId}/threaddump")
@RolesAllowed("read")
@Blocking
@GET
public List<ThreadDump> getThreadDumps(@RestPath long targetId) {
log.warn("Fetching thread dumps for target: " + targetId);
log.warn("Thread dumps: " + helper.getThreadDumps(targetId));
log.warn("Storage bucket: " + bucket);
Copy link
Member

@andrewazores andrewazores Jun 4, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar for .warnv() here, especially since there is the getThreadDumps() call which will be pretty expensive for a simple logging call.

return helper.getThreadDumps(targetId);
}

@Path("/gc")
@DELETE
@Blocking
@Path("targets/{targetId}/threaddump/{threadDumpId}")
@RolesAllowed("write")
public void deleteThreadDump(@RestPath String threadDumpId) {
try {
log.warn("Deleting thread dump with ID: " + threadDumpId);
storage.headObject(
HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build());
} catch (NoSuchKeyException e) {
throw new NotFoundException(e);
}
storage.deleteObject(
DeleteObjectRequest.builder().bucket(bucket).key(threadDumpId).build());
}

@Path("/threaddump/download/{encodedKey}")
@RolesAllowed("read")
@Blocking
@GET
public RestResponse<Object> handleStorageDownload(
@RestPath String encodedKey, @RestQuery String query) throws URISyntaxException {
Pair<String, String> decodedKey = helper.decodedKey(encodedKey);
var threadDumpId = decodedKey.getValue().strip();
log.warn("Handling download Request for encodedKey: " + encodedKey);
log.warn("Handling download Request for query: " + query);
log.warn("Decoded key: " + decodedKey.toString());
log.warn("UUID: " + threadDumpId);
log.warn("Bucket: " + bucket);
storage.headObject(HeadObjectRequest.builder().bucket(bucket).key(threadDumpId).build())
.sdkHttpResponse();

if (!presignedDownloadsEnabled) {
return ResponseBuilder.ok()
.header(
HttpHeaders.CONTENT_DISPOSITION,
String.format("attachment; filename=\"%s\"", decodedKey.getValue()))
.header(HttpHeaders.CONTENT_TYPE, HttpMimeType.OCTET_STREAM.mime())
.entity(helper.getThreadDumpStream(encodedKey))
.build();
}

log.tracev("Handling presigned download request for {0}", decodedKey);
GetObjectRequest getRequest =
GetObjectRequest.builder().bucket(bucket).key(threadDumpId).build();
GetObjectPresignRequest presignRequest =
GetObjectPresignRequest.builder()
.signatureDuration(Duration.ofMinutes(1))
.getObjectRequest(getRequest)
.build();
PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest);
URI uri = presignedRequest.url().toURI();
if (externalStorageUrl.isPresent()) {
String extUrl = externalStorageUrl.get();
if (StringUtils.isNotBlank(extUrl)) {
URI extUri = new URI(extUrl);
uri =
new URI(
extUri.getScheme(),
extUri.getAuthority(),
URI.create(String.format("%s/%s", extUri.getPath(), uri.getPath()))
.normalize()
.getPath(),
uri.getQuery(),
uri.getFragment());
}
}
ResponseBuilder<Object> response =
ResponseBuilder.create(RestResponse.Status.PERMANENT_REDIRECT);
if (StringUtils.isNotBlank(query)) {
response =
response.header(
HttpHeaders.CONTENT_DISPOSITION,
String.format(
"attachment; filename=\"%s\"",
new String(base64Url.decode(query), StandardCharsets.UTF_8)));
}
return response.location(uri).build();
}

@Path("targets/{targetId}/gc")
@RolesAllowed("write")
@Blocking
@POST
Expand All @@ -41,4 +204,14 @@ public void gc(@RestPath long targetId) {
conn.invokeMBeanOperation(
"java.lang:type=Memory", "gc", null, null, Void.class));
}

public record ThreadDump(String content, String jvmId, String downloadUrl, String uuid) {

public ThreadDump {
Objects.requireNonNull(content);
Objects.requireNonNull(jvmId);
Objects.requireNonNull(downloadUrl);
Objects.requireNonNull(uuid);
}
}
}
Loading
Loading