From 02a170ec6dd4a2c834212bb43605005676793e7d Mon Sep 17 00:00:00 2001 From: zkofiro Date: Thu, 23 May 2024 15:58:53 -0700 Subject: [PATCH 1/6] send slow RT to CW by default --- agent/http_client_3/pom.xml | 7 ++- .../tank/httpclient3/TankHttpClient3.java | 55 ++++++++++++++++++- agent/http_client_4/pom.xml | 5 ++ .../tank/httpclient4/TankHttpClient4.java | 55 +++++++++++++++++++ 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/agent/http_client_3/pom.xml b/agent/http_client_3/pom.xml index d347ab9b7..1825c9db8 100644 --- a/agent/http_client_3/pom.xml +++ b/agent/http_client_3/pom.xml @@ -28,7 +28,12 @@ agent-common ${project.version} - + + + software.amazon.awssdk + cloudwatch + + commons-httpclient commons-httpclient diff --git a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java index 1938c4f6c..357d32404 100644 --- a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java +++ b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java @@ -20,6 +20,7 @@ import java.io.UnsupportedEncodingException; import java.net.SocketException; import java.net.UnknownHostException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -30,6 +31,7 @@ import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; +import com.intuit.tank.harness.AmazonUtil; import jakarta.annotation.Nonnull; import org.apache.commons.httpclient.Cookie; @@ -56,6 +58,7 @@ import org.apache.commons.httpclient.methods.multipart.StringPart; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.lang3.time.DateUtils; import org.apache.http.HttpHeaders; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -69,6 +72,11 @@ import com.intuit.tank.http.TankHttpUtil.PartHolder; import com.intuit.tank.logging.LogEventType; import com.intuit.tank.vm.settings.AgentConfig; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.StandardUnit; public class TankHttpClient3 implements TankHttpClient { @@ -76,6 +84,12 @@ public class TankHttpClient3 implements TankHttpClient { private HttpClient httpclient; + private static final String namespace = "Intuit/Tank"; + private CloudWatchAsyncClient cloudWatchClient; + private Dimension instanceId; + private Dimension jobId; + private Dimension requestPath; + /** * no-arg constructor for client */ @@ -86,6 +100,8 @@ public TankHttpClient3() { httpclient.getParams().setBooleanParameter("http.protocol.allow-circular-redirects", true); httpclient.getParams().setIntParameter("http.protocol.max-redirects", 100); httpclient.setState(new HttpState()); + + this.cloudWatchClient = CloudWatchAsyncClient.builder().build(); } public Object createHttpClient() { return null; } @@ -318,6 +334,7 @@ private void doWaitDueToLongResponse(BaseRequest request, long responseTime, Str if (maxAgentResponseTime < responseTime) { long waitTime = Math.min(config.getMaxAgentWaitTime(), responseTime); LOG.warn(request.getLogUtil().getLogMessage("Response time to slow | delaying " + waitTime + " ms | url --> " + uri, LogEventType.Script)); + sendRTCloudWatchMetrics(responseTime, uri); Thread.sleep(waitTime); } } catch (InterruptedException e) { @@ -393,7 +410,43 @@ private void setHeaders(BaseRequest request, HttpMethod method, HashMap datumList = new ArrayList<>(); + + this.instanceId = Dimension.builder() + .name("InstanceId") + .value(AmazonUtil.getInstanceId()) + .build(); + this.jobId = Dimension.builder() + .name("JobId") + .value(AmazonUtil.getJobId()) + .build(); + this.requestPath = Dimension.builder() + .name("request") + .value(uri) + .build(); + + datumList.add(MetricDatum.builder() + .metricName("responseTime") + .unit(StandardUnit.COUNT) + .value((double) responseTime) + .timestamp(timestamp) + .dimensions(requestPath, instanceId, jobId) + .build()); + datumList.add(MetricDatum.builder() + .metricName("requestPath") + .unit(StandardUnit.COUNT) + .value((double) responseTime) + .timestamp(timestamp) + .dimensions(requestPath, instanceId, jobId) + .build()); + PutMetricDataRequest request = PutMetricDataRequest.builder() + .namespace(namespace) + .metricData(datumList) + .build(); + cloudWatchClient.putMetricData(request); + } private List buildParts(BaseRequest request) { List parts = new ArrayList(); diff --git a/agent/http_client_4/pom.xml b/agent/http_client_4/pom.xml index 8fe3e1d3e..869f76a00 100644 --- a/agent/http_client_4/pom.xml +++ b/agent/http_client_4/pom.xml @@ -29,6 +29,11 @@ ${project.version} + + software.amazon.awssdk + cloudwatch + + org.apache.httpcomponents httpclient diff --git a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java index 788139bf8..22e550897 100644 --- a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java +++ b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java @@ -17,10 +17,12 @@ import java.io.InputStream; import java.net.SocketException; import java.net.UnknownHostException; +import java.time.Instant; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.intuit.tank.harness.AmazonUtil; import jakarta.annotation.Nonnull; import org.apache.commons.lang3.StringUtils; @@ -64,6 +66,11 @@ import com.intuit.tank.http.TankHttpUtil.PartHolder; import com.intuit.tank.logging.LogEventType; import com.intuit.tank.vm.settings.AgentConfig; +import software.amazon.awssdk.services.cloudwatch.CloudWatchAsyncClient; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.MetricDatum; +import software.amazon.awssdk.services.cloudwatch.model.PutMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.StandardUnit; public class TankHttpClient4 implements TankHttpClient { @@ -72,6 +79,12 @@ public class TankHttpClient4 implements TankHttpClient { private CloseableHttpClient httpclient; private HttpClientContext context; + private static final String namespace = "Intuit/Tank"; + private CloudWatchAsyncClient cloudWatchClient; + private Dimension instanceId; + private Dimension jobId; + private Dimension requestPath; + /** * no-arg constructor for client */ @@ -93,6 +106,8 @@ public TankHttpClient4() { context.setUserToken(UUID.randomUUID()); context.setCookieStore(new BasicCookieStore()); context.setRequestConfig(requestConfig); + + this.cloudWatchClient = CloudWatchAsyncClient.builder().build(); } public Object createHttpClient() { @@ -342,6 +357,7 @@ private void doWaitDueToLongResponse(BaseRequest request, long responseTime, Str long maxAgentResponseTime = config.getMaxAgentResponseTime(); if (maxAgentResponseTime < responseTime) { long waitTime = Math.min(config.getMaxAgentWaitTime(), responseTime); + sendRTCloudWatchMetrics(responseTime, uri); LOG.warn(request.getLogUtil().getLogMessage("Response time to slow | delaying " + waitTime + " ms | url --> " + uri, LogEventType.Script)); Thread.sleep(waitTime); } @@ -392,6 +408,45 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques } } + private void sendRTCloudWatchMetrics(long responseTime, String uri) { + Instant timestamp = new Date().toInstant(); + List datumList = new ArrayList<>(); + + this.instanceId = Dimension.builder() + .name("InstanceId") + .value(AmazonUtil.getInstanceId()) + .build(); + this.jobId = Dimension.builder() + .name("JobId") + .value(AmazonUtil.getJobId()) + .build(); + + this.requestPath = Dimension.builder() + .name("request") + .value(uri) + .build(); + + datumList.add(MetricDatum.builder() + .metricName("responseTime") + .unit(StandardUnit.COUNT) + .value((double) responseTime) + .timestamp(timestamp) + .dimensions(requestPath, instanceId, jobId) + .build()); + datumList.add(MetricDatum.builder() + .metricName("requestPath") + .unit(StandardUnit.COUNT) + .value((double) responseTime) + .timestamp(timestamp) + .dimensions(requestPath, instanceId, jobId) + .build()); + PutMetricDataRequest request = PutMetricDataRequest.builder() + .namespace(namespace) + .metricData(datumList) + .build(); + cloudWatchClient.putMetricData(request); + } + /** * Set all the header keys * From b6e28a27ef3dce49d1cc7f7164a25c6dbb52b652 Mon Sep 17 00:00:00 2001 From: zkofiro Date: Thu, 23 May 2024 17:14:51 -0700 Subject: [PATCH 2/6] update metric --- .../com/intuit/tank/httpclient3/TankHttpClient3.java | 9 +-------- .../com/intuit/tank/httpclient4/TankHttpClient4.java | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java index 357d32404..7595ecbc5 100644 --- a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java +++ b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java @@ -423,7 +423,7 @@ private void sendRTCloudWatchMetrics(long responseTime, String uri) { .value(AmazonUtil.getJobId()) .build(); this.requestPath = Dimension.builder() - .name("request") + .name("slowRequestPath") .value(uri) .build(); @@ -434,13 +434,6 @@ private void sendRTCloudWatchMetrics(long responseTime, String uri) { .timestamp(timestamp) .dimensions(requestPath, instanceId, jobId) .build()); - datumList.add(MetricDatum.builder() - .metricName("requestPath") - .unit(StandardUnit.COUNT) - .value((double) responseTime) - .timestamp(timestamp) - .dimensions(requestPath, instanceId, jobId) - .build()); PutMetricDataRequest request = PutMetricDataRequest.builder() .namespace(namespace) .metricData(datumList) diff --git a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java index 22e550897..5524f3529 100644 --- a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java +++ b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java @@ -422,7 +422,7 @@ private void sendRTCloudWatchMetrics(long responseTime, String uri) { .build(); this.requestPath = Dimension.builder() - .name("request") + .name("slowRequestPath") .value(uri) .build(); @@ -433,13 +433,6 @@ private void sendRTCloudWatchMetrics(long responseTime, String uri) { .timestamp(timestamp) .dimensions(requestPath, instanceId, jobId) .build()); - datumList.add(MetricDatum.builder() - .metricName("requestPath") - .unit(StandardUnit.COUNT) - .value((double) responseTime) - .timestamp(timestamp) - .dimensions(requestPath, instanceId, jobId) - .build()); PutMetricDataRequest request = PutMetricDataRequest.builder() .namespace(namespace) .metricData(datumList) From 22178ea3ba7eb11c38c40c805eed142eb1be3a8e Mon Sep 17 00:00:00 2001 From: zkofiro Date: Tue, 3 Dec 2024 18:11:25 -0800 Subject: [PATCH 3/6] resetting - moving to correct branch --- agent/http_client_3/pom.xml | 7 +------ agent/http_client_4/pom.xml | 5 ----- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/agent/http_client_3/pom.xml b/agent/http_client_3/pom.xml index 2d0d9f835..10b545b36 100644 --- a/agent/http_client_3/pom.xml +++ b/agent/http_client_3/pom.xml @@ -28,12 +28,7 @@ agent-common ${project.version} - - - software.amazon.awssdk - cloudwatch - - + commons-httpclient commons-httpclient diff --git a/agent/http_client_4/pom.xml b/agent/http_client_4/pom.xml index 8d2ade3f9..5a2d8907d 100644 --- a/agent/http_client_4/pom.xml +++ b/agent/http_client_4/pom.xml @@ -29,11 +29,6 @@ ${project.version} - - software.amazon.awssdk - cloudwatch - - org.apache.httpcomponents httpclient From 2e95f52f4d82ddf75810d24e30ba0e1a1f66968d Mon Sep 17 00:00:00 2001 From: zkofiro Date: Tue, 3 Dec 2024 20:37:02 -0800 Subject: [PATCH 4/6] capture and log istio response/service time + csv export enhacements --- .../com/intuit/tank/http/BaseResponse.java | 11 + .../intuit/tank/http/BaseResponseTest.java | 3 + .../tank/httpclient3/TankHttpClient3.java | 14 ++ .../tank/httpclient4/TankHttpClient4.java | 14 ++ .../tank/httpclientjdk/TankHttpClientJDK.java | 14 ++ .../tank/tools/debugger/ActionProducer.java | 2 +- .../tools/debugger/AgentDebuggerFrame.java | 17 +- .../dependency-reduced-pom.xml | 218 +++++++++++++++++ .../dependency-reduced-pom.xml | 224 ++++++++++++++++++ 9 files changed, 513 insertions(+), 4 deletions(-) create mode 100644 tools/agent_debugger_pkg/dependency-reduced-pom.xml create mode 100644 tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java b/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java index ea11fb8af..dad7a6277 100644 --- a/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java +++ b/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java @@ -29,6 +29,7 @@ public abstract class BaseResponse { protected String response; protected long responseTime = -1; + protected long istioResponseTime = -1; protected int httpCode = -1; protected String rspMessage = ""; protected HashMap headers = new HashMap(); @@ -52,6 +53,7 @@ public String getLogMsg() { sb.append("RESPONSE HTTP CODE: ").append(this.httpCode).append(NEWLINE) .append("RESPONSE HTTP MSG: ").append(this.rspMessage).append(NEWLINE) .append("RESPONSE TIME: ").append(responseTime).append(NEWLINE) + .append("ISTIO RESPONSE TIME: ").append(istioResponseTime).append(NEWLINE) .append("RESPONSE SIZE: ").append(getResponseSize()).append(NEWLINE); for (Entry mapEntry : headers.entrySet()) { sb.append("RESPONSE HEADER: ") @@ -84,6 +86,7 @@ public String convertToCSV() { sb.append(this.httpCode).append(","); sb.append(this.rspMessage).append(","); sb.append(responseTime).append(","); + sb.append(istioResponseTime).append(","); sb.append(getResponseSize()).append(","); headers.forEach((key, value) -> sb.append(key).append(" = ").append(value.replace(",", "")).append(",")); cookies.forEach((key, value) -> sb.append(key).append(" = ").append(value).append(",")); @@ -171,6 +174,14 @@ public void setResponseBody(String body) { this.response = body; } + public long getIstioResponseTime() { + return this.istioResponseTime; + } + + public void setIstioResponseTime(long istioResponseTime) { + this.istioResponseTime = istioResponseTime; + } + /** * * @param byteArray diff --git a/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java b/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java index 9519d414f..d0e23002e 100644 --- a/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java +++ b/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java @@ -443,6 +443,7 @@ public void testLogResponse_1() throws Exception { BaseResponse fixture = new MockResponse(); fixture.responseTime = 1L; + fixture.istioResponseTime = 1L; fixture.httpCode = 1; fixture.response = ""; fixture.responseByteArray = new byte[] {}; @@ -451,6 +452,7 @@ public void testLogResponse_1() assertEquals("RESPONSE HTTP CODE: 1\n" + "RESPONSE HTTP MSG: \n" + "RESPONSE TIME: 1\n" + + "ISTIO RESPONSE TIME: 1\n" + "RESPONSE SIZE: 0\n" + "RESPONSE BODY: not logged because null is not a content-type.\n", fixture.getLogMsg()); } @@ -466,6 +468,7 @@ public void testLogResponse_2() { assertEquals("RESPONSE HTTP CODE: -1\n" + "RESPONSE HTTP MSG: \n" + "RESPONSE TIME: -1\n" + + "ISTIO RESPONSE TIME: -1\n" + "RESPONSE SIZE: 12\n" + "RESPONSE HEADER: content-type = text/html\n" + "RESPONSE COOKIE: testHeadersKey = testHeadersValue\n" + diff --git a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java index eafc9c8d5..33322c90b 100644 --- a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java +++ b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java @@ -361,6 +361,20 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques response.setHeader(header.getName(), header.getValue()); } + // Extract Istio response/service time header + String istioResponseTimeHeader = response.getHttpHeader("x-envoy-upstream-service-time"); + if (istioResponseTimeHeader != null) { + try { + long istioResponseTime = Long.parseLong(istioResponseTimeHeader); + response.setIstioResponseTime(istioResponseTime); + } catch (NumberFormatException e) { + LOG.warn("could not parse istio service time header: " + istioResponseTimeHeader); + response.setIstioResponseTime(-1); + } + } else { + response.setIstioResponseTime(-1); + } + Cookie[] cookies = httpstate.getCookies(); if (cookies != null) { for (Cookie cookie : cookies) { diff --git a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java index 592bfca46..bf439d49c 100644 --- a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java +++ b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java @@ -389,6 +389,20 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques response.setHeader(header.getName(), header.getValue()); } + // Extract Istio response/service time header + String istioResponseTimeHeader = response.getHttpHeader("x-envoy-upstream-service-time"); + if (istioResponseTimeHeader != null) { + try { + long istioResponseTime = Long.parseLong(istioResponseTimeHeader); + response.setIstioResponseTime(istioResponseTime); + } catch (NumberFormatException e) { + LOG.warn("could not parse istio service time header: " + istioResponseTimeHeader); + response.setIstioResponseTime(-1); + } + } else { + response.setIstioResponseTime(-1); + } + if (context.getCookieStore().getCookies() != null) { for (Cookie cookie : context.getCookieStore().getCookies()) { response.setCookie(cookie.getName(), cookie.getValue()); diff --git a/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java b/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java index 90cebca50..ed3a15ff3 100644 --- a/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java +++ b/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java @@ -314,6 +314,20 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques } } + // Extract Istio response/service time header + List istioResponseTimeHeaders = headers.map().get("x-envoy-upstream-service-time"); + if (istioResponseTimeHeaders != null && !istioResponseTimeHeaders.isEmpty()) { + try { + long istioResponseTime = Long.parseLong(istioResponseTimeHeaders.get(0)); + response.setIstioResponseTime(istioResponseTime); + } catch (NumberFormatException e) { + LOG.warn("could not parse istio service time header: " + istioResponseTimeHeaders.get(0)); + response.setIstioResponseTime(-1); + } + } else { + response.setIstioResponseTime(-1); + } + List cookies = cookieManager.getCookieStore().getCookies(); if (!cookies.isEmpty()) { for (HttpCookie cookie : cookies) { diff --git a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java index 7c245f71c..37741d1ae 100644 --- a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java +++ b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/ActionProducer.java @@ -441,7 +441,7 @@ public void setChoiceComboBoxOptions(JComboBox cb) { cb.addItem(new TankClientChoice("Apache HttpClient 3.1", "com.intuit.tank.httpclient3.TankHttpClient3")); cb.addItem(new TankClientChoice("Apache HttpClient 4.5", "com.intuit.tank.httpclient4.TankHttpClient4")); cb.addItem(new TankClientChoice("JDK Http Client", "com.intuit.tank.httpclientjdk.TankHttpClientJDK")); - cb.setSelectedIndex(2); + cb.setSelectedIndex(1); } } diff --git a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java index 2231fd13d..3e93d5da5 100644 --- a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java +++ b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java @@ -1014,14 +1014,25 @@ public RequestResponsePanel getRequestResponsePanel() { public void exportCSV(File csvOutputFile) { try (PrintWriter pw = new PrintWriter(csvOutputFile)) { - pw.println("URL,HTTP Code,HTTP Msg,Response Time,Response Size,Headers...,Cookies..."); + pw.println("URL,HTTP Code,HTTP Msg,Response Time,Istio Response Time,Response Size,Headers...,Cookies..."); steps.stream() .filter(step -> step.getResponse() != null) - .map(step -> step.getRequest().getRequestUrl() + "," + step.getResponse().convertToCSV()) - .forEach(pw::println); + .forEach(step -> { + String escapedUrl = escapeCSV(step.getRequest().getRequestUrl()); + pw.println(escapedUrl + "," + step.getResponse().convertToCSV()); + }); } catch ( FileNotFoundException e) { LOG.error("Error exporting CSV: {}", String.valueOf(e)); } } + private String escapeCSV(String value) { + if (value == null) { + return ""; + } + if (value.contains(",") || value.contains("\"") || value.contains("\n")) { + return "\"" + value.replace("\"", "\"\"") + "\""; + } + return value; + } } diff --git a/tools/agent_debugger_pkg/dependency-reduced-pom.xml b/tools/agent_debugger_pkg/dependency-reduced-pom.xml new file mode 100644 index 000000000..9aa4372c2 --- /dev/null +++ b/tools/agent_debugger_pkg/dependency-reduced-pom.xml @@ -0,0 +1,218 @@ + + + + tools-parent + com.intuit.tank + 4.1.1-SNAPSHOT + + 4.0.0 + agent-debugger-package + pom + Agent Debugger Packaging + + + + maven-shade-plugin + 3.5.1 + + + package + + shade + + + target/Tank-Debugger-all.jar + + + com.intuit.tank.tools.debugger.AgentDebuggerFrame + + + + + + + + + + org.apache.logging.log4j + log4j-transform-maven-shade-plugin-extensions + 0.1.0 + + + + + maven-jarsigner-plugin + + + + + + com.google.code.findbugs + jsr305 + 3.0.2 + provided + + + org.junit.jupiter + junit-jupiter-api + 5.11.1 + test + + + opentest4j + org.opentest4j + + + junit-platform-commons + org.junit.platform + + + apiguardian-api + org.apiguardian + + + + + org.junit.jupiter + junit-jupiter-engine + 5.11.1 + test + + + junit-platform-engine + org.junit.platform + + + apiguardian-api + org.apiguardian + + + + + org.junit.jupiter + junit-jupiter-params + 5.11.1 + test + + + apiguardian-api + org.apiguardian + + + + + com.intuit.tank + test-support + 4.1.1-SNAPSHOT + test + + + org.mockito + mockito-junit-jupiter + 5.13.0 + test + + + mockito-core + org.mockito + + + + + org.jboss.weld + weld-junit5 + 4.0.1.Final + test + + + weld-se-core + org.jboss.weld.se + + + weld-junit-common + org.jboss.weld + + + + + org.jboss.weld.servlet + weld-servlet-core + 5.1.3.Final + test + + + weld-environment-common + org.jboss.weld.environment + + + weld-spi + org.jboss.weld + + + weld-lite-extension-translator + org.jboss.weld + + + weld-jsf + org.jboss.weld.module + + + weld-web + org.jboss.weld.module + + + jakarta.el-api + jakarta.el + + + jboss-logging + org.jboss.logging + + + jboss-classfilewriter + org.jboss.classfilewriter + + + + + org.apache.tomcat.embed + tomcat-embed-core + 11.0.1 + test + + + tomcat-annotations-api + org.apache.tomcat + + + + + org.apache.tomcat + tomcat-catalina + 11.0.1 + test + + + tomcat-jaspic-api + org.apache.tomcat + + + tomcat-annotations-api + org.apache.tomcat + + + + + org.apache.tomcat + tomcat-coyote + 11.0.1 + provided + + + org.apache.tomcat + tomcat-jasper + 11.0.1 + provided + + + diff --git a/tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml b/tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml new file mode 100644 index 000000000..62408d245 --- /dev/null +++ b/tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml @@ -0,0 +1,224 @@ + + + + tools-parent + com.intuit.tank + 4.0.1-SNAPSHOT + + 4.0.0 + headless-agent-debugger-package + pom + Headless Agent Debugger Packaging + + + + maven-shade-plugin + 3.5.1 + + + package + + shade + + + target/Headless-Debugger-all.jar + + + com.intuit.tank.tools.headless.HeadlessDebuggerSetup + + + + + + + + + + org.apache.logging.log4j + log4j-transform-maven-shade-plugin-extensions + 0.1.0 + + + + + maven-jarsigner-plugin + + + + + + com.google.code.findbugs + jsr305 + 3.0.2 + provided + + + org.junit.jupiter + junit-jupiter-api + 5.10.2 + test + + + opentest4j + org.opentest4j + + + junit-platform-commons + org.junit.platform + + + apiguardian-api + org.apiguardian + + + + + org.junit.jupiter + junit-jupiter-engine + 5.10.2 + test + + + junit-platform-engine + org.junit.platform + + + apiguardian-api + org.apiguardian + + + + + org.junit.jupiter + junit-jupiter-params + 5.10.2 + test + + + apiguardian-api + org.apiguardian + + + + + com.intuit.tank + test-support + 4.0.1-SNAPSHOT + test + + + org.mockito + mockito-junit-jupiter + 5.10.0 + test + + + mockito-core + org.mockito + + + + + org.jboss.weld + weld-junit5 + 4.0.1.Final + test + + + weld-se-core + org.jboss.weld.se + + + weld-junit-common + org.jboss.weld + + + + + org.jboss.weld.servlet + weld-servlet-core + 5.1.2.Final + test + + + weld-environment-common + org.jboss.weld.environment + + + weld-spi + org.jboss.weld + + + weld-lite-extension-translator + org.jboss.weld + + + weld-jsf + org.jboss.weld.module + + + weld-web + org.jboss.weld.module + + + jakarta.el-api + jakarta.el + + + jboss-logging + org.jboss.logging + + + jboss-classfilewriter + org.jboss.classfilewriter + + + + + org.apache.tomcat.embed + tomcat-embed-core + 10.1.25 + test + + + tomcat-annotations-api + org.apache.tomcat + + + + + org.apache.tomcat + tomcat-catalina + 10.1.25 + test + + + tomcat-jaspic-api + org.apache.tomcat + + + tomcat-annotations-api + org.apache.tomcat + + + + + org.apache.tomcat + tomcat-coyote + 10.1.25 + provided + + + org.apache.tomcat + tomcat-jasper + 10.1.25 + provided + + + jakarta.servlet.jsp + jakarta.servlet.jsp-api + 3.1.0 + test + + + From 8eaa9d7d52c572f2865e0dda79b47980cbcdeb23 Mon Sep 17 00:00:00 2001 From: zkofiro Date: Tue, 3 Dec 2024 20:42:44 -0800 Subject: [PATCH 5/6] unneeded files --- .../dependency-reduced-pom.xml | 218 ----------------- .../dependency-reduced-pom.xml | 224 ------------------ 2 files changed, 442 deletions(-) delete mode 100644 tools/agent_debugger_pkg/dependency-reduced-pom.xml delete mode 100644 tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml diff --git a/tools/agent_debugger_pkg/dependency-reduced-pom.xml b/tools/agent_debugger_pkg/dependency-reduced-pom.xml deleted file mode 100644 index 9aa4372c2..000000000 --- a/tools/agent_debugger_pkg/dependency-reduced-pom.xml +++ /dev/null @@ -1,218 +0,0 @@ - - - - tools-parent - com.intuit.tank - 4.1.1-SNAPSHOT - - 4.0.0 - agent-debugger-package - pom - Agent Debugger Packaging - - - - maven-shade-plugin - 3.5.1 - - - package - - shade - - - target/Tank-Debugger-all.jar - - - com.intuit.tank.tools.debugger.AgentDebuggerFrame - - - - - - - - - - org.apache.logging.log4j - log4j-transform-maven-shade-plugin-extensions - 0.1.0 - - - - - maven-jarsigner-plugin - - - - - - com.google.code.findbugs - jsr305 - 3.0.2 - provided - - - org.junit.jupiter - junit-jupiter-api - 5.11.1 - test - - - opentest4j - org.opentest4j - - - junit-platform-commons - org.junit.platform - - - apiguardian-api - org.apiguardian - - - - - org.junit.jupiter - junit-jupiter-engine - 5.11.1 - test - - - junit-platform-engine - org.junit.platform - - - apiguardian-api - org.apiguardian - - - - - org.junit.jupiter - junit-jupiter-params - 5.11.1 - test - - - apiguardian-api - org.apiguardian - - - - - com.intuit.tank - test-support - 4.1.1-SNAPSHOT - test - - - org.mockito - mockito-junit-jupiter - 5.13.0 - test - - - mockito-core - org.mockito - - - - - org.jboss.weld - weld-junit5 - 4.0.1.Final - test - - - weld-se-core - org.jboss.weld.se - - - weld-junit-common - org.jboss.weld - - - - - org.jboss.weld.servlet - weld-servlet-core - 5.1.3.Final - test - - - weld-environment-common - org.jboss.weld.environment - - - weld-spi - org.jboss.weld - - - weld-lite-extension-translator - org.jboss.weld - - - weld-jsf - org.jboss.weld.module - - - weld-web - org.jboss.weld.module - - - jakarta.el-api - jakarta.el - - - jboss-logging - org.jboss.logging - - - jboss-classfilewriter - org.jboss.classfilewriter - - - - - org.apache.tomcat.embed - tomcat-embed-core - 11.0.1 - test - - - tomcat-annotations-api - org.apache.tomcat - - - - - org.apache.tomcat - tomcat-catalina - 11.0.1 - test - - - tomcat-jaspic-api - org.apache.tomcat - - - tomcat-annotations-api - org.apache.tomcat - - - - - org.apache.tomcat - tomcat-coyote - 11.0.1 - provided - - - org.apache.tomcat - tomcat-jasper - 11.0.1 - provided - - - diff --git a/tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml b/tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml deleted file mode 100644 index 62408d245..000000000 --- a/tools/headless_agent_debugger_pkg/dependency-reduced-pom.xml +++ /dev/null @@ -1,224 +0,0 @@ - - - - tools-parent - com.intuit.tank - 4.0.1-SNAPSHOT - - 4.0.0 - headless-agent-debugger-package - pom - Headless Agent Debugger Packaging - - - - maven-shade-plugin - 3.5.1 - - - package - - shade - - - target/Headless-Debugger-all.jar - - - com.intuit.tank.tools.headless.HeadlessDebuggerSetup - - - - - - - - - - org.apache.logging.log4j - log4j-transform-maven-shade-plugin-extensions - 0.1.0 - - - - - maven-jarsigner-plugin - - - - - - com.google.code.findbugs - jsr305 - 3.0.2 - provided - - - org.junit.jupiter - junit-jupiter-api - 5.10.2 - test - - - opentest4j - org.opentest4j - - - junit-platform-commons - org.junit.platform - - - apiguardian-api - org.apiguardian - - - - - org.junit.jupiter - junit-jupiter-engine - 5.10.2 - test - - - junit-platform-engine - org.junit.platform - - - apiguardian-api - org.apiguardian - - - - - org.junit.jupiter - junit-jupiter-params - 5.10.2 - test - - - apiguardian-api - org.apiguardian - - - - - com.intuit.tank - test-support - 4.0.1-SNAPSHOT - test - - - org.mockito - mockito-junit-jupiter - 5.10.0 - test - - - mockito-core - org.mockito - - - - - org.jboss.weld - weld-junit5 - 4.0.1.Final - test - - - weld-se-core - org.jboss.weld.se - - - weld-junit-common - org.jboss.weld - - - - - org.jboss.weld.servlet - weld-servlet-core - 5.1.2.Final - test - - - weld-environment-common - org.jboss.weld.environment - - - weld-spi - org.jboss.weld - - - weld-lite-extension-translator - org.jboss.weld - - - weld-jsf - org.jboss.weld.module - - - weld-web - org.jboss.weld.module - - - jakarta.el-api - jakarta.el - - - jboss-logging - org.jboss.logging - - - jboss-classfilewriter - org.jboss.classfilewriter - - - - - org.apache.tomcat.embed - tomcat-embed-core - 10.1.25 - test - - - tomcat-annotations-api - org.apache.tomcat - - - - - org.apache.tomcat - tomcat-catalina - 10.1.25 - test - - - tomcat-jaspic-api - org.apache.tomcat - - - tomcat-annotations-api - org.apache.tomcat - - - - - org.apache.tomcat - tomcat-coyote - 10.1.25 - provided - - - org.apache.tomcat - tomcat-jasper - 10.1.25 - provided - - - jakarta.servlet.jsp - jakarta.servlet.jsp-api - 3.1.0 - test - - - From 337f0cdadfb83f73f52f457e44fa0935aa888fbf Mon Sep 17 00:00:00 2001 From: zkofiro Date: Thu, 23 Jan 2025 10:41:50 -0800 Subject: [PATCH 6/6] generalize to proxy --- .../java/com/intuit/tank/http/BaseResponse.java | 14 +++++++------- .../com/intuit/tank/http/BaseResponseTest.java | 6 +++--- .../intuit/tank/httpclient3/TankHttpClient3.java | 16 ++++++++-------- .../intuit/tank/httpclient4/TankHttpClient4.java | 16 ++++++++-------- .../tank/httpclientjdk/TankHttpClientJDK.java | 16 ++++++++-------- .../tank/tools/debugger/AgentDebuggerFrame.java | 2 +- 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java b/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java index dad7a6277..5e7d5b42a 100644 --- a/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java +++ b/agent/agent_common/src/main/java/com/intuit/tank/http/BaseResponse.java @@ -29,7 +29,7 @@ public abstract class BaseResponse { protected String response; protected long responseTime = -1; - protected long istioResponseTime = -1; + protected long proxyResponseTime = -1; protected int httpCode = -1; protected String rspMessage = ""; protected HashMap headers = new HashMap(); @@ -53,7 +53,7 @@ public String getLogMsg() { sb.append("RESPONSE HTTP CODE: ").append(this.httpCode).append(NEWLINE) .append("RESPONSE HTTP MSG: ").append(this.rspMessage).append(NEWLINE) .append("RESPONSE TIME: ").append(responseTime).append(NEWLINE) - .append("ISTIO RESPONSE TIME: ").append(istioResponseTime).append(NEWLINE) + .append("PROXY RESPONSE TIME: ").append(proxyResponseTime).append(NEWLINE) .append("RESPONSE SIZE: ").append(getResponseSize()).append(NEWLINE); for (Entry mapEntry : headers.entrySet()) { sb.append("RESPONSE HEADER: ") @@ -86,7 +86,7 @@ public String convertToCSV() { sb.append(this.httpCode).append(","); sb.append(this.rspMessage).append(","); sb.append(responseTime).append(","); - sb.append(istioResponseTime).append(","); + sb.append(proxyResponseTime).append(","); sb.append(getResponseSize()).append(","); headers.forEach((key, value) -> sb.append(key).append(" = ").append(value.replace(",", "")).append(",")); cookies.forEach((key, value) -> sb.append(key).append(" = ").append(value).append(",")); @@ -174,12 +174,12 @@ public void setResponseBody(String body) { this.response = body; } - public long getIstioResponseTime() { - return this.istioResponseTime; + public long getProxyResponseTime() { + return this.proxyResponseTime; } - public void setIstioResponseTime(long istioResponseTime) { - this.istioResponseTime = istioResponseTime; + public void setProxyResponseTime(long proxyResponseTime) { + this.proxyResponseTime = proxyResponseTime; } /** diff --git a/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java b/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java index d0e23002e..3d411d0a2 100644 --- a/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java +++ b/agent/agent_common/src/test/java/com/intuit/tank/http/BaseResponseTest.java @@ -443,7 +443,7 @@ public void testLogResponse_1() throws Exception { BaseResponse fixture = new MockResponse(); fixture.responseTime = 1L; - fixture.istioResponseTime = 1L; + fixture.proxyResponseTime = 1L; fixture.httpCode = 1; fixture.response = ""; fixture.responseByteArray = new byte[] {}; @@ -452,7 +452,7 @@ public void testLogResponse_1() assertEquals("RESPONSE HTTP CODE: 1\n" + "RESPONSE HTTP MSG: \n" + "RESPONSE TIME: 1\n" + - "ISTIO RESPONSE TIME: 1\n" + + "PROXY RESPONSE TIME: 1\n" + "RESPONSE SIZE: 0\n" + "RESPONSE BODY: not logged because null is not a content-type.\n", fixture.getLogMsg()); } @@ -468,7 +468,7 @@ public void testLogResponse_2() { assertEquals("RESPONSE HTTP CODE: -1\n" + "RESPONSE HTTP MSG: \n" + "RESPONSE TIME: -1\n" + - "ISTIO RESPONSE TIME: -1\n" + + "PROXY RESPONSE TIME: -1\n" + "RESPONSE SIZE: 12\n" + "RESPONSE HEADER: content-type = text/html\n" + "RESPONSE COOKIE: testHeadersKey = testHeadersValue\n" + diff --git a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java index c1b07de09..6d069ca96 100644 --- a/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java +++ b/agent/http_client_3/src/main/java/com/intuit/tank/httpclient3/TankHttpClient3.java @@ -386,18 +386,18 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques response.setHeader(header.getName(), header.getValue()); } - // Extract Istio response/service time header - String istioResponseTimeHeader = response.getHttpHeader("x-envoy-upstream-service-time"); - if (istioResponseTimeHeader != null) { + // Extract proxy response/service time header + String proxyResponseTimeHeader = response.getHttpHeader("x-envoy-upstream-service-time"); + if (proxyResponseTimeHeader != null) { try { - long istioResponseTime = Long.parseLong(istioResponseTimeHeader); - response.setIstioResponseTime(istioResponseTime); + long proxyResponseTime = Long.parseLong(proxyResponseTimeHeader); + response.setProxyResponseTime(proxyResponseTime); } catch (NumberFormatException e) { - LOG.warn("could not parse istio service time header: " + istioResponseTimeHeader); - response.setIstioResponseTime(-1); + LOG.warn("could not parse proxy service time header: " + proxyResponseTimeHeader); + response.setProxyResponseTime(-1); } } else { - response.setIstioResponseTime(-1); + response.setProxyResponseTime(-1); } Cookie[] cookies = httpstate.getCookies(); diff --git a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java index 05bbc72bc..0d4151786 100644 --- a/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java +++ b/agent/http_client_4/src/main/java/com/intuit/tank/httpclient4/TankHttpClient4.java @@ -410,18 +410,18 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques response.setHeader(header.getName(), header.getValue()); } - // Extract Istio response/service time header - String istioResponseTimeHeader = response.getHttpHeader("x-envoy-upstream-service-time"); - if (istioResponseTimeHeader != null) { + // Extract proxy response/service time header + String proxyResponseTimeHeader = response.getHttpHeader("x-envoy-upstream-service-time"); + if (proxyResponseTimeHeader != null) { try { - long istioResponseTime = Long.parseLong(istioResponseTimeHeader); - response.setIstioResponseTime(istioResponseTime); + long proxyResponseTime = Long.parseLong(proxyResponseTimeHeader); + response.setProxyResponseTime(proxyResponseTime); } catch (NumberFormatException e) { - LOG.warn("could not parse istio service time header: " + istioResponseTimeHeader); - response.setIstioResponseTime(-1); + LOG.warn("could not parse proxy service time header: " + proxyResponseTimeHeader); + response.setProxyResponseTime(-1); } } else { - response.setIstioResponseTime(-1); + response.setProxyResponseTime(-1); } if (context.getCookieStore().getCookies() != null) { diff --git a/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java b/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java index cd9667cde..13cedf28f 100644 --- a/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java +++ b/agent/http_client_jdk/src/main/java/com/intuit/tank/httpclientjdk/TankHttpClientJDK.java @@ -343,18 +343,18 @@ private void processResponse(byte[] bResponse, long waitTime, BaseRequest reques } } - // Extract Istio response/service time header - List istioResponseTimeHeaders = headers.map().get("x-envoy-upstream-service-time"); - if (istioResponseTimeHeaders != null && !istioResponseTimeHeaders.isEmpty()) { + // Extract Proxy response/service time header + List proxyResponseTimeHeaders = headers.map().get("x-envoy-upstream-service-time"); + if (proxyResponseTimeHeaders != null && !proxyResponseTimeHeaders.isEmpty()) { try { - long istioResponseTime = Long.parseLong(istioResponseTimeHeaders.get(0)); - response.setIstioResponseTime(istioResponseTime); + long proxyResponseTime = Long.parseLong(proxyResponseTimeHeaders.get(0)); + response.setProxyResponseTime(proxyResponseTime); } catch (NumberFormatException e) { - LOG.warn("could not parse istio service time header: " + istioResponseTimeHeaders.get(0)); - response.setIstioResponseTime(-1); + LOG.warn("could not parse proxy service time header: " + proxyResponseTimeHeaders.get(0)); + response.setProxyResponseTime(-1); } } else { - response.setIstioResponseTime(-1); + response.setProxyResponseTime(-1); } List cookies = cookieManager.getCookieStore().getCookies(); diff --git a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java index 3e93d5da5..96f8670fa 100644 --- a/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java +++ b/tools/agent_debugger/src/main/java/com/intuit/tank/tools/debugger/AgentDebuggerFrame.java @@ -1014,7 +1014,7 @@ public RequestResponsePanel getRequestResponsePanel() { public void exportCSV(File csvOutputFile) { try (PrintWriter pw = new PrintWriter(csvOutputFile)) { - pw.println("URL,HTTP Code,HTTP Msg,Response Time,Istio Response Time,Response Size,Headers...,Cookies..."); + pw.println("URL,HTTP Code,HTTP Msg,Response Time,Proxy Response Time,Response Size,Headers...,Cookies..."); steps.stream() .filter(step -> step.getResponse() != null) .forEach(step -> {