From 6acb031d5cd67b0432347c60f02a6ccbfd2f7952 Mon Sep 17 00:00:00 2001 From: "Sameeksha Vaity (from Dev Box)" Date: Fri, 25 Apr 2025 14:38:43 -0700 Subject: [PATCH] Apply VSCode agent changes --- sdk/clientcore/http-stress/README.md | 204 ++--------- sdk/clientcore/http-stress/pom.xml | 71 ++-- .../java/io/clientcore/http/stress/App.java | 4 +- .../io/clientcore/http/stress/HttpGet.java | 4 +- .../io/clientcore/http/stress/HttpPatch.java | 4 +- .../clientcore/http/stress/ScenarioBase.java | 5 +- .../clientcore/http/stress/StressOptions.java | 2 +- .../clientcore/http/stress/package-info.java | 2 +- .../http/stress/util/TelemetryHelper.java | 2 +- .../http/stress/util/package-info.java | 2 +- sdk/core-v2/http-stress/.gitignore | 3 + sdk/core-v2/http-stress/.helmignore | 4 + sdk/core-v2/http-stress/CHANGELOG.md | 3 + sdk/core-v2/http-stress/Chart.yaml | 13 + sdk/core-v2/http-stress/README.md | 131 +++++++ .../http-stress/checkstyle-suppressions.xml | 8 + sdk/core-v2/http-stress/dockerfiles/java17 | 47 +++ sdk/core-v2/http-stress/dockerfiles/java21 | 47 +++ sdk/core-v2/http-stress/pom.xml | 135 ++++++++ sdk/core-v2/http-stress/scenarios-matrix.yaml | 16 + sdk/core-v2/http-stress/spotbugs-exclude.xml | 28 ++ .../java/io/clientcore/http/stress/App.java | 29 ++ .../io/clientcore/http/stress/HttpGet.java | 153 ++++++++ .../io/clientcore/http/stress/HttpPatch.java | 99 ++++++ .../clientcore/http/stress/ScenarioBase.java | 42 +++ .../clientcore/http/stress/StressOptions.java | 31 ++ .../clientcore/http/stress/package-info.java | 7 + .../http/stress/util/TelemetryHelper.java | 327 ++++++++++++++++++ .../http/stress/util/package-info.java | 7 + .../src/main/java/module-info.java | 26 ++ .../src/main/resources/logback.xml | 16 + .../src/main/resources/simplehttpserver.crt | 19 + .../http-stress/stress-test-resources.bicep | 2 + sdk/core-v2/http-stress/templates/job.yaml | 57 +++ .../http-stress/workbooks/runDetails.json | 310 +++++++++++++++++ sdk/core-v2/pom.xml | 1 + 36 files changed, 1649 insertions(+), 212 deletions(-) create mode 100644 sdk/core-v2/http-stress/.gitignore create mode 100644 sdk/core-v2/http-stress/.helmignore create mode 100644 sdk/core-v2/http-stress/CHANGELOG.md create mode 100644 sdk/core-v2/http-stress/Chart.yaml create mode 100644 sdk/core-v2/http-stress/README.md create mode 100644 sdk/core-v2/http-stress/checkstyle-suppressions.xml create mode 100644 sdk/core-v2/http-stress/dockerfiles/java17 create mode 100644 sdk/core-v2/http-stress/dockerfiles/java21 create mode 100644 sdk/core-v2/http-stress/pom.xml create mode 100644 sdk/core-v2/http-stress/scenarios-matrix.yaml create mode 100644 sdk/core-v2/http-stress/spotbugs-exclude.xml create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/App.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/package-info.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java create mode 100644 sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java create mode 100644 sdk/core-v2/http-stress/src/main/java/module-info.java create mode 100644 sdk/core-v2/http-stress/src/main/resources/logback.xml create mode 100644 sdk/core-v2/http-stress/src/main/resources/simplehttpserver.crt create mode 100644 sdk/core-v2/http-stress/stress-test-resources.bicep create mode 100644 sdk/core-v2/http-stress/templates/job.yaml create mode 100644 sdk/core-v2/http-stress/workbooks/runDetails.json diff --git a/sdk/clientcore/http-stress/README.md b/sdk/clientcore/http-stress/README.md index 357b190e149f..fb772e37667f 100644 --- a/sdk/clientcore/http-stress/README.md +++ b/sdk/clientcore/http-stress/README.md @@ -1,6 +1,6 @@ -# Stress tests for Azure client library for Java +# Stress tests for Azure Core v2 HTTP client library for Java -This package contains template project for stress tests and recommendations on how to create them for your library. +This package contains the stress test project for Azure Core v2 HTTP stack. It demonstrates how to create and run stress tests for the Azure SDK for Java core HTTP pipeline and client infrastructure. ## Getting started @@ -18,15 +18,15 @@ Check out [Azure SDK Stress Test Wiki][azure_sdk_stress_test] for general inform ### Deploy Stress Test -cd into `azure-sdk-for-java` root folder and run command to deploy the package to cluster: +Change directory to the Azure SDK for Java root and deploy the package to your cluster: ```shell -./eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -MatrixSelection all -SearchDirectory ./sdk/ +./eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -MatrixSelection all -SearchDirectory ./sdk/core-v2 ``` ### Check Status -Only the most frequently used commands are listed below. See [Deploying A Stress Test][deploy_stress_test] for more details. +See [Deploying A Stress Test][deploy_stress_test] for more details. List deployed packages: @@ -34,38 +34,16 @@ List deployed packages: helm list -n ``` -the namespace usually matches your username. - Get stress test pods and status: ```shell kubectl get pods -n ``` -To get readable metadata for pods and/or containers use - -```shell -kubectl describe pod -n -c -``` - Get stress test pod logs: ```shell kubectl logs -n -# Note that we may define multiple containers (for example, `fault-injector` and `main`) -kubectl logs -n -c -``` - -If stress test pod is in `Error` status, check logs from containers: - -```shell -kubectl logs -n -``` - -You may also get logs for specific containers: - -```shell -kubectl logs -n -c ``` Stop and remove deployed package: @@ -80,28 +58,21 @@ Execute commands in the container: ```shell kubectl exec --stdin --tty -n -c -- /bin/bash -```` - -### Share data from within the container - -Stress containers run with `$DEBUG_SHARE` environment variable set to the location of the shared folder. You can put anything you want to share there and access it - check out https://aka.ms/azsdk/stress/fileshare. +``` ## Key concepts ### Project Structure -See [Layout][stress_test_layout] section for details. - -Below is the current structure of project: ``` . ├── src/ # Test code -├── templates/ # A directory of helm templates that will generate Kubernetes manifest files. -├── workbooks/ # A directory of Azure Monitor workbooks for analyzing stress test results. -├── Chart.yaml # A YAML file containing information about the helm chart and its dependencies -├── scenarios-matrix.yaml # A YAML file containing configuration and custom values for stress test(s) -├── Dockerfile # A Dockerfile for building the stress test image -├── stress-test-resources.bicep # An Azure Bicep for deploying stress test azure resources +├── templates/ # Helm templates for Kubernetes manifests +├── workbooks/ # Azure Monitor workbooks for analyzing stress test results +├── Chart.yaml # Helm chart metadata +├── scenarios-matrix.yaml # Configuration for stress test scenarios +├── Dockerfile # Dockerfile for building the stress test image +├── stress-test-resources.bicep # Azure Bicep for deploying stress test resources ├── pom.xml └── README.md ``` @@ -110,139 +81,43 @@ Below is the current structure of project: Start with [Azure SDK stress Wiki](https://aka.ms/azsdk/stress) to learn about stress tests. -1. Copy `src/main/java/com/azure/sdk/clientcore/http-stress` folder to your service folder. -2. Update the code - - Update `pom.xml` to change artifact name and add dependencies on your service. - - Implement your first stress test instead of `HttpGet` and make sure to update `StressTestOptions` to include important parameters for your tests. - -Now you can run stress tests locally. Remaining steps are required to run tests on a stress cluster. - -3. Update `dockerfiles` to build your service artifacts and any dependencies of current version. -4. Describe Azure resources necessary for your tests in `stress-test-resources.bicep` -5. Update `Chart.yaml`: - - change chart `name` to include your service name. Please keep `java-` prefix. - - change `annotations.stressTest` to `true` to enable auto-discovery -5. Update `templates/job.yaml` - - remove `server` container as you probably don't need it - - replace occurrences of `java-template` to match name in the `Chart.yaml` - - update test parameters for `test` container, feel free to rename the container as you see fit -6. Define scenarios and parameters in `scenarios-matrix.yaml` - -Now you're ready to run tests with `./eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -SearchDirectory ./sdk/`. -See [Deploying A Stress Test][deploy_stress_test] for more details. - -Let's see how we can check test results. - -### Checking test results +1. Copy `src/main/java/com/azure/core/http/stress` to your service folder. +2. Update the code: + - Update `pom.xml` to change artifact name and add dependencies on your service. + - Implement your first stress test instead of `HttpGet` and update `StressOptions` for your test parameters. -#### Stress Test Dashboard +Now you can run stress tests locally. Remaining steps are required to run tests on a stress cluster. -General-purpose stress test dashboard is available at https://aka.ms/azsdk/stress/dashboard. It shows: -- Pod status events -- CPU and memory utilization of the stress test pods -- Container logs and events +3. Update `dockerfiles` to build your service artifacts and dependencies. +4. Describe Azure resources in `stress-test-resources.bicep`. +5. Update `Chart.yaml` and `templates/job.yaml` for your service. +6. Define scenarios and parameters in `scenarios-matrix.yaml`. -Stress test dashboard does not know about local stress test runs. +Now you're ready to run tests with `./eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -SearchDirectory ./sdk/core-v2`. -#### Application Insights - -Stress test template comes with OpenTelemetry and rich monitoring experience including: -- resource utilization metrics (CPU, memory, GC, threads, etc.) -- live metrics, performance overview, etc -- distributed tracing and dependency calls (HTTP, Azure SDK calls) -- exceptions and logs -- profiling in production - -The telemetry is sent to Application Insights where it's useful to: -- monitor and compare throughput and latency across runs -- investigate issues and find bottlenecks - -You may choose to use [ApplicationInsights Java agent](https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-enable?tabs=java#install-the-client-library) if -your test throughput (and amount of telemetry it generates) is relatively low. -Since agent does a lot of things, it might create some noise during performance analysis and micro-optimizations. - -Execute the perf test with Application Insights enabled: -`$env:APPLICATIONINSIGHTS_CONNECTION_STRING="value"; java -jar "/path to/your file.jar" ` - ->Note: If you're running tests locally, you need to provide `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable, -skip setting the `javaagent` explicitly to send telemetry to Application Insights. - -### Logging - -We use [logback.xml][logback_xml] to configure the logging. By default, the stress test run on cluster will output -`WARN` level log which you may adjust based on your needs. -You may also control the verbosity of logs that go to Application Insights - see [OpenTelemetry logback appender][opentelemetry-logback] for more details. - -Since logs are hard to query and are extremely verbose (in case of high-scale stress tests), we're relying on metrics and workbooks for test result analysis. - -See also [Logging in Azure SDK][logging-azure-sdk]. - -### Metrics - -While some Azure SDKs provide custom metrics, we're going to collect generic test metrics and build queries/workbooks on top of them, -so it's important to reuse the same metric across different tests whenever possible. - -We need just one generic metric for basic analysis - the one that measures duration of one test execution (with additional dimensions). -It's implemented in `io.clientcore.http.stress.util.TelemetryHelper` and has the following semantic: -- name: `test.run.duration` - it is used in the stress workbook, so make sure to use the same name when applicable -- unit: seconds -- customDimensions: - - `error.type` - The low-cardinality type of error describing what happened (eg. exception class name). - -The metric should measure exactly one test operation, so we'll be able to derive the key performance indicators from it such as: -- throughput (rate of operations per period of time) -- duration of one operation -- error rate (how frequently errors of different types occur) - -Each metric collected with OpenTelemetry (and exported to Application Insights) also has the following dimensions: -- `cloud_RoleName` - in case of stress tests, it matches value of `otel.service.name` property configured in `Chart.yaml` to `{{ .Release.Name }}-{{ .Stress.BaseName }}`. -- `cloud_RoleInstance` - in case of k8s it matches pod name and is auto-detected. - -When running multiple test containers, make sure to assign different role instances to them, for example use `{{ .Stress.BaseName }}-consumer` and `{{ .Stress.BaseName }}-producer`. -This would allow you to distinguish telemetry coming from different containers. - -You would need to adjust the workbook to accommodate those changes. - -In addition to `test.run_duration`, we're also collecting: -- [JVM metrics](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/runtime-telemetry/runtime-telemetry-java8/library/README.md) measured by OpenTelemetry: - - CPU and memory usage - - GC stats - - Thread count - - Class stats - - See [JVM metrics semantic conventions for the details](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/runtime/jvm-metrics.md) +### Checking test results -You can also enable [reactor schedulers metrics](https://github.com/reactor/reactor-core/blob/962aeb77a09088fa2a7bac6d814c2b35220b1d35/docs/modules/ROOT/pages/metrics.adoc) collection by installing `micrometer-core` and -[OpenTelemetry micrometer bridge](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/micrometer/micrometer-1.5/library). +See [Stress Test Dashboard](https://aka.ms/azsdk/stress/dashboard) and [Application Insights](https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-enable?tabs=java#install-the-client-library) for monitoring and telemetry. -### Stress test workbook +### Logging and Metrics -[Stress test workbook](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/faa080af-c1d8-40ad-9cce-e1a450ca5b57/resourceGroups/rg-stress-cluster-pg/providers/Microsoft.Insights/components/stress-pg-ai-s7b6dif73rup6/workbooks) -shows a summary of a test run. +- Logging is configured via `logback.xml`. +- Metrics are collected using OpenTelemetry and exported to Application Insights. +- The main metric is `test.run.duration` (seconds), implemented in `com.azure.core.http.stress.util.TelemetryHelper`. -First, select a time range and run from the list, then check the report: -- `Test summary` contains key test parameters and key counters (total number of operations, errors, etc.) -- Tst operation success rate, latency and error rate -- CPU and memory utilization, number of threads and time spent in GC -- Warnings, errors, and exceptions in logs. Note logs and traces are sampled (at 1%) rate, so you won't see every error there +### Example: Running a Stress Test -Since you're changing the chart name, you would need to update the workbook to use `java-your-service-name` instead of `java-template`. -Then you'd need to create a new workbook for your service, follow -[Azure Monitor workbook documentation](https://learn.microsoft.com/azure/azure-monitor/visualize/workbooks-create-workbook) for more details. -Then you can import json file from `workbooks` folder. +```java readme-sample-runStressTest +public class RunStressTest { + public static void main(String[] args) { + com.azure.core.http.stress.App.main(args); + } +} +``` ## Writing useful tests -Stress tests are intended to detect reliability and resiliency issues: -- bugs in retry policy -- graceful degradation under high load and transient failures -- memory and connection leaks, thread pool starvation, etc - -To explore fault injection options, check out [Chaos mesh](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md#chaos-manifest) and [Http Fault injector](https://github.com/Azure/azure-sdk-tools/tree/main/tools/http-fault-injector). - -> Note: [Azure Chaos Studio](https://azure.microsoft.com/products/chaos-studio) is not currently supported by the stress test infra. - -Even without fault injection, by applying maximum load to the service, we can detect memory leaks, extensive allocations, -thread pool issues, or other performance issues in the code. So make sure to configure resource limits and apply the maximum load you can get under them. +Stress tests are intended to detect reliability and resiliency issues, such as retry policy bugs, resource leaks, and performance bottlenecks. For fault injection, see [Chaos mesh](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md#chaos-manifest). [azure_sdk_stress_test]: https://aka.ms/azsdk/stress @@ -253,9 +128,4 @@ thread pool issues, or other performance issues in the code. So make sure to con [helm]: https://helm.sh/docs/intro/install/ [azure_cli]: https://learn.microsoft.com/cli/azure/install-azure-cli [powershell]: https://learn.microsoft.com/powershell/scripting/install/installing-powershell?view=powershell-7 -[enable_application_insights]: https://learn.microsoft.com/en-us/azure/azure-monitor/app/opentelemetry-enable?tabs=java#enable-azure-monitor-application-insights -[logback_xml]: https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/servicebus/azure-messaging-servicebus-stress/src/main/resources/logback.xml [deploy_stress_test]: https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md#deploying-a-stress-test -[stress_test_layout]: https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md#layout -[opentelemetry-logback]: https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/logback/logback-appender-1.0/library -[logging-azure-sdk]: https://github.com/Azure/azure-sdk-for-java/wiki/Logging-in-Azure-SDK diff --git a/sdk/clientcore/http-stress/pom.xml b/sdk/clientcore/http-stress/pom.xml index 06c38ed74343..481da652fd26 100644 --- a/sdk/clientcore/http-stress/pom.xml +++ b/sdk/clientcore/http-stress/pom.xml @@ -1,20 +1,20 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - io.clientcore - clientcore-parent - 1.0.0-beta.3 - ../../parents/clientcore-parent + com.azure.v2 + azure-client-sdk-parent + 2.0.0-beta.1 + ../../parents/azure-client-sdk-parent-v2/pom.xml - io.clientcore + com.azure.v2 http-stress - 1.0.0-beta.1 + 1.0.0-beta.1 jar @@ -34,6 +34,11 @@ core 1.0.0-beta.9 + + com.azure + azure-core + 2.0.0 + io.clientcore http-okhttp3 @@ -97,32 +102,32 @@ 3.6.0 - - package - - shade - - - - - io.clientcore.http.stress.App - - - - ${project.artifactId}-${project.version}-jar-with-dependencies - - - *:* - - META-INF/maven/** - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - + + package + + shade + + + + + com.azure.core.http.stress.App + + + + ${project.artifactId}-${project.version}-jar-with-dependencies + + + *:* + + META-INF/maven/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/App.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/App.java index 5832931adebf..568f777db3d3 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/App.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/App.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.clientcore.http.stress; +package com.azure.core.http.stress; import com.azure.perf.test.core.PerfStressProgram; -import io.clientcore.http.stress.util.TelemetryHelper; +import com.azure.core.http.stress.util.TelemetryHelper; /** * Stress test application diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java index 0513db6b12d2..9e1c77f7654f 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.clientcore.http.stress; +package com.azure.core.http.stress; import com.azure.perf.test.core.PerfStressOptions; import io.clientcore.core.http.client.JdkHttpClientBuilder; @@ -17,7 +17,7 @@ import io.clientcore.core.instrumentation.logging.ClientLogger; import io.clientcore.core.models.binarydata.BinaryData; import io.clientcore.http.okhttp3.OkHttpHttpClientProvider; -import io.clientcore.http.stress.util.TelemetryHelper; +import com.azure.core.http.stress.util.TelemetryHelper; import reactor.core.publisher.Mono; import java.net.URI; diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java index 62292297a4f0..b563ca2962e9 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.clientcore.http.stress; +package com.azure.core.http.stress; import com.azure.perf.test.core.PerfStressOptions; import io.clientcore.core.http.client.JdkHttpClientBuilder; @@ -17,7 +17,7 @@ import io.clientcore.core.instrumentation.logging.ClientLogger; import io.clientcore.core.models.binarydata.BinaryData; import io.clientcore.http.okhttp3.OkHttpHttpClientProvider; -import io.clientcore.http.stress.util.TelemetryHelper; +import com.azure.core.http.stress.util.TelemetryHelper; import reactor.core.publisher.Mono; import java.net.URI; diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java index 079c98850de8..3a63bd7e00f3 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java @@ -1,10 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.clientcore.http.stress; +package com.azure.core.http.stress; + +import com.azure.core.http.stress.util.TelemetryHelper; import com.azure.perf.test.core.PerfStressTest; -import io.clientcore.http.stress.util.TelemetryHelper; import reactor.core.publisher.Mono; /** diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java index dcdbe3b62f78..42ee312ac911 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.clientcore.http.stress; +package com.azure.core.http.stress; import com.azure.perf.test.core.PerfStressOptions; import com.beust.jcommander.Parameter; diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/package-info.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/package-info.java index bada1ab1446e..f32ed301db0b 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/package-info.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/package-info.java @@ -4,4 +4,4 @@ /** * Contains classes for stress tests. */ -package io.clientcore.http.stress; +package com.azure.core.http.stress; diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java index 5a962e57c9eb..d9b25c33fd23 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -package io.clientcore.http.stress.util; +package com.azure.core.http.stress.util; import com.azure.monitor.opentelemetry.autoconfigure.AzureMonitorAutoConfigure; import io.clientcore.core.instrumentation.logging.ClientLogger; diff --git a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java index e1970e002e69..e458400df84d 100644 --- a/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java +++ b/sdk/clientcore/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java @@ -4,4 +4,4 @@ /** * Contains classes for stress tests utils. */ -package io.clientcore.http.stress.util; +package com.azure.core.http.stress.util; diff --git a/sdk/core-v2/http-stress/.gitignore b/sdk/core-v2/http-stress/.gitignore new file mode 100644 index 000000000000..56647ec6534b --- /dev/null +++ b/sdk/core-v2/http-stress/.gitignore @@ -0,0 +1,3 @@ +**/stress-test-resources.json +Chart.lock +charts/ \ No newline at end of file diff --git a/sdk/core-v2/http-stress/.helmignore b/sdk/core-v2/http-stress/.helmignore new file mode 100644 index 000000000000..a6f2989376b2 --- /dev/null +++ b/sdk/core-v2/http-stress/.helmignore @@ -0,0 +1,4 @@ +target/ +src/ +README.md +CHANGELOG.md diff --git a/sdk/core-v2/http-stress/CHANGELOG.md b/sdk/core-v2/http-stress/CHANGELOG.md new file mode 100644 index 000000000000..4144f75694a0 --- /dev/null +++ b/sdk/core-v2/http-stress/CHANGELOG.md @@ -0,0 +1,3 @@ +# Release History + +## 1.0.0-beta.1 (Unreleased) diff --git a/sdk/core-v2/http-stress/Chart.yaml b/sdk/core-v2/http-stress/Chart.yaml new file mode 100644 index 000000000000..e8a48cfc2441 --- /dev/null +++ b/sdk/core-v2/http-stress/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: java-clientcore-http +description: Stress tests for clientcore HTTP implementations +version: 0.1.1 +appVersion: v0.1 +annotations: + stressTest: 'true' # change it to true. This enables auto-discovery of this test via `find-all-stress-packages.ps1` + namespace: 'java-clientcore-http' + +dependencies: +- name: stress-test-addons + version: ~0.3.0 + repository: "@stress-test-charts" diff --git a/sdk/core-v2/http-stress/README.md b/sdk/core-v2/http-stress/README.md new file mode 100644 index 000000000000..fb772e37667f --- /dev/null +++ b/sdk/core-v2/http-stress/README.md @@ -0,0 +1,131 @@ +# Stress tests for Azure Core v2 HTTP client library for Java + +This package contains the stress test project for Azure Core v2 HTTP stack. It demonstrates how to create and run stress tests for the Azure SDK for Java core HTTP pipeline and client infrastructure. + +## Getting started + +Check out [Azure SDK Stress Test Wiki][azure_sdk_stress_test] for general information about stress tests. + +### Prerequisites + +- [Java Development Kit (JDK)][jdk_link], version 11 or later. +- [Maven][maven] +- [Docker][docker] +- [Kubectl][kubectl] +- [Helm][helm] +- [Azure CLI][azure_cli] +- [Powershell 7.0+][powershell] + +### Deploy Stress Test + +Change directory to the Azure SDK for Java root and deploy the package to your cluster: + +```shell +./eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -MatrixSelection all -SearchDirectory ./sdk/core-v2 +``` + +### Check Status + +See [Deploying A Stress Test][deploy_stress_test] for more details. + +List deployed packages: + +```shell +helm list -n +``` + +Get stress test pods and status: + +```shell +kubectl get pods -n +``` + +Get stress test pod logs: + +```shell +kubectl logs -n +``` + +Stop and remove deployed package: + +```shell +helm uninstall -n +``` + +### Other useful commands + +Execute commands in the container: + +```shell +kubectl exec --stdin --tty -n -c -- /bin/bash +``` + +## Key concepts + +### Project Structure + +``` +. +├── src/ # Test code +├── templates/ # Helm templates for Kubernetes manifests +├── workbooks/ # Azure Monitor workbooks for analyzing stress test results +├── Chart.yaml # Helm chart metadata +├── scenarios-matrix.yaml # Configuration for stress test scenarios +├── Dockerfile # Dockerfile for building the stress test image +├── stress-test-resources.bicep # Azure Bicep for deploying stress test resources +├── pom.xml +└── README.md +``` + +### How to create your own tests + +Start with [Azure SDK stress Wiki](https://aka.ms/azsdk/stress) to learn about stress tests. + +1. Copy `src/main/java/com/azure/core/http/stress` to your service folder. +2. Update the code: + - Update `pom.xml` to change artifact name and add dependencies on your service. + - Implement your first stress test instead of `HttpGet` and update `StressOptions` for your test parameters. + +Now you can run stress tests locally. Remaining steps are required to run tests on a stress cluster. + +3. Update `dockerfiles` to build your service artifacts and dependencies. +4. Describe Azure resources in `stress-test-resources.bicep`. +5. Update `Chart.yaml` and `templates/job.yaml` for your service. +6. Define scenarios and parameters in `scenarios-matrix.yaml`. + +Now you're ready to run tests with `./eng/common/scripts/stress-testing/deploy-stress-tests.ps1 -SearchDirectory ./sdk/core-v2`. + +### Checking test results + +See [Stress Test Dashboard](https://aka.ms/azsdk/stress/dashboard) and [Application Insights](https://learn.microsoft.com/azure/azure-monitor/app/opentelemetry-enable?tabs=java#install-the-client-library) for monitoring and telemetry. + +### Logging and Metrics + +- Logging is configured via `logback.xml`. +- Metrics are collected using OpenTelemetry and exported to Application Insights. +- The main metric is `test.run.duration` (seconds), implemented in `com.azure.core.http.stress.util.TelemetryHelper`. + +### Example: Running a Stress Test + +```java readme-sample-runStressTest +public class RunStressTest { + public static void main(String[] args) { + com.azure.core.http.stress.App.main(args); + } +} +``` + +## Writing useful tests + +Stress tests are intended to detect reliability and resiliency issues, such as retry policy bugs, resource leaks, and performance bottlenecks. For fault injection, see [Chaos mesh](https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md#chaos-manifest). + + +[azure_sdk_stress_test]: https://aka.ms/azsdk/stress +[jdk_link]: https://learn.microsoft.com/java/azure/jdk/?view=azure-java-stable +[maven]: https://maven.apache.org/ +[docker]: https://docs.docker.com/get-docker/ +[kubectl]: https://kubernetes.io/docs/tasks/tools/#kubectl +[helm]: https://helm.sh/docs/intro/install/ +[azure_cli]: https://learn.microsoft.com/cli/azure/install-azure-cli +[powershell]: https://learn.microsoft.com/powershell/scripting/install/installing-powershell?view=powershell-7 +[deploy_stress_test]: https://github.com/Azure/azure-sdk-tools/blob/main/tools/stress-cluster/chaos/README.md#deploying-a-stress-test diff --git a/sdk/core-v2/http-stress/checkstyle-suppressions.xml b/sdk/core-v2/http-stress/checkstyle-suppressions.xml new file mode 100644 index 000000000000..bea803f0c7f7 --- /dev/null +++ b/sdk/core-v2/http-stress/checkstyle-suppressions.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/sdk/core-v2/http-stress/dockerfiles/java17 b/sdk/core-v2/http-stress/dockerfiles/java17 new file mode 100644 index 000000000000..e1c2ef68a800 --- /dev/null +++ b/sdk/core-v2/http-stress/dockerfiles/java17 @@ -0,0 +1,47 @@ +ARG REGISTRY="azsdkengsys.azurecr.io" +ARG JRE_VERSION="17" +FROM ${REGISTRY}/java/jdk-mariner-mvn:jdk17-latest as builder + +# Do not remove this line. Update ensures container images do not get flagged for out of date and vulnerable distro packages. +RUN yum -y update + +# Add necessary files to the image +RUN mkdir /stress +WORKDIR /stress + +ADD ./.vscode /stress/.vscode +ADD ./sdk/tools /stress/sdk/tools +ADD ./eng /stress/eng +ADD ./common /stress/common +ADD ./sdk/parents /stress/sdk/parents +ADD ./sdk/clientcore /stress/sdk/clientcore + +ARG SKIP_CHECKS="-Dcheckstyle.skip -Dgpg.skip -Dmaven.javadoc.skip -Drevapi.skip -Dspotbugs.skip -Djacoco.skip -DskipTests -Dcodesnippet.skip -Dspotless.skip" +ARG MAVEN_ARGS="-B -V -U -Dhttp.keepAlive=true" + +# Build dependencies and stress tests +RUN --mount=type=cache,target=/root/.m2 \ +mvn -f /stress/eng/code-quality-reports/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/common/perf-test-core/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/parents/azure-perf-test-parent/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/tools/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/parents/clientcore-parent/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/clientcore/core/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/clientcore/http-okhttp3/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/clientcore/http-stress/pom.xml clean install ${SKIP_CHECKS} + +FROM mcr.microsoft.com/openjdk/jdk:${JRE_VERSION}-mariner + +# Do not remove this line. Update ensures container images do not get flagged for out of date and vulnerable distro packages. +RUN yum -y update + +# Copy target files from builder image +WORKDIR /app +COPY --from=builder /stress/sdk/clientcore/http-stress/target . + +# Import test server self-signed certificate +COPY --from=builder /stress/sdk/clientcore/http-stress/src/main/resources/simplehttpserver.crt ./simplehttpserver.crt +RUN keytool -import -alias test -file ./simplehttpserver.crt -keystore ${JAVA_HOME}/lib/security/cacerts -noprompt -keypass changeit -storepass changeit + +# This is never executed (since job yaml overrides it) +ENTRYPOINT ["bash"] diff --git a/sdk/core-v2/http-stress/dockerfiles/java21 b/sdk/core-v2/http-stress/dockerfiles/java21 new file mode 100644 index 000000000000..137eaa0a24c8 --- /dev/null +++ b/sdk/core-v2/http-stress/dockerfiles/java21 @@ -0,0 +1,47 @@ +ARG REGISTRY="azsdkengsys.azurecr.io" +ARG JRE_VERSION="21" +FROM ${REGISTRY}/java/jdk-mariner-mvn:jdk11-latest as builder + +# Do not remove this line. Update ensures container images do not get flagged for out of date and vulnerable distro packages. +RUN yum -y update + +# Add necessary files to the image +RUN mkdir /stress +WORKDIR /stress + +ADD ./.vscode /stress/.vscode +ADD ./sdk/tools /stress/sdk/tools +ADD ./eng /stress/eng +ADD ./common /stress/common +ADD ./sdk/parents /stress/sdk/parents +ADD ./sdk/clientcore /stress/sdk/clientcore + +ARG SKIP_CHECKS="-Dcheckstyle.skip -Dgpg.skip -Dmaven.javadoc.skip -Drevapi.skip -Dspotbugs.skip -Djacoco.skip -DskipTests -Dcodesnippet.skip -Dspotless.skip" +ARG MAVEN_ARGS="-B -V -U -Dhttp.keepAlive=true -Djdk.virtualThreadScheduler.parallelism=1" + +# Build dependencies and stress tests +RUN --mount=type=cache,target=/root/.m2 \ +mvn -f /stress/eng/code-quality-reports/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/common/perf-test-core/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/parents/azure-perf-test-parent/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/tools/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/parents/clientcore-parent/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/clientcore/core/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/clientcore/http-okhttp3/pom.xml clean install ${SKIP_CHECKS} && \ +mvn -f /stress/sdk/clientcore/http-stress/pom.xml clean install ${SKIP_CHECKS} + +FROM mcr.microsoft.com/openjdk/jdk:${JRE_VERSION}-mariner + +# Do not remove this line. Update ensures container images do not get flagged for out of date and vulnerable distro packages. +RUN yum -y update + +# Copy target files from builder image +WORKDIR /app +COPY --from=builder /stress/sdk/clientcore/http-stress/target . + +# Import test server self-signed certificate +COPY --from=builder /stress/sdk/clientcore/http-stress/src/main/resources/simplehttpserver.crt ./simplehttpserver.crt +RUN keytool -import -alias test -file ./simplehttpserver.crt -keystore ${JAVA_HOME}/lib/security/cacerts -noprompt -keypass changeit -storepass changeit + +# This is never executed (since job yaml overrides it) +ENTRYPOINT ["bash"] diff --git a/sdk/core-v2/http-stress/pom.xml b/sdk/core-v2/http-stress/pom.xml new file mode 100644 index 000000000000..481da652fd26 --- /dev/null +++ b/sdk/core-v2/http-stress/pom.xml @@ -0,0 +1,135 @@ + + + + 4.0.0 + + + com.azure.v2 + azure-client-sdk-parent + 2.0.0-beta.1 + ../../parents/azure-client-sdk-parent-v2/pom.xml + + + com.azure.v2 + http-stress + 1.0.0-beta.1 + jar + + + 1.8 + 1.8 + all,-missing + + + + + ch.qos.logback + logback-classic + 1.3.14 + + + io.clientcore + core + 1.0.0-beta.9 + + + com.azure + azure-core + 2.0.0 + + + io.clientcore + http-okhttp3 + 1.0.0-beta.3 + + + + io.opentelemetry + opentelemetry-api + 1.49.0 + + + + com.azure + perf-test-core + 1.0.0-beta.1 + + + + com.azure + azure-monitor-opentelemetry-autoconfigure + 1.2.0 + + + io.opentelemetry.instrumentation + opentelemetry-runtime-telemetry-java8 + 2.14.0-alpha + + + io.opentelemetry.instrumentation + opentelemetry-logback-appender-1.0 + 2.14.0-alpha + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 + + + + + com.azure:perf-test-core:[1.0.0-beta.1] + com.azure:azure-monitor-opentelemetry-autoconfigure:[1.2.0] + io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:[2.14.0-alpha] + io.opentelemetry.instrumentation:opentelemetry-logback-appender-1.0:[2.14.0-alpha] + ch.qos.logback:logback-classic:[1.3.14] + io.opentelemetry:opentelemetry-api:[1.49.0] + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + + package + + shade + + + + + com.azure.core.http.stress.App + + + + ${project.artifactId}-${project.version}-jar-with-dependencies + + + *:* + + META-INF/maven/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/sdk/core-v2/http-stress/scenarios-matrix.yaml b/sdk/core-v2/http-stress/scenarios-matrix.yaml new file mode 100644 index 000000000000..5a6500668ca2 --- /dev/null +++ b/sdk/core-v2/http-stress/scenarios-matrix.yaml @@ -0,0 +1,16 @@ +displayNames: + java-template: "" + dockerfiles/java21: jre21 + dockerfiles/java17: jre17 +matrix: + image: + - dockerfiles/java17 + - dockerfiles/java21 + httpClient: [default, okhttp] + scenarios: + get: + imageBuildDir: ..\..\..\ + testDurationMin: 30 + testScenario: httpget + concurrency: 75 + diff --git a/sdk/core-v2/http-stress/spotbugs-exclude.xml b/sdk/core-v2/http-stress/spotbugs-exclude.xml new file mode 100644 index 000000000000..b5d5b3aa70f3 --- /dev/null +++ b/sdk/core-v2/http-stress/spotbugs-exclude.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/App.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/App.java new file mode 100644 index 000000000000..568f777db3d3 --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/App.java @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.core.http.stress; + +import com.azure.perf.test.core.PerfStressProgram; +import com.azure.core.http.stress.util.TelemetryHelper; + +/** + * Stress test application + */ +public final class App { + /** + * Main method to invoke other stress tests. + * + * @param args the input arguments + */ + public static void main(String[] args) { + TelemetryHelper.init(); + + PerfStressProgram.run(new Class[] { HttpGet.class, + // HttpPatch.class, + // add other stress tests here + }, args); + } + + private App() { + } +} diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java new file mode 100644 index 000000000000..9e1c77f7654f --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpGet.java @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.core.http.stress; + +import com.azure.perf.test.core.PerfStressOptions; +import io.clientcore.core.http.client.JdkHttpClientBuilder; +import io.clientcore.core.http.models.HttpHeaderName; +import io.clientcore.core.http.pipeline.HttpInstrumentationOptions; +import io.clientcore.core.http.models.HttpMethod; +import io.clientcore.core.http.models.HttpRequest; +import io.clientcore.core.http.models.Response; +import io.clientcore.core.http.pipeline.HttpInstrumentationPolicy; +import io.clientcore.core.http.pipeline.HttpPipeline; +import io.clientcore.core.http.pipeline.HttpPipelineBuilder; +import io.clientcore.core.http.pipeline.HttpRetryPolicy; +import io.clientcore.core.instrumentation.logging.ClientLogger; +import io.clientcore.core.models.binarydata.BinaryData; +import io.clientcore.http.okhttp3.OkHttpHttpClientProvider; +import com.azure.core.http.stress.util.TelemetryHelper; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Performance test for simple HTTP GET against test server. + */ +public class HttpGet extends ScenarioBase { + // there will be multiple instances of scenario + private static final TelemetryHelper TELEMETRY_HELPER = new TelemetryHelper(HttpGet.class); + private static final ClientLogger LOGGER = new ClientLogger(HttpGet.class); + private final HttpPipeline pipeline; + private final URI uri; + final ExecutorService executorService = Executors.newFixedThreadPool(options.getParallel()); + + // This is almost-unique-id generator. We could use UUID, but it's a bit more expensive to use. + private final AtomicLong clientRequestId = new AtomicLong(Instant.now().getEpochSecond()); + + /** + * Creates an instance of performance test. + * + * @param options stress test options + */ + public HttpGet(StressOptions options) { + super(options, TELEMETRY_HELPER); + pipeline = getPipelineBuilder().build(); + try { + uri = new URI(options.getServiceEndpoint()); + } catch (URISyntaxException ex) { + throw LOGGER.logThrowableAsError(new IllegalArgumentException("'uri' must be a valid URI.", ex)); + } + } + + @Override + public void run() { + TELEMETRY_HELPER.instrumentRun(this::runInternal); + } + + private void runInternal() { + // no need to handle exceptions here, they will be handled (and recorded) by the telemetry helper + HttpRequest request = createRequest(); + Response response = pipeline.send(request); + response.getValue().toBytes(); + } + + @Override + public Mono runAsync() { + return TELEMETRY_HELPER.instrumentRunAsync(runInternalAsync()); + } + + @Override + public CompletableFuture runAsyncWithCompletableFuture() { + return TELEMETRY_HELPER.instrumentRunAsyncWithCompletableFuture(runAsyncWithCompletableFutureInternal()); + } + + @Override + public Runnable runAsyncWithExecutorService() { + return TELEMETRY_HELPER.instrumentRunAsyncWithRunnable(runAsyncWithExecutorServiceInternal()); + } + + @Override + public Runnable runAsyncWithVirtualThread() { + return TELEMETRY_HELPER.instrumentRunAsyncWithRunnable(runAsyncWithVirtualThreadInternal()); + } + + private Mono runInternalAsync() { + return Mono.usingWhen(Mono.fromCallable(() -> pipeline.send(createRequest())), response -> { + response.getValue().toBytes(); + return Mono.empty(); + }, response -> Mono.fromRunnable(response::close)); + } + + // Method to run using CompletableFuture + private CompletableFuture runAsyncWithCompletableFutureInternal() { + return CompletableFuture.supplyAsync(() -> { + try (Response response = pipeline.send(createRequest())) { + response.getValue().toBytes(); + } catch (Exception e) { + LOGGER.logThrowableAsError(e); + } + return null; + }, executorService); + } + + // Method to run using ExecutorService + private Runnable runAsyncWithExecutorServiceInternal() { + return () -> { + try (Response response = pipeline.send(createRequest())) { + response.getValue().toBytes(); + } catch (Exception e) { + LOGGER.logThrowableAsError(e); + } + }; + } + + // Method to run using Virtual Threads + private Runnable runAsyncWithVirtualThreadInternal() { + return () -> { + try (Response response = pipeline.send(createRequest())) { + response.getValue().toBytes(); + } catch (Exception e) { + LOGGER.logThrowableAsError(e); + } + }; + } + + private HttpRequest createRequest() { + HttpRequest request = new HttpRequest().setMethod(HttpMethod.GET).setUri(uri); + request.getHeaders().set(HttpHeaderName.USER_AGENT, "clientcore-stress"); + request.getHeaders() + .set(HttpHeaderName.fromString("x-client-id"), String.valueOf(clientRequestId.incrementAndGet())); + return request; + } + + private HttpPipelineBuilder getPipelineBuilder() { + HttpPipelineBuilder builder = new HttpPipelineBuilder().addPolicy(new HttpRetryPolicy()) + .addPolicy(new HttpInstrumentationPolicy( + new HttpInstrumentationOptions().setHttpLogLevel(HttpInstrumentationOptions.HttpLogLevel.HEADERS))); + + if (options.getHttpClient() == PerfStressOptions.HttpClientType.OKHTTP) { + builder.httpClient(new OkHttpHttpClientProvider().getSharedInstance()); + } else { + builder.httpClient(new JdkHttpClientBuilder().build()); + } + return builder; + } +} diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java new file mode 100644 index 000000000000..b563ca2962e9 --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/HttpPatch.java @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.core.http.stress; + +import com.azure.perf.test.core.PerfStressOptions; +import io.clientcore.core.http.client.JdkHttpClientBuilder; +import io.clientcore.core.http.models.HttpHeaderName; +import io.clientcore.core.http.pipeline.HttpInstrumentationOptions; +import io.clientcore.core.http.models.HttpMethod; +import io.clientcore.core.http.models.HttpRequest; +import io.clientcore.core.http.models.Response; +import io.clientcore.core.http.pipeline.HttpInstrumentationPolicy; +import io.clientcore.core.http.pipeline.HttpPipeline; +import io.clientcore.core.http.pipeline.HttpPipelineBuilder; +import io.clientcore.core.http.pipeline.HttpRetryPolicy; +import io.clientcore.core.instrumentation.logging.ClientLogger; +import io.clientcore.core.models.binarydata.BinaryData; +import io.clientcore.http.okhttp3.OkHttpHttpClientProvider; +import com.azure.core.http.stress.util.TelemetryHelper; +import reactor.core.publisher.Mono; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Performance test for simple HTTP GET against test server. + */ +public class HttpPatch extends ScenarioBase { + // there will be multiple instances of scenario + private static final TelemetryHelper TELEMETRY_HELPER = new TelemetryHelper(HttpPatch.class); + private static final ClientLogger LOGGER = new ClientLogger(HttpPatch.class); + private final HttpPipeline pipeline; + private final URI uri; + + // This is almost-unique-id generator. We could use UUID, but it's a bit more expensive to use. + private final AtomicLong clientRequestId = new AtomicLong(Instant.now().getEpochSecond()); + + /** + * Creates an instance of performance test. + * @param options stress test options + */ + public HttpPatch(StressOptions options) { + super(options, TELEMETRY_HELPER); + pipeline = getPipelineBuilder().build(); + try { + uri = new URI(options.getServiceEndpoint()); + } catch (URISyntaxException ex) { + throw LOGGER.logThrowableAsError(new IllegalArgumentException("'uri' must be a valid URI.", ex)); + } + } + + @Override + public void run() { + TELEMETRY_HELPER.instrumentRun(this::runInternal); + } + + private void runInternal() { + // no need to handle exceptions here, they will be handled (and recorded) by the telemetry helper + try (Response response = pipeline.send(createRequest())) { + int responseCode = response.getStatusCode(); + assert responseCode == 200 : "Unexpected response code: " + responseCode; + response.getValue().close(); + } + } + + @Override + public Mono runAsync() { + return Mono.error(new UnsupportedOperationException("Not implemented")); + } + + private HttpRequest createRequest() { + String body = "{\"id\": \"1\", \"name\": \"test\"}"; + HttpRequest request + = new HttpRequest().setMethod(HttpMethod.PATCH).setUri(uri).setBody(BinaryData.fromString(body)); + request.getHeaders().set(HttpHeaderName.CONTENT_LENGTH, String.valueOf(body.length())); + request.getHeaders().set(HttpHeaderName.USER_AGENT, "clientcore-stress"); + request.getHeaders() + .set(HttpHeaderName.fromString("x-client-id"), String.valueOf(clientRequestId.incrementAndGet())); + request.getHeaders().set(HttpHeaderName.CONTENT_TYPE, "application/json"); + request.getHeaders().set(HttpHeaderName.ACCEPT, "application/json"); + return request; + } + + private HttpPipelineBuilder getPipelineBuilder() { + HttpPipelineBuilder builder = new HttpPipelineBuilder().addPolicy(new HttpRetryPolicy()) + .addPolicy(new HttpInstrumentationPolicy( + new HttpInstrumentationOptions().setHttpLogLevel(HttpInstrumentationOptions.HttpLogLevel.HEADERS))); + + if (options.getHttpClient() == PerfStressOptions.HttpClientType.OKHTTP) { + builder.httpClient(new OkHttpHttpClientProvider().getSharedInstance()); + } else { + builder.httpClient(new JdkHttpClientBuilder().build()); + } + return builder; + } +} diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java new file mode 100644 index 000000000000..3a63bd7e00f3 --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/ScenarioBase.java @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.core.http.stress; + +import com.azure.core.http.stress.util.TelemetryHelper; + +import com.azure.perf.test.core.PerfStressTest; +import reactor.core.publisher.Mono; + +/** + * Performance test for getting messages. + * + * @param The options configured for the test. + */ +public abstract class ScenarioBase extends PerfStressTest { + private final TelemetryHelper telemetryHelper; + private final long startTime = System.currentTimeMillis(); + + /** + * Creates a stress test. + * + * @param options Performance test configuration options. + * @param telemetryHelper Telemetry helper to monitor test execution and record stats. + */ + public ScenarioBase(TOptions options, TelemetryHelper telemetryHelper) { + super(options); + this.telemetryHelper = telemetryHelper; + } + + @Override + public Mono globalSetupAsync() { + telemetryHelper.recordStart(options); + return super.globalSetupAsync(); + } + + @Override + public Mono globalCleanupAsync() { + telemetryHelper.recordEnd(startTime); + return super.globalCleanupAsync(); + } +} diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java new file mode 100644 index 000000000000..42ee312ac911 --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/StressOptions.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.core.http.stress; + +import com.azure.perf.test.core.PerfStressOptions; +import com.beust.jcommander.Parameter; + +/** + * Options to be used by your stress tests. +*/ +public class StressOptions extends PerfStressOptions { + @Parameter(names = { "--endpoint" }, description = "Service endpoint") + private String serviceEndpoint; + + /** + * Creates a new instance of {@link StressOptions}. + */ + public StressOptions() { + } + + /** + * Gets the service endpoint. + * @return the service endpoint. + */ + public String getServiceEndpoint() { + return serviceEndpoint; + } + + // When adding new test parameters, consider adding them to TelemetryHelper.recordStart() +} diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/package-info.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/package-info.java new file mode 100644 index 000000000000..f32ed301db0b --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/package-info.java @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Contains classes for stress tests. + */ +package com.azure.core.http.stress; diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java new file mode 100644 index 000000000000..d9b25c33fd23 --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/TelemetryHelper.java @@ -0,0 +1,327 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.core.http.stress.util; + +import com.azure.monitor.opentelemetry.autoconfigure.AzureMonitorAutoConfigure; +import io.clientcore.core.instrumentation.logging.ClientLogger; +import io.clientcore.http.stress.StressOptions; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; +import io.opentelemetry.instrumentation.runtimemetrics.java8.Classes; +import io.opentelemetry.instrumentation.runtimemetrics.java8.Cpu; +import io.opentelemetry.instrumentation.runtimemetrics.java8.GarbageCollector; +import io.opentelemetry.instrumentation.runtimemetrics.java8.MemoryPools; +import io.opentelemetry.instrumentation.runtimemetrics.java8.Threads; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import reactor.core.Exceptions; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import java.io.UncheckedIOException; +import java.util.AbstractMap; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; + +import static com.azure.perf.test.core.PerfStressOptions.HttpClientType.JDK; +import static com.azure.perf.test.core.PerfStressOptions.HttpClientType.OKHTTP; + +/** + * Telemetry helper is used to monitor test execution and record stats. + */ +public class TelemetryHelper { + private final Tracer tracer; + private final ClientLogger logger; + private static final AttributeKey ERROR_TYPE_ATTRIBUTE = AttributeKey.stringKey("error.type"); + private static final AttributeKey SAMPLE_IN_ATTRIBUTE = AttributeKey.booleanKey("sample.in"); + private static final AttributeKey DURATION_SEC_ATTRIBUTE = AttributeKey.longKey("durationSec"); + private static final AttributeKey DURATION_MS_ATTRIBUTE = AttributeKey.longKey("durationMs"); + private static final AttributeKey SCENARIO_NAME_ATTRIBUTE = AttributeKey.stringKey("scenarioName"); + private static final AttributeKey CONCURRENCY_ATTRIBUTE = AttributeKey.longKey("concurrency"); + private static final AttributeKey SYNC_ATTRIBUTE = AttributeKey.booleanKey("sync"); + private static final AttributeKey SIZE_ATTRIBUTE = AttributeKey.longKey("size"); + private static final AttributeKey HOSTNAME_ATTRIBUTE = AttributeKey.stringKey("hostname"); + private static final AttributeKey SERVICE_ENDPOINT_ATTRIBUTE = AttributeKey.stringKey("serviceEndpoint"); + private static final AttributeKey HTTP_CLIENT_ATTRIBUTE = AttributeKey.stringKey("httpClient"); + private static final AttributeKey JRE_VERSION_ATTRIBUTE = AttributeKey.stringKey("jreVersion"); + private static final AttributeKey JRE_VENDOR_ATTRIBUTE = AttributeKey.stringKey("jreVendor"); + private static final AttributeKey GIT_COMMIT_ATTRIBUTE = AttributeKey.stringKey("gitCommit"); + private static final AttributeKey COMPLETEABLE_FUTURE_ATTRIBUTE + = AttributeKey.booleanKey("completeableFuture"); + private static final AttributeKey EXECUTOR_SERVICE_ATTRIBUTE = AttributeKey.booleanKey("executorService"); + private static final AttributeKey VIRTUAL_THREAD_ATTRIBUTE = AttributeKey.booleanKey("virtualThread"); + private final Attributes commonAttributes; + private final Attributes canceledAttributes; + + private final String scenarioName; + private final DoubleHistogram runDuration; + + static { + // enables micrometer metrics from Reactor schedulers allowing to monitor thread pool usage and starvation + Schedulers.enableMetrics(); + } + + /** + * Creates a telemetry helper for a given scenario class. + * + * @param scenarioClass the scenario class + */ + public TelemetryHelper(Class scenarioClass) { + this.scenarioName = scenarioClass.getName(); + this.commonAttributes = Attributes.of(SCENARIO_NAME_ATTRIBUTE, scenarioName); + this.canceledAttributes + = Attributes.of(SCENARIO_NAME_ATTRIBUTE, scenarioName, ERROR_TYPE_ATTRIBUTE, "cancelled"); + this.tracer = GlobalOpenTelemetry.getTracer(scenarioName); + Meter meter = GlobalOpenTelemetry.getMeter(scenarioName); + this.logger = new ClientLogger(scenarioName); + this.runDuration = meter.histogramBuilder("test.run.duration").setUnit("s").build(); + } + + /** + * Initializes telemetry helper: sets up Azure Monitor exporter, enables JVM metrics collection. + */ + public static void init() { + AutoConfiguredOpenTelemetrySdkBuilder sdkBuilder = AutoConfiguredOpenTelemetrySdk.builder(); + String applicationInsightsConnectionString = System.getenv("APPLICATIONINSIGHTS_CONNECTION_STRING"); + if (applicationInsightsConnectionString != null) { + AzureMonitorAutoConfigure.customize(sdkBuilder, applicationInsightsConnectionString); + } else { + System.setProperty("otel.traces.exporter", "none"); + System.setProperty("otel.logs.exporter", "none"); + System.setProperty("otel.metrics.exporter", "none"); + } + + OpenTelemetry otel = sdkBuilder + // in case of multi-container test, customize instance id to distinguish telemetry from different containers + //.addResourceCustomizer((resource, props) -> resource.toBuilder().put(AttributeKey.stringKey("service.instance.id"), "container-name-1").build()) + .addSamplerCustomizer((sampler, props) -> new Sampler() { + @Override + public SamplingResult shouldSample(Context parentContext, String traceId, String name, + SpanKind spanKind, Attributes attributes, List parentLinks) { + if (Boolean.TRUE.equals(attributes.get(SAMPLE_IN_ATTRIBUTE))) { + return SamplingResult.recordAndSample(); + } + return sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + } + + @Override + public String getDescription() { + return sampler.getDescription(); + } + }) + .setResultAsGlobal() + .build() + .getOpenTelemetrySdk(); + Classes.registerObservers(otel); + Cpu.registerObservers(otel); + MemoryPools.registerObservers(otel); + Threads.registerObservers(otel); + GarbageCollector.registerObservers(otel); + OpenTelemetryAppender.install(otel); + } + + /** + * Instruments a runnable: records runnable duration along with the status (success, error, cancellation), + * + * @param oneRun the runnable to instrument + */ + @SuppressWarnings("try") + public void instrumentRun(Runnable oneRun) { + long start = System.currentTimeMillis(); + Span span = tracer.spanBuilder("run").startSpan(); + try (Scope ignored = span.makeCurrent()) { + oneRun.run(); + trackSuccess(start, span); + } catch (Throwable e) { + if (e.getMessage().contains("Timeout on blocking read") + || e instanceof InterruptedException + || e instanceof TimeoutException) { + trackCancellation(start, span); + } else { + trackFailure(start, e, span); + } + } + } + + /** + * Instruments a Mono: records mono duration along with the status (success, error, cancellation), + * + * @param runAsync the mono to instrument + * @return the instrumented mono + */ + @SuppressWarnings("try") + public Mono instrumentRunAsync(Mono runAsync) { + return Mono.defer(() -> { + long start = System.currentTimeMillis(); + Span span = tracer.spanBuilder("runAsync").startSpan(); + try (Scope ignored = span.makeCurrent()) { + return runAsync.doOnError(e -> trackFailure(start, e, span)) + .doOnCancel(() -> trackCancellation(start, span)) + .doOnSuccess(v -> trackSuccess(start, span)) + .contextWrite( + reactor.util.context.Context.of(com.azure.core.util.tracing.Tracer.PARENT_TRACE_CONTEXT_KEY, + io.opentelemetry.context.Context.current())); + } + }); + } + + /** + * Instruments a CompletableFuture: records future duration along with the status (success, error, cancellation), + * + * @param runAsyncFuture the future to instrument + * @return the instrumented future + */ + @SuppressWarnings("try") + public CompletableFuture instrumentRunAsyncWithCompletableFuture(CompletableFuture runAsyncFuture) { + return CompletableFuture.supplyAsync(() -> { + long start = System.currentTimeMillis(); + Span span = tracer.spanBuilder("runAsyncCompletableFuture").startSpan(); + + return new AbstractMap.SimpleImmutableEntry<>(start, span); + }).thenCompose(startAndSpan -> { + long start = startAndSpan.getKey(); + Span span = startAndSpan.getValue(); + + return runAsyncFuture.whenComplete((result, throwable) -> { + try (Scope ignored = span.makeCurrent()) { + if (throwable != null) { + trackFailure(start, throwable, span); + } else { + trackSuccess(start, span); + } + } finally { + span.end(); + } + }); + }); + } + + /** + * Instruments a Runnable: records runnable duration along with the status (success, error, cancellation). + * + * @param task the runnable to instrument + * @return A {@link Runnable} with instrumentation wrapping the {@code task}. + */ + @SuppressWarnings("try") + public Runnable instrumentRunAsyncWithRunnable(Runnable task) { + return () -> { + long start = System.currentTimeMillis(); + Span span = tracer.spanBuilder("runAsyncRunnable").startSpan(); + try (Scope ignored = span.makeCurrent()) { + try { + task.run(); + trackSuccess(start, span); + } catch (Exception e) { + trackFailure(start, e, span); + } finally { + span.end(); + } + } + }; + } + + private void trackSuccess(long start, Span span) { + logger.atInfo().log("run ended"); + + runDuration.record(getDuration(start), commonAttributes); + span.end(); + } + + private void trackCancellation(long start, Span span) { + logger.atWarning().addKeyValue("error.type", "cancelled").log("run ended"); + + runDuration.record(getDuration(start), canceledAttributes); + span.setAttribute(ERROR_TYPE_ATTRIBUTE, "cancelled"); + span.setStatus(StatusCode.ERROR); + span.end(); + } + + private void trackFailure(long start, Throwable e, Span span) { + Throwable unwrapped = Exceptions.unwrap(e); + if (unwrapped instanceof UncheckedIOException) { + unwrapped = unwrapped.getCause(); + } + + span.recordException(unwrapped); + span.setStatus(StatusCode.ERROR, unwrapped.getMessage()); + + String errorType = unwrapped.getClass().getName(); + logger.atError().addKeyValue("error.type", errorType).setThrowable(unwrapped).log("run ended"); + + Attributes errorAttributes + = Attributes.of(SCENARIO_NAME_ATTRIBUTE, scenarioName, ERROR_TYPE_ATTRIBUTE, errorType); + runDuration.record(getDuration(start), errorAttributes, io.opentelemetry.context.Context.current().with(span)); + span.end(); + } + + /** + * Records an event representing the start of a test along with test options. + * + * @param options test parameters + */ + public void recordStart(StressOptions options) { + Span before = startSampledInSpan("before run"); + before.setAttribute(DURATION_SEC_ATTRIBUTE, options.getDuration()); + before.setAttribute(SCENARIO_NAME_ATTRIBUTE, scenarioName); + before.setAttribute(CONCURRENCY_ATTRIBUTE, options.getParallel()); + + before.setAttribute(SYNC_ATTRIBUTE, options.isSync()); + before.setAttribute(SIZE_ATTRIBUTE, options.getSize()); + before.setAttribute(HOSTNAME_ATTRIBUTE, System.getenv().get("HOSTNAME")); + before.setAttribute(SERVICE_ENDPOINT_ATTRIBUTE, options.getServiceEndpoint()); + if (options.getHttpClient() == JDK) { + before.setAttribute(HTTP_CLIENT_ATTRIBUTE, "jdk"); + } else if (options.getHttpClient() == OKHTTP) { + before.setAttribute(HTTP_CLIENT_ATTRIBUTE, "okhttp"); + } else { + before.setAttribute(HTTP_CLIENT_ATTRIBUTE, "default"); + } + before.setAttribute(JRE_VERSION_ATTRIBUTE, System.getProperty("java.version")); + before.setAttribute(JRE_VENDOR_ATTRIBUTE, System.getProperty("java.vendor")); + before.setAttribute(GIT_COMMIT_ATTRIBUTE, System.getenv("GIT_COMMIT")); + before.setAttribute(COMPLETEABLE_FUTURE_ATTRIBUTE, options.isCompletableFuture()); + before.setAttribute(EXECUTOR_SERVICE_ATTRIBUTE, options.isExecutorService()); + before.setAttribute(VIRTUAL_THREAD_ATTRIBUTE, options.isVirtualThread()); + + before.end(); + } + + /** + * Records an event representing the end of the test. + * + * @param startTime the start time of the test + */ + public void recordEnd(long startTime) { + Span after = startSampledInSpan("after run"); + after.setAttribute(DURATION_MS_ATTRIBUTE, System.currentTimeMillis() - startTime); + after.end(); + } + + private Span startSampledInSpan(String name) { + return tracer.spanBuilder(name) + // guarantee that we have before/after spans sampled in + // and record duration/result of the test + .setAttribute(SAMPLE_IN_ATTRIBUTE, true) + .startSpan(); + } + + private static double getDuration(long start) { + return Math.max(0L, System.currentTimeMillis() - start) / 1000.0D; + } +} diff --git a/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java new file mode 100644 index 000000000000..e458400df84d --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/io/clientcore/http/stress/util/package-info.java @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Contains classes for stress tests utils. + */ +package com.azure.core.http.stress.util; diff --git a/sdk/core-v2/http-stress/src/main/java/module-info.java b/sdk/core-v2/http-stress/src/main/java/module-info.java new file mode 100644 index 000000000000..0c6289081e8a --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/java/module-info.java @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Contains classes for stress tests. + */ +module io.clientcore.http.stress { + requires transitive com.azure.core.test.perf; + requires transitive io.clientcore.core; + requires transitive io.clientcore.http.okhttp3; + + requires com.azure.monitor.opentelemetry.autoconfigure; + requires com.azure.core; + requires jcommander; + requires io.opentelemetry.api; + requires io.opentelemetry.context; + requires io.opentelemetry.instrumentation.logback_appender_1_0; + requires io.opentelemetry.instrumentation.runtime_telemetry_java8; + requires io.opentelemetry.sdk; + requires io.opentelemetry.sdk.autoconfigure; + requires io.opentelemetry.sdk.autoconfigure.spi; + requires io.opentelemetry.sdk.trace; + + exports io.clientcore.http.stress; + exports io.clientcore.http.stress.util; +} diff --git a/sdk/core-v2/http-stress/src/main/resources/logback.xml b/sdk/core-v2/http-stress/src/main/resources/logback.xml new file mode 100644 index 000000000000..a4643c0ce304 --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/resources/logback.xml @@ -0,0 +1,16 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + diff --git a/sdk/core-v2/http-stress/src/main/resources/simplehttpserver.crt b/sdk/core-v2/http-stress/src/main/resources/simplehttpserver.crt new file mode 100644 index 000000000000..1fb68bb575ee --- /dev/null +++ b/sdk/core-v2/http-stress/src/main/resources/simplehttpserver.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDDTCCAfWgAwIBAgIJAJUP82tfx9kZMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMTCWxvY2FsaG9zdDAeFw0yNDAxMDQyMzExMzdaFw0yNTAxMDMyMzExMzdaMBQx +EjAQBgNVBAMTCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAM4xzCbl8XG120Ns3zunVMjOeldEWarequhVaUMAC9Yx+6VpVLLpH8qwSFS8 +Cwj9ePtd5m+BulfPCZV0sfUgijfG53kov+O3ri7uFxR5mpO3JRlCITEnIJ+S0AJ5 +bbrPW285PgFQzzSIE7zT449A1mIv0aZOxsv+Tl2UHTZeCD7+fEMQBeMoxy5eE1Tl +Jejq0Anm2DJJsBG11pB2ehVhCec4N91LkSCVFywzuQT1A/QJPfNzHotdbSxYqbzy +F4laJJUxWRcgtedCSjYjt4//fkkq0sZOgCJfQvCV2loEf+6cGzYgkM3sdSICqdIj +uHZ0BJJBD6gcKfbQARLhlMjBYkkCAwEAAaNiMGAwDAYDVR0TAQH/BAIwADAOBgNV +HQ8BAf8EBAMCBaAwFgYDVR0lAQH/BAwwCgYIKwYBBQUHAwEwFwYDVR0RAQH/BA0w +C4IJbG9jYWxob3N0MA8GCisGAQQBgjdUAQEEAQIwDQYJKoZIhvcNAQELBQADggEB +AL3uWKII2qUpny9wxc43NYAEyjaMnSUrMoWs15bc94ikjMbWYOnCUtfpdspfM71P +Wsu4Xcb+BBxK0gzEq46nkC5g1712hgae/+PxKf4DmarB1YT7nWM9jVhYCyL+VhfQ +7B7QX7Qp0sXx6JjtbuJnKNRVjS4Rtn3O6fnF6EGlmxz7X3KJ/odQAmUHkUwuALom +f4qVRREJtDNrOzFVEo9mKZNv+S3duCco3gNLeDlFqT01Ph7P+qiqmEUN/6rUrB8A +1cvldYM829wP5izqgSPGnA6UjIh1BFnsThJoNit1IFVUhrmbwrujzjNj+N6SDxcM +NAsWNoChEk2kINynt0Pk2ww= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/sdk/core-v2/http-stress/stress-test-resources.bicep b/sdk/core-v2/http-stress/stress-test-resources.bicep new file mode 100644 index 000000000000..a9cb5bc8f989 --- /dev/null +++ b/sdk/core-v2/http-stress/stress-test-resources.bicep @@ -0,0 +1,2 @@ +@description('The base resource name.') +param baseName string = resourceGroup().name diff --git a/sdk/core-v2/http-stress/templates/job.yaml b/sdk/core-v2/http-stress/templates/job.yaml new file mode 100644 index 000000000000..ca332eacffa4 --- /dev/null +++ b/sdk/core-v2/http-stress/templates/job.yaml @@ -0,0 +1,57 @@ +{{- include "stress-test-addons.deploy-job-template.from-pod" (list . "stress.java-clientcore-http") -}} +{{- define "stress.java-clientcore-http" -}} +metadata: + labels: + testName: "{{ .Release.Name }}" +spec: + containers: + # simple and fast .NET Core HTTP server to run tests against + # When writing real stress test, you probably won't need it and would use corresponding Azure Service. + - name: server + image: stresspgs7b6dif73rup6.azurecr.io/stress/simplehttpserver + imagePullPolicy: Always + command: ['sh', '-c'] + args: + - | + set -a && + export ASPNETCORE_URLS="http://localhost:8080;https://localhost:8081" && + export Test__DurationInSec={{ mul ( add .Stress.testDurationMin 1) 60 }} && + dotnet /app/simple-server.dll + resources: + limits: + memory: "400Mi" + cpu: "2" + {{- include "stress-test-addons.container-env" . | nindent 6 }} + - name: test + image: {{ .Stress.imageTag }} + imagePullPolicy: Always + command: ['sh', '-c'] + args: + - | + set -a && + source $ENV_FILE && + java \ + -Dotel.service.name={{ .Release.Name }}-{{ .Stress.BaseName }} \ + -Dotel.traces.sampler=traceidratio \ + -Dotel.traces.sampler.arg=0.00001 \ + -Dhttp.maxConnections=100 \ + -Dhttp.keepAlive=true \ + -XX:InitialRAMPercentage=75.0 \ + -XX:MaxRAMPercentage=75.0 \ + -jar /app/http-stress-1.0.0-beta.1-jar-with-dependencies.jar \ + {{ .Stress.testScenario }} \ + --parallel {{ .Stress.concurrency }} \ + --duration {{ mul .Stress.testDurationMin 60 }} \ + --endpoint https://localhost:8081 \ + {{ if ne .Stress.httpClient "default" }}--http-client {{ .Stress.httpClient }}{{ end }} \ + --completeablefuture \ + --warmup 0 + --no-cleanup + # add your test parameters here + resources: + # make sure to configure resource limits for your test + limits: + memory: "1Gi" + cpu: "1" + {{- include "stress-test-addons.container-env" . | nindent 6 }} +{{- end -}} diff --git a/sdk/core-v2/http-stress/workbooks/runDetails.json b/sdk/core-v2/http-stress/workbooks/runDetails.json new file mode 100644 index 000000000000..46583554864b --- /dev/null +++ b/sdk/core-v2/http-stress/workbooks/runDetails.json @@ -0,0 +1,310 @@ +{ + "version": "Notebook/1.0", + "items": [ + { + "type": 1, + "content": { + "json": "## Workbook for java clientcore HTTP stress tests.\n\nSelect the run from the following list." + }, + "name": "text - 2" + }, + { + "type": 9, + "content": { + "version": "KqlParameterItem/1.0", + "parameters": [ + { + "id": "ab5bb927-f8a4-4491-8621-d300820d2ff3", + "version": "KqlParameterItem/1.0", + "name": "timeRange", + "label": "Time Range", + "type": 4, + "typeSettings": { + "selectableValues": [ + { + "durationMs": 900000 + }, + { + "durationMs": 1800000 + }, + { + "durationMs": 3600000 + }, + { + "durationMs": 14400000 + }, + { + "durationMs": 43200000 + }, + { + "durationMs": 86400000 + }, + { + "durationMs": 172800000 + }, + { + "durationMs": 259200000 + }, + { + "durationMs": 604800000 + } + ], + "allowCustom": true + }, + "value": { + "durationMs": 1800000 + } + }, + { + "id": "1b563bbe-70e0-48e6-ae33-d71d97ab8332", + "version": "KqlParameterItem/1.0", + "name": "runId", + "label": "Pod name", + "type": 2, + "isRequired": true, + "query": "dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and name == \"before run\"\r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| summarize start=min(timestamp) by runId, pod = tostring(customDimensions[\"hostname\"])\r\n| order by start desc\r\n| project runId, pod", + "typeSettings": { + "additionalResourceOptions": [], + "showDefault": false + }, + "queryType": 0, + "resourceType": "microsoft.insights/components", + "value": null + } + ], + "style": "pills", + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "parameters - 2", + "styleSettings": { + "maxWidth": "30" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let runId = \"{runId}\";\r\nlet roleName = strcat(\"java-clientcore-http-\", runId);\r\nlet metrics = customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName == roleName;\r\nlet testSpans = dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName == roleName;\r\nlet errors = metrics\r\n| where name == \"test.run.duration\"\r\n| extend errorType = tostring(customDimensions[\"error.type\"])\r\n| summarize error_by_type=sum(valueCount) by errorType\r\n| summarize test_errors=make_bag(bag_pack(errorType, error_by_type))\r\n| evaluate narrow();\r\nlet runs = metrics \r\n| where name == \"test.run.duration\" \r\n| summarize successful_runs=sumif(valueCount, customDimensions[\"error.type\"] == \"\"), total_runs=sum(valueCount)\r\n| evaluate narrow();\r\nlet parameters = testSpans \r\n| where name == \"before run\"\r\n| project params_pod=customDimensions[\"hostname\"], params_scenarioName=customDimensions[\"scenarioName\"], params_durationSec=customDimensions[\"durationSec\"], params_concurrency=customDimensions[\"concurrency\"], params_sync=customDimensions[\"sync\"], params_httpClient=strcat(tostring(customDimensions[\"httpClientPackage\"]), \":\", tostring(customDimensions[\"httpClientPackageVersion\"])), params_JRE=strcat(tostring(customDimensions[\"jreVendor\"]), \" \", tostring(customDimensions[\"jreVersion\"]))\r\n| evaluate narrow();\r\nlet actualDuration = metrics\r\n| where name == \"test.run.duration\"\r\n| summarize maxTs = max(timestamp), minTs = min(timestamp)\r\n| project actual_durationSec=(maxTs-minTs)/1s\r\n| evaluate narrow();\r\nlet avgThroughput = metrics \r\n| where name == \"test.run.duration\" \r\n| summarize throughputPerMin=sum(valueCount) by bin(timestamp, 1m) // in case AppInsights ingestion drops something\r\n| summarize avg_throughtputPerSec=avg(throughputPerMin/60)\r\n| evaluate narrow();\r\nparameters \r\n| union runs, errors, actualDuration, avgThroughput\r\n| project Property = Column, Value\r\n", + "size": 0, + "showAnalytics": true, + "title": "Test summary", + "noDataMessageStyle": 5, + "queryType": 0, + "resourceType": "microsoft.insights/components", + "gridSettings": { + "sortBy": [ + { + "itemKey": "Property", + "sortOrder": 1 + } + ] + }, + "sortBy": [ + { + "itemKey": "Property", + "sortOrder": 1 + } + ] + }, + "customWidth": "30", + "name": "query - 9", + "styleSettings": { + "maxWidth": "30" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "let runs = customMetrics \r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" \r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| summarize start=min(timestamp), end=max(timestamp) by runId\r\n| project start, duration= end-start, runId;\r\nlet runSpans = dependencies\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and name == \"before run\"\r\n| extend runId = tostring(split(cloud_RoleName, \"-\")[2])\r\n| distinct runId, pod=tostring(customDimensions[\"hostname\"]), client=strcat(tostring(customDimensions[\"httpClientPackage\"]), \":\", tostring(customDimensions[\"httpClientPackageVersion\"]));\r\nruns \r\n| join kind = innerunique runSpans on runId\r\n| order by start desc\r\n| project-away runId1\r\n", + "size": 0, + "title": "Runs in {timeRange:label}", + "noDataMessageStyle": 5, + "queryType": 0, + "resourceType": "microsoft.insights/components", + "gridSettings": { + "sortBy": [ + { + "itemKey": "start", + "sortOrder": 2 + } + ] + }, + "sortBy": [ + { + "itemKey": "start", + "sortOrder": 2 + } + ] + }, + "customWidth": "40", + "name": "query - 8", + "styleSettings": { + "maxWidth": "40" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"test.run.duration\" and cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| summarize successful_runs=sum(valueCount) by bin(timestamp, 1m)\r\n| render timechart", + "size": 0, + "aggregation": 3, + "title": "Test run success rate (per minute)", + "noDataMessageStyle": 5, + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "query - 3", + "styleSettings": { + "maxWidth": "30", + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where name == \"test.run.duration\"\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where customDimensions[\"error.type\"] == \"\"\r\n| summarize avg_duration = avg(valueSum/valueCount) * 1000 by bin(timestamp, 1m)\r\n| render timechart", + "size": 0, + "aggregation": 3, + "title": "Duration of successfull operation (ms)", + "noDataMessageStyle": 5, + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "query - 5", + "styleSettings": { + "maxWidth": "30", + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"test.run.duration\"\r\n| extend status = tostring(customDimensions[\"error.type\"])\r\n| where status != \"\"\r\n| summarize test_errors = sum(valueCount) by status, bin(timestamp, 1m)\r\n| render linechart", + "size": 0, + "aggregation": 3, + "title": "Error rate (per minute)", + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "query - 3 - Copy", + "styleSettings": { + "maxWidth": "30", + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.memory.used\" and customDimensions[\"jvm.memory.type\"]==\"heap\"\r\n| summarize heap_memory_used=sum(valueSum/valueCount) by bin(timestamp, 1m)\r\n| render areachart", + "size": 0, + "aggregation": 3, + "title": "Heap memory used (MB)", + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "query - 8", + "styleSettings": { + "maxWidth": "30" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.cpu.recent_utilization\"\r\n| summarize cpu_time_percent=avg(value) * 100 by bin(timestamp, 1m)\r\n| render timechart\r\n", + "size": 0, + "aggregation": 3, + "title": "CPU %", + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "query - 9", + "styleSettings": { + "maxWidth": "30" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.thread.count\"\r\n| summarize max_thread_count=max(valueMax) by bin(timestamp, 1m)\r\n| render timechart\r\n", + "size": 0, + "aggregation": 3, + "title": "Thread count", + "noDataMessageStyle": 5, + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "15", + "name": "query - 11", + "styleSettings": { + "maxWidth": "15" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "customMetrics\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\"\r\n| where name == \"jvm.gc.duration\" \r\n| extend gc_type=tostring(customDimensions[\"jvm.gc.name\"])\r\n| summarize gc_percentage=sum(valueSum) / 60 * 100 by gc_type, bin(timestamp, 1m)\r\n| render timechart\r\n", + "size": 0, + "aggregation": 3, + "title": "% of time spent in GC", + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "15", + "name": "query - 11", + "styleSettings": { + "maxWidth": "15" + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "traces \r\n| union exceptions\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\" and severityLevel > 1\r\n| extend category = tostring(customDimensions[\"LoggerName\"])\r\n| extend logOrExceptionMessage = coalesce(message, outerMessage) \r\n| extend message = case(logOrExceptionMessage startswith \"{\\\"az.sdk.message\", azSdkContext=parse_json(logOrExceptionMessage)[\"az.sdk.message\"], substring(logOrExceptionMessage, 0, 48))\r\n| project timestamp, category, message, severity = case(severityLevel == 2, \"Warning\", severityLevel == 3, \"Error\", severityLevel == 1, \"Info\", \"\")\r\n| summarize occurences = count() by severity, category, message\r\n| order by occurences desc\r\n", + "size": 0, + "title": "Warnings and errors in logs (sampled, 0.001%)", + "queryType": 0, + "resourceType": "microsoft.insights/components", + "sortBy": [] + }, + "customWidth": "60", + "name": "query - 6", + "styleSettings": { + "maxWidth": "60", + "showBorder": true + } + }, + { + "type": 3, + "content": { + "version": "KqlItem/1.0", + "query": "traces \r\n| union exceptions\r\n| where timestamp >= {timeRange:start} and timestamp <= {timeRange:end}\r\n| where cloud_RoleName startswith \"java-clientcore-http\" and cloud_RoleName endswith \"{runId}\" and severityLevel > 1\r\n| extend severity = case(severityLevel == 2, \"Warning\", severityLevel == 3, \"Error\", \"\")\r\n| summarize warnings = countif(severityLevel==2), errors = countif(severityLevel==3) by bin(timestamp, 1m)\r\n| render timechart\r\n", + "size": 0, + "title": "Errors and warnings in logs (sampled, 0.001%) over time", + "queryType": 0, + "resourceType": "microsoft.insights/components" + }, + "customWidth": "30", + "name": "query - 12", + "styleSettings": { + "maxWidth": "30", + "showBorder": true + } + } + ], + "$schema": "https://github.com/Microsoft/Application-Insights-Workbooks/blob/master/schema/workbook.json" +} diff --git a/sdk/core-v2/pom.xml b/sdk/core-v2/pom.xml index 6068d2f634ab..59b8de306903 100644 --- a/sdk/core-v2/pom.xml +++ b/sdk/core-v2/pom.xml @@ -12,5 +12,6 @@ azure-core azure-core-test + http-stress