diff --git a/README.md b/README.md index ebd214940..595935afa 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,9 @@ The Azure DevOps pipelines that drive the daily benchmark sweep are generated from JSON configs by the [pod-scheduler](scripts/pod-scheduler/README.md). See [`build/README.md`](build/README.md) for the mapping between configs and generated YAML files, and how to regenerate them after editing. + +# PerfLab export + +The external [Crank PerfLab exporter](src/Crank.PerfLabExporter/README.md) +converts raw Crank JSON to canonical PerfLab JSON and optionally publishes it to +the existing results blob and queue protocol. diff --git a/build/README.md b/build/README.md index 205883ac0..35377ebe3 100644 --- a/build/README.md +++ b/build/README.md @@ -69,3 +69,27 @@ After any edit, regenerate the YAML and commit both the JSON and the YAML together. The snapshot tests in `scripts/pod-scheduler/tests/test_snapshots.py` will fail in CI if the generated YAML drifts from the configs. + +## Trend PerfLab lanes + +[`trend-perflab-lanes.json`](trend-perflab-lanes.json) is the source of truth +for Trend's stable PerfLab lane identity. Its queue convention is +`....Perf`. The pod scheduler copies the +matching lane into every Trend template call. Service Bus queue names such as +`citrine1`, `azure`, and `cobalthosted` remain worker-routing details and must +never be used as PerfLab `Run.Queue`. + +Trend scenarios declare `testName`, `family`, and `categories` directly in the +scenario templates. Plaintext variants use `aspnet-plaintext`, JSON variants +use `aspnet-json`, and the remaining families are antiforgery, TLS, request +rejection, fortunes, single query, multiple queries, updates, and caching. +`testName` remains the individual PerfLab `Test.Name`; `family` becomes +`Run.Name`. + +Trend jobs load the canonical repository configs through a raw GitHub base URL +pinned to `$(Build.SourceVersion)`. Benchmarks-owned source revisions and raw +assets use `{{benchmarksCommit}}`, whose shared default is `main`; Trend +overrides it with `--variable benchmarksCommit=$(Build.SourceVersion)`. +Crank resolves `imports` before applying command-line variables, so existing +Benchmarks-owned import URLs continue to use their declared revisions rather +than being duplicated into a generated config tree. diff --git a/build/benchmarks-ci-01.yml b/build/benchmarks-ci-01.yml index 42688238f..58750125f 100644 --- a/build/benchmarks-ci-01.yml +++ b/build/benchmarks-ci-01.yml @@ -146,7 +146,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-gold-windows-x64 + perfLabQueue: Windows.Server2022.Amd64.AspNetGold.Perf + perfLabOs: "Windows Server 2022" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetGold + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile gold-win-app --profile gold-load2-load " # GROUP 5 @@ -252,7 +262,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-gold-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetGold.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetGold + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile gold-lin-app --profile gold-load-load --profile gold-db-db " # GROUP 9 diff --git a/build/benchmarks-ci-02.yml b/build/benchmarks-ci-02.yml index 1eab26370..840315ea9 100644 --- a/build/benchmarks-ci-02.yml +++ b/build/benchmarks-ci-02.yml @@ -252,7 +252,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine1 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-lin-app --profile gold-load-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-gold-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetGold.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetGold + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile gold-lin-app --profile gold-load-load " - job: Trends_Database_gold_win displayName: 8- Trends Database gold-win @@ -266,7 +276,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: citrine2 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-gold-windows-x64 + perfLabQueue: Windows.Server2022.Amd64.AspNetGold.Perf + perfLabOs: "Windows Server 2022" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetGold + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile gold-win-app --profile gold-load2-load --profile gold-db-db " # GROUP 9 diff --git a/build/benchmarks-ci-azure.yml b/build/benchmarks-ci-azure.yml index 2406d43af..8eacc4120 100644 --- a/build/benchmarks-ci-azure.yml +++ b/build/benchmarks-ci-azure.yml @@ -236,7 +236,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-azure-arm64 + perfLabQueue: Ubuntu.2204.Arm64.AspNetAzure.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: arm64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetAzure + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile azure-server-arm64-app --profile azure-client-load " # GROUP 8 @@ -266,7 +276,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-azure-amd64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetAzure2.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetAzure2 + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile azure2-server-amd64-app --profile azure2-client-load " # GROUP 9 @@ -282,7 +302,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-cloud-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetCobaltCloud.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "16" + perfLabHardware: AspNetCobaltCloud + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load " - job: Trends_cobalt_cloud_lin_azl3_dual displayName: 9- Trends cobalt-cloud-lin-azl3-dual @@ -296,7 +326,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-cloud-azurelinux3-x64 + perfLabQueue: AzureLinux.3.Amd64.AspNetCobaltCloud.Perf + perfLabOs: "Azure Linux 3.0" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "16" + perfLabHardware: AspNetCobaltCloud + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-db-load " # GROUP 10 @@ -312,7 +352,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-lin-app --profile idna-intel-lin-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-idna-amd-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetIdnaAmd.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetIdnaAmd + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile idna-amd-lin-app --profile idna-intel-lin-load " - job: Trends_idna_amd_win displayName: 10- Trends idna-amd-win @@ -326,7 +376,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-amd-win-app --profile azure2-db-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-idna-amd-windows-x64 + perfLabQueue: Windows.Server2022.Amd64.AspNetIdnaAmd.Perf + perfLabOs: "Windows Server 2022" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetIdnaAmd + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile idna-amd-win-app --profile azure2-db-load " # GROUP 11 @@ -342,7 +402,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-lin-app --profile idna-amd-lin-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-idna-intel-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetIdnaIntel.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetIdnaIntel + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile idna-intel-lin-app --profile idna-amd-lin-load " - job: Trends_idna_intel_win displayName: 11- Trends idna-intel-win @@ -356,7 +426,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile idna-intel-win-app --profile azure2-db-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-idna-intel-windows-x64 + perfLabQueue: Windows.Server2022.Amd64.AspNetIdnaIntel.Perf + perfLabOs: "Windows Server 2022" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetIdnaIntel + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile idna-intel-win-app --profile azure2-db-load " # GROUP 12 @@ -372,7 +452,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-azure-arm64 + perfLabQueue: Ubuntu.2204.Arm64.AspNetAzure.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: arm64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetAzure + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile azure-server-arm64-app --profile azure-client-load --profile azure-db-db " - job: Trends_Database_azure2_amd64 displayName: 12- Trends Database azure2-amd64 @@ -386,7 +476,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azurearm64 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-azure-amd64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetAzure2.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "4" + perfLabHardware: AspNetAzure2 + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile azure2-server-amd64-app --profile azure2-client-load --profile azure2-db-db " # GROUP 13 @@ -402,7 +502,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-cloud-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetCobaltCloud.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "16" + perfLabHardware: AspNetCobaltCloud + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-cloud-lin-server-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " # GROUP 14 @@ -418,5 +528,15 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: azure serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-cloud-azurelinux3-x64 + perfLabQueue: AzureLinux.3.Amd64.AspNetCobaltCloud.Perf + perfLabOs: "Azure Linux 3.0" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "16" + perfLabHardware: AspNetCobaltCloud + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-cloud-lin-server-azure-linux3-app --profile cobalt-cloud-lin-client-load --profile cobalt-cloud-lin-db-db " diff --git a/build/benchmarks-ci-cobalt.yml b/build/benchmarks-ci-cobalt.yml index 1d0c2d0b5..9972154a0 100644 --- a/build/benchmarks-ci-cobalt.yml +++ b/build/benchmarks-ci-cobalt.yml @@ -402,7 +402,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load " - job: Trends_cobalt_hosted_lin_azl3 displayName: 13- Trends cobalt-hosted-lin-azl3 @@ -416,7 +426,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-azurelinux3-x64 + perfLabQueue: AzureLinux.3.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Azure Linux 3.0" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load " # GROUP 14 @@ -432,7 +452,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "28" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load " - job: Trends_cobalt_hosted_lin_azl3_28 displayName: 14- Trends cobalt-hosted-lin-azl3-28 @@ -446,7 +476,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-azurelinux3-x64 + perfLabQueue: AzureLinux.3.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Azure Linux 3.0" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "28" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load " # GROUP 15 @@ -582,7 +622,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - job: Trends_Database_cobalt_hosted_lin_azl3 displayName: 19- Trends Database cobalt-hosted-lin-azl3 @@ -596,7 +646,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-azurelinux3-x64 + perfLabQueue: AzureLinux.3.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Azure Linux 3.0" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "56" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-azure-linux3-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " # GROUP 20 @@ -612,7 +672,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-linux-x64 + perfLabQueue: Ubuntu.2204.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Ubuntu 22.04" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "28" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-28-app --profile cobalt-hosted-lin-client-load --profile cobalt-hosted-lin-db-db " - job: Trends_Database_cobalt_hosted_lin_azl3_28 displayName: 20- Trends Database cobalt-hosted-lin-azl3-28 @@ -626,7 +696,17 @@ jobs: connection: ASPNET Benchmarks Service Bus serviceBusQueueName: cobalthosted_azurelinux3 serviceBusNamespace: aspnetbenchmarks - arguments: "$(ciProfile) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " + benchmarksRawBaseUrl: "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + enablePerfLabPublication: false + perfLabLaneName: aspnet-cobalt-hosted-azurelinux3-x64 + perfLabQueue: AzureLinux.3.Amd64.AspNetCobaltHosted.Perf + perfLabOs: "Azure Linux 3.0" + perfLabArchitecture: x64 + perfLabLocale: en-US + perfLabCores: "28" + perfLabHardware: AspNetCobaltHosted + perfLabTopology: "SUT+Load+DB" + arguments: "--config https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)/build/ci.profile.yml --variable benchmarksCommit=$(Build.SourceVersion) --profile cobalt-hosted-lin-server-azure-linux3-28-app --profile cobalt-hosted-lin-client-azure-linux3-load --profile cobalt-hosted-lin-db-azure-linux3-db " # GROUP 21 diff --git a/build/benchmarks_ci_azure_pods.json b/build/benchmarks_ci_azure_pods.json index fead30052..90e20e7eb 100644 --- a/build/benchmarks_ci_azure_pods.json +++ b/build/benchmarks_ci_azure_pods.json @@ -8,6 +8,10 @@ "azure", "azurearm64" ], + "trend_lane_registry": "trend-perflab-lanes.json", + "pipeline": { + "trend_benchmarks_raw_base_url": "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + }, "yaml_generation": { "target_yaml_count": 1, "schedule_offset_hours": 0 @@ -94,6 +98,7 @@ "template": "trend-scenarios.yml", "type": 2, "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3-dual", "idna-amd-lin", "idna-amd-win", "idna-intel-lin", "idna-intel-win"], + "enable_perf_lab_publication": false, "estimated_runtime": 20.0 }, { @@ -101,6 +106,7 @@ "template": "trend-database-scenarios.yml", "type": 3, "pods": ["azure-arm64", "azure2-amd64", "cobalt-cloud-lin", "cobalt-cloud-lin-azl3"], + "enable_perf_lab_publication": false, "estimated_runtime": 15.0 } ] diff --git a/build/benchmarks_ci_cobalt_pods.json b/build/benchmarks_ci_cobalt_pods.json index aed674790..e4a12c6bc 100644 --- a/build/benchmarks_ci_cobalt_pods.json +++ b/build/benchmarks_ci_cobalt_pods.json @@ -8,6 +8,10 @@ "cobalthosted", "cobalthosted_azurelinux3" ], + "trend_lane_registry": "trend-perflab-lanes.json", + "pipeline": { + "trend_benchmarks_raw_base_url": "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + }, "yaml_generation": { "target_yaml_count": 1, "schedule_offset_hours": 0 @@ -90,6 +94,7 @@ "template": "trend-scenarios.yml", "type": 2, "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "enable_perf_lab_publication": false, "estimated_runtime": 20.0 }, { @@ -97,6 +102,7 @@ "template": "trend-database-scenarios.yml", "type": 3, "pods": ["cobalt-hosted-lin", "cobalt-hosted-lin-azl3", "cobalt-hosted-lin-28", "cobalt-hosted-lin-azl3-28"], + "enable_perf_lab_publication": false, "estimated_runtime": 15.0 }, { diff --git a/build/benchmarks_ci_pods.json b/build/benchmarks_ci_pods.json index 3486ca0b6..ce7f0c6ed 100644 --- a/build/benchmarks_ci_pods.json +++ b/build/benchmarks_ci_pods.json @@ -10,6 +10,10 @@ "citrine3", "mono" ], + "trend_lane_registry": "trend-perflab-lanes.json", + "pipeline": { + "trend_benchmarks_raw_base_url": "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + }, "yaml_generation": { "target_yaml_count": 2, "schedule_offset_hours": 6 @@ -173,6 +177,7 @@ "template": "trend-scenarios.yml", "type": 2, "pods": ["gold-lin", "gold-win"], + "enable_perf_lab_publication": false, "estimated_runtime": 20.0 }, { @@ -180,6 +185,7 @@ "template": "trend-database-scenarios.yml", "type": 3, "pods": ["gold-lin", "gold-win"], + "enable_perf_lab_publication": false, "estimated_runtime": 15.0 }, { diff --git a/build/crank-perflab-counter-policy.json b/build/crank-perflab-counter-policy.json new file mode 100644 index 000000000..c06ed9618 --- /dev/null +++ b/build/crank-perflab-counter-policy.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "mappings": [ + { + "path": "jobs.load.results['http/rps/mean']", + "name": "Requests/sec", + "metricName": "requests/sec", + "higherIsBetter": true, + "topCounter": true, + "defaultCounter": true, + "regressionThreshold": 0.02 + }, + { + "path": "jobs.load.results['http/latency/mean']", + "name": "Mean latency", + "metricName": "ms", + "higherIsBetter": false, + "topCounter": true, + "defaultCounter": false, + "regressionThreshold": 0.05, + "excludedScenarios": [ + "RejectionInvalidHeaderHttpSys", + "RejectionInvalidHeaderKestrel" + ] + }, + { + "path": "jobs.load.results['http/latency/99']", + "name": "P99 latency", + "metricName": "ms", + "higherIsBetter": false, + "topCounter": true, + "defaultCounter": false, + "regressionThreshold": 0.05, + "excludedScenarios": [ + "RejectionInvalidHeaderHttpSys", + "RejectionInvalidHeaderKestrel" + ] + }, + { + "path": "jobs.application.results['benchmarks/start-time']", + "name": "Startup time", + "metricName": "ms", + "higherIsBetter": false, + "topCounter": true, + "defaultCounter": false, + "regressionThreshold": 0.05 + }, + { + "path": "jobs.application.results['benchmarks/published-size']", + "name": "Published size", + "metricName": "bytes", + "higherIsBetter": false, + "topCounter": true, + "defaultCounter": false, + "regressionThreshold": 0.02, + "scale": 1024 + } + ] +} diff --git a/build/dependencies.props b/build/dependencies.props index ccbddc40f..259663559 100644 --- a/build/dependencies.props +++ b/build/dependencies.props @@ -48,5 +48,13 @@ 2.0.0-beta4.22272.1 0.2.0-alpha.24114.2 1.9.0 + 1.21.0 + 12.29.1 + 12.27.1 + 6.0.2 + 3.0.5 + 17.12.0 + 2.9.2 + 2.8.2 diff --git a/build/trend-database-scenarios.yml b/build/trend-database-scenarios.yml index 7ee92d86e..b6892349c 100644 --- a/build/trend-database-scenarios.yml +++ b/build/trend-database-scenarios.yml @@ -14,6 +14,69 @@ parameters: - name: condition type: string default: 'true' +- name: benchmarksRawBaseUrl + type: string +- name: enablePerfLabPublication + type: boolean + default: false +- name: perfLabLaneName + type: string +- name: perfLabQueue + type: string +- name: perfLabOs + type: string +- name: perfLabArchitecture + type: string +- name: perfLabLocale + type: string +- name: perfLabCores + type: string +- name: perfLabHardware + type: string +- name: perfLabTopology + type: string +- name: runtimeRepository + type: string + default: https://github.com/dotnet/runtime +- name: runtimeBranch + type: string + default: main +- name: framework + type: string + default: net11.0 +- name: runtime + type: string + default: CoreCLR +- name: configuration + type: string + default: Release +- name: adapterPolicyPath + type: string + default: crank-perflab-counter-policy.json +- name: perfLabStorageAccount + type: string + default: pvscmdupload +- name: perfLabContainer + type: string + default: results +- name: perfLabResultsQueue + type: string + default: resultsqueue +- name: crankVersionEnvironmentVariable + type: string + default: CRANK_VERSION +- name: perfLabTenantIdEnvironmentVariable + type: string + default: PERFLAB_UPLOAD_TENANT_ID +- name: perfLabClientIdEnvironmentVariable + type: string + default: PERFLAB_UPLOAD_CLIENT_ID +- name: perfLabCertificateBase64EnvironmentVariable + type: string + default: PERFLAB_UPLOAD_CERTIFICATE_BASE64 +- name: perfLabCertificatePasswordEnvironmentVariable + type: string + default: PERFLAB_UPLOAD_CERTIFICATE_PASSWORD # Scenarios - name: scenarios @@ -23,49 +86,97 @@ parameters: # Platform - displayName: Fortunes Platform - arguments: --scenario fortunes $(platformJobs) --property scenario=FortunesPlatform --property protocol=http + testName: FortunesPlatform + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=FortunesPlatform --property protocol=http - displayName: Fortunes Platform EF - arguments: --scenario fortunes_ef $(platformJobs) --property scenario=FortunesPlatformEF --property protocol=http + testName: FortunesPlatformEF + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes_ef --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=FortunesPlatformEF --property protocol=http - displayName: Fortunes Platform Dapper - arguments: --scenario fortunes_dapper $(platformJobs) --property scenario=FortunesPlatformDapper --property protocol=http + testName: FortunesPlatformDapper + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes_dapper --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=FortunesPlatformDapper --property protocol=http - displayName: Single Query Platform - arguments: --scenario single_query $(platformJobs) --property scenario=SingleQueryPlatform --property protocol=http + testName: SingleQueryPlatform + family: aspnet-single-query + categories: aspnet,database,single-query + arguments: --scenario single_query --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=SingleQueryPlatform --property protocol=http - displayName: Multiple Queries Platform - arguments: --scenario multiple_queries $(platformJobs) --property scenario=MultipleQueriesPlatform --property protocol=http + testName: MultipleQueriesPlatform + family: aspnet-multiple-queries + categories: aspnet,database,multiple-queries + arguments: --scenario multiple_queries --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=MultipleQueriesPlatform --property protocol=http - displayName: Updates Platform - arguments: --scenario updates $(platformJobs) --property scenario=UpdatesPlatform --property protocol=http --variable duration=30 + testName: UpdatesPlatform + family: aspnet-updates + categories: aspnet,database,updates + arguments: --scenario updates --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=UpdatesPlatform --property protocol=http --variable duration=30 - displayName: Caching Platform - arguments: --scenario caching $(platformJobs) --property scenario=CachingPlatform --property protocol=http + testName: CachingPlatform + family: aspnet-caching + categories: aspnet,database,caching + arguments: --scenario caching --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --property scenario=CachingPlatform --property protocol=http # Middleware - displayName: Fortunes - arguments: --scenario fortunes $(databaseJobs) --property scenario=Fortunes --property protocol=http + testName: Fortunes + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/database.benchmarks.yml --property scenario=Fortunes --property protocol=http - displayName: Fortunes EF - arguments: --scenario fortunes_ef $(databaseJobs) --property scenario=FortunesEf --property protocol=http + testName: FortunesEf + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes_ef --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/database.benchmarks.yml --property scenario=FortunesEf --property protocol=http - displayName: Fortunes Dapper - arguments: --scenario fortunes_dapper $(databaseJobs) --property scenario=FortunesDapper --property protocol=http + testName: FortunesDapper + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes_dapper --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/database.benchmarks.yml --property scenario=FortunesDapper --property protocol=http # Minimal APIS - displayName: Fortunes Minimal APIs - arguments: --scenario fortunes $(minimalJobs) --property scenario=FortunesMinimalApis --property protocol=http + testName: FortunesMinimalApis + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml --property scenario=FortunesMinimalApis --property protocol=http - displayName: Single Query Minimal APIs - arguments: --scenario single_query $(minimalJobs) --property scenario=SingleQueryMinimalApis --property protocol=http + testName: SingleQueryMinimalApis + family: aspnet-single-query + categories: aspnet,database,single-query + arguments: --scenario single_query --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml --property scenario=SingleQueryMinimalApis --property protocol=http - displayName: Multiple Queries Minimal APIs - arguments: --scenario multiple_queries $(minimalJobs) --property scenario=MultipleQueriesMinimalApis --property protocol=http + testName: MultipleQueriesMinimalApis + family: aspnet-multiple-queries + categories: aspnet,database,multiple-queries + arguments: --scenario multiple_queries --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml --property scenario=MultipleQueriesMinimalApis --property protocol=http - displayName: Updates Minimal APIs - arguments: --scenario updates $(minimalJobs) --property scenario=UpdatesMinimalApis --property protocol=http --variable duration=30 + testName: UpdatesMinimalApis + family: aspnet-updates + categories: aspnet,database,updates + arguments: --scenario updates --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml --property scenario=UpdatesMinimalApis --property protocol=http --variable duration=30 # Blazor - displayName: Fortunes Blazor SSR - arguments: --scenario fortunes $(blazorSsrJobs) --property scenario=FortunesBlazorSsr --property protocol=http + testName: FortunesBlazorSsr + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/BlazorSSR/blazorssr.benchmarks.yml --property scenario=FortunesBlazorSsr --property protocol=http # Razor Pages - displayName: Fortunes Razor Pages - arguments: --scenario fortunes $(razorPagesJobs) --property scenario=FortunesRazorPages --property protocol=http + testName: FortunesRazorPages + family: aspnet-fortunes + categories: aspnet,database,fortunes + arguments: --scenario fortunes --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/RazorPages/razorpages.benchmarks.yml --property scenario=FortunesRazorPages --property protocol=http steps: - ${{ each s in parameters.scenarios }}: @@ -84,5 +195,25 @@ steps: "name": "crank", "condition": "(${{ parameters.condition }})", "retries": 1, - "args": [ "${{ s.arguments }} $(azureProfile) --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml --application.framework net11.0 ${{ parameters.arguments }} --application.options.collectCounters true --application.collectDependencies true --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] + "args": [ "${{ s.arguments }} --config ${{ parameters.benchmarksRawBaseUrl }}/build/azure.profile.yml --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/steadystate.profile.yml --application.framework ${{ parameters.framework }} ${{ parameters.arguments }} --application.options.collectCounters true --application.collectDependencies true --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --property perflab.build.repo=\"${{ parameters.runtimeRepository }}\" --property perflab.build.branch=\"${{ parameters.runtimeBranch }}\" --property perflab.lane.name=\"${{ parameters.perfLabLaneName }}\" --property perflab.lane.queue=\"${{ parameters.perfLabQueue }}\" --property perflab.lane.os.name=\"${{ parameters.perfLabOs }}\" --property perflab.lane.os.architecture=\"${{ parameters.perfLabArchitecture }}\" --property perflab.lane.os.locale=\"${{ parameters.perfLabLocale }}\" --property perflab.lane.additionalData.hardware=\"${{ parameters.perfLabHardware }}\" --property perflab.configuration.Framework=\"${{ parameters.framework }}\" --property perflab.configuration.Runtime=\"${{ parameters.runtime }}\" --property perflab.configuration.Cores=\"${{ parameters.perfLabCores }}\" --property perflab.configuration.Topology=\"${{ parameters.perfLabTopology }}\" --property perflab.configuration.Configuration=\"${{ parameters.configuration }}\" --property perflab.scenario.name=\"${{ s.testName }}\" --property perflab.scenario.family=\"${{ s.family }}\" --property perflab.scenario.categories=\"${{ s.categories }}\" --property perflab.perfRepoHash=\"$(Build.SourceVersion)\" --property perflab.azureDevOps.project=\"$(System.TeamProject)\" --property perflab.azureDevOps.pipeline=\"$(Build.DefinitionName)\" --property perflab.azureDevOps.buildId=\"$(Build.BuildId)\" --property perflab.azureDevOps.buildNumber=\"$(Build.BuildNumber)\" --property perflab.azureDevOps.buildUrl=\"$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)\" --property perflab.sql.session=\"$(session)\" --property perflab.sql.table=\"TrendBenchmarks\" --property perflab.policy.path=\"${{ parameters.adapterPolicyPath }}\" --property perflab.additionalData.producer=\"TrendBenchmarks\" --json crank-results.json --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ], + "postProcess": { + "enabled": ${{ lower(parameters.enablePerfLabPublication) }}, + "name": "Crank PerfLab export", + "args": [ + "upload", + "--crank-json", "crank-results.json", + "--counter-policy", "${{ parameters.adapterPolicyPath }}", + "--identity-source", "crank", + "--identity-property-prefix", "perflab.", + "--crank-version-environment-variable", "${{ parameters.crankVersionEnvironmentVariable }}", + "--storage-account", "${{ parameters.perfLabStorageAccount }}", + "--container", "${{ parameters.perfLabContainer }}", + "--queue", "${{ parameters.perfLabResultsQueue }}", + "--storage-authentication", "certificate", + "--tenant-id-environment-variable", "${{ parameters.perfLabTenantIdEnvironmentVariable }}", + "--client-id-environment-variable", "${{ parameters.perfLabClientIdEnvironmentVariable }}", + "--certificate-base64-environment-variable", "${{ parameters.perfLabCertificateBase64EnvironmentVariable }}", + "--certificate-password-environment-variable", "${{ parameters.perfLabCertificatePasswordEnvironmentVariable }}" + ] + } } diff --git a/build/trend-perflab-lanes.json b/build/trend-perflab-lanes.json new file mode 100644 index 000000000..b3f30e60d --- /dev/null +++ b/build/trend-perflab-lanes.json @@ -0,0 +1,141 @@ +{ + "schemaVersion": 1, + "queueNamingConvention": "....Perf", + "lanes": { + "gold-lin": { + "name": "aspnet-gold-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetGold.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 56, + "hardware": "AspNetGold" + }, + "gold-win": { + "name": "aspnet-gold-windows-x64", + "queue": "Windows.Server2022.Amd64.AspNetGold.Perf", + "os": "Windows Server 2022", + "architecture": "x64", + "locale": "en-US", + "cores": 56, + "hardware": "AspNetGold" + }, + "azure-arm64": { + "name": "aspnet-azure-arm64", + "queue": "Ubuntu.2204.Arm64.AspNetAzure.Perf", + "os": "Ubuntu 22.04", + "architecture": "arm64", + "locale": "en-US", + "cores": 4, + "hardware": "AspNetAzure" + }, + "azure2-amd64": { + "name": "aspnet-azure-amd64", + "queue": "Ubuntu.2204.Amd64.AspNetAzure2.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "AspNetAzure2" + }, + "cobalt-cloud-lin": { + "name": "aspnet-cobalt-cloud-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetCobaltCloud.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 16, + "hardware": "AspNetCobaltCloud" + }, + "cobalt-cloud-lin-azl3": { + "name": "aspnet-cobalt-cloud-azurelinux3-x64", + "queue": "AzureLinux.3.Amd64.AspNetCobaltCloud.Perf", + "os": "Azure Linux 3.0", + "architecture": "x64", + "locale": "en-US", + "cores": 16, + "hardware": "AspNetCobaltCloud" + }, + "cobalt-cloud-lin-azl3-dual": { + "name": "aspnet-cobalt-cloud-azurelinux3-x64", + "queue": "AzureLinux.3.Amd64.AspNetCobaltCloud.Perf", + "os": "Azure Linux 3.0", + "architecture": "x64", + "locale": "en-US", + "cores": 16, + "hardware": "AspNetCobaltCloud" + }, + "idna-amd-lin": { + "name": "aspnet-idna-amd-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetIdnaAmd.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "AspNetIdnaAmd" + }, + "idna-amd-win": { + "name": "aspnet-idna-amd-windows-x64", + "queue": "Windows.Server2022.Amd64.AspNetIdnaAmd.Perf", + "os": "Windows Server 2022", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "AspNetIdnaAmd" + }, + "idna-intel-lin": { + "name": "aspnet-idna-intel-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetIdnaIntel.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "AspNetIdnaIntel" + }, + "idna-intel-win": { + "name": "aspnet-idna-intel-windows-x64", + "queue": "Windows.Server2022.Amd64.AspNetIdnaIntel.Perf", + "os": "Windows Server 2022", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "AspNetIdnaIntel" + }, + "cobalt-hosted-lin": { + "name": "aspnet-cobalt-hosted-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetCobaltHosted.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 56, + "hardware": "AspNetCobaltHosted" + }, + "cobalt-hosted-lin-azl3": { + "name": "aspnet-cobalt-hosted-azurelinux3-x64", + "queue": "AzureLinux.3.Amd64.AspNetCobaltHosted.Perf", + "os": "Azure Linux 3.0", + "architecture": "x64", + "locale": "en-US", + "cores": 56, + "hardware": "AspNetCobaltHosted" + }, + "cobalt-hosted-lin-28": { + "name": "aspnet-cobalt-hosted-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetCobaltHosted.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 28, + "hardware": "AspNetCobaltHosted" + }, + "cobalt-hosted-lin-azl3-28": { + "name": "aspnet-cobalt-hosted-azurelinux3-x64", + "queue": "AzureLinux.3.Amd64.AspNetCobaltHosted.Perf", + "os": "Azure Linux 3.0", + "architecture": "x64", + "locale": "en-US", + "cores": 28, + "hardware": "AspNetCobaltHosted" + } + } +} diff --git a/build/trend-scenarios.yml b/build/trend-scenarios.yml index a48edb096..39b109708 100644 --- a/build/trend-scenarios.yml +++ b/build/trend-scenarios.yml @@ -14,6 +14,69 @@ parameters: - name: condition type: string default: 'true' +- name: benchmarksRawBaseUrl + type: string +- name: enablePerfLabPublication + type: boolean + default: false +- name: perfLabLaneName + type: string +- name: perfLabQueue + type: string +- name: perfLabOs + type: string +- name: perfLabArchitecture + type: string +- name: perfLabLocale + type: string +- name: perfLabCores + type: string +- name: perfLabHardware + type: string +- name: perfLabTopology + type: string +- name: runtimeRepository + type: string + default: https://github.com/dotnet/runtime +- name: runtimeBranch + type: string + default: main +- name: framework + type: string + default: net11.0 +- name: runtime + type: string + default: CoreCLR +- name: configuration + type: string + default: Release +- name: adapterPolicyPath + type: string + default: crank-perflab-counter-policy.json +- name: perfLabStorageAccount + type: string + default: pvscmdupload +- name: perfLabContainer + type: string + default: results +- name: perfLabResultsQueue + type: string + default: resultsqueue +- name: crankVersionEnvironmentVariable + type: string + default: CRANK_VERSION +- name: perfLabTenantIdEnvironmentVariable + type: string + default: PERFLAB_UPLOAD_TENANT_ID +- name: perfLabClientIdEnvironmentVariable + type: string + default: PERFLAB_UPLOAD_CLIENT_ID +- name: perfLabCertificateBase64EnvironmentVariable + type: string + default: PERFLAB_UPLOAD_CERTIFICATE_BASE64 +- name: perfLabCertificatePasswordEnvironmentVariable + type: string + default: PERFLAB_UPLOAD_CERTIFICATE_PASSWORD # Scenarios - name: scenarios @@ -23,129 +86,276 @@ parameters: # Platform - displayName: Plaintext Platform - arguments: --scenario plaintext $(platformJobs) --load.connections 1024 --property scenario=PlaintextPlatform --property protocol=http + testName: PlaintextPlatform + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plaintext --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --load.connections 1024 --property scenario=PlaintextPlatform --property protocol=http - displayName: Json Platform - arguments: --scenario json $(platformJobs) --load.connections 512 --property scenario=JsonPlatform --property protocol=http + testName: JsonPlatform + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --load.connections 512 --property scenario=JsonPlatform --property protocol=http - displayName: Plaintext Platform Inline - arguments: --scenario plaintext $(platformJobs) --load.connections 1024 --application.environmentVariables DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS=1 --property scenario=PlaintextPlatformInline --property protocol=http + testName: PlaintextPlatformInline + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plaintext --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --load.connections 1024 --application.environmentVariables DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS=1 --property scenario=PlaintextPlatformInline --property protocol=http - displayName: Json Platform Inline - arguments: --scenario json $(platformJobs) --load.connections 512 --application.environmentVariables DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS=1 --property scenario=JsonPlatformInline --property protocol=http + testName: JsonPlatformInline + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/platform.benchmarks.yml --load.connections 512 --application.environmentVariables DOTNET_SYSTEM_NET_SOCKETS_INLINE_COMPLETIONS=1 --property scenario=JsonPlatformInline --property protocol=http # Plaintext - displayName: Plaintext - arguments: --scenario plaintext $(plaintextJobs) --load.connections 1024 --property scenario=Plaintext --property protocol=http + testName: Plaintext + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plaintext --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --load.connections 1024 --property scenario=Plaintext --property protocol=http - displayName: Plaintext Query String - arguments: --scenario plaintext $(plaintextJobs) --property scenario=PlaintextQueryString --property protocol=http --variable path=/plaintext?query1=lorem&query2=ipsum&query3=dolor%20est + testName: PlaintextQueryString + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plaintext --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextQueryString --property protocol=http --variable path=/plaintext?query1=lorem&query2=ipsum&query3=dolor%20est - displayName: Plaintext Mvc - arguments: --scenario mvc $(plaintextJobs) --property scenario=PlaintextMvc --property protocol=http + testName: PlaintextMvc + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario mvc --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextMvc --property protocol=http - displayName: Plaintext Minimal APIs - arguments: --scenario plaintext $(minimalJobs) --load.connections 1024 --property scenario=PlaintextMinimalApis --property protocol=http + testName: PlaintextMinimalApis + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plaintext --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml --load.connections 1024 --property scenario=PlaintextMinimalApis --property protocol=http - displayName: Plaintext Endpoint - arguments: --scenario endpoint $(plaintextJobs) --property scenario=PlaintextEndpoint --property protocol=http + testName: PlaintextEndpoint + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario endpoint --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextEndpoint --property protocol=http - displayName: Plaintext (No Filters) - arguments: --scenario plainTextSansFilter $(plaintextJobs) --property scenario=PlaintextWithParametersNoFilter --property protocol=http + testName: PlaintextWithParametersNoFilter + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plainTextSansFilter --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextWithParametersNoFilter --property protocol=http - displayName: Plaintext (Empty Filters) - arguments: --scenario plainTextEmptyFilter $(plaintextJobs) --property scenario=PlaintextWithParametersEmptyFilter --property protocol=http + testName: PlaintextWithParametersEmptyFilter + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plainTextEmptyFilter --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextWithParametersEmptyFilter --property protocol=http - displayName: Plaintext (Empty Filters, No Parameters) - arguments: --scenario plainTextNoParamsEmptyFilter $(plaintextJobs) --property scenario=PlaintextNoParametersEmptyFilter --property protocol=http + testName: PlaintextNoParametersEmptyFilter + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plainTextNoParamsEmptyFilter --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextNoParametersEmptyFilter --property protocol=http - displayName: Plaintext (With Filters) - arguments: --scenario plainTextWithFilter $(plaintextJobs) --property scenario=PlaintextWithParameterstWithFilter --property protocol=http + testName: PlaintextWithParameterstWithFilter + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario plainTextWithFilter --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=PlaintextWithParameterstWithFilter --property protocol=http - displayName: Plaintext Connection Close - arguments: --scenario connectionclose $(plaintextJobs) --property scenario=ConnectionClose --property protocol=http --variable connections=32 --property connections=32 + testName: ConnectionClose + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario connectionclose --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=ConnectionClose --property protocol=http --variable connections=32 --property connections=32 - displayName: Plaintext Connection Close Https - arguments: --scenario connectionclosehttps $(plaintextJobs) --property scenario=ConnectionCloseHttps --property protocol=https --variable connections=32 --property connections=32 + testName: ConnectionCloseHttps + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario connectionclosehttps --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=ConnectionCloseHttps --property protocol=https --variable connections=32 --property connections=32 - displayName: Plaintext Connection Close HttpSys - arguments: --scenario connectionclose $(plaintextJobs) --property scenario=ConnectionCloseHttpSys --property protocol=http --variable connections=32 --property connections=32 --variable server=HttpSys --application.options.requiredOperatingSystem windows + testName: ConnectionCloseHttpSys + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario connectionclose --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=ConnectionCloseHttpSys --property protocol=http --variable connections=32 --property connections=32 --variable server=HttpSys --application.options.requiredOperatingSystem windows - displayName: Plaintext Connection Close Https HttpSys - arguments: --scenario connectionclosehttps $(plaintextJobs) --property scenario=ConnectionCloseHttpsHttpSys --property protocol=https --variable connections=32 --property connections=32 --variable server=HttpSys --application.options.requiredOperatingSystem windows + testName: ConnectionCloseHttpsHttpSys + family: aspnet-plaintext + categories: aspnet,plaintext + arguments: --scenario connectionclosehttps --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/plaintext.benchmarks.yml --property scenario=ConnectionCloseHttpsHttpSys --property protocol=https --variable connections=32 --property connections=32 --variable server=HttpSys --application.options.requiredOperatingSystem windows # Json - displayName: Json - arguments: --scenario json $(jsonJobs) --load.connections 512 --property scenario=Json --property protocol=http + testName: Json + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --load.connections 512 --property scenario=Json --property protocol=http - displayName: Json HttpSys - arguments: --scenario json $(jsonJobs) --property protocol=http --property scenario=JsonHttpSys --variable server=HttpSys --application.options.requiredOperatingSystem windows + testName: JsonHttpSys + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property protocol=http --property scenario=JsonHttpSys --variable server=HttpSys --application.options.requiredOperatingSystem windows - displayName: Json IIS InProc - arguments: --scenario json $(jsonJobs) --property protocol=http --property scenario=JsonIISInProc --variable server=IISInProcess --application.WebHost IISInProcess --application.options.requiredOperatingSystem windows + testName: JsonIISInProc + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property protocol=http --property scenario=JsonIISInProc --variable server=IISInProcess --application.WebHost IISInProcess --application.options.requiredOperatingSystem windows - displayName: Json IIS OutOfProc - arguments: --scenario json $(jsonJobs) --property protocol=http --property scenario=JsonIISOutOfProc --variable server=IISOutOfProcess --application.WebHost IISOutOfProcess --application.options.requiredOperatingSystem windows + testName: JsonIISOutOfProc + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property protocol=http --property scenario=JsonIISOutOfProc --variable server=IISOutOfProcess --application.WebHost IISOutOfProcess --application.options.requiredOperatingSystem windows - displayName: Json Https - arguments: --scenario https $(jsonJobs) --property scenario=JsonHttps --property protocol=https + testName: JsonHttps + family: aspnet-json + categories: aspnet,json + arguments: --scenario https --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=JsonHttps --property protocol=https - displayName: Json Https HttpSys - arguments: --scenario https $(jsonJobs) --property scenario=JsonHttpsHttpSys --property protocol=https --variable server=HttpSys --application.options.requiredOperatingSystem windows + testName: JsonHttpsHttpSys + family: aspnet-json + categories: aspnet,json + arguments: --scenario https --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=JsonHttpsHttpSys --property protocol=https --variable server=HttpSys --application.options.requiredOperatingSystem windows - displayName: Json Https IIS InProc - arguments: --scenario https $(jsonJobs) --property protocol=https --property scenario=JsonHttpsIISInProc --variable server=IISInProcess --application.WebHost IISInProcess --application.options.requiredOperatingSystem windows + testName: JsonHttpsIISInProc + family: aspnet-json + categories: aspnet,json + arguments: --scenario https --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property protocol=https --property scenario=JsonHttpsIISInProc --variable server=IISInProcess --application.WebHost IISInProcess --application.options.requiredOperatingSystem windows - displayName: Json Https IIS OutOfProc - arguments: --scenario https $(jsonJobs) --property protocol=https --property scenario=JsonHttpsIISOutOfProc --variable server=IISOutOfProcess --application.WebHost IISOutOfProcess --application.options.requiredOperatingSystem windows + testName: JsonHttpsIISOutOfProc + family: aspnet-json + categories: aspnet,json + arguments: --scenario https --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property protocol=https --property scenario=JsonHttpsIISOutOfProc --variable server=IISOutOfProcess --application.WebHost IISOutOfProcess --application.options.requiredOperatingSystem windows - displayName: Json Crossgen2 - arguments: --scenario json $(jsonJobs) --property scenario=JsonCrossgen2 --property protocol=http --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishReadyToRunUseCrossgen2=true \" --application.options.requiredOperatingSystem windows + testName: JsonCrossgen2 + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=JsonCrossgen2 --property protocol=http --application.buildArguments \"/p:PublishReadyToRun=true /p:PublishReadyToRunUseCrossgen2=true \" --application.options.requiredOperatingSystem windows - displayName: Json Mvc - arguments: --scenario mvc $(jsonJobs) --property scenario=JsonMvc --property protocol=http + testName: JsonMvc + family: aspnet-json + categories: aspnet,json + arguments: --scenario mvc --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=JsonMvc --property protocol=http - displayName: Json Minimal APIs - arguments: --scenario json $(minimalJobs) --load.connections 512 --property scenario=JsonMinimalApis --property protocol=http + testName: JsonMinimalApis + family: aspnet-json + categories: aspnet,json + arguments: --scenario json --config ${{ parameters.benchmarksRawBaseUrl }}/src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml --load.connections 512 --property scenario=JsonMinimalApis --property protocol=http - displayName: Json 5M - arguments: --scenario json_middleware --variable singleStringJsonSize=5000000 --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --property scenario=Json5M + testName: Json5M + family: aspnet-json + categories: aspnet,json + arguments: --scenario json_middleware --variable singleStringJsonSize=5000000 --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=Json5M - displayName: Json 400k - arguments: --scenario json_middleware --variable singleStringJsonSize=400000 --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --property scenario=Json400k + testName: Json400k + family: aspnet-json + categories: aspnet,json + arguments: --scenario json_middleware --variable singleStringJsonSize=400000 --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=Json400k - displayName: Json 10k - arguments: --scenario json_middleware --variable singleStringJsonSize=10000 --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --property scenario=Json10k + testName: Json10k + family: aspnet-json + categories: aspnet,json + arguments: --scenario json_middleware --variable singleStringJsonSize=10000 --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/json.benchmarks.yml --property scenario=Json10k # Antiforgery - displayName: Antiforgery No Operation - arguments: --scenario antiforgery-noop $(antiforgeryJobs) --property scenario=AntiforgeryNoOp + testName: AntiforgeryNoOp + family: aspnet-antiforgery + categories: aspnet,antiforgery + arguments: --scenario antiforgery-noop --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/antiforgery.benchmarks.yml --property scenario=AntiforgeryNoOp - displayName: Antiforgery Token Validation - arguments: --scenario antiforgery-validation $(antiforgeryJobs) --property scenario=AntiforgeryTokenValidation + testName: AntiforgeryTokenValidation + family: aspnet-antiforgery + categories: aspnet,antiforgery + arguments: --scenario antiforgery-validation --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/antiforgery.benchmarks.yml --property scenario=AntiforgeryTokenValidation - displayName: Antiforgery Token Generation - arguments: --scenario antiforgery-generation $(antiforgeryJobs) --property scenario=AntiforgeryTokenGeneration + testName: AntiforgeryTokenGeneration + family: aspnet-antiforgery + categories: aspnet,antiforgery + arguments: --scenario antiforgery-generation --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/antiforgery.benchmarks.yml --property scenario=AntiforgeryTokenGeneration - displayName: CSRF Accepted - arguments: --scenario csrf-accepted $(antiforgeryJobs) --property scenario=AntiforgeryCsrfAccepted + testName: AntiforgeryCsrfAccepted + family: aspnet-antiforgery + categories: aspnet,antiforgery + arguments: --scenario csrf-accepted --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/antiforgery.benchmarks.yml --property scenario=AntiforgeryCsrfAccepted - displayName: CSRF Rejected - arguments: --scenario csrf-rejected $(antiforgeryJobs) --property scenario=AntiforgeryCsrfRejected + testName: AntiforgeryCsrfRejected + family: aspnet-antiforgery + categories: aspnet,antiforgery + arguments: --scenario csrf-rejected --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/antiforgery.benchmarks.yml --property scenario=AntiforgeryCsrfRejected # TLS - displayName: "HttpSys Windows: TLS Handshakes" - arguments: --scenario tls-handshakes-httpsys $(tlsJobs) --property scenario=HttpSysTLSHandshakes --application.options.requiredOperatingSystem windows + testName: HttpSysTLSHandshakes + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario tls-handshakes-httpsys --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=HttpSysTLSHandshakes --application.options.requiredOperatingSystem windows - displayName: "HttpSys Windows: mTLS Handshakes" - arguments: --scenario mTls-handshakes-httpsys $(tlsJobs) --property scenario=HttpSysMutualTLSHandshakes --application.options.requiredOperatingSystem windows + testName: HttpSysMutualTLSHandshakes + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario mTls-handshakes-httpsys --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=HttpSysMutualTLSHandshakes --application.options.requiredOperatingSystem windows - displayName: "HttpSys Windows: TLS Renegotiation" - arguments: --scenario tls-renegotiation-httpsys $(tlsJobs) --property scenario=HttpSysTLSRenegotiation --application.options.requiredOperatingSystem windows + testName: HttpSysTLSRenegotiation + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario tls-renegotiation-httpsys --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=HttpSysTLSRenegotiation --application.options.requiredOperatingSystem windows - displayName: "Kestrel Linux: TLS Handshakes" - arguments: --scenario tls-handshakes-kestrel $(tlsJobs) --property scenario=KestrelTLSHandshakes --application.options.requiredOperatingSystem linux + testName: KestrelTLSHandshakes + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario tls-handshakes-kestrel --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=KestrelTLSHandshakes --application.options.requiredOperatingSystem linux - displayName: "Nginx: TLS Handshakes" - arguments: --scenario tls-handshakes-docker-nginx $(tlsJobs) --property scenario=NginxTLSHandshakes --application.options.requiredOperatingSystem linux + testName: NginxTLSHandshakes + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario tls-handshakes-docker-nginx --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=NginxTLSHandshakes --application.options.requiredOperatingSystem linux - displayName: "Kestrel Linux: mTLS Handshakes" - arguments: --scenario mTls-handshakes-kestrel $(tlsJobs) --property scenario=KestrelMutualTLSHandshakes --application.options.requiredOperatingSystem linux + testName: KestrelMutualTLSHandshakes + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario mTls-handshakes-kestrel --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=KestrelMutualTLSHandshakes --application.options.requiredOperatingSystem linux - displayName: "Kestrel Linux: TLS Renegotiation" - arguments: --scenario tls-renegotiation-kestrel $(tlsJobs) --property scenario=KestrelTLSRenegotiation --application.options.requiredOperatingSystem linux + testName: KestrelTLSRenegotiation + family: aspnet-tls + categories: aspnet,tls + arguments: --scenario tls-renegotiation-kestrel --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/tls.benchmarks.yml --property scenario=KestrelTLSRenegotiation --application.options.requiredOperatingSystem linux # Rejection - displayName: "HttpSys Windows: Encoded URL symbols" - arguments: --scenario httpsys-encoded-url $(rejectionJobs) --property scenario=RejectionEncodedUrlHttpSys --application.options.requiredOperatingSystem windows + testName: RejectionEncodedUrlHttpSys + family: aspnet-request-rejection + categories: aspnet,request-rejection + arguments: --scenario httpsys-encoded-url --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/rejection.benchmarks.yml --property scenario=RejectionEncodedUrlHttpSys --application.options.requiredOperatingSystem windows - displayName: "HttpSys Windows: Invalid Header" - arguments: --scenario httpsys-header-symbols $(rejectionJobs) --property scenario=RejectionInvalidHeaderHttpSys --application.options.requiredOperatingSystem windows + testName: RejectionInvalidHeaderHttpSys + family: aspnet-request-rejection + categories: aspnet,request-rejection + arguments: --scenario httpsys-header-symbols --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/rejection.benchmarks.yml --property scenario=RejectionInvalidHeaderHttpSys --application.options.requiredOperatingSystem windows - displayName: "HttpSys Windows: Host Header Mismatch" - arguments: --scenario httpsys-hostheader-mismatch $(rejectionJobs) --property scenario=RejectionHostHeaderMismatchHttpSys --application.options.requiredOperatingSystem windows + testName: RejectionHostHeaderMismatchHttpSys + family: aspnet-request-rejection + categories: aspnet,request-rejection + arguments: --scenario httpsys-hostheader-mismatch --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/rejection.benchmarks.yml --property scenario=RejectionHostHeaderMismatchHttpSys --application.options.requiredOperatingSystem windows - displayName: "Kestrel Linux: Encoded URL symbols" - arguments: --scenario kestrel-encoded-url $(rejectionJobs) --property scenario=RejectionEncodedUrlKestrel --application.options.requiredOperatingSystem linux + testName: RejectionEncodedUrlKestrel + family: aspnet-request-rejection + categories: aspnet,request-rejection + arguments: --scenario kestrel-encoded-url --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/rejection.benchmarks.yml --property scenario=RejectionEncodedUrlKestrel --application.options.requiredOperatingSystem linux - displayName: "Kestrel Linux: Invalid Header" - arguments: --scenario kestrel-header-symbols $(rejectionJobs) --property scenario=RejectionInvalidHeaderKestrel --application.options.requiredOperatingSystem linux + testName: RejectionInvalidHeaderKestrel + family: aspnet-request-rejection + categories: aspnet,request-rejection + arguments: --scenario kestrel-header-symbols --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/rejection.benchmarks.yml --property scenario=RejectionInvalidHeaderKestrel --application.options.requiredOperatingSystem linux - displayName: "Kestrel Linux: Host Header Mismatch" - arguments: --scenario kestrel-hostheader-mismatch $(rejectionJobs) --property scenario=RejectionHostHeaderMismatchKestrel --application.options.requiredOperatingSystem linux + testName: RejectionHostHeaderMismatchKestrel + family: aspnet-request-rejection + categories: aspnet,request-rejection + arguments: --scenario kestrel-hostheader-mismatch --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/rejection.benchmarks.yml --property scenario=RejectionHostHeaderMismatchKestrel --application.options.requiredOperatingSystem linux steps: - ${{ each s in parameters.scenarios }}: @@ -164,5 +374,25 @@ steps: "name": "crank", "condition": "(${{ parameters.condition }})", "retries": 1, - "args": [ "${{ s.arguments }} $(azureProfile) --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml --application.framework net11.0 --application.collectDependencies true ${{ parameters.arguments }} --application.options.collectCounters true --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ] + "args": [ "${{ s.arguments }} --config ${{ parameters.benchmarksRawBaseUrl }}/build/azure.profile.yml --config ${{ parameters.benchmarksRawBaseUrl }}/scenarios/steadystate.profile.yml --application.framework ${{ parameters.framework }} --application.collectDependencies true ${{ parameters.arguments }} --application.options.collectCounters true --no-metadata --no-measurements --load.options.reuseBuild true --session $(session) --description \"${{ s.displayName }} $(System.JobDisplayName)\" --property buildId=\"$(buildId)\" --property buildNumber=\"$(buildNumber)\" --property perflab.build.repo=\"${{ parameters.runtimeRepository }}\" --property perflab.build.branch=\"${{ parameters.runtimeBranch }}\" --property perflab.lane.name=\"${{ parameters.perfLabLaneName }}\" --property perflab.lane.queue=\"${{ parameters.perfLabQueue }}\" --property perflab.lane.os.name=\"${{ parameters.perfLabOs }}\" --property perflab.lane.os.architecture=\"${{ parameters.perfLabArchitecture }}\" --property perflab.lane.os.locale=\"${{ parameters.perfLabLocale }}\" --property perflab.lane.additionalData.hardware=\"${{ parameters.perfLabHardware }}\" --property perflab.configuration.Framework=\"${{ parameters.framework }}\" --property perflab.configuration.Runtime=\"${{ parameters.runtime }}\" --property perflab.configuration.Cores=\"${{ parameters.perfLabCores }}\" --property perflab.configuration.Topology=\"${{ parameters.perfLabTopology }}\" --property perflab.configuration.Configuration=\"${{ parameters.configuration }}\" --property perflab.scenario.name=\"${{ s.testName }}\" --property perflab.scenario.family=\"${{ s.family }}\" --property perflab.scenario.categories=\"${{ s.categories }}\" --property perflab.perfRepoHash=\"$(Build.SourceVersion)\" --property perflab.azureDevOps.project=\"$(System.TeamProject)\" --property perflab.azureDevOps.pipeline=\"$(Build.DefinitionName)\" --property perflab.azureDevOps.buildId=\"$(Build.BuildId)\" --property perflab.azureDevOps.buildNumber=\"$(Build.BuildNumber)\" --property perflab.azureDevOps.buildUrl=\"$(System.TeamFoundationCollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)\" --property perflab.sql.session=\"$(session)\" --property perflab.sql.table=\"TrendBenchmarks\" --property perflab.policy.path=\"${{ parameters.adapterPolicyPath }}\" --property perflab.additionalData.producer=\"TrendBenchmarks\" --json crank-results.json --command-line-property --table TrendBenchmarks --sql SQL_CONNECTION_STRING --cert-tenant-id SQL_SERVER_TENANTID --cert-client-id SQL_SERVER_CLIENTID --cert-path SQL_SERVER_CERT_PATH --cert-sni --chart" ], + "postProcess": { + "enabled": ${{ lower(parameters.enablePerfLabPublication) }}, + "name": "Crank PerfLab export", + "args": [ + "upload", + "--crank-json", "crank-results.json", + "--counter-policy", "${{ parameters.adapterPolicyPath }}", + "--identity-source", "crank", + "--identity-property-prefix", "perflab.", + "--crank-version-environment-variable", "${{ parameters.crankVersionEnvironmentVariable }}", + "--storage-account", "${{ parameters.perfLabStorageAccount }}", + "--container", "${{ parameters.perfLabContainer }}", + "--queue", "${{ parameters.perfLabResultsQueue }}", + "--storage-authentication", "certificate", + "--tenant-id-environment-variable", "${{ parameters.perfLabTenantIdEnvironmentVariable }}", + "--client-id-environment-variable", "${{ parameters.perfLabClientIdEnvironmentVariable }}", + "--certificate-base64-environment-variable", "${{ parameters.perfLabCertificateBase64EnvironmentVariable }}", + "--certificate-password-environment-variable", "${{ parameters.perfLabCertificatePasswordEnvironmentVariable }}" + ] + } } diff --git a/scenarios/antiforgery.benchmarks.yml b/scenarios/antiforgery.benchmarks.yml index eb6bdc864..0244e1b90 100644 --- a/scenarios/antiforgery.benchmarks.yml +++ b/scenarios/antiforgery.benchmarks.yml @@ -12,7 +12,7 @@ jobs: antiforgerybenchmarks: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/Antiforgery/Antiforgery.csproj readyStateText: Application started. arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}} --scenario {{scenario}} --debug {{debug}}" @@ -27,7 +27,7 @@ scenarios: load: job: wrk variables: - script: https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Antiforgery/scripts/wrk-auth-token.lua + script: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/Antiforgery/scripts/wrk-auth-token.lua antiforgery-generation: application: @@ -58,7 +58,7 @@ scenarios: job: wrk variables: path: /csrf - script: https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Antiforgery/scripts/wrk-csrf.lua + script: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/Antiforgery/scripts/wrk-csrf.lua customHeaders: - "Sec-Fetch-Site: same-origin" @@ -71,6 +71,6 @@ scenarios: job: wrk variables: path: /csrf - script: https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Antiforgery/scripts/wrk-csrf.lua + script: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/Antiforgery/scripts/wrk-csrf.lua customHeaders: - "Sec-Fetch-Site: cross-site" \ No newline at end of file diff --git a/scenarios/database.benchmarks.yml b/scenarios/database.benchmarks.yml index 974f9670e..fad564644 100644 --- a/scenarios/database.benchmarks.yml +++ b/scenarios/database.benchmarks.yml @@ -14,7 +14,7 @@ jobs: aspnetbenchmarks: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/Benchmarks/Benchmarks.csproj readyStateText: Application started. variables: @@ -38,7 +38,7 @@ jobs: mssql: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" dockerFile: docker/mssql-techempower/Dockerfile dockerImageName: mssql_te dockerContextDirectory: docker/mssql-techempower diff --git a/scenarios/json.benchmarks.yml b/scenarios/json.benchmarks.yml index 161ad9a5f..bced271d8 100644 --- a/scenarios/json.benchmarks.yml +++ b/scenarios/json.benchmarks.yml @@ -15,7 +15,7 @@ jobs: aspnetbenchmarks: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/Benchmarks/Benchmarks.csproj readyStateText: Application started. variables: @@ -29,7 +29,7 @@ jobs: mapaction: source: repository: https://github.com/aspnet/Benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/MapAction/MapAction.csproj readyStateText: Application started. arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}}" @@ -37,7 +37,7 @@ jobs: jsonbenchmarks: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/Json/Json.csproj readyStateText: Application started. variables: diff --git a/scenarios/packages.yml b/scenarios/packages.yml index ae1cac504..9c6fefab3 100644 --- a/scenarios/packages.yml +++ b/scenarios/packages.yml @@ -1,3 +1,6 @@ +variables: + benchmarksCommit: main + profiles: latest-extensions-packages: diff --git a/scenarios/plaintext.benchmarks.yml b/scenarios/plaintext.benchmarks.yml index 626003ac9..dee8fd5c9 100644 --- a/scenarios/plaintext.benchmarks.yml +++ b/scenarios/plaintext.benchmarks.yml @@ -15,7 +15,7 @@ jobs: aspnetbenchmarks: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/Benchmarks/Benchmarks.csproj readyStateText: Application started. variables: @@ -29,7 +29,7 @@ jobs: mapaction: source: repository: https://github.com/aspnet/Benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/MapAction/MapAction.csproj readyStateText: Application started. arguments: "--urls {{serverScheme}}://{{serverAddress}}:{{serverPort}}" diff --git a/scenarios/platform.benchmarks.yml b/scenarios/platform.benchmarks.yml index 0f3c03d5a..7c7a39208 100644 --- a/scenarios/platform.benchmarks.yml +++ b/scenarios/platform.benchmarks.yml @@ -15,7 +15,7 @@ jobs: platformbenchmarks: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/TechEmpower/PlatformBenchmarks/PlatformBenchmarks.csproj readyStateText: Application started. framework: net7.0 diff --git a/scenarios/rejection.benchmarks.yml b/scenarios/rejection.benchmarks.yml index cd0f7eeb9..26f43e260 100644 --- a/scenarios/rejection.benchmarks.yml +++ b/scenarios/rejection.benchmarks.yml @@ -16,7 +16,7 @@ jobs: httpSysServer: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/TLS/HttpSys/HttpSys.csproj readyStateText: Application started. variables: @@ -36,7 +36,7 @@ jobs: kestrelServer: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/TLS/Kestrel/Kestrel.csproj readyStateText: Application started. variables: diff --git a/scenarios/tls.benchmarks.yml b/scenarios/tls.benchmarks.yml index 462a5e10a..5373a0e76 100644 --- a/scenarios/tls.benchmarks.yml +++ b/scenarios/tls.benchmarks.yml @@ -11,7 +11,7 @@ jobs: httpSysServer: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/TLS/HttpSys/HttpSys.csproj readyStateText: Application started. variables: @@ -28,7 +28,7 @@ jobs: kestrelServer: source: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" project: src/BenchmarksApps/TLS/Kestrel/Kestrel.csproj readyStateText: Application started. variables: @@ -47,7 +47,7 @@ jobs: sources: dockerKestrel: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" dockerFile: dockerKestrel/src/BenchmarksApps/TLS/Kestrel/Dockerfile dockerImageName: dockerKestrel dockerContextDirectory: dockerKestrel/src/BenchmarksApps/TLS/Kestrel @@ -64,7 +64,7 @@ jobs: sources: dockerNginx: repository: https://github.com/aspnet/benchmarks.git - branchOrCommit: main + branchOrCommit: "{{benchmarksCommit}}" dockerFile: dockerNginx/src/BenchmarksApps/TLS/Nginx/Dockerfile dockerImageName: dockerNginx dockerContextDirectory: dockerNginx/src/BenchmarksApps/TLS @@ -106,7 +106,7 @@ scenarios: connections: 32 serverScheme: https # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy cert for testing")] - certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/refs/heads/main/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx + certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy password for testing")] certPwd: testPassword sslProtocol: tls12 @@ -125,7 +125,7 @@ scenarios: connections: 32 serverScheme: https # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy cert for testing")] - certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/refs/heads/main/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx + certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy password for testing")] certPwd: testPassword sslProtocol: tls12 @@ -178,7 +178,7 @@ scenarios: connections: 32 serverScheme: https # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy cert for testing")] - certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/refs/heads/main/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx + certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy password for testing")] certPwd: testPassword sslProtocol: tls12 @@ -198,7 +198,7 @@ scenarios: connections: 32 serverScheme: https # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy cert for testing")] - certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/refs/heads/main/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx + certPath: https://raw.githubusercontent.com/aspnet/Benchmarks/{{benchmarksCommit}}/src/BenchmarksApps/TLS/Certificates/2048/testCert-2048.pfx # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="This is a dummy password for testing")] certPwd: testPassword sslProtocol: tls12 diff --git a/scripts/pod-scheduler/README.md b/scripts/pod-scheduler/README.md index 4e46f0b62..cd0b540ba 100644 --- a/scripts/pod-scheduler/README.md +++ b/scripts/pod-scheduler/README.md @@ -51,7 +51,8 @@ cannot silently drop scenarios from the pipeline. Pass `--lenient` to fall back to the previous warn-and-skip behavior. Output is **deterministic**: identical input JSON always produces identical -YAML, so regenerations diff cleanly. To verify, run the snapshot tests: +YAML, so regenerations diff cleanly. To verify, run the snapshot and Trend +configuration contract tests: ```bash cd scripts/pod-scheduler @@ -73,8 +74,10 @@ python -m unittest discover tests "pipeline": { "pool": "server", "service_bus_connection": "ASPNET Benchmarks Service Bus", - "service_bus_namespace": "aspnetbenchmarks" - } + "service_bus_namespace": "aspnetbenchmarks", + "trend_benchmarks_raw_base_url": "https://raw.githubusercontent.com/aspnet/Benchmarks/$(Build.SourceVersion)" + }, + "trend_lane_registry": "trend-perflab-lanes.json" }, "pods": [ { @@ -96,7 +99,28 @@ python -m unittest discover tests } ``` -The `pipeline` block is optional; defaults match the legacy hardcoded values. +The `pipeline` block is optional for non-Trend scenarios; its +`trend_benchmarks_raw_base_url` field is required for Trend and must use +`$(Build.SourceVersion)`, never `/main`. +`trend_lane_registry` is required when a config schedules either Trend +template. It maps pod names to stable PerfLab lane names, canonical PerfLab +queues, OS/architecture/locale, core count, and hardware identity. These +values are intentionally independent of the Service Bus worker queues. + +Every Trend scenario source entry must explicitly set +`"enable_perf_lab_publication": false`. The generator passes this value as the +boolean `enablePerfLabPublication` template parameter, and the worker receives +`postProcess.enabled: false`; raw JSON and existing SQL writes continue, but no +PerfLab storage or queue publication occurs. Enabling publication requires +explicit approval and a separate single-pod canary scenario. The loader rejects +broader or non-Trend opt-ins. + +Trend references the canonical repository configs through a raw GitHub base +URL pinned to `$(Build.SourceVersion)`. Benchmarks-owned source revisions and +raw assets use `{{benchmarksCommit}}`; the shared default is `main`, and Trend +overrides it with `--variable benchmarksCommit=$(Build.SourceVersion)`. +Benchmarks-owned `imports` retain their existing URLs because Crank resolves +imports before applying command-line variables. The `schedule` field's **hour** must be a `H` or `H/N` cron expression (e.g. `3` or `3/12`). Lists, ranges, and `*` are rejected at load time so the @@ -124,6 +148,7 @@ hour-offset used for split YAMLs cannot silently no-op. | `pods` | List of pod names this scenario targets (no duplicates) | | `estimated_runtime` | Runtime estimate in minutes; defaults per type if omitted | | `timeout` | Optional explicit AzDO `timeoutInMinutes` override. When unset, the generator picks `max(120, min(240, ceil(2 * estimated_runtime)))` | +| `enable_perf_lab_publication` | Required for Trend; keep `false` unless an explicitly approved single-pod canary is being configured | ### Scenario Types diff --git a/scripts/pod-scheduler/config_loader.py b/scripts/pod-scheduler/config_loader.py index f5b1d6377..d9fd9e6f9 100644 --- a/scripts/pod-scheduler/config_loader.py +++ b/scripts/pod-scheduler/config_loader.py @@ -3,10 +3,12 @@ """ import json +import os import re from typing import Any, Dict from models import ( + PerfLabLane, PipelineSettings, Pod, Scenario, @@ -20,6 +22,10 @@ class ConfigError(ValueError): _CRON_HOUR_RE = re.compile(r"^\d+(/\d+)?$") +_TREND_TEMPLATES = { + "trend-scenarios.yml", + "trend-database-scenarios.yml", +} def _require(node: Dict[str, Any], key: str, context: str) -> Any: @@ -43,12 +49,85 @@ def _validate_cron(schedule: str) -> None: ) +def _load_trend_lanes( + config_path: str, + metadata: Dict[str, Any], +) -> Dict[str, PerfLabLane]: + registry_name = metadata.get("trend_lane_registry") + if not registry_name: + return {} + + registry_path = os.path.join(os.path.dirname(config_path), registry_name) + try: + with open(registry_path, "r", encoding="utf-8") as f: + registry = json.load(f) + except (OSError, json.JSONDecodeError) as error: + raise ConfigError( + f"Could not load Trend lane registry {registry_path!r}: {error}" + ) from error + + if registry.get("schemaVersion") != 1: + raise ConfigError( + f"Trend lane registry {registry_path!r} must use schemaVersion 1" + ) + + lanes: Dict[str, PerfLabLane] = {} + raw_lanes = _require(registry, "lanes", "Trend lane registry") + for pod_name, lane_data in raw_lanes.items(): + if not isinstance(lane_data, dict): + raise ConfigError( + f"Trend lane registry entry {pod_name!r} must be an object" + ) + try: + cores = int( + _require(lane_data, "cores", f"Trend lane '{pod_name}'") + ) + except (TypeError, ValueError) as error: + raise ConfigError( + f"Trend lane '{pod_name}' cores must be an integer" + ) from error + if cores <= 0: + raise ConfigError( + f"Trend lane '{pod_name}' has non-positive cores {cores}" + ) + lanes[pod_name] = PerfLabLane( + name=_require(lane_data, "name", f"Trend lane '{pod_name}'"), + queue=_require(lane_data, "queue", f"Trend lane '{pod_name}'"), + os=_require(lane_data, "os", f"Trend lane '{pod_name}'"), + architecture=_require( + lane_data, "architecture", f"Trend lane '{pod_name}'" + ), + locale=_require(lane_data, "locale", f"Trend lane '{pod_name}'"), + cores=cores, + hardware=_require( + lane_data, "hardware", f"Trend lane '{pod_name}'" + ), + ) + if any( + not str(value).strip() + for value in ( + lanes[pod_name].name, + lanes[pod_name].queue, + lanes[pod_name].os, + lanes[pod_name].architecture, + lanes[pod_name].locale, + lanes[pod_name].hardware, + ) + ): + raise ConfigError( + f"Trend lane '{pod_name}' contains an empty identity value" + ) + + return lanes + + def load_config(path: str) -> ScheduleConfig: """Load and validate a pod-scheduler JSON configuration file.""" with open(path, "r", encoding="utf-8") as f: data = json.load(f) metadata = _require(data, "metadata", "config root") + trend_lanes = _load_trend_lanes(path, metadata) schedule = _require(metadata, "schedule", "metadata") _validate_cron(schedule) @@ -57,6 +136,14 @@ def load_config(path: str) -> ScheduleConfig: raise ConfigError("metadata.queues must be a non-empty list") if len(queues) != len(set(queues)): raise ConfigError(f"metadata.queues contains duplicates: {queues}") + routing_lane_overlap = sorted({ + lane.queue for lane in trend_lanes.values() if lane.queue in queues + }) + if routing_lane_overlap: + raise ConfigError( + "Service Bus routing queues cannot be used as PerfLab queues: " + f"{routing_lane_overlap}" + ) yaml_gen = metadata.get("yaml_generation", {}) @@ -71,6 +158,10 @@ def load_config(path: str) -> ScheduleConfig: "service_bus_namespace", PipelineSettings.service_bus_namespace, ), + trend_benchmarks_raw_base_url=pipeline_meta.get( + "trend_benchmarks_raw_base_url", + PipelineSettings.trend_benchmarks_raw_base_url, + ), ) pods: Dict[str, Pod] = {} @@ -89,12 +180,14 @@ def load_config(path: str) -> ScheduleConfig: sut_profile=_require(profiles, "sut", f"pod '{pod_name}'.profiles"), load_profile=profiles.get("load"), db_profile=profiles.get("db"), + perf_lab_lane=trend_lanes.get(pod_name), ) scenarios = [] raw_scenarios = _require(data, "scenarios", "config root") for sc_data in raw_scenarios: name = _require(sc_data, "name", "scenario entry") + template = _require(sc_data, "template", f"scenario '{name}'") scenario_pods = _require(sc_data, "pods", f"scenario '{name}'") if not scenario_pods: raise ConfigError(f"scenario '{name}' has empty pods list") @@ -105,6 +198,30 @@ def load_config(path: str) -> ScheduleConfig: raise ConfigError( f"scenario '{name}' lists duplicate pods: {dupes}" ) + if ( + template in _TREND_TEMPLATES + and "enable_perf_lab_publication" not in sc_data + ): + raise ConfigError( + f"Trend scenario '{name}' must explicitly set " + "enable_perf_lab_publication" + ) + enable_publication = sc_data.get( + "enable_perf_lab_publication", + False, + ) + if not isinstance(enable_publication, bool): + raise ConfigError( + f"scenario '{name}' enable_perf_lab_publication must be " + "a boolean" + ) + if enable_publication and ( + template not in _TREND_TEMPLATES or len(scenario_pods) != 1 + ): + raise ConfigError( + "PerfLab publication may only be enabled for an explicit " + "single-pod Trend canary" + ) runtime_raw = sc_data.get("estimated_runtime") or 0 timeout = sc_data.get("timeout") if timeout is not None: @@ -115,13 +232,39 @@ def load_config(path: str) -> ScheduleConfig: ) scenarios.append(Scenario( name=name, - template=_require(sc_data, "template", f"scenario '{name}'"), + template=template, type=ScenarioType(_require(sc_data, "type", f"scenario '{name}'")), pods=list(scenario_pods), estimated_runtime=float(runtime_raw) if runtime_raw else 0.0, + enable_perf_lab_publication=enable_publication, timeout=timeout, )) + for scenario in scenarios: + if scenario.template not in _TREND_TEMPLATES: + continue + raw_base_url = pipeline.trend_benchmarks_raw_base_url + if not raw_base_url: + raise ConfigError( + "Trend scenarios require " + "metadata.pipeline.trend_benchmarks_raw_base_url" + ) + if ( + "raw.githubusercontent.com/aspnet/Benchmarks/main" + in raw_base_url + or "$(Build.SourceVersion)" not in raw_base_url + ): + raise ConfigError( + "Trend Benchmarks raw base URL must be pinned to " + "$(Build.SourceVersion)" + ) + for pod_name in scenario.pods: + if pod_name in pods and pods[pod_name].perf_lab_lane is None: + raise ConfigError( + f"Trend scenario '{scenario.name}' pod '{pod_name}' has " + "no entry in metadata.trend_lane_registry" + ) + return ScheduleConfig( name=metadata.get("name", ""), schedule=schedule, diff --git a/scripts/pod-scheduler/generator.py b/scripts/pod-scheduler/generator.py index efde9d19a..b04a7b71c 100644 --- a/scripts/pod-scheduler/generator.py +++ b/scripts/pod-scheduler/generator.py @@ -20,6 +20,10 @@ _CRON_HOUR_RE = re.compile(r"^(\d+)(/\d+)?$") +_TREND_TEMPLATES = { + "trend-scenarios.yml", + "trend-database-scenarios.yml", +} class GeneratorError(ValueError): @@ -78,12 +82,29 @@ def schedule_to_template_data( for stage in schedule.stages: jobs = [] for run in stage.runs: + lane = run.pod.perf_lab_lane jobs.append({ "name": run.name, "job_id": run.job_name, "template": run.scenario.template, "profiles": run.profiles, "timeout": _job_timeout(run), + "enable_perf_lab_publication": ( + run.scenario.enable_perf_lab_publication + ), + "perf_lab_lane": None if lane is None else { + "name": lane.name, + "queue": lane.queue, + "os": lane.os, + "architecture": lane.architecture, + "locale": lane.locale, + "cores": lane.cores, + "hardware": lane.hardware, + }, + "perf_lab_topology": { + 2: "SUT+Load", + 3: "SUT+Load+DB", + }.get(int(run.scenario.type)), }) groups.append({"jobs": jobs}) @@ -186,8 +207,56 @@ def _render_yaml( lines.append( f" serviceBusNamespace: {pipeline.service_bus_namespace}" ) + if job["template"] in _TREND_TEMPLATES: + raw_base_url = pipeline.trend_benchmarks_raw_base_url + if not raw_base_url: + raise GeneratorError( + f"Trend job {job['name']!r} has no pinned " + "Benchmarks raw base URL" + ) + lane = job["perf_lab_lane"] + if lane is None: + raise GeneratorError( + f"Trend job {job['name']!r} has no PerfLab lane mapping" + ) + lines.append( + " benchmarksRawBaseUrl: " + f'"{raw_base_url}"' + ) + publication_enabled = str( + job.get("enable_perf_lab_publication", False) + ).lower() + lines.append( + " enablePerfLabPublication: " + f"{publication_enabled}" + ) + lines.append(f" perfLabLaneName: {lane['name']}") + lines.append(f" perfLabQueue: {lane['queue']}") + lines.append(f' perfLabOs: "{lane["os"]}"') + lines.append( + f" perfLabArchitecture: {lane['architecture']}" + ) + lines.append(f" perfLabLocale: {lane['locale']}") + lines.append(f' perfLabCores: "{lane["cores"]}"') + lines.append(f" perfLabHardware: {lane['hardware']}") + lines.append( + f' perfLabTopology: "{job["perf_lab_topology"]}"' + ) + ci_profile = ( + "--config " + f"{pipeline.trend_benchmarks_raw_base_url}/" + "build/ci.profile.yml" + if job["template"] in _TREND_TEMPLATES + else "$(ciProfile)" + ) + pin_argument = ( + "--variable benchmarksCommit=$(Build.SourceVersion) " + if job["template"] in _TREND_TEMPLATES + else "" + ) lines.append( - f' arguments: "$(ciProfile) {profiles_args} "' + f' arguments: "{ci_profile} ' + f'{pin_argument}{profiles_args} "' ) lines.append("") diff --git a/scripts/pod-scheduler/main.py b/scripts/pod-scheduler/main.py index 00cdc0236..58a325c23 100644 --- a/scripts/pod-scheduler/main.py +++ b/scripts/pod-scheduler/main.py @@ -223,7 +223,11 @@ def main(argv: List[str] = None) -> int: ) print("Done!") return 0 - except (ConfigError, SchedulerError, GeneratorError) as exc: + except ( + ConfigError, + SchedulerError, + GeneratorError, + ) as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 diff --git a/scripts/pod-scheduler/models.py b/scripts/pod-scheduler/models.py index 22ec8503f..c915859aa 100644 --- a/scripts/pod-scheduler/models.py +++ b/scripts/pod-scheduler/models.py @@ -58,6 +58,18 @@ def sanitize_job_id(raw: str) -> str: return cleaned +@dataclass +class PerfLabLane: + """Stable PerfLab identity for a Trend SUT machine class.""" + name: str + queue: str + os: str + architecture: str + locale: str + cores: int + hardware: str + + @dataclass class Pod: """A fixed group of machines that run scenarios together.""" @@ -70,6 +82,7 @@ class Pod: sut_profile: str = "" load_profile: Optional[str] = None db_profile: Optional[str] = None + perf_lab_lane: Optional[PerfLabLane] = None def machines_for_type(self, scenario_type: ScenarioType) -> Set[str]: """Return the set of physical machines used for a given scenario type.""" @@ -109,6 +122,7 @@ class Scenario: # Optional explicit timeout (minutes) for the generated AzDO job. When # None, the generator derives one from estimated_runtime. timeout: Optional[int] = None + enable_perf_lab_publication: bool = False @dataclass @@ -179,6 +193,7 @@ class PipelineSettings: pool: str = DEFAULT_PIPELINE_POOL service_bus_connection: str = DEFAULT_PIPELINE_CONNECTION service_bus_namespace: str = DEFAULT_PIPELINE_NAMESPACE + trend_benchmarks_raw_base_url: str = "" @dataclass diff --git a/scripts/pod-scheduler/tests/test_config_loader.py b/scripts/pod-scheduler/tests/test_config_loader.py index 9dd698939..f2bb82cce 100644 --- a/scripts/pod-scheduler/tests/test_config_loader.py +++ b/scripts/pod-scheduler/tests/test_config_loader.py @@ -39,6 +39,38 @@ def _write(tmp, payload): return path +def _enable_trend(payload): + payload["metadata"]["trend_lane_registry"] = "lanes.json" + payload["metadata"]["pipeline"] = { + "trend_benchmarks_raw_base_url": ( + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + "$(Build.SourceVersion)" + ) + } + payload["scenarios"][0]["template"] = "trend-scenarios.yml" + payload["scenarios"][0]["enable_perf_lab_publication"] = False + + +def _write_lane_registry(tmp): + with open( + os.path.join(tmp, "lanes.json"), "w", encoding="utf-8" + ) as f: + json.dump({ + "schemaVersion": 1, + "lanes": { + "p1": { + "name": "stable-lane", + "queue": "Ubuntu.2204.Amd64.Test.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "Test", + } + }, + }, f) + + class TestLoadConfig(unittest.TestCase): def test_happy_path(self): with tempfile.TemporaryDirectory() as tmp: @@ -112,6 +144,170 @@ def test_negative_timeout_rejected(self): with self.assertRaises(ConfigError): load_config(path) + def test_trend_scenario_requires_lane_registry_entry(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + payload["scenarios"][0]["template"] = "trend-scenarios.yml" + payload["scenarios"][0][ + "enable_perf_lab_publication" + ] = False + payload["metadata"]["pipeline"] = { + "trend_benchmarks_raw_base_url": ( + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + "$(Build.SourceVersion)" + ) + } + path = _write(tmp, payload) + with self.assertRaisesRegex(ConfigError, "no entry"): + load_config(path) + + def test_trend_lane_registry_is_loaded(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + with open( + os.path.join(tmp, "lanes.json"), "w", encoding="utf-8" + ) as f: + json.dump({ + "schemaVersion": 1, + "lanes": { + "p1": { + "name": "stable-lane", + "queue": "Ubuntu.2204.Amd64.Test.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "Test", + } + }, + }, f) + path = _write(tmp, payload) + + cfg = load_config(path) + + lane = cfg.pods["p1"].perf_lab_lane + self.assertIsNotNone(lane) + self.assertEqual(lane.queue, "Ubuntu.2204.Amd64.Test.Perf") + + def test_trend_lane_cannot_reuse_service_bus_routing_queue(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + with open( + os.path.join(tmp, "lanes.json"), "w", encoding="utf-8" + ) as f: + json.dump({ + "schemaVersion": 1, + "lanes": { + "p1": { + "name": "stable-lane", + "queue": "a", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "Test", + } + }, + }, f) + path = _write(tmp, payload) + + with self.assertRaisesRegex( + ConfigError, + "routing queues cannot be used", + ): + load_config(path) + + def test_trend_raw_base_url_must_pin_build_source_version(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + payload["metadata"]["pipeline"][ + "trend_benchmarks_raw_base_url" + ] = "https://raw.githubusercontent.com/aspnet/Benchmarks/main" + with open( + os.path.join(tmp, "lanes.json"), "w", encoding="utf-8" + ) as f: + json.dump({ + "schemaVersion": 1, + "lanes": { + "p1": { + "name": "stable-lane", + "queue": "Ubuntu.2204.Amd64.Test.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 4, + "hardware": "Test", + } + }, + }, f) + path = _write(tmp, payload) + + with self.assertRaisesRegex( + ConfigError, + "must be pinned", + ): + load_config(path) + + def test_trend_publication_setting_is_required(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + del payload["scenarios"][0]["enable_perf_lab_publication"] + _write_lane_registry(tmp) + path = _write(tmp, payload) + + with self.assertRaisesRegex( + ConfigError, + "must explicitly set", + ): + load_config(path) + + def test_trend_publication_setting_must_be_boolean(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + payload["scenarios"][0][ + "enable_perf_lab_publication" + ] = "false" + _write_lane_registry(tmp) + path = _write(tmp, payload) + + with self.assertRaisesRegex(ConfigError, "must be a boolean"): + load_config(path) + + def test_publication_opt_in_requires_single_pod_trend_canary(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + payload["scenarios"][0][ + "enable_perf_lab_publication" + ] = True + payload["scenarios"][0]["pods"] = ["p1", "p2"] + _write_lane_registry(tmp) + path = _write(tmp, payload) + + with self.assertRaisesRegex(ConfigError, "single-pod Trend canary"): + load_config(path) + + def test_single_pod_trend_canary_may_opt_in(self): + with tempfile.TemporaryDirectory() as tmp: + payload = json.loads(json.dumps(_BASE)) + _enable_trend(payload) + payload["scenarios"][0][ + "enable_perf_lab_publication" + ] = True + _write_lane_registry(tmp) + path = _write(tmp, payload) + + cfg = load_config(path) + + self.assertTrue( + cfg.scenarios[0].enable_perf_lab_publication + ) + if __name__ == "__main__": unittest.main() diff --git a/scripts/pod-scheduler/tests/test_generator.py b/scripts/pod-scheduler/tests/test_generator.py index 91a59f617..56c631476 100644 --- a/scripts/pod-scheduler/tests/test_generator.py +++ b/scripts/pod-scheduler/tests/test_generator.py @@ -3,9 +3,14 @@ import tests # noqa: F401 # ensures sys.path is set up -from generator import GeneratorError, _job_timeout, _offset_cron +from generator import ( + GeneratorError, + _job_timeout, + _offset_cron, + _render_yaml, +) from main import _format_source_path -from models import Pod, Run, Scenario, ScenarioType +from models import PipelineSettings, Pod, Run, Scenario, ScenarioType class TestOffsetCron(unittest.TestCase): @@ -77,5 +82,68 @@ def test_outside_repo_falls_back_to_basename(self): ) +class TestTrendLaneRendering(unittest.TestCase): + def test_routing_queue_and_perflab_queue_are_distinct(self): + data = { + "schedule": "0 3/12 * * *", + "queues": ["citrine1"], + "groups": [{ + "jobs": [{ + "name": "Trends gold-lin", + "job_id": "Trends_gold_lin", + "template": "trend-scenarios.yml", + "profiles": ["gold-lin-app", "gold-load-load"], + "timeout": 120, + "enable_perf_lab_publication": False, + "perf_lab_lane": { + "name": "aspnet-gold-linux-x64", + "queue": "Ubuntu.2204.Amd64.AspNetGold.Perf", + "os": "Ubuntu 22.04", + "architecture": "x64", + "locale": "en-US", + "cores": 56, + "hardware": "AspNetGold", + }, + "perf_lab_topology": "SUT+Load", + }] + }], + } + + yaml = _render_yaml( + data, + PipelineSettings( + trend_benchmarks_raw_base_url=( + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + "$(Build.SourceVersion)" + ) + ), + ) + + self.assertIn("serviceBusQueueName: citrine1", yaml) + self.assertIn( + 'benchmarksRawBaseUrl: "' + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + '$(Build.SourceVersion)"', + yaml, + ) + self.assertIn("enablePerfLabPublication: false", yaml) + self.assertNotIn("enablePerfLabPublication: true", yaml) + self.assertIn( + 'arguments: "--config ' + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + "$(Build.SourceVersion)/build/ci.profile.yml " + "--variable benchmarksCommit=$(Build.SourceVersion) " + '--profile gold-lin-app --profile gold-load-load "', + yaml, + ) + self.assertNotIn("trend-configs", yaml) + self.assertNotIn("$(ciProfile)", yaml) + self.assertIn( + "perfLabQueue: Ubuntu.2204.Amd64.AspNetGold.Perf", yaml + ) + self.assertNotIn("perfLabQueue: citrine1", yaml) + self.assertIn('perfLabTopology: "SUT+Load"', yaml) + + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/scripts/pod-scheduler/tests/test_trend_canonical_configs.py b/scripts/pod-scheduler/tests/test_trend_canonical_configs.py new file mode 100644 index 000000000..8139bd6de --- /dev/null +++ b/scripts/pod-scheduler/tests/test_trend_canonical_configs.py @@ -0,0 +1,134 @@ +import os +import re +import unittest + +import tests # noqa: F401 # ensures sys.path is set up + + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) +_BUILD = os.path.join(_REPO, "build") +_TREND_CONFIGS = ( + "scenarios/platform.benchmarks.yml", + "scenarios/plaintext.benchmarks.yml", + "scenarios/json.benchmarks.yml", + "scenarios/antiforgery.benchmarks.yml", + "scenarios/tls.benchmarks.yml", + "scenarios/rejection.benchmarks.yml", + "scenarios/database.benchmarks.yml", + "src/BenchmarksApps/TechEmpower/Minimal/minimal.benchmarks.yml", + "src/BenchmarksApps/TechEmpower/BlazorSSR/blazorssr.benchmarks.yml", + "src/BenchmarksApps/TechEmpower/RazorPages/razorpages.benchmarks.yml", +) +_SHARED_CONFIGS = ( + "build/ci.profile.yml", + "scenarios/aspnet.profiles.yml", + "scenarios/aspnet.profiles.standard.yml", +) +_PACKAGES_IMPORT = ( + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + "main/scenarios/packages.yml" +) +_SHARED_PROFILE_IMPORT_RE = re.compile( + r"https://raw\.githubusercontent\.com/aspnet/Benchmarks/" + r"main/scenarios/aspnet\.profiles(?:\.standard)?\.yml", + re.IGNORECASE, +) +_BENCHMARKS_REPOSITORY_RE = re.compile( + r"repository:\s*https://github\.com/aspnet/benchmarks(?:\.git)?", + re.IGNORECASE, +) +_PINNED_BENCHMARKS_SOURCE_RE = re.compile( + r"repository:\s*https://github\.com/aspnet/benchmarks(?:\.git)?\s*" + r"\n\s*branchOrCommit:\s*[\"']?\{\{benchmarksCommit\}\}[\"']?", + re.IGNORECASE, +) +_FLOATING_BENCHMARKS_RAW_RE = re.compile( + r"https://raw\.githubusercontent\.com/aspnet/Benchmarks/" + r"(?:refs/heads/)?main/(?P[^\s]+)", + re.IGNORECASE, +) +_PINNED_BENCHMARKS_RAW_RE = re.compile( + r"https://raw\.githubusercontent\.com/aspnet/Benchmarks/" + r"\{\{benchmarksCommit\}\}/", + re.IGNORECASE, +) + + +def _read_repo(path): + with open( + os.path.join(_REPO, path.replace("/", os.sep)), + encoding="utf-8-sig", + ) as file: + return file.read() + + +class TestTrendCanonicalConfigs(unittest.TestCase): + def test_shared_benchmarks_commit_defaults_to_main(self): + packages = _read_repo("scenarios/packages.yml") + self.assertRegex( + packages, + r"(?m)^variables:\s*\n\s+benchmarksCommit:\s+main$", + ) + for config in _SHARED_CONFIGS: + with self.subTest(config=config): + self.assertIn(_PACKAGES_IMPORT, _read_repo(config)) + + def test_templates_reference_existing_canonical_configs(self): + roots = {"build/ci.profile.yml"} + for template in ( + "trend-scenarios.yml", + "trend-database-scenarios.yml", + ): + roots.update(re.findall( + r"\$\{\{ parameters\.benchmarksRawBaseUrl \}\}/" + r"([^\s\"\\]+\.ya?ml)", + _read_repo(f"build/{template}"), + )) + + for root in roots: + with self.subTest(root=root): + self.assertNotIn("trend-configs", root) + self.assertTrue( + os.path.isfile( + os.path.join(_REPO, root.replace("/", os.sep)) + ), + root, + ) + + self.assertFalse( + os.path.exists(os.path.join(_BUILD, "trend-configs")) + ) + + def test_workload_sources_and_assets_use_benchmarks_commit(self): + source_count = 0 + asset_count = 0 + for config in _TREND_CONFIGS: + with self.subTest(config=config): + text = _read_repo(config) + self.assertRegex(text, _SHARED_PROFILE_IMPORT_RE) + repositories = _BENCHMARKS_REPOSITORY_RE.findall(text) + pinned_sources = _PINNED_BENCHMARKS_SOURCE_RE.findall(text) + self.assertEqual( + len(repositories), + len(pinned_sources), + config, + ) + source_count += len(pinned_sources) + + for match in _FLOATING_BENCHMARKS_RAW_RE.finditer(text): + self.assertRegex( + match.group("path"), + r"^scenarios/aspnet\.profiles" + r"(?:\.standard)?\.yml$", + ) + + pinned_assets = _PINNED_BENCHMARKS_RAW_RE.findall(text) + asset_count += len(pinned_assets) + + self.assertGreater(source_count, 0) + self.assertGreater(asset_count, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/pod-scheduler/tests/test_trend_contract.py b/scripts/pod-scheduler/tests/test_trend_contract.py new file mode 100644 index 000000000..73504678d --- /dev/null +++ b/scripts/pod-scheduler/tests/test_trend_contract.py @@ -0,0 +1,488 @@ +import json +import os +import re +import textwrap +import unittest + + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.abspath(os.path.join(_HERE, "..", "..", "..")) +_BUILD = os.path.join(_REPO, "build") +_TEMPLATES = [ + "trend-scenarios.yml", + "trend-database-scenarios.yml", +] +_PIPELINES = [ + "benchmarks-ci-01.yml", + "benchmarks-ci-02.yml", + "benchmarks-ci-azure.yml", + "benchmarks-ci-cobalt.yml", +] +_SOURCE_CONFIGS = [ + "benchmarks_ci_pods.json", + "benchmarks_ci_azure_pods.json", + "benchmarks_ci_cobalt_pods.json", +] +_ROUTING_QUEUES = { + "citrine1", + "citrine2", + "citrine3", + "mono", + "azure", + "azurearm64", + "cobalthosted", + "cobalthosted_azurelinux3", +} +_PINNED_RAW_BASE_URL = ( + "https://raw.githubusercontent.com/aspnet/Benchmarks/" + "$(Build.SourceVersion)" +) +_CANONICAL_CONFIG_BASE_URL = _PINNED_RAW_BASE_URL + + +def _read(name): + with open(os.path.join(_BUILD, name), encoding="utf-8-sig") as file: + return file.read() + + +_BENCHMARKS_MAIN_MACROS = { + name + for name, value in re.findall( + r"^- name: ([A-Za-z0-9]+)\n value: (.+)$", + _read("job-variables.yml"), + re.MULTILINE, + ) + if "raw.githubusercontent.com/aspnet/Benchmarks/main" in value +} +_BENCHMARKS_MACRO_RE = re.compile( + r"\$\((?:" + "|".join(sorted(_BENCHMARKS_MAIN_MACROS)) + r")\)" +) + + +def _scenarios(template): + text = _read(template) + default = text.index(" default:") + steps = text.index("\nsteps:", default) + block = text[default:steps] + entries = [] + current = None + for line in block.splitlines(): + match = re.match(r"^ - displayName: (.+)$", line) + if match: + if current: + entries.append(current) + current = {"displayName": match.group(1)} + continue + match = re.match( + r"^ (testName|family|categories|arguments): (.*)$", line + ) + if match and current is not None: + current[match.group(1)] = match.group(2) + if current: + entries.append(current) + return entries + + +def _expected_family(test_name): + if test_name.startswith("Plaintext") or test_name.startswith( + "ConnectionClose" + ): + return "aspnet-plaintext" + if test_name.startswith("Json"): + return "aspnet-json" + if test_name.startswith("Antiforgery"): + return "aspnet-antiforgery" + if "TLS" in test_name: + return "aspnet-tls" + if test_name.startswith("Rejection"): + return "aspnet-request-rejection" + if test_name.startswith("Fortunes"): + return "aspnet-fortunes" + if test_name.startswith("SingleQuery"): + return "aspnet-single-query" + if test_name.startswith("MultipleQueries"): + return "aspnet-multiple-queries" + if test_name.startswith("Updates"): + return "aspnet-updates" + if test_name.startswith("Caching"): + return "aspnet-caching" + raise AssertionError(f"No expected family rule for {test_name}") + + +def _payload(template, publication_enabled=False): + text = _read(template) + message_body = text[text.index(" {\n"):] + message_body = message_body.replace( + "${{ lower(parameters.enablePerfLabPublication) }}", + str(publication_enabled).lower(), + ) + return json.loads(textwrap.dedent(message_body)) + + +def _trend_calls(): + pattern = re.compile( + r" - template: " + r"(?P