From 337265678af17abce6d7c2d36f9fd47e408011e6 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:39:59 -0700 Subject: [PATCH 01/15] benchmark template --- .../templates/jobs/benchmark-job.yml | 84 +++++++++++++++++++ tools/azsdk-cli/ci.yml | 4 + 2 files changed, 88 insertions(+) create mode 100644 eng/common/pipelines/templates/jobs/benchmark-job.yml diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml new file mode 100644 index 00000000000..e5371b7c88d --- /dev/null +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -0,0 +1,84 @@ +parameters: + - name: BenchmarkProject + type: string + default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' + - name: ScenarioName + type: string + default: '' + - name: Tags + type: string + default: '' + - name: Model + type: string + default: '' + - name: Parallelism + type: number + default: 5 + +jobs: + - job: Run_Benchmark + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - group: 'AzSDK_Eval_Variable_group' + displayName: 'Run Benchmark' + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - checkout: none + - task: UseDotNet@2 + displayName: "Use .NET SDK 8.0.x" + retryCountOnTaskFailure: 3 + inputs: + packageType: sdk + version: 8.0.x + performMultiLevelLookup: true + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Repositories: + - Name: $(Build.Repository.Name) + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) + Paths: + - 'tools/azsdk-cli/**' + - 'eng/common/**' + + - task: PowerShell@2 + displayName: 'Run benchmarks' + condition: succeeded() + inputs: + targetType: 'inline' + workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' + pwsh: true + script: | + $args = @("run") + + # Determine scenario selection: name, tags, or all + if ("${{ parameters.ScenarioName }}") { + $args += "${{ parameters.ScenarioName }}" + } elseif ("${{ parameters.Tags }}") { + $args += "--tags", "${{ parameters.Tags }}" + } else { + $args += "--all" + } + + # Optional model override + if ("${{ parameters.Model }}") { + $args += "--model", "${{ parameters.Model }}" + } + + $args += "--parallel", "${{ parameters.Parallelism }}" + $args += "--cleanup", "always" + + Write-Host "Running: dotnet run -- $($args -join ' ')" + dotnet run -- @args + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_MULTILEVEL_LOOKUP: 0 + COPILOT_GITHUB_TOKEN: $(azuresdk-copilot-github-pat) diff --git a/tools/azsdk-cli/ci.yml b/tools/azsdk-cli/ci.yml index 178e679d1ff..9ac2c3c5a53 100644 --- a/tools/azsdk-cli/ci.yml +++ b/tools/azsdk-cli/ci.yml @@ -60,4 +60,8 @@ extends: - template: /tools/azsdk-cli/auto-documentation.yml parameters: ToolDirectory: $(ToolDir) + - template: /eng/common/pipelines/templates/jobs/benchmark-job.yml + parameters: + BenchmarkProject: tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks + Parallelism: 5 From d5a2a7f7d3c893009a2b08ca4c940816d2dfb921 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:01:14 -0700 Subject: [PATCH 02/15] cli args instead of args --- .../pipelines/templates/jobs/benchmark-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index e5371b7c88d..4b4701a8cfa 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -56,27 +56,27 @@ jobs: workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' pwsh: true script: | - $args = @("run") + $cliArgs = @("run") # Determine scenario selection: name, tags, or all if ("${{ parameters.ScenarioName }}") { - $args += "${{ parameters.ScenarioName }}" + $cliArgs += "${{ parameters.ScenarioName }}" } elseif ("${{ parameters.Tags }}") { - $args += "--tags", "${{ parameters.Tags }}" + $cliArgs += "--tags", "${{ parameters.Tags }}" } else { - $args += "--all" + $cliArgs += "--all" } # Optional model override if ("${{ parameters.Model }}") { - $args += "--model", "${{ parameters.Model }}" + $cliArgs += "--model", "${{ parameters.Model }}" } - $args += "--parallel", "${{ parameters.Parallelism }}" - $args += "--cleanup", "always" + $cliArgs += "--parallel", "${{ parameters.Parallelism }}" + $cliArgs += "--cleanup", "always" - Write-Host "Running: dotnet run -- $($args -join ' ')" - dotnet run -- @args + Write-Host "Running: dotnet run -- $($cliArgs -join ' ')" + dotnet run -- @cliArgs env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_CLI_TELEMETRY_OPTOUT: 1 From 0176aeed26e92b84010475704438ef58f218379c Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:05:10 -0700 Subject: [PATCH 03/15] target path --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 4b4701a8cfa..2beba3f43f8 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -45,7 +45,7 @@ jobs: Commitish: $(Build.SourceVersion) WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) Paths: - - 'tools/azsdk-cli/**' + - '${{ parameters.BenchmarkProject }}/**' - 'eng/common/**' - task: PowerShell@2 From 771339449bc83017c5038a18b36087efbc7a0c78 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:06:58 -0700 Subject: [PATCH 04/15] .net 9 --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 6 +++--- .../Azure.Sdk.Tools.Cli.Benchmarks.csproj | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 2beba3f43f8..99f6cb3c267 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -30,14 +30,14 @@ jobs: steps: - checkout: none - task: UseDotNet@2 - displayName: "Use .NET SDK 8.0.x" + displayName: "Use .NET SDK 9.0.x" retryCountOnTaskFailure: 3 inputs: packageType: sdk - version: 8.0.x + version: 9.0.x performMultiLevelLookup: true - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - template:/eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: SkipCheckoutNone: true Repositories: diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj index ed558fdc5b9..7ad673ef77a 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net8.0 + net9.0 enable enable From 92a5b9e8809b3867faf52f448fa47ddb75795ba4 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:08:33 -0700 Subject: [PATCH 05/15] fix --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 99f6cb3c267..d7d15bc90a7 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -37,7 +37,7 @@ jobs: version: 9.0.x performMultiLevelLookup: true - - template:/eng/common/pipelines/templates/steps/sparse-checkout.yml + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: SkipCheckoutNone: true Repositories: From 4248b531c075d4208243fdc8214ff2368f714ab5 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:16:21 -0700 Subject: [PATCH 06/15] blank defaults --- .../pipelines/templates/jobs/benchmark-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index d7d15bc90a7..ef6ce173958 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -4,13 +4,13 @@ parameters: default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' - name: ScenarioName type: string - default: '' + default: ' ' - name: Tags type: string - default: '' + default: ' ' - name: Model type: string - default: '' + default: ' ' - name: Parallelism type: number default: 5 @@ -59,17 +59,17 @@ jobs: $cliArgs = @("run") # Determine scenario selection: name, tags, or all - if ("${{ parameters.ScenarioName }}") { - $cliArgs += "${{ parameters.ScenarioName }}" - } elseif ("${{ parameters.Tags }}") { - $cliArgs += "--tags", "${{ parameters.Tags }}" + if ("${{ parameters.ScenarioName }}".Trim()) { + $cliArgs += "${{ parameters.ScenarioName }}".Trim() + } elseif ("${{ parameters.Tags }}".Trim()) { + $cliArgs += "--tags", "${{ parameters.Tags }}".Trim() } else { $cliArgs += "--all" } # Optional model override - if ("${{ parameters.Model }}") { - $cliArgs += "--model", "${{ parameters.Model }}" + if ("${{ parameters.Model }}".Trim()) { + $cliArgs += "--model", "${{ parameters.Model }}".Trim() } $cliArgs += "--parallel", "${{ parameters.Parallelism }}" From 95fc9ed3988f8e207eef080afe3ad452fe298207 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:32:08 -0700 Subject: [PATCH 07/15] back to 8 --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 8 +------- .../Azure.Sdk.Tools.Cli.Benchmarks.csproj | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index ef6ce173958..97fc8c7d2ff 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -29,13 +29,7 @@ jobs: steps: - checkout: none - - task: UseDotNet@2 - displayName: "Use .NET SDK 9.0.x" - retryCountOnTaskFailure: 3 - inputs: - packageType: sdk - version: 9.0.x - performMultiLevelLookup: true + - template: /eng/pipelines/templates/steps/install-dotnet.yml - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj index 7ad673ef77a..ed558fdc5b9 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Azure.Sdk.Tools.Cli.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net8.0 enable enable From b631840ffed68755c420e8822416d07519a8aa47 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:48:58 -0700 Subject: [PATCH 08/15] install after checkout --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 97fc8c7d2ff..d490d897792 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -29,7 +29,6 @@ jobs: steps: - checkout: none - - template: /eng/pipelines/templates/steps/install-dotnet.yml - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -42,6 +41,8 @@ jobs: - '${{ parameters.BenchmarkProject }}/**' - 'eng/common/**' + - template: /eng/pipelines/templates/steps/install-dotnet.yml + - task: PowerShell@2 displayName: 'Run benchmarks' condition: succeeded() From 77d9c7057f603d448f09a393ef683f3792a56f6f Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:51:38 -0700 Subject: [PATCH 09/15] tsp command failing in ci --- .../Scenarios/Typespec/AddArmResourceScenario.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs index d8471d24a93..5b3f06885d5 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs @@ -88,7 +88,7 @@ public override async Task SetupAsync(Workspace workspace) // Verify the project compiles successfully new CommandValidator("tsp compile succeeds", - command: "tsp", - arguments: ["compile", "./specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp"]) + command: "npx", + arguments: ["tsp", "compile", "./specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp"]) ]; } From 28f81a010e415e47e2c089384977f1c7ed18c49f Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Tue, 17 Mar 2026 11:19:51 -0700 Subject: [PATCH 10/15] optional --- tools/azsdk-cli/ci.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/azsdk-cli/ci.yml b/tools/azsdk-cli/ci.yml index 9ac2c3c5a53..89a5ea9ec0b 100644 --- a/tools/azsdk-cli/ci.yml +++ b/tools/azsdk-cli/ci.yml @@ -1,4 +1,10 @@ # NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. +parameters: + - name: RunBenchmarks + displayName: Run Benchmarks + type: boolean + default: false + trigger: branches: include: @@ -60,8 +66,9 @@ extends: - template: /tools/azsdk-cli/auto-documentation.yml parameters: ToolDirectory: $(ToolDir) - - template: /eng/common/pipelines/templates/jobs/benchmark-job.yml - parameters: - BenchmarkProject: tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks - Parallelism: 5 + - ${{ if eq(parameters.RunBenchmarks, true) }}: + - template: /eng/common/pipelines/templates/jobs/benchmark-job.yml + parameters: + BenchmarkProject: tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks + Parallelism: 5 From 37e82541925c130b33874e33f154e7cc27b90e6f Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:32:09 -0700 Subject: [PATCH 11/15] under eng instead of eng/common --- eng/{common => }/pipelines/templates/jobs/benchmark-job.yml | 0 tools/azsdk-cli/ci.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename eng/{common => }/pipelines/templates/jobs/benchmark-job.yml (100%) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/pipelines/templates/jobs/benchmark-job.yml similarity index 100% rename from eng/common/pipelines/templates/jobs/benchmark-job.yml rename to eng/pipelines/templates/jobs/benchmark-job.yml diff --git a/tools/azsdk-cli/ci.yml b/tools/azsdk-cli/ci.yml index 531f18bdc36..de32e9b1c0e 100644 --- a/tools/azsdk-cli/ci.yml +++ b/tools/azsdk-cli/ci.yml @@ -78,7 +78,7 @@ extends: parameters: ToolDirectory: $(ToolDir) - ${{ if eq(parameters.RunBenchmarks, true) }}: - - template: /eng/common/pipelines/templates/jobs/benchmark-job.yml + - template: /eng/pipelines/templates/jobs/benchmark-job.yml parameters: BenchmarkProject: tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks Parallelism: 5 From 2f6630a0cacb82ebd035cdc04c21284bf2106c9c Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:29:26 -0700 Subject: [PATCH 12/15] simplify --- .../templates/jobs/benchmark-job.yml | 79 ------------------- tools/azsdk-cli/ci.yml | 27 ++++++- 2 files changed, 23 insertions(+), 83 deletions(-) delete mode 100644 eng/pipelines/templates/jobs/benchmark-job.yml diff --git a/eng/pipelines/templates/jobs/benchmark-job.yml b/eng/pipelines/templates/jobs/benchmark-job.yml deleted file mode 100644 index d490d897792..00000000000 --- a/eng/pipelines/templates/jobs/benchmark-job.yml +++ /dev/null @@ -1,79 +0,0 @@ -parameters: - - name: BenchmarkProject - type: string - default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' - - name: ScenarioName - type: string - default: ' ' - - name: Tags - type: string - default: ' ' - - name: Model - type: string - default: ' ' - - name: Parallelism - type: number - default: 5 - -jobs: - - job: Run_Benchmark - variables: - - template: /eng/pipelines/templates/variables/globals.yml - - template: /eng/pipelines/templates/variables/image.yml - - group: 'AzSDK_Eval_Variable_group' - displayName: 'Run Benchmark' - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - - steps: - - checkout: none - - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: $(Build.Repository.Name) - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) - Paths: - - '${{ parameters.BenchmarkProject }}/**' - - 'eng/common/**' - - - template: /eng/pipelines/templates/steps/install-dotnet.yml - - - task: PowerShell@2 - displayName: 'Run benchmarks' - condition: succeeded() - inputs: - targetType: 'inline' - workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' - pwsh: true - script: | - $cliArgs = @("run") - - # Determine scenario selection: name, tags, or all - if ("${{ parameters.ScenarioName }}".Trim()) { - $cliArgs += "${{ parameters.ScenarioName }}".Trim() - } elseif ("${{ parameters.Tags }}".Trim()) { - $cliArgs += "--tags", "${{ parameters.Tags }}".Trim() - } else { - $cliArgs += "--all" - } - - # Optional model override - if ("${{ parameters.Model }}".Trim()) { - $cliArgs += "--model", "${{ parameters.Model }}".Trim() - } - - $cliArgs += "--parallel", "${{ parameters.Parallelism }}" - $cliArgs += "--cleanup", "always" - - Write-Host "Running: dotnet run -- $($cliArgs -join ' ')" - dotnet run -- @cliArgs - env: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 - DOTNET_CLI_TELEMETRY_OPTOUT: 1 - DOTNET_MULTILEVEL_LOOKUP: 0 - COPILOT_GITHUB_TOKEN: $(azuresdk-copilot-github-pat) diff --git a/tools/azsdk-cli/ci.yml b/tools/azsdk-cli/ci.yml index de32e9b1c0e..fe28ec9f3fe 100644 --- a/tools/azsdk-cli/ci.yml +++ b/tools/azsdk-cli/ci.yml @@ -78,8 +78,27 @@ extends: parameters: ToolDirectory: $(ToolDir) - ${{ if eq(parameters.RunBenchmarks, true) }}: - - template: /eng/pipelines/templates/jobs/benchmark-job.yml - parameters: - BenchmarkProject: tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks - Parallelism: 5 + - job: Run_Benchmark + displayName: 'Run Benchmark' + variables: + - group: 'AzSDK_Eval_Variable_group' + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + steps: + - checkout: self + - template: /eng/pipelines/templates/steps/install-dotnet.yml + - task: PowerShell@2 + displayName: 'Run benchmarks' + inputs: + targetType: 'inline' + workingDirectory: '$(Build.SourcesDirectory)/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' + pwsh: true + script: dotnet run -- run --all --parallel 5 --cleanup "always" + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_MULTILEVEL_LOOKUP: 0 + COPILOT_GITHUB_TOKEN: $(azuresdk-copilot-github-pat) From 75edefcd3cb558f220263dc7292884b08bfce4a4 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 18 Mar 2026 09:56:53 -0700 Subject: [PATCH 13/15] in ci instead of release --- tools/azsdk-cli/ci.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/azsdk-cli/ci.yml b/tools/azsdk-cli/ci.yml index fe28ec9f3fe..024b2316171 100644 --- a/tools/azsdk-cli/ci.yml +++ b/tools/azsdk-cli/ci.yml @@ -73,10 +73,6 @@ extends: TargetRepoName: azure-rest-api-specs TargetRepoOwner: Azure TargetRepoCommit: main - CustomReleaseJobs: - - template: /tools/azsdk-cli/auto-documentation.yml - parameters: - ToolDirectory: $(ToolDir) - ${{ if eq(parameters.RunBenchmarks, true) }}: - job: Run_Benchmark displayName: 'Run Benchmark' @@ -87,13 +83,23 @@ extends: image: $(LINUXVMIMAGE) os: linux steps: - - checkout: self + - checkout: none + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Repositories: + - Name: $(Build.Repository.Name) + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) + Paths: + - 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/**' + - 'eng/common/**' - template: /eng/pipelines/templates/steps/install-dotnet.yml - task: PowerShell@2 displayName: 'Run benchmarks' inputs: targetType: 'inline' - workingDirectory: '$(Build.SourcesDirectory)/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' + workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' pwsh: true script: dotnet run -- run --all --parallel 5 --cleanup "always" env: @@ -101,4 +107,7 @@ extends: DOTNET_CLI_TELEMETRY_OPTOUT: 1 DOTNET_MULTILEVEL_LOOKUP: 0 COPILOT_GITHUB_TOKEN: $(azuresdk-copilot-github-pat) - + CustomReleaseJobs: + - template: /tools/azsdk-cli/auto-documentation.yml + parameters: + ToolDirectory: $(ToolDir) \ No newline at end of file From 2783fdd269429218d5d7ea61680b3b4110755c4b Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:11:20 -0700 Subject: [PATCH 14/15] comments --- tools/azsdk-cli/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/azsdk-cli/ci.yml b/tools/azsdk-cli/ci.yml index 024b2316171..48d68bf02c9 100644 --- a/tools/azsdk-cli/ci.yml +++ b/tools/azsdk-cli/ci.yml @@ -73,7 +73,7 @@ extends: TargetRepoName: azure-rest-api-specs TargetRepoOwner: Azure TargetRepoCommit: main - - ${{ if eq(parameters.RunBenchmarks, true) }}: + - ${{ if and(eq(parameters.RunBenchmarks, true), ne(variables['Build.Reason'], 'PullRequest')) }}: - job: Run_Benchmark displayName: 'Run Benchmark' variables: From 1fba2328904e46c7fea4205f6e9b81155186af1c Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 18 Mar 2026 10:13:16 -0700 Subject: [PATCH 15/15] merge conflict --- .../Scenarios/Typespec/AddArmResourceScenario.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs index 5b3f06885d5..d8471d24a93 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks/Scenarios/Typespec/AddArmResourceScenario.cs @@ -88,7 +88,7 @@ public override async Task SetupAsync(Workspace workspace) // Verify the project compiles successfully new CommandValidator("tsp compile succeeds", - command: "npx", - arguments: ["tsp", "compile", "./specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp"]) + command: "tsp", + arguments: ["compile", "./specification/widget/resource-manager/Microsoft.Widget/Widget/main.tsp"]) ]; }