Port tool-scenario benchmarks from Azure.Sdk.Tools.Cli.Evaluations to Vally (#15124)

.github/skills/.vally.yaml

@@ -12,12 +12,15 @@ paths:
   evalFilenames: ["eval.yaml", "*.eval.yaml"]
 
 environments:
+  # Launch the pre-built DLLs via `dotnet <dll>`, NOT `dotnet run` — avoids the
+  # MSBuild boot race under parallel workers. See issue #15948.
+  # CI builds the DLLs in the 'Build MCP servers' step of skill-eval.yml.
   azsdk-mcp:
     mcpServers:
       azure-sdk-mcp:
         type: stdio
         command: dotnet
-        args: ["run", "--project", "../../tools/azsdk-cli/Azure.Sdk.Tools.Cli", "--", "start"]
+        args: ["../../artifacts/bin/Azure.Sdk.Tools.Cli/Debug/net8.0/azsdk.dll", "start"]
         timeout: "60s"
         env:
           AZSDKTOOLS_AGENT_TESTING: "true"
@@ -27,5 +30,5 @@ environments:
       azure-sdk-mcp:
         type: stdio
         command: dotnet
-        args: ["run", "--project", "../../tools/azsdk-cli/Azure.Sdk.Tools.Mock"]
+        args: ["../../artifacts/bin/Azure.Sdk.Tools.Mock/Debug/net8.0/azsdk-mock.dll"]
         timeout: "60s"

eng/pipelines/skill-eval.yml

@@ -42,6 +42,14 @@ jobs:
       - script: npm install -g @github/copilot-sdk
         displayName: 'Install Copilot SDK'
 
+      # Pre-build the MCP servers so vally launches `dotnet <dll>` instead of
+      # `dotnet run` — avoids the MSBuild boot race under parallel workers.
+      # See issue #15948.
+      - script: |
+          dotnet build tools/azsdk-cli/Azure.Sdk.Tools.Cli  -c Debug --nologo
+          dotnet build tools/azsdk-cli/Azure.Sdk.Tools.Mock -c Debug --nologo
+        displayName: 'Build MCP servers'
+
       - script: |
           input_areas=$(echo "${{ parameters.areas }}" | xargs)
           if [ -n "$input_areas" ]; then

tools/azsdk-cli/Azure.Sdk.Tools.Vally/.gitignore

@@ -0,0 +1,2 @@
+vally-results/
+results/

tools/azsdk-cli/Azure.Sdk.Tools.Vally/.vally.yaml

@@ -0,0 +1,92 @@
+# Vally configuration for Azure SDK Tools MCP tool / scenario evaluations.
+# See: https://vally.dev/reference/vally-config
+#
+# These are scenario evals (does the agent invoke the right MCP tool(s) for a
+# given prompt?) and are intentionally separate from the per-skill evals under
+# .github/skills/. See README.md for context.
+
+paths:
+  evals: [evals/]
+  evalFilenames: ["*.eval.yaml"]
+  results: results/
+
+environments:
+  # Launch the pre-built DLL via `dotnet <dll>`, NOT `dotnet run` — avoids the
+  # MSBuild boot race under parallel workers. See issue #15948.
+  # Run `dotnet build ../Azure.Sdk.Tools.Mock -c Debug` once before vally.
+  azsdk-mcp-mock:
+    mcpServers:
+      azure-sdk-mcp:
+        type: stdio
+        command: dotnet
+        args: ["../../../artifacts/bin/Azure.Sdk.Tools.Mock/Debug/net8.0/azsdk-mock.dll"]
+        timeout: "30s"
+
+  # Live MCP. AZSDKTOOLS_AGENT_TESTING=true keeps write tools inside the test
+  # area. Pre-built DLL pattern — see issue #15948.
+  # Run `dotnet build ../Azure.Sdk.Tools.Cli -c Debug` once before vally.
+  azsdk-mcp-live:
+    mcpServers:
+      azure-sdk-mcp:
+        type: stdio
+        command: dotnet
+        args: ["../../../artifacts/bin/Azure.Sdk.Tools.Cli/Debug/net8.0/azsdk.dll", "start"]
+        timeout: "5m"
+        env:
+          AZSDKTOOLS_AGENT_TESTING: "true"
+          AZSDKTOOLS_COLLECT_TELEMETRY: "false"
+
+# Suites group evals for selective execution.
+#
+# Layout maps directly to suites — no tag-based mock/live filtering. Vally's
+# suite filter is positive-match only (AND across keys, OR within values),
+# so subfolders are the cleanest way to split mock vs live. See
+# https://github.com/microsoft/vally suite-filter source.
+suites:
+  # ---- by tier ----
+  unit:
+    description: |
+      Hermetic single-tool / trigger evals. No external I/O. Fast; the
+      foundation of the PR gate.
+    evals: ["evals/tools/*.eval.yaml"]
+
+  scenarios-mock:
+    description: |
+      Multi-tool scenarios against the mock MCP environment. Hermetic; safe
+      for PR gate.
+    evals: ["evals/workflow-scenarios/mock/*.eval.yaml"]
+
+  scenarios-live:
+    description: |
+      Scenarios against live MCP — real DevOps / GitHub / pipelines. Slow;
+      nightly only. Prime any required clones first via
+      `evals/setup/ensure-specs-clone.ps1`.
+    evals: ["evals/workflow-scenarios/live/*.eval.yaml"]
+
+  # ---- composite suites ----
+  pr-gate:
+    description: Hermetic tiers only (unit + scenarios-mock). Target for CI PR check.
+    evals:
+      - "evals/tools/*.eval.yaml"
+      - "evals/workflow-scenarios/mock/*.eval.yaml"
+  nightly:
+    description: All tiers including live scenarios.
+    evals: ["evals/**/*.eval.yaml"]
+
+  # ---- by feature area (tag-filtered) ----
+  release-plan:
+    description: All evals tagged area=release-plan.
+    filter: { area: release-plan }
+    evals: ["evals/**/*.eval.yaml"]
+  typespec:
+    description: All evals tagged area=typespec.
+    filter: { area: typespec }
+    evals: ["evals/**/*.eval.yaml"]
+  pipeline:
+    description: All evals tagged area=pipeline.
+    filter: { area: pipeline }
+    evals: ["evals/**/*.eval.yaml"]
+  github:
+    description: All evals tagged area=github.
+    filter: { area: github }
+    evals: ["evals/**/*.eval.yaml"]