azure-sdk-tools/eng/pipelines/skill-eval.yml at 12714ae43c1cb8dbc079a3c1fa50458bdb584ba7 · Azure/azure-sdk-tools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
trigger:
  branches:
    include:
      - main
  paths:
    include:
      - .github/skills/**

pr: none

parameters:
  - name: runAll
    displayName: 'Run all skill evals (ignore change detection)'
    type: boolean
    default: false
  - name: areas
    displayName: 'Skill areas to eval (comma-separated, e.g. sensei,skill-authoring)'
    type: string
    default: ' '

variables:
  - group: 'AzSDK_Eval_Variable_group'

jobs:
  - job: Evaluate
    displayName: 'Vally Eval'
    pool:
      vmImage: 'ubuntu-latest'
    steps:
      - checkout: self
        fetchDepth: 2

      - task: NodeTool@0
        displayName: 'Use Node.js 22'
        inputs:
          versionSpec: '22.x'

      - script: npm ci
        displayName: 'Install Vally CLI (pinned via lockfile)'
        workingDirectory: eng/skill-eval

      - script: npm install -g @github/copilot-sdk
        displayName: 'Install Copilot SDK'

      # Pre-build the MCP servers so vally launches `dotnet <dll>` instead of
      # `dotnet run` — avoids the MSBuild boot race under parallel workers.
      # See issue #15948.
      - script: |
          dotnet build tools/azsdk-cli/Azure.Sdk.Tools.Cli  -c Debug --nologo
          dotnet build tools/azsdk-cli/Azure.Sdk.Tools.Mock -c Debug --nologo
        displayName: 'Build MCP servers'

      - script: |
          input_areas=$(echo "${{ parameters.areas }}" | xargs)
          if [ -n "$input_areas" ]; then
            areas="$input_areas"
          elif [ "${{ parameters.runAll }}" = "True" ]; then
            areas=""
          else
            areas=$(git diff --name-only HEAD~1 HEAD -- .github/skills/ \
              | grep -oP '\.github/skills/\K[^/]+(?=/)' \
              | sort -u \
              | grep -v '^\.' \
              | paste -sd, -)
          fi

          if [ -n "$areas" ]; then
            echo "Evaluating areas: $areas"
            echo "##vso[task.setvariable variable=EVAL_ARGS]--tag area=$areas"
          elif [ "${{ parameters.runAll }}" = "True" ]; then
            echo "Evaluating all skills"
            echo "##vso[task.setvariable variable=EVAL_ARGS] "
          else
            echo "No skill changes detected — skipping evals"
            echo "##vso[task.setvariable variable=EVAL_ARGS]SKIP"
          fi
        displayName: 'Detect changed skills'

      - script: |
          cd .github/skills
          npm exec --no --prefix "$(Build.SourcesDirectory)/eng/skill-eval" -- vally eval --tag "type=ci-gate" $(EVAL_ARGS) --output-dir ".vally/results"
        displayName: 'Run evaluations'
        condition: and(succeeded(), ne(variables['EVAL_ARGS'], 'SKIP'))
        continueOnError: true
        env:
          GITHUB_TOKEN: $(azuresdk-copilot-github-pat)

      - task: PublishPipelineArtifact@1
        displayName: 'Upload eval results'
        condition: and(always(), ne(variables['EVAL_ARGS'], 'SKIP'))
        inputs:
          targetPath: '.github/skills/.vally/results'
          artifactName: 'eval-results-$(Build.BuildId)'