add tags for all single-case eval files and test workaround in no-skill pipeline

Author: JoyerJin

.github/skills/azure-typespec-author/evaluate/evals/001001.eval.yaml

@@ -16,61 +16,99 @@ config:
 # Test cases
 stimuli:
 - name: 001001-version-spread-property
-  prompt: I added `...Azure.ResourceManager.ManagedServiceIdentityProperty;` which updates all my existing API versions and introduces a breaking change. I want to introduce the properties of the spread model '...ManagedServiceIdentityProperty' in API version 2025-05-04-preview only.
-  environment:
+  tags:
+    - 'main'
+  prompt: 'I added `...Azure.ResourceManager.ManagedServiceIdentityProperty;` which updates all my existing API versions and introduces a breaking change. I want to introduce the properties of the spread model ''...ManagedServiceIdentityProperty'' in API version 2025-05-04-preview only.'
+  environment: &shared_environment
     files:
-    - src: ../fixtures/001-share-version-new-feature/employee.tsp
-      dest: employee.tsp
-    - src: ../fixtures/001-share-version-new-feature/main.tsp
-      dest: main.tsp
-    - src: ../fixtures/Microsoft.Widget/Widget/package-lock.json
-      dest: package-lock.json
-    - src: ../fixtures/Microsoft.Widget/Widget/package.json
-      dest: package.json
-    - src: ../fixtures/001-share-version-new-feature/readme.md
-      dest: readme.md
-    - src: ../fixtures/001-share-version-new-feature/shared.tsp
-      dest: shared.tsp
-    - src: ../fixtures/001-share-version-new-feature/tspconfig.yaml
-      dest: tspconfig.yaml
+      - src: ../fixtures/001-share-version-new-feature/employee.tsp
+        dest: employee.tsp
+      - src: ../fixtures/001-share-version-new-feature/main.tsp
+        dest: main.tsp
+      - src: ../fixtures/Microsoft.Widget/Widget/package-lock.json
+        dest: package-lock.json
+      - src: ../fixtures/Microsoft.Widget/Widget/package.json
+        dest: package.json
+      - src: ../fixtures/001-share-version-new-feature/readme.md
+        dest: readme.md
+      - src: ../fixtures/001-share-version-new-feature/shared.tsp
+        dest: shared.tsp
+      - src: ../fixtures/001-share-version-new-feature/tspconfig.yaml
+        dest: tspconfig.yaml
     commands:
-      - node -e "const fs=require('fs');const t=process.env.FIXTURE_NODE_MODULES;if(t&&fs.existsSync(t)&&!fs.existsSync('node_modules')){fs.symlinkSync(t,'node_modules','junction')}"
+      - 'node -e "const fs=require(''fs'');const t=process.env.FIXTURE_NODE_MODULES;if(t&&fs.existsSync(t)&&!fs.existsSync(''node_modules'')){fs.symlinkSync(t,''node_modules'',''junction'')}"'
   graders:
-  - type: tool-calls
+  - type: 'tool-calls'
     config:
       required:
-        - edit
-        - azure-sdk-mcp-azsdk_typespec_generate_authoring_plan
-        - azure-sdk-mcp-azsdk_run_typespec_validation
-  - type: skill-invocation
+        - 'edit'
+        - 'azure-sdk-mcp-azsdk_typespec_generate_authoring_plan'
+        - 'azure-sdk-mcp-azsdk_run_typespec_validation'
+  - type: 'skill-invocation'
     config:
       required:
-        - azure-typespec-author
-  - type: file-matches
+        - 'azure-typespec-author'
+  - type: 'file-matches'
     config:
-      path: main.tsp
-      pattern: 2025-05-04-preview
-  - type: file-matches
+      pattern: '2025-05-04-preview'
+      path: 'main.tsp'
+  - type: 'file-matches'
     config:
-      path: employee.tsp
-      pattern: "\\.\\.\\.ManagedServiceIdentityProperty;"
-  - type: file-matches
+      pattern: '\.\.\.ManagedServiceIdentityProperty;'
+      path: 'employee.tsp'
+  - type: 'file-matches'
     config:
-      path: employee.tsp
-      pattern: "@@added\\(Employee\\.identity, (Microsoft\\.Widget\\.)?Versions\\.(v2025_05_04_preview|`2025-05-04-preview`)\\);"
-  - type: prompt
+      pattern: '@@added\(Employee\.identity, (Microsoft\.Widget\.)?Versions\.(v2025_05_04_preview|`2025-05-04-preview`)\);'
+      path: 'employee.tsp'
+  - type: 'prompt'
     config:
-      prompt: Verify changes are scoped to this task only, with no unrelated edits beyond introducing preview-only identity via versioning.
-      model: claude-opus-4.6
-      scoring: scale_1_5
-      threshold: 1.0
+      prompt: 'Verify changes are scoped to this task only, with no unrelated edits beyond introducing preview-only identity via versioning.'
+      model: 'claude-opus-4.6'
+      threshold: 1
+      scoring: 'scale_1_5'
+  constraints:
+    max_turns: 5
+    max_tokens: 50000
+- name: 001001-version-spread-property-trigger
+  tags:
+    - 'trigger'
+  prompt: 'I added `...Azure.ResourceManager.ManagedServiceIdentityProperty;` which updates all my existing API versions and introduces a breaking change. I want to introduce the properties of the spread model ''...ManagedServiceIdentityProperty'' in API version 2025-05-04-preview only.'
+  environment: *shared_environment
+  graders:
+  - type: 'tool-calls'
+    config:
+      required:
+        - 'edit'
+        - 'azure-sdk-mcp-azsdk_run_typespec_validation'
+  - type: 'skill-invocation'
+    config:
+      required:
+        - 'azure-typespec-author'
+  constraints:
+    max_turns: 5
+    max_tokens: 50000
+- name: 001001-version-spread-property-forced
+  tags:
+    - 'forced'
+  prompt: '@azure-typespec-author I added `...Azure.ResourceManager.ManagedServiceIdentityProperty;` which updates all my existing API versions and introduces a breaking change. I want to introduce the properties of the spread model ''...ManagedServiceIdentityProperty'' in API version 2025-05-04-preview only.'
+  environment: *shared_environment
+  graders:
+  - type: 'tool-calls'
+    config:
+      required:
+        - 'edit'
+        - 'azure-sdk-mcp-azsdk_run_typespec_validation'
+  - type: 'skill-invocation'
+    config:
+      required:
+        - 'azure-typespec-author'
   constraints:
     max_turns: 5
     max_tokens: 50000
 scoring:
   weights:
-    file-matches: 3
+    prompt: 1
     tool-calls: 1
     skill-invocation: 1
-    prompt: 1
-  threshold: 1.0
+    file-matches: 3
+  threshold: 1

.github/skills/azure-typespec-author/evaluate/evals/001002.eval.yaml

@@ -16,66 +16,109 @@ config:
 # Test cases
 stimuli:
 - name: 001002-version-default-value
+  tags:
+    - 'main'
   prompt: |
     add a default value `21` for property age in model EmployeeProperties only for the new api version 2025-11-01
-  constraints:
-    max_turns: 6
-    max_tokens: 100000
-  environment:
+    
+  environment: &shared_environment
     files:
-    - src: ../fixtures/001002-version-default-value/employee.tsp
-      dest: employee.tsp
-    - src: ../fixtures/001002-version-default-value/main.tsp
-      dest: main.tsp
-    - src: ../fixtures/Microsoft.Widget/Widget/package-lock.json
-      dest: package-lock.json
-    - src: ../fixtures/Microsoft.Widget/Widget/package.json
-      dest: package.json
-    - src: ../fixtures/001002-version-default-value/readme.md
-      dest: readme.md
-    - src: ../fixtures/001002-version-default-value/shared.tsp
-      dest: shared.tsp
-    - src: ../fixtures/001002-version-default-value/tspconfig.yaml
-      dest: tspconfig.yaml
+      - src: ../fixtures/001002-version-default-value/employee.tsp
+        dest: employee.tsp
+      - src: ../fixtures/001002-version-default-value/main.tsp
+        dest: main.tsp
+      - src: ../fixtures/Microsoft.Widget/Widget/package-lock.json
+        dest: package-lock.json
+      - src: ../fixtures/Microsoft.Widget/Widget/package.json
+        dest: package.json
+      - src: ../fixtures/001002-version-default-value/readme.md
+        dest: readme.md
+      - src: ../fixtures/001002-version-default-value/shared.tsp
+        dest: shared.tsp
+      - src: ../fixtures/001002-version-default-value/tspconfig.yaml
+        dest: tspconfig.yaml
     commands:
-      - node -e "const fs=require('fs');const t=process.env.FIXTURE_NODE_MODULES;if(t&&fs.existsSync(t)&&!fs.existsSync('node_modules')){fs.symlinkSync(t,'node_modules','junction')}"
+      - 'node -e "const fs=require(''fs'');const t=process.env.FIXTURE_NODE_MODULES;if(t&&fs.existsSync(t)&&!fs.existsSync(''node_modules'')){fs.symlinkSync(t,''node_modules'',''junction'')}"'
   graders:
-  - type: tool-calls
+  - type: 'tool-calls'
     config:
       required:
-        - edit
-        - azure-sdk-mcp-azsdk_typespec_generate_authoring_plan
-        - azure-sdk-mcp-azsdk_run_typespec_validation
-  - type: skill-invocation
+        - 'edit'
+        - 'azure-sdk-mcp-azsdk_typespec_generate_authoring_plan'
+        - 'azure-sdk-mcp-azsdk_run_typespec_validation'
+  - type: 'skill-invocation'
     config:
       required:
-        - azure-typespec-author
-  - type: file-matches
+        - 'azure-typespec-author'
+  - type: 'file-matches'
     config:
-      path: employee.tsp
       pattern: '@removed\(Versions\.v2025_11_01\)'
-  - type: file-matches
+      path: 'employee.tsp'
+  - type: 'file-matches'
     config:
-      path: employee.tsp
       pattern: '@renamedFrom\(Versions\.v2025_11_01, "age"\)'
-  - type: file-matches
+      path: 'employee.tsp'
+  - type: 'file-matches'
     config:
-      path: employee.tsp
       pattern: '@added\(Versions\.v2025_11_01\)'
-  - type: file-matches
+      path: 'employee.tsp'
+  - type: 'file-matches'
     config:
-      path: employee.tsp
       pattern: 'age\?: int32 = 21;'
-  - type: prompt
+      path: 'employee.tsp'
+  - type: 'prompt'
     config:
-      scoring: scale_1_5
-      threshold: 1.0
+      threshold: 1
+      model: 'claude-opus-4.6'
+      scoring: 'scale_1_5'
       prompt: 'Verify change scope and quality only: add a default value of 21 for the `age` property in `EmployeeProperties` that only applies to the 2025-11-01 API version. The old age renamed and removed for the new version.'
-      model: claude-opus-4.6
+  constraints:
+    max_turns: 6
+    max_tokens: 100000
+- name: 001002-version-default-value-trigger
+  tags:
+    - 'trigger'
+  prompt: |
+    add a default value `21` for property age in model EmployeeProperties only for the new api version 2025-11-01
+    
+  environment: *shared_environment
+  graders:
+  - type: 'tool-calls'
+    config:
+      required:
+        - 'edit'
+        - 'azure-sdk-mcp-azsdk_run_typespec_validation'
+  - type: 'skill-invocation'
+    config:
+      required:
+        - 'azure-typespec-author'
+  constraints:
+    max_turns: 5
+    max_tokens: 50000
+- name: 001002-version-default-value-forced
+  tags:
+    - 'forced'
+  prompt: |
+    @azure-typespec-author add a default value `21` for property age in model EmployeeProperties only for the new api version 2025-11-01
+    
+  environment: *shared_environment
+  graders:
+  - type: 'tool-calls'
+    config:
+      required:
+        - 'edit'
+        - 'azure-sdk-mcp-azsdk_run_typespec_validation'
+  - type: 'skill-invocation'
+    config:
+      required:
+        - 'azure-typespec-author'
+  constraints:
+    max_turns: 5
+    max_tokens: 50000
 scoring:
   weights:
+    prompt: 1
     tool-calls: 1
     skill-invocation: 1
-    prompt: 1
     file-matches: 4
-  threshold: 1.0
+  threshold: 1