Skip to content

Commit b333b91

Browse files
AditiGoyalUipathAditiGoyalUipath
authored
fix: Add (?s) DOTALL flag to data-fabric coder_eval command patterns (#447)
* feat: Added Codel Eval tests for Datafabric Cli
1 parent 0126731 commit b333b91

14 files changed

Lines changed: 36 additions & 35 deletions

‎skills/uipath-data-fabric/SKILL.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ uip df records query <entity-id> \
9393
| Explore only native entities | `entities list --native-only` |
9494
| Create a new entity | `entities create <name> --body '{"fields":[{"fieldName":"Title","type":"STRING"}]}'` |
9595
| Update entity / add fields | `entities update <id> --body '{"addFields":[{"fieldName":"NewField","type":"STRING"}]}'` |
96+
| Update existing field metadata | `entities update <id> --body '{"updateFields":[{"id":"<field-uuid>","displayName":"New Label","isRequired":true}]}'` — `id` is the field UUID from `entities get Fields[].ID` |
9697
| Update entity metadata | `entities update <id> --body '{"displayName":"New Name","description":"desc"}'` |
9798
| Read records (first page) | `records list <entity-id> --limit 50` |
9899
| Read records (next page) | `records list <entity-id> --cursor <NextCursor>` |

‎tests/tasks/uipath-data-fabric/e2e_employee_directory.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,31 +104,31 @@ success_criteria:
104104
- type: command_executed
105105
description: "Agent used fieldName key in entity create body"
106106
tool_name: "Bash"
107-
command_pattern: 'uip\s+df\s+entities\s+create.*fieldName'
107+
command_pattern: '(?s)uip\s+df\s+entities\s+create.*fieldName'
108108
min_count: 1
109109
weight: 2.0
110110
pass_threshold: 1.0
111111

112112
- type: command_executed
113113
description: "Agent used addFields key when evolving schema"
114114
tool_name: "Bash"
115-
command_pattern: 'uip\s+df\s+entities\s+update.*addFields'
115+
command_pattern: '(?s)uip\s+df\s+entities\s+update.*addFields'
116116
min_count: 1
117117
weight: 2.0
118118
pass_threshold: 1.0
119119

120120
- type: command_executed
121121
description: "Agent used filterGroup key in records query body"
122122
tool_name: "Bash"
123-
command_pattern: 'uip\s+df\s+records\s+query.*filterGroup'
123+
command_pattern: '(?s)uip\s+df\s+records\s+query.*filterGroup'
124124
min_count: 1
125125
weight: 2.5
126126
pass_threshold: 1.0
127127

128128
- type: command_executed
129129
description: "Agent included Id key in records update body"
130130
tool_name: "Bash"
131-
command_pattern: 'uip\s+df\s+records\s+update.*\"Id\"'
131+
command_pattern: '(?s)uip\s+df\s+records\s+update.*\"Id\"'
132132
min_count: 1
133133
weight: 2.5
134134
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/e2e_product_catalogue.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -87,31 +87,31 @@ success_criteria:
8787
- type: command_executed
8888
description: "Agent uploaded product image"
8989
tool_name: "Bash"
90-
command_pattern: 'uip\s+df\s+files\s+upload\s+.*--file'
90+
command_pattern: '(?s)uip\s+df\s+files\s+upload\s+.*--file'
9191
min_count: 1
9292
weight: 2.0
9393
pass_threshold: 1.0
9494

9595
- type: command_executed
9696
description: "Agent used fieldName key in entity create body"
9797
tool_name: "Bash"
98-
command_pattern: 'uip\s+df\s+entities\s+create.*fieldName'
98+
command_pattern: '(?s)uip\s+df\s+entities\s+create.*fieldName'
9999
min_count: 1
100100
weight: 2.0
101101
pass_threshold: 1.0
102102

103103
- type: command_executed
104104
description: "Agent used filterGroup key in records query body"
105105
tool_name: "Bash"
106-
command_pattern: 'uip\s+df\s+records\s+query.*filterGroup'
106+
command_pattern: '(?s)uip\s+df\s+records\s+query.*filterGroup'
107107
min_count: 1
108108
weight: 2.5
109109
pass_threshold: 1.0
110110

111111
- type: command_executed
112112
description: "Agent included Id key in records update body"
113113
tool_name: "Bash"
114-
command_pattern: 'uip\s+df\s+records\s+update.*\"Id\"'
114+
command_pattern: '(?s)uip\s+df\s+records\s+update.*\"Id\"'
115115
min_count: 1
116116
weight: 2.5
117117
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_csv_import.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ success_criteria:
7676
- type: command_executed
7777
description: "Agent ran records import with --file"
7878
tool_name: "Bash"
79-
command_pattern: 'uip\s+df\s+records\s+import\s+.*--file'
79+
command_pattern: '(?s)uip\s+df\s+records\s+import\s+.*--file'
8080
min_count: 1
8181
weight: 3.0
8282
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_entity_scope.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ success_criteria:
9090
- type: command_executed
9191
description: "Agent used fieldName key in entities create body"
9292
tool_name: "Bash"
93-
command_pattern: 'uip\s+df\s+entities\s+create.*fieldName'
93+
command_pattern: '(?s)uip\s+df\s+entities\s+create.*fieldName'
9494
min_count: 1
9595
weight: 2.0
9696
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_error_paths.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ success_criteria:
158158
- type: command_executed
159159
description: "Agent attempted file upload (error scenarios)"
160160
tool_name: "Bash"
161-
command_pattern: 'uip\s+df\s+files\s+upload\s+.*--file'
161+
command_pattern: '(?s)uip\s+df\s+files\s+upload\s+.*--file'
162162
min_count: 3
163163
weight: 2.0
164164
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_files.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,15 +90,15 @@ success_criteria:
9090
- type: command_executed
9191
description: "Agent uploaded file at least twice (v1 and v2 overwrite)"
9292
tool_name: "Bash"
93-
command_pattern: 'uip\s+df\s+files\s+upload\s+.*--file'
93+
command_pattern: '(?s)uip\s+df\s+files\s+upload\s+.*--file'
9494
min_count: 2
9595
weight: 3.0
9696
pass_threshold: 1.0
9797

9898
- type: command_executed
9999
description: "Agent downloaded file at least twice (after v1 and after v2)"
100100
tool_name: "Bash"
101-
command_pattern: 'uip\s+df\s+files\s+download\s+.*--destination'
101+
command_pattern: '(?s)uip\s+df\s+files\s+download\s+.*--destination'
102102
min_count: 2
103103
weight: 2.5
104104
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_full_lifecycle.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -147,39 +147,39 @@ success_criteria:
147147
- type: command_executed
148148
description: "Agent used fieldName (not name) in entity create body"
149149
tool_name: "Bash"
150-
command_pattern: 'uip\s+df\s+entities\s+create.*fieldName'
150+
command_pattern: '(?s)uip\s+df\s+entities\s+create.*fieldName'
151151
min_count: 1
152152
weight: 2.0
153153
pass_threshold: 1.0
154154

155155
- type: command_executed
156156
description: "Agent used addFields key in entities update body"
157157
tool_name: "Bash"
158-
command_pattern: 'uip\s+df\s+entities\s+update.*addFields'
158+
command_pattern: '(?s)uip\s+df\s+entities\s+update.*addFields'
159159
min_count: 1
160160
weight: 2.0
161161
pass_threshold: 1.0
162162

163163
- type: command_executed
164164
description: "Agent used updateFields with lowercase id key"
165165
tool_name: "Bash"
166-
command_pattern: 'uip\s+df\s+entities\s+update.*updateFields.*\"id\"'
166+
command_pattern: '(?s)uip\s+df\s+entities\s+update.*updateFields.*\"id\"'
167167
min_count: 1
168168
weight: 2.0
169169
pass_threshold: 1.0
170170

171171
- type: command_executed
172172
description: "Agent used filterGroup key in records query body"
173173
tool_name: "Bash"
174-
command_pattern: 'uip\s+df\s+records\s+query.*filterGroup'
174+
command_pattern: '(?s)uip\s+df\s+records\s+query.*filterGroup'
175175
min_count: 1
176176
weight: 2.0
177177
pass_threshold: 1.0
178178

179179
- type: command_executed
180180
description: "Agent included Id key in records update body"
181181
tool_name: "Bash"
182-
command_pattern: 'uip\s+df\s+records\s+update.*\"Id\"'
182+
command_pattern: '(?s)uip\s+df\s+records\s+update.*\"Id\"'
183183
min_count: 1
184184
weight: 2.0
185185
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_preseeded_entity.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,23 +184,23 @@ success_criteria:
184184
- type: command_executed
185185
description: "Agent used filterGroup key in records query body"
186186
tool_name: "Bash"
187-
command_pattern: 'uip\s+df\s+records\s+query.*filterGroup'
187+
command_pattern: '(?s)uip\s+df\s+records\s+query.*filterGroup'
188188
min_count: 4
189189
weight: 2.5
190190
pass_threshold: 1.0
191191

192192
- type: command_executed
193193
description: "Agent used logicalOperator in AND filterGroup query (step 4e)"
194194
tool_name: "Bash"
195-
command_pattern: 'uip\s+df\s+records\s+query.*logicalOperator'
195+
command_pattern: '(?s)uip\s+df\s+records\s+query.*logicalOperator'
196196
min_count: 1
197197
weight: 2.0
198198
pass_threshold: 1.0
199199

200200
- type: command_executed
201201
description: "Agent included Id key in records update body"
202202
tool_name: "Bash"
203-
command_pattern: 'uip\s+df\s+records\s+update.*\"Id\"'
203+
command_pattern: '(?s)uip\s+df\s+records\s+update.*\"Id\"'
204204
min_count: 1
205205
weight: 2.0
206206
pass_threshold: 1.0

‎tests/tasks/uipath-data-fabric/integration_query_filters.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ success_criteria:
105105
- type: command_executed
106106
description: "Agent ran filter queries with --body (AND, OR, nested, sort, projection)"
107107
tool_name: "Bash"
108-
command_pattern: 'uip\s+df\s+records\s+query\s+.*--body'
108+
command_pattern: '(?s)uip\s+df\s+records\s+query\s+.*--body'
109109
min_count: 4
110110
weight: 3.0
111111
pass_threshold: 1.0
@@ -129,31 +129,31 @@ success_criteria:
129129
- type: command_executed
130130
description: "Agent used filterGroup key in records query body"
131131
tool_name: "Bash"
132-
command_pattern: 'uip\s+df\s+records\s+query.*filterGroup'
132+
command_pattern: '(?s)uip\s+df\s+records\s+query.*filterGroup'
133133
min_count: 3
134134
weight: 2.5
135135
pass_threshold: 1.0
136136

137137
- type: command_executed
138138
description: "Agent used logicalOperator in filterGroup body (AND=0 / OR=1)"
139139
tool_name: "Bash"
140-
command_pattern: 'uip\s+df\s+records\s+query.*logicalOperator'
140+
command_pattern: '(?s)uip\s+df\s+records\s+query.*logicalOperator'
141141
min_count: 2
142142
weight: 2.0
143143
pass_threshold: 1.0
144144

145145
- type: command_executed
146146
description: "Agent used sortOptions with isDescending in query body"
147147
tool_name: "Bash"
148-
command_pattern: 'uip\s+df\s+records\s+query.*sortOptions'
148+
command_pattern: '(?s)uip\s+df\s+records\s+query.*sortOptions'
149149
min_count: 1
150150
weight: 2.0
151151
pass_threshold: 1.0
152152

153153
- type: command_executed
154154
description: "Agent used selectedFields projection in query body"
155155
tool_name: "Bash"
156-
command_pattern: 'uip\s+df\s+records\s+query.*selectedFields'
156+
command_pattern: '(?s)uip\s+df\s+records\s+query.*selectedFields'
157157
min_count: 1
158158
weight: 2.0
159159
pass_threshold: 1.0

0 commit comments

Comments
 (0)