Skip to content

Commit 0924c60

Browse files
andrebanandreban
authored
Merge pull request #17 from marisviana/feat/industry-vertical-evals
Add healthcare, real-estate, and commerce evals; improve travel entity mapping
2 parents 8dce8a6 + 5f59cdf commit 0924c60

File tree

9 files changed

+1169
-2
lines changed

9 files changed

+1169
-2
lines changed

‎evals-cli/examples/commerce/retail/evals.json‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
[
2+
{
3+
"messages": [
4+
{
5+
"role": "user",
6+
"type": "message",
7+
"content": "I'm looking for Nike running shoes under $120."
8+
}
9+
],
10+
"expectedCall": {
11+
"functionName": "searchProducts",
12+
"arguments": {
13+
"query": "running shoes",
14+
"brand": "Nike",
15+
"maxPrice": 120,
16+
"category": "Footwear"
17+
}
18+
}
19+
},
20+
{
21+
"messages": [
22+
{
23+
"role": "user",
24+
"type": "message",
25+
"content": "Find me the cheapest iPhone 15."
26+
}
27+
],
28+
"expectedCall": {
29+
"functionName": "searchProducts",
30+
"arguments": {
31+
"query": "iPhone 15",
32+
"brand": "Apple",
33+
"category": "Electronics",
34+
"sortBy": "price_low_to_high"
35+
}
36+
}
37+
},
38+
{
39+
"messages": [
40+
{
41+
"role": "user",
42+
"type": "message",
43+
"content": "I need 10.5 inch blue sneakers available for pickup in 90210."
44+
}
45+
],
46+
"expectedCall": {
47+
"functionName": "searchProducts",
48+
"arguments": {
49+
"query": "sneakers",
50+
"category": "Footwear",
51+
"location": "90210",
52+
"attributes": {
53+
"color": "blue",
54+
"size": "10.5 inch"
55+
}
56+
}
57+
}
58+
},
59+
{
60+
"messages": [
61+
{
62+
"role": "user",
63+
"type": "message",
64+
"content": "Find a gaming laptop under $1500, but don't show me any Dell models."
65+
}
66+
],
67+
"expectedCall": {
68+
"functionName": "searchProducts",
69+
"arguments": {
70+
"query": "gaming laptop",
71+
"maxPrice": 1500,
72+
"category": "Electronics"
73+
}
74+
}
75+
},
76+
{
77+
"messages": [
78+
{
79+
"role": "user",
80+
"type": "message",
81+
"content": "Amazon has the Sony WH-1000XM5 for $299. Can you match that for ID 'SONY-XM5'?"
82+
}
83+
],
84+
"expectedCall": {
85+
"functionName": "priceMatch",
86+
"arguments": {
87+
"productId": "SONY-XM5",
88+
"competitorPrice": 299
89+
}
90+
}
91+
},
92+
{
93+
"messages": [
94+
{
95+
"role": "user",
96+
"type": "message",
97+
"content": "Show me top-rated 4K TV from Samsung."
98+
}
99+
],
100+
"expectedCall": {
101+
"functionName": "searchProducts",
102+
"arguments": {
103+
"query": "4K TV",
104+
"brand": "Samsung",
105+
"category": "Electronics",
106+
"sortBy": "rating"
107+
}
108+
}
109+
},
110+
{
111+
"messages": [
112+
{
113+
"role": "user",
114+
"type": "message",
115+
"content": "I need some black ink cartridges for my HP printer."
116+
}
117+
],
118+
"expectedCall": {
119+
"functionName": "searchProducts",
120+
"arguments": {
121+
"query": "ink cartridges",
122+
"brand": "HP",
123+
"category": "Office Supplies",
124+
"attributes": {
125+
"color": "black"
126+
}
127+
}
128+
}
129+
},
130+
{
131+
"messages": [
132+
{
133+
"role": "user",
134+
"type": "message",
135+
"content": "Is there a brand new MacBook Pro available at the store in Austin?"
136+
}
137+
],
138+
"expectedCall": {
139+
"functionName": "searchProducts",
140+
"arguments": {
141+
"query": "MacBook Pro",
142+
"brand": "Apple",
143+
"category": "Electronics",
144+
"location": "Austin",
145+
"attributes": {
146+
"isNew": true
147+
}
148+
}
149+
}
150+
},
151+
{
152+
"messages": [
153+
{
154+
"role": "user",
155+
"type": "message",
156+
"content": "I want to buy a yacht for $50. What can you find?"
157+
}
158+
],
159+
"expectedCall": null
160+
},
161+
{
162+
"messages": [
163+
{
164+
"role": "user",
165+
"type": "message",
166+
"content": "Do you have the Dyson V15 in stock at 10001? I want to pick it up today."
167+
}
168+
],
169+
"expectedCall": {
170+
"functionName": "checkInventory",
171+
"arguments": {
172+
"productId": "Dyson V15",
173+
"zipCode": "10001"
174+
}
175+
}
176+
}
177+
]

‎evals-cli/examples/commerce/retail/schema.json‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
{
2+
"tools": [
3+
{
4+
"name": "searchProducts",
5+
"description": "Searches for items. MUST map 'iPhone/MacBook' to brand 'Apple'. Adjectives (e.g., 'black', 'new') must go in 'attributes'. Ignore negative constraints (e.g., 'not Dell')—do not populate the brand field for them.",
6+
"inputSchema": {
7+
"type": "object",
8+
"properties": {
9+
"query": {
10+
"type": "string",
11+
"description": "Base noun (e.g., 'laptop', 'shoes')."
12+
},
13+
"category": {
14+
"type": "string",
15+
"enum": [
16+
"Electronics",
17+
"Footwear",
18+
"Apparel",
19+
"Home Goods",
20+
"Office Supplies"
21+
]
22+
},
23+
"maxPrice": {
24+
"type": "number"
25+
},
26+
"brand": {
27+
"type": "string"
28+
},
29+
"location": {
30+
"type": "string",
31+
"description": "City or Store location."
32+
},
33+
"sortBy": {
34+
"type": "string",
35+
"enum": [
36+
"price_low_to_high",
37+
"price_high_to_low",
38+
"rating"
39+
]
40+
},
41+
"attributes": {
42+
"type": "object",
43+
"properties": {
44+
"color": {
45+
"type": "string"
46+
},
47+
"size": {
48+
"type": "string",
49+
"description": "Include units if mentioned (e.g., '10.5 inch')."
50+
},
51+
"isNew": {
52+
"type": "boolean"
53+
}
54+
}
55+
}
56+
},
57+
"required": [
58+
"query",
59+
"category"
60+
]
61+
}
62+
},
63+
{
64+
"name": "priceMatch",
65+
"description": "Compares prices.",
66+
"inputSchema": {
67+
"type": "object",
68+
"properties": {
69+
"productId": {
70+
"type": "string"
71+
},
72+
"competitorPrice": {
73+
"type": "number"
74+
}
75+
},
76+
"required": [
77+
"productId",
78+
"competitorPrice"
79+
]
80+
}
81+
},
82+
{
83+
"name": "checkInventory",
84+
"description": "Checks stock levels.",
85+
"inputSchema": {
86+
"type": "object",
87+
"properties": {
88+
"productId": {
89+
"type": "string"
90+
},
91+
"zipCode": {
92+
"type": "string"
93+
}
94+
},
95+
"required": [
96+
"productId"
97+
]
98+
}
99+
}
100+
]
101+
}

0 commit comments

Comments
 (0)