Skip to content

Commit c97c78e

Browse files
charlyautomatizacharlyautomatiza
committed
chore(evals): add skill eval manifests
1 parent 88c4276 commit c97c78e

3 files changed

Lines changed: 827 additions & 0 deletions

File tree

‎skills/k6-builder/evals/evals.json‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
{
2+
"skill_name": "k6-builder",
3+
"evals": [
4+
{
5+
"id": 4,
6+
"prompt": "Create a runnable k6 browser script for checkout flow in https://shop.example.com (login -> search -> add to cart -> checkout). SLA: checkout completion p95<4s.",
7+
"expected_output": "browser-checkout-runnable",
8+
"files": [],
9+
"assertions": [
10+
{
11+
"name": "browser_module",
12+
"text": "Must use k6 browser capabilities (browser/page APIs)"
13+
},
14+
{
15+
"name": "flow_steps",
16+
"text": "Must model the requested multi-step browser flow"
17+
},
18+
{
19+
"name": "checkout_sla",
20+
"text": "Must include a threshold aligned with checkout p95<4s"
21+
},
22+
{
23+
"name": "runnable_artifact",
24+
"text": "Must produce runnable script artifact and execution hints"
25+
}
26+
]
27+
},
28+
{
29+
"id": 0,
30+
"prompt": "I need a runnable k6 script for https://api.example.com with load scenario and SLA p95<300ms,error<1%.",
31+
"expected_output": "load-test-basic",
32+
"files": [],
33+
"assertions": [
34+
{
35+
"name": "runnable_artifact",
36+
"text": "Must deliver a runnable k6 artifact, not just a conceptual explanation"
37+
},
38+
{
39+
"name": "load_executor",
40+
"text": "Must recommend a coherent executor for load test, preferably ramping-vus"
41+
},
42+
{
43+
"name": "sla_thresholds",
44+
"text": "Must include thresholds derived from the SLA: p95<300ms and error<1%"
45+
},
46+
{
47+
"name": "named_default_function",
48+
"text": "The generated artifact must have a named default function, not an anonymous export default function ()"
49+
},
50+
{
51+
"name": "env_var_explicit",
52+
"text": "Must include an explicit required environment variables block with __ENV"
53+
}
54+
]
55+
},
56+
{
57+
"id": 1,
58+
"prompt": "Generate dev, staging, and prod configuration for a stress test with SLA p99<1s and no hard-coded secrets.",
59+
"expected_output": "multi-env-stress-config",
60+
"files": [],
61+
"assertions": [
62+
{
63+
"name": "multi_env_config",
64+
"text": "Must generate configuration for dev, staging, and prod with explicit differences per environment"
65+
},
66+
{
67+
"name": "safe_placeholders",
68+
"text": "Must not hard-code secrets and must use placeholders or __ENV for sensitive configuration"
69+
},
70+
{
71+
"name": "explicit_per_env_vus",
72+
"text": "Must show distinct and explicit VU counts for dev, staging, and prod"
73+
},
74+
{
75+
"name": "env_placeholders_present",
76+
"text": "Must use __ENV or .env.example for all URLs and environment-sensitive configuration"
77+
},
78+
{
79+
"name": "sla_threshold_all_envs",
80+
"text": "The SLA threshold p99<1s must appear in the configuration of all environments"
81+
}
82+
]
83+
},
84+
{
85+
"id": 5,
86+
"prompt": "Generate a runnable k6 test for transaction chain: authenticate -> create order -> pay order -> verify order status. API base URL via env var. SLA: end-to-end transaction p95<900ms.",
87+
"expected_output": "multi-step-transaction-runnable",
88+
"files": [],
89+
"assertions": [
90+
{
91+
"name": "multi_step_chain",
92+
"text": "Must model the full multi-step transaction sequence"
93+
},
94+
{
95+
"name": "correlation_handling",
96+
"text": "Must handle data correlation between steps (token/orderId)"
97+
},
98+
{
99+
"name": "e2e_threshold",
100+
"text": "Must include threshold aligned to end-to-end SLA"
101+
},
102+
{
103+
"name": "env_usage",
104+
"text": "Must use __ENV for base URL or credentials"
105+
}
106+
]
107+
},
108+
{
109+
"id": 2,
110+
"prompt": "I have a smoke test plan and want the final runnable artifact plus a recommendation to validate at the end.",
111+
"expected_output": "smoke-artifact",
112+
"files": [],
113+
"assertions": [
114+
{
115+
"name": "smoke_artifact",
116+
"text": "Must deliver a minimal runnable artifact for smoke test"
117+
},
118+
{
119+
"name": "smoke_executor",
120+
"text": "Must use a configuration coherent with smoke test, preferably constant-vus"
121+
},
122+
{
123+
"name": "validate_handoff",
124+
"text": "Must recommend k6-validate as the next step"
125+
},
126+
{
127+
"name": "named_default_function",
128+
"text": "The generated smoke artifact must have a named default function"
129+
}
130+
]
131+
},
132+
{
133+
"id": 3,
134+
"prompt": "Build a runnable k6 script to load test my WebSocket chat backend at wss://chat.example.com/socket. Need 200 concurrent users and SLA message latency p95<150ms, error<1%.",
135+
"expected_output": "websocket-chat-runnable",
136+
"files": [],
137+
"assertions": [
138+
{
139+
"name": "ws_protocol_usage",
140+
"text": "Must use WebSocket protocol primitives (ws.connect or equivalent)"
141+
},
142+
{
143+
"name": "ws_latency_threshold",
144+
"text": "Must include thresholds for WebSocket latency/SLA"
145+
},
146+
{
147+
"name": "concurrency_model",
148+
"text": "Must configure a coherent concurrency model for 200 users"
149+
},
150+
{
151+
"name": "runnable_artifact",
152+
"text": "Must deliver runnable artifact code with imports/options/default flow"
153+
}
154+
]
155+
},
156+
{
157+
"id": 6,
158+
"prompt": "Build a runnable k6 gRPC stress script for OrderService/CreateOrder at grpc://perf-orders.internal:8443. Keep auth token in env vars only. SLA: p99<250ms and error<1%.",
159+
"assertions": [
160+
{
161+
"name": "grpc_runnable_artifact",
162+
"text": "Must provide runnable gRPC artifact with options and executable flow"
163+
},
164+
{
165+
"name": "grpc_metric_threshold",
166+
"text": "Must include grpc_req_duration threshold aligned with p99<250ms"
167+
},
168+
{
169+
"name": "secure_env_usage",
170+
"text": "Must use __ENV for endpoint/auth and avoid hard-coded secrets"
171+
},
172+
{
173+
"name": "named_default_function",
174+
"text": "Must use a named export default function"
175+
}
176+
],
177+
"files": [],
178+
"expected_output": "grpc-stress-runnable"
179+
},
180+
{
181+
"id": 7,
182+
"prompt": "Create a runnable k6 browser smoke script for https://shop.example.net login and product page render. Keep it minimal (5 VUs, 1m) with SLA p95<2s.",
183+
"assertions": [
184+
{
185+
"name": "browser_module_usage",
186+
"text": "Must use k6 browser APIs for the flow"
187+
},
188+
{
189+
"name": "smoke_profile",
190+
"text": "Must align with smoke/minimal profile"
191+
},
192+
{
193+
"name": "sla_in_thresholds",
194+
"text": "Must include threshold aligned with p95<2s"
195+
},
196+
{
197+
"name": "validate_handoff",
198+
"text": "Must include a next step recommending validation"
199+
}
200+
],
201+
"files": [],
202+
"expected_output": "browser-smoke-runnable"
203+
}
204+
]
205+
}

0 commit comments

Comments
 (0)