Skip to content

Commit 076b7c0

Browse files
suchintansuchintan
authored
ci(release-benchmark): deep workload suite in every language; restore the reverted evaluate() probe (#207)
Skyvern-AI/rustwright-cloud#190 --------- Co-authored-by: suchintan <3853670+suchintan@users.noreply.github.com>
1 parent f0a2635 commit 076b7c0

20 files changed

Lines changed: 1460 additions & 229 deletions

File tree

‎bindings/cases/DEEP-CONTRACT.md‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Deep benchmark contract
2+
3+
The deep manifests extend benchmark manifest version 1 with one optional
4+
case-level field:
5+
6+
{ "repeat": 25 }
7+
8+
repeat is an integer in the inclusive range 1 through 1000. Values outside
9+
that domain are invalid. If repeat is absent, its value is 1.
10+
11+
## Execution semantics
12+
13+
For repeat 1, the runner executes every step once in array order. A goto is not
14+
required, and behavior is exactly the legacy manifest-v1 behavior.
15+
16+
For repeat greater than 1:
17+
18+
1. The runner finds the first goto step. A repeated case without a goto is
19+
invalid and must be rejected before browser launch.
20+
2. Every step before and including that first goto is the setup prefix. The
21+
runner executes the setup prefix once.
22+
3. Every step after the first goto is the repeated block. The runner executes
23+
that block repeat times, in order, against the same page. Iterations are
24+
numbered from 1. Any later goto belongs to the repeated block and therefore
25+
runs once per iteration.
26+
27+
Assertions in the repeated block run in every iteration. The first failure
28+
stops the case. A repeated-block failure has this exact prefix:
29+
30+
iteration N: <error>
31+
32+
The Rust reference retains the original 1-based manifest step number inside
33+
the error, for example: iteration 3: step 7: title mismatch: ... . A setup
34+
failure is not iteration-prefixed.
35+
36+
Captures made by the setup prefix are retained. Captures made by the repeated
37+
block are iteration-local while an iteration runs; after each iteration they
38+
replace captures with the same names from earlier iterations. Consequently,
39+
the reported capture values are those from the last iteration. Duplicate
40+
capture names within one setup prefix or within one iteration remain errors.
41+
If an iteration fails, values captured before its failing step replace values
42+
from earlier iterations before the runner records the error.
43+
44+
The per-case ms value covers page creation, the one-time setup prefix, every
45+
iteration of the repeated block, and page close.
46+
47+
## Result compatibility
48+
49+
repeat does not change the result schema and must not add iteration metadata:
50+
51+
{
52+
"lang": "<language>",
53+
"results": [
54+
{
55+
"id": "<case-id>",
56+
"ok": true,
57+
"captures": {},
58+
"ms": 0.0
59+
}
60+
]
61+
}
62+
63+
Failed cases add only the existing string error field. Wave 2 language
64+
bindings and Playwright baselines must match the Rust reference byte-for-byte
65+
on this schema: the same keys, value types, optional-error rule, case order,
66+
capture names, and structural capture values.
67+
68+
Wave 2 must also add repeat to the machine-readable manifest schema and to
69+
each language runner's manifest validator. Those validators must apply the
70+
same default, inclusive 1-through-1000 domain, and
71+
repeat-greater-than-1/missing-goto rejection rule before browser launch.

‎bindings/cases/deep-smoke.json‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{
2+
"version": 1,
3+
"cases": [
4+
{
5+
"id": "dense-form-submit",
6+
"description": "Fill 36 fields, submit the form, and verify every controlled value on each iteration.",
7+
"html": "<!doctype html><html><head><title>Dense Form</title><style>body{font:14px sans-serif}#fields{display:grid;grid-template-columns:repeat(4,1fr);gap:8px}label{display:flex;flex-direction:column}output{display:block;margin-top:16px;font-weight:bold}</style></head><body><main><h1>Account migration</h1><form id='bulk-form'><div id='fields'></div><button id='submit' type='button'>Submit migration</button><output id='status'>pending</output></form></main><script>const fields=document.querySelector('#fields');fields.innerHTML=Array.from({length:36},(_,index)=>'<label>Field '+String(index+1).padStart(2,'0')+'<input id=\\'field-'+String(index+1).padStart(2,'0')+'\\' name=\\'field-'+String(index+1).padStart(2,'0')+'\\'></label>').join('');document.querySelector('#submit').addEventListener('click',()=>{const values=Array.from(document.querySelectorAll('#fields input'),input=>input.value);document.querySelector('#status').textContent='submitted:'+values.filter(Boolean).length;document.body.dataset.submissions=String(Number(document.body.dataset.submissions||0)+1);});</script></body></html>",
8+
"repeat": 3,
9+
"steps": [
10+
{ "op": "goto", "useCaseHtml": true, "waitUntil": "load" },
11+
{ "op": "fill", "selector": "#field-01", "value": "value-01" },
12+
{ "op": "fill", "selector": "#field-02", "value": "value-02" },
13+
{ "op": "fill", "selector": "#field-03", "value": "value-03" },
14+
{ "op": "fill", "selector": "#field-04", "value": "value-04" },
15+
{ "op": "fill", "selector": "#field-05", "value": "value-05" },
16+
{ "op": "fill", "selector": "#field-06", "value": "value-06" },
17+
{ "op": "fill", "selector": "#field-07", "value": "value-07" },
18+
{ "op": "fill", "selector": "#field-08", "value": "value-08" },
19+
{ "op": "fill", "selector": "#field-09", "value": "value-09" },
20+
{ "op": "fill", "selector": "#field-10", "value": "value-10" },
21+
{ "op": "fill", "selector": "#field-11", "value": "value-11" },
22+
{ "op": "fill", "selector": "#field-12", "value": "value-12" },
23+
{ "op": "fill", "selector": "#field-13", "value": "value-13" },
24+
{ "op": "fill", "selector": "#field-14", "value": "value-14" },
25+
{ "op": "fill", "selector": "#field-15", "value": "value-15" },
26+
{ "op": "fill", "selector": "#field-16", "value": "value-16" },
27+
{ "op": "fill", "selector": "#field-17", "value": "value-17" },
28+
{ "op": "fill", "selector": "#field-18", "value": "value-18" },
29+
{ "op": "fill", "selector": "#field-19", "value": "value-19" },
30+
{ "op": "fill", "selector": "#field-20", "value": "value-20" },
31+
{ "op": "fill", "selector": "#field-21", "value": "value-21" },
32+
{ "op": "fill", "selector": "#field-22", "value": "value-22" },
33+
{ "op": "fill", "selector": "#field-23", "value": "value-23" },
34+
{ "op": "fill", "selector": "#field-24", "value": "value-24" },
35+
{ "op": "fill", "selector": "#field-25", "value": "value-25" },
36+
{ "op": "fill", "selector": "#field-26", "value": "value-26" },
37+
{ "op": "fill", "selector": "#field-27", "value": "value-27" },
38+
{ "op": "fill", "selector": "#field-28", "value": "value-28" },
39+
{ "op": "fill", "selector": "#field-29", "value": "value-29" },
40+
{ "op": "fill", "selector": "#field-30", "value": "value-30" },
41+
{ "op": "fill", "selector": "#field-31", "value": "value-31" },
42+
{ "op": "fill", "selector": "#field-32", "value": "value-32" },
43+
{ "op": "fill", "selector": "#field-33", "value": "value-33" },
44+
{ "op": "fill", "selector": "#field-34", "value": "value-34" },
45+
{ "op": "fill", "selector": "#field-35", "value": "value-35" },
46+
{ "op": "fill", "selector": "#field-36", "value": "value-36" },
47+
{ "op": "click", "selector": "#submit" },
48+
{ "op": "textContent", "selector": "#status", "capture": "status" },
49+
{ "op": "assertText", "selector": "#status", "equals": "submitted:36" },
50+
{ "op": "assertEval", "expression": "Array.from(document.querySelectorAll('#fields input')).every((input,index)=>input.value==='value-'+String(index+1).padStart(2,'0'))", "equals": true }
51+
]
52+
},
53+
{
54+
"id": "tab-navigation-loop",
55+
"description": "Navigate a large in-page tab workspace repeatedly and assert the selected panel.",
56+
"html": "<!doctype html><html><head><title>Tab Workspace</title><style>[role=tab]{padding:8px}.panel[hidden]{display:none}.panel{padding:16px;border:1px solid #888}</style></head><body><main><h1>Operations workspace</h1><nav id='tabs' role='tablist'></nav><div id='panels'></div><output id='active'></output></main><script>const tabs=document.querySelector('#tabs');const panels=document.querySelector('#panels');tabs.innerHTML=Array.from({length:8},(_,index)=>'<button id=\\'tab-'+index+'\\' role=\\'tab\\'>Section '+index+'</button>').join('');panels.innerHTML=Array.from({length:8},(_,index)=>'<section id=\\'panel-'+index+'\\' class=\\'panel\\'>'+Array.from({length:40},(_,row)=>'<p>Panel '+index+' deterministic content '+row+'</p>').join('')+'</section>').join('');function select(index){document.querySelectorAll('[role=tab]').forEach((tab,item)=>tab.setAttribute('aria-selected',String(item===index)));document.querySelectorAll('.panel').forEach((panel,item)=>panel.hidden=item!==index);document.querySelector('#active').textContent='active:panel-'+index;}tabs.addEventListener('click',event=>select(Number(event.target.id.split('-')[1])));select(0);</script></body></html>",
57+
"repeat": 3,
58+
"steps": [
59+
{ "op": "goto", "useCaseHtml": true },
60+
{ "op": "click", "selector": "#tab-1" },
61+
{ "op": "click", "selector": "#tab-4" },
62+
{ "op": "click", "selector": "#tab-2" },
63+
{ "op": "click", "selector": "#tab-7" },
64+
{ "op": "textContent", "selector": "#active", "capture": "activePanel" },
65+
{ "op": "assertText", "selector": "#active", "equals": "active:panel-7" },
66+
{ "op": "evaluate", "expression": "()=>({selected:document.querySelector('[aria-selected=true]').id,visible:document.querySelector('.panel:not([hidden])').id})", "capture": "tabState" },
67+
{ "op": "assertEval", "expression": "({selected:document.querySelector('[aria-selected=true]').id,visible:document.querySelector('.panel:not([hidden])').id})", "equals": { "selected": "tab-7", "visible": "panel-7" } }
68+
]
69+
},
70+
{
71+
"id": "large-table-evaluate",
72+
"description": "Run repeated aggregate computations over a deterministic 400-row ledger.",
73+
"html": "<!doctype html><html><head><title>Ledger Aggregate</title><style>table{border-collapse:collapse}td,th{padding:3px 8px;border:1px solid #ccc}</style></head><body><main><h1>Regional ledger</h1><table id='ledger'><thead><tr><th>ID</th><th>Region</th><th>Amount</th></tr></thead><tbody></tbody></table></main><script>document.querySelector('#ledger tbody').innerHTML=Array.from({length:400},(_,index)=>'<tr data-region=\\'r'+index%5+'\\'><td>TX-'+String(index).padStart(4,'0')+'</td><td>r'+index%5+'</td><td class=\\'amount\\'>'+((index%25)+1)+'</td></tr>').join('');</script></body></html>",
74+
"repeat": 3,
75+
"steps": [
76+
{ "op": "goto", "useCaseHtml": true },
77+
{ "op": "evaluate", "expression": "()=>{const rows=Array.from(document.querySelectorAll('#ledger tbody tr'));const amounts=rows.map(row=>Number(row.querySelector('.amount').textContent));return {rows:rows.length,total:amounts.reduce((sum,value)=>sum+value,0),high:amounts.filter(value=>value>=20).length};}", "capture": "aggregate" },
78+
{ "op": "evaluate", "expression": "()=>Object.fromEntries(Array.from({length:5},(_,index)=>['r'+index,document.querySelectorAll('tr[data-region=r'+index+']').length]))", "capture": "regions" },
79+
{ "op": "evaluate", "expression": "()=>Array.from(document.querySelectorAll('#ledger tbody tr')).reduce((sum,row,index)=>sum+Number(row.children[2].textContent)*(index+1),0)", "capture": "weightedChecksum" },
80+
{ "op": "assertEval", "expression": "(()=>{const rows=Array.from(document.querySelectorAll('#ledger tbody tr'));const amounts=rows.map(row=>Number(row.querySelector('.amount').textContent));return {rows:rows.length,total:amounts.reduce((sum,value)=>sum+value,0),high:amounts.filter(value=>value>=20).length};})()", "equals": { "rows": 400, "total": 5200, "high": 96 } },
81+
{ "op": "assertEval", "expression": "Object.fromEntries(Array.from({length:5},(_,index)=>['r'+index,document.querySelectorAll('tr[data-region=r'+index+']').length]))", "equals": { "r0": 80, "r1": 80, "r2": 80, "r3": 80, "r4": 80 } }
82+
]
83+
}
84+
]
85+
}

0 commit comments

Comments
 (0)