Skip to content

Commit fcf2bcc

Browse files
pgherveouclaude
pgherveou
and
claude
committed
Update benchmark reports and build tooling
Enhanced benchmark reporting with opcode gas analysis, improved contract comparison metrics, and updated build configurations. Added cache and output directories to gitignore. Co-Authored-By: Claude Sonnet 4.5 <[email protected]>
1 parent 7452869 commit fcf2bcc

File tree

11 files changed

+489
-155
lines changed

11 files changed

+489
-155
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22
logs
33
*.log
44
.env
5+
out/
6+
contracts/out-evm/
7+
contracts/out-pvm/
8+
cache/
59

610
npm-debug.log*
711
yarn-debug.log*

‎benchmark/reports.ts‎

Lines changed: 131 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -204,18 +204,22 @@ async function generateContractComparison() {
204204
// Get all transactions with contract_id extracted in SQL
205205
const txRows = db.prepare(`
206206
SELECT
207-
contract_id,
208-
contract_name,
209-
transaction_name,
210-
gas_used,
211-
weight_consumed_ref_time,
212-
weight_consumed_proof_size,
213-
base_call_weight_ref_time,
214-
base_call_weight_proof_size
215-
FROM transactions
216-
WHERE chain_name == 'eth-rpc'
217-
ORDER BY contract_id, transaction_name, (base_call_weight_ref_time + weight_consumed_ref_time) ASC
207+
t.chain_name,
208+
t.contract_id,
209+
t.contract_name,
210+
t.transaction_name,
211+
t.gas_used,
212+
t.weight_consumed_ref_time,
213+
t.weight_consumed_proof_size,
214+
t.base_call_weight_ref_time,
215+
t.base_call_weight_proof_size,
216+
SUM(s.gas_cost) as total_opcode_gas
217+
FROM transactions t
218+
LEFT JOIN transaction_steps s ON s.hash = t.hash AND s.chain_name = t.chain_name
219+
GROUP BY t.hash, t.chain_name
220+
ORDER BY t.chain_name, t.contract_id, t.transaction_name, t.gas_used ASC
218221
`).all() as Array<{
222+
chain_name: string
219223
contract_name: string
220224
transaction_name: string
221225
gas_used: number
@@ -224,68 +228,134 @@ async function generateContractComparison() {
224228
weight_consumed_proof_size: number | null
225229
base_call_weight_ref_time: number | null
226230
base_call_weight_proof_size: number | null
231+
total_opcode_gas: number | null
227232
}>
228233

229-
const groupedByContractAndTx = Object.groupBy(
230-
txRows,
231-
(row) => `${row.contract_id}:${row.transaction_name}`,
232-
)
234+
// Group by chain first
235+
const byChain = Object.groupBy(txRows, (row) => row.chain_name)
236+
237+
for (const [chainName, chainRows] of Object.entries(byChain)) {
238+
if (!chainRows) continue
239+
240+
markdown += `## Chain: ${chainName}\n\n`
233241

234-
// Generate one table per (contract_id, operation)
235-
for (
236-
const [groupKey, implementations] of Object.entries(
237-
groupedByContractAndTx,
242+
const groupedByContractAndTx = Object.groupBy(
243+
chainRows,
244+
(row) => `${row.contract_id}:${row.transaction_name}`,
238245
)
239-
) {
240-
if (!implementations) continue
241246

242-
const [contract_id, transaction_name] = groupKey.split(':')
243-
markdown += `## ${contract_id} - ${transaction_name}\n\n`
247+
// Generate one table per (contract_id, operation)
248+
for (
249+
const [groupKey, implementations] of Object.entries(
250+
groupedByContractAndTx,
251+
)
252+
) {
253+
if (!implementations) continue
244254

245-
// Find best (lowest) total weight ref_time
246-
const bestRefTime = Math.min(
247-
...implementations
248-
.filter((row) =>
255+
const [contract_id, transaction_name] = groupKey.split(':')
256+
markdown += `### ${contract_id} - ${transaction_name}\n\n`
257+
258+
// Check if this chain has weight data (eth-rpc) or just gas_used (Geth)
259+
const hasWeightData = implementations.some(
260+
(row) =>
249261
row.weight_consumed_ref_time !== null &&
250-
row.base_call_weight_ref_time !== null
262+
row.base_call_weight_ref_time !== null,
263+
)
264+
265+
if (hasWeightData) {
266+
// For eth-rpc: show ref_time, vs Best, % metered, pov
267+
// Find best (lowest) total weight ref_time
268+
const bestRefTime = Math.min(
269+
...implementations
270+
.filter((row) =>
271+
row.weight_consumed_ref_time !== null &&
272+
row.base_call_weight_ref_time !== null
273+
)
274+
.map((row) =>
275+
row.base_call_weight_ref_time! +
276+
row.weight_consumed_ref_time!
277+
),
251278
)
252-
.map((row) =>
253-
row.base_call_weight_ref_time! +
254-
row.weight_consumed_ref_time!
255-
),
256-
)
257279

258-
const tableData = implementations.map((row) => {
259-
const result: Record<string, string> = {
260-
'Implementation': row.contract_name,
261-
}
280+
// Sort by total ref_time
281+
const sorted = [...implementations].sort((a, b) => {
282+
const aTotal = (a.base_call_weight_ref_time ?? 0) +
283+
(a.weight_consumed_ref_time ?? 0)
284+
const bTotal = (b.base_call_weight_ref_time ?? 0) +
285+
(b.weight_consumed_ref_time ?? 0)
286+
return aTotal - bTotal
287+
})
288+
289+
const tableData = sorted.map((row) => {
290+
const result: Record<string, string> = {
291+
'Implementation': row.contract_name,
292+
}
293+
294+
if (
295+
row.weight_consumed_ref_time !== null &&
296+
row.base_call_weight_ref_time !== null
297+
) {
298+
const totalRefTime = row.base_call_weight_ref_time +
299+
row.weight_consumed_ref_time
300+
const totalProofSize =
301+
(row.base_call_weight_proof_size ?? 0) +
302+
(row.weight_consumed_proof_size ?? 0)
303+
const meterPercent =
304+
((row.weight_consumed_ref_time / totalRefTime) *
305+
100)
306+
.toFixed(1)
307+
const vsBest = totalRefTime === bestRefTime
308+
? '-'
309+
: `+${
310+
((totalRefTime / bestRefTime - 1) * 100)
311+
.toFixed(1)
312+
}%`
313+
314+
result['ref_time'] = totalRefTime
315+
.toLocaleString()
316+
result['vs Best'] = vsBest
317+
result['% metered'] = `${meterPercent}%`
318+
result['pov'] = totalProofSize.toLocaleString()
319+
}
320+
321+
return result
322+
})
323+
324+
markdown += table(tableData) + '\n\n'
325+
} else {
326+
// For Geth: show gas_used, vs Best, % metered
327+
const bestGasUsed = Math.min(
328+
...implementations.map((row) => row.gas_used),
329+
)
262330

263-
if (
264-
row.weight_consumed_ref_time !== null &&
265-
row.base_call_weight_ref_time !== null
266-
) {
267-
const totalRefTime = row.base_call_weight_ref_time +
268-
row.weight_consumed_ref_time
269-
const totalProofSize = (row.base_call_weight_proof_size ?? 0) +
270-
(row.weight_consumed_proof_size ?? 0)
271-
const meterPercent =
272-
((row.weight_consumed_ref_time / totalRefTime) * 100)
273-
.toFixed(1)
274-
const vsBest = totalRefTime === bestRefTime
275-
? '-'
276-
: `+${((totalRefTime / bestRefTime - 1) * 100).toFixed(1)}%`
277-
278-
result['ref_time'] = totalRefTime
279-
.toLocaleString()
280-
result['vs Best'] = vsBest
281-
result['% metered'] = `${meterPercent}%`
282-
result['pov'] = totalProofSize.toLocaleString()
283-
}
331+
// Sort by gas_used
332+
const sorted = [...implementations].sort((a, b) =>
333+
a.gas_used - b.gas_used
334+
)
284335

285-
return result
286-
})
336+
const tableData = sorted.map((row) => {
337+
const vsBest = row.gas_used === bestGasUsed
338+
? '-'
339+
: `+${
340+
((row.gas_used / bestGasUsed - 1) * 100).toFixed(1)
341+
}%`
342+
const meterPercent = row.total_opcode_gas
343+
? ((row.total_opcode_gas / row.gas_used) * 100).toFixed(
344+
1,
345+
)
346+
: '0.0'
287347

288-
markdown += table(tableData) + '\n\n'
348+
return {
349+
'Implementation': row.contract_name,
350+
'gas_used': row.gas_used.toLocaleString(),
351+
'vs Best': vsBest,
352+
'% metered': `${meterPercent}%`,
353+
}
354+
})
355+
356+
markdown += table(tableData) + '\n\n'
357+
}
358+
}
289359
}
290360

291361
await Deno.writeTextFile(

‎benchmark/reports/bytecode_size_comparison.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ Generated on: 2025-12-19
66

77
| Contract | VM Type | Size (bytes) | vs Smallest |
88
| ------------------- | ------- | ------------ | ----------- |
9-
| fibonacci_rust | PVM | 220 | - |
9+
| fibonacci_u32_rust | PVM | 220 | - |
1010
| fibonacci_u128_rust | PVM | 296 | +34.5% |
1111
| Fibonacci_evm | EVM | 391 | +77.7% |
1212
| fibonacci_u256_rust | PVM | 698 | +217.3% |

‎benchmark/reports/contract_comparison.md‎

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,49 @@
11
# Revive Contract Comparison
22

3-
Generated on: 2025-12-19
3+
Generated on: 2026-01-13
44

55
Comparison of gas usage across different contract implementations.
66

7-
## Fibonacci - deploy
7+
## Chain: Geth
8+
9+
### Fibonacci - deploy
10+
11+
| Implementation | gas_used | vs Best | % metered |
12+
| -------------- | -------- | ------- | --------- |
13+
| Fibonacci_evm | 112,683 | - | 0.1% |
14+
15+
16+
### Fibonacci - fib_10
17+
18+
| Implementation | gas_used | vs Best | % metered |
19+
| -------------- | -------- | ------- | --------- |
20+
| Fibonacci_evm | 47,664 | - | 0.2% |
21+
22+
23+
### SimpleToken - deploy
24+
25+
| Implementation | gas_used | vs Best | % metered |
26+
| --------------- | -------- | ------- | --------- |
27+
| SimpleToken_evm | 152,967 | - | 0.1% |
28+
29+
30+
### SimpleToken - mint
31+
32+
| Implementation | gas_used | vs Best | % metered |
33+
| --------------- | -------- | ------- | --------- |
34+
| SimpleToken_evm | 68,105 | - | 35.3% |
35+
36+
37+
### SimpleToken - transfer
38+
39+
| Implementation | gas_used | vs Best | % metered |
40+
| --------------- | -------- | ------- | --------- |
41+
| SimpleToken_evm | 46,257 | - | 51.9% |
42+
43+
44+
## Chain: eth-rpc
45+
46+
### Fibonacci - deploy
847

948
| Implementation | ref_time | vs Best | % metered | pov |
1049
| ------------------- | ------------- | ------- | --------- | ------ |
@@ -15,7 +54,8 @@ Comparison of gas usage across different contract implementations.
1554
| Fibonacci_pvm | 1,557,918,056 | +2.4% | 0.7% | 6,934 |
1655
| fibonacci_ink | 1,718,162,422 | +13.0% | 10.8% | 17,191 |
1756

18-
## Fibonacci - fib_10
57+
58+
### Fibonacci - fib_10
1959

2060
| Implementation | ref_time | vs Best | % metered | pov |
2161
| ------------------- | ------------- | ------- | --------- | ------ |
@@ -26,7 +66,8 @@ Comparison of gas usage across different contract implementations.
2666
| Fibonacci_pvm | 1,587,091,712 | +58.1% | 42.5% | 10,322 |
2767
| fibonacci_u256_rust | 2,512,824,851 | +150.3% | 63.7% | 9,130 |
2868

29-
## SimpleToken - deploy
69+
70+
### SimpleToken - deploy
3071

3172
| Implementation | ref_time | vs Best | % metered | pov |
3273
| -------------------------- | ------------- | ------- | --------- | ------ |
@@ -35,7 +76,8 @@ Comparison of gas usage across different contract implementations.
3576
| SimpleToken_pvm | 1,610,852,522 | +5.4% | 0.6% | 6,934 |
3677
| simple_token_ink | 1,843,184,400 | +20.6% | 11.7% | 17,191 |
3778

38-
## SimpleToken - mint
79+
80+
### SimpleToken - mint
3981

4082
| Implementation | ref_time | vs Best | % metered | pov |
4183
| -------------------------- | ------------- | ------- | --------- | ------ |
@@ -44,11 +86,14 @@ Comparison of gas usage across different contract implementations.
4486
| simple_token_no_alloc_rust | 1,505,513,738 | +5.2% | 39.3% | 53,929 |
4587
| simple_token_ink | 1,687,565,485 | +17.9% | 45.9% | 56,631 |
4688

47-
## SimpleToken - transfer
89+
90+
### SimpleToken - transfer
4891

4992
| Implementation | ref_time | vs Best | % metered | pov |
5093
| -------------------------- | ------------- | ------- | --------- | ------ |
5194
| SimpleToken_evm | 1,461,245,351 | - | 37.5% | 49,655 |
5295
| SimpleToken_pvm | 1,554,253,981 | +6.4% | 41.2% | 54,961 |
5396
| simple_token_no_alloc_rust | 1,562,060,164 | +6.9% | 41.5% | 53,961 |
5497
| simple_token_ink | 2,012,500,642 | +37.7% | 54.6% | 77,209 |
98+
99+

0 commit comments

Comments
 (0)