Skip to content

Commit e5d7c59

Browse files
xlcxlc
committed
feat: Add benchmark suite for core JAM components
Adds a `JAMBenchmarks` executable using `swift-benchmark` to test the performance of erasure coding, shuffling, and STF application. This change includes necessary helpers, public API adjustments, and VS Code launch configurations.
1 parent cfe98d5 commit e5d7c59

File tree

6 files changed

+165
-7
lines changed

6 files changed

+165
-7
lines changed

‎.vscode/launch.json‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,24 @@
7171
"name": "Release BokaFuzzer (Fuzzing)",
7272
"program": "${workspaceFolder:boka}/Fuzzing/.build/release/BokaFuzzer",
7373
"preLaunchTask": "swift: Build Release BokaFuzzer (Fuzzing)"
74+
},
75+
{
76+
"type": "swift",
77+
"request": "launch",
78+
"args": [],
79+
"cwd": "${workspaceFolder:boka}/JAMTests",
80+
"name": "Debug JAMBenchmarks (JAMTests)",
81+
"program": "${workspaceFolder:boka}/JAMTests/.build/debug/JAMBenchmarks",
82+
"preLaunchTask": "swift: Build Debug JAMBenchmarks (JAMTests)"
83+
},
84+
{
85+
"type": "swift",
86+
"request": "launch",
87+
"args": [],
88+
"cwd": "${workspaceFolder:boka}/JAMTests",
89+
"name": "Release JAMBenchmarks (JAMTests)",
90+
"program": "${workspaceFolder:boka}/JAMTests/.build/release/JAMBenchmarks",
91+
"preLaunchTask": "swift: Build Release JAMBenchmarks (JAMTests)"
7492
}
7593
]
76-
}
94+
}

‎Blockchain/Sources/Blockchain/RuntimeProtocols/Accumulation.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -569,7 +569,7 @@ extension Accumulation {
569569

570570
let rightQueueItems = accumulationQueue.array[index...]
571571
let leftQueueItems = accumulationQueue.array[0 ..< index]
572-
var allQueueItems = rightQueueItems.flatMap { $0 } + leftQueueItems.flatMap { $0 } + newQueueItems
572+
var allQueueItems = rightQueueItems.flatMap(\.self) + leftQueueItems.flatMap(\.self) + newQueueItems
573573

574574
editQueue(items: &allQueueItems, accumulatedPackages: Set(zeroPrereqReports.map(\.packageSpecification.workPackageHash)))
575575

@@ -697,7 +697,7 @@ extension Accumulation {
697697
for (service, _) in accumulateOutput.gasUsed {
698698
if accumulateStats[service] != nil { continue }
699699

700-
let digests = accumulated.compactMap(\.digests).flatMap { $0 }
700+
let digests = accumulated.compactMap(\.digests).flatMap(\.self)
701701
let num = digests.filter { $0.serviceIndex == service }.count
702702

703703
if num == 0 { continue }

‎JAMTests/Benchmarks/JAMBenchmarks/JAMBenchmarks.swift‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import Benchmark
2+
import Blockchain
3+
import Codec
4+
import Foundation
5+
import JAMTests
6+
import Utils
7+
8+
@inline(__always) private func w3f(_ path: String, ext: String = "bin") -> [Testcase] {
9+
(try? JAMBenchSupport.w3fTestcases(at: path, ext: ext)) ?? []
10+
}
11+
12+
// Simple sync runner for async operations (Swift 6-safe)
13+
private final class _UnsafeBox<T>: @unchecked Sendable { var value: T? }
14+
@inline(__always) private func blockOn<T>(_ work: @escaping @Sendable () async -> T) -> T {
15+
let g = DispatchGroup(); g.enter()
16+
let box = _UnsafeBox<T>()
17+
Task.detached { box.value = await work(); g.leave() }
18+
g.wait()
19+
return box.value!
20+
}
21+
22+
let benchmarks: @Sendable () -> Void = {
23+
// W3F Erasure (full): encode + reconstruct
24+
struct ErasureCodingTestcase: Codable { let data: Data; let shards: [Data] }
25+
let erasureCases = w3f("erasure/full")
26+
if !erasureCases.isEmpty {
27+
let cfg = TestVariants.full.config
28+
let basicSize = cfg.value.erasureCodedPieceSize
29+
let recoveryCount = cfg.value.totalNumberOfValidators
30+
let originalCount = basicSize / 2
31+
Benchmark("w3f.erasure.full.encode+reconstruct") { _ in
32+
for c in erasureCases {
33+
if let t = try? JamDecoder.decode(ErasureCodingTestcase.self, from: c.data, withConfig: cfg),
34+
let shards = try? ErasureCoding.chunk(data: t.data, basicSize: basicSize, recoveryCount: recoveryCount)
35+
{
36+
let typed = shards.enumerated().map { ErasureCoding.Shard(data: $0.element, index: UInt32($0.offset)) }
37+
_ = try? ErasureCoding.reconstruct(
38+
shards: Array(typed.prefix(originalCount)),
39+
basicSize: basicSize,
40+
originalCount: originalCount,
41+
recoveryCount: recoveryCount
42+
)
43+
}
44+
}
45+
}
46+
}
47+
48+
// W3F Shuffle (JSON + compute)
49+
struct ShuffleTestCase: Codable { let input: Int; let entropy: String; let output: [Int] }
50+
if let data = try? JAMBenchSupport.w3fFile(at: "shuffle/shuffle_tests", ext: "json"),
51+
let tests = try? JSONDecoder().decode([ShuffleTestCase].self, from: data),
52+
!tests.isEmpty
53+
{
54+
Benchmark("w3f.shuffle") { _ in
55+
for t in tests {
56+
var input = Array(0 ..< t.input)
57+
if let e = Data32(fromHexString: t.entropy) { input.shuffle(randomness: e); blackHole(input) }
58+
}
59+
}
60+
}
61+
62+
// JamTestnet traces (full): decode + touch roots
63+
let tracePaths = ["traces/fallback", "traces/safrole", "traces/storage", "traces/preimages"]
64+
let traces = tracePaths.flatMap { w3f($0) }
65+
if !traces.isEmpty {
66+
Benchmark("w3f.jamtestnet.apply.full") { benchmark in
67+
for _ in benchmark.scaledIterations {
68+
for c in traces {
69+
let tc = try! JamTestnet.decodeTestcase(c, config: TestVariants.full.config)
70+
// Run full STF apply and touch resulting state root
71+
let result = blockOn {
72+
await (try? JamTestnet.runSTF(tc, config: TestVariants.full.config))
73+
}
74+
switch result {
75+
case let .success(stateRef):
76+
let root = blockOn { await stateRef.value.stateRoot }
77+
blackHole(root)
78+
default:
79+
blackHole(c.description)
80+
}
81+
}
82+
}
83+
}
84+
}
85+
86+
// RecentHistory STF (full): updatePartial + update
87+
struct ReportedWorkPackage: Codable { let hash: Data32; let exportsRoot: Data32 }
88+
struct RecentHistoryInput: Codable {
89+
let headerHash: Data32; let parentStateRoot: Data32; let accumulateRoot: Data32; let workPackages: [ReportedWorkPackage]
90+
}
91+
struct RecentHistoryTestcase: Codable { let input: RecentHistoryInput; let preState: RecentHistory; let postState: RecentHistory }
92+
let historyCases = w3f("stf/history/full")
93+
if !historyCases.isEmpty {
94+
let cfg = TestVariants.full.config
95+
Benchmark("w3f.history.full.update") { _ in
96+
for c in historyCases {
97+
let tc = try! JamDecoder.decode(RecentHistoryTestcase.self, from: c.data, withConfig: cfg)
98+
var state = tc.preState
99+
state.updatePartial(parentStateRoot: tc.input.parentStateRoot)
100+
let lookup = Dictionary(uniqueKeysWithValues: tc.input.workPackages.map { ($0.hash, $0.exportsRoot) })
101+
state.update(headerHash: tc.input.headerHash, accumulateRoot: tc.input.accumulateRoot, lookup: lookup)
102+
blackHole(state)
103+
}
104+
}
105+
}
106+
}

‎JAMTests/Package.swift‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ let package = Package(
2222
.package(path: "../Blockchain"),
2323
.package(path: "../PolkaVM"),
2424
.package(url: "https://github.com/apple/swift-testing.git", branch: "6.0.0"),
25+
.package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.29.4")),
2526
],
2627
targets: [
2728
// Targets are the basic building blocks of a package, defining a module or a test suite.
@@ -56,6 +57,25 @@ let package = Package(
5657
.interoperabilityMode(.Cxx),
5758
]
5859
),
60+
.executableTarget(
61+
name: "JAMBenchmarks",
62+
dependencies: [
63+
.product(name: "Benchmark", package: "package-benchmark"),
64+
"JAMTests",
65+
"Utils",
66+
"Codec",
67+
],
68+
path: "Benchmarks/JAMBenchmarks",
69+
cxxSettings: [
70+
.unsafeFlags(["-Wno-incomplete-umbrella"]),
71+
],
72+
swiftSettings: [
73+
.interoperabilityMode(.Cxx),
74+
],
75+
plugins: [
76+
.plugin(name: "BenchmarkPlugin", package: "package-benchmark"),
77+
],
78+
),
5979
],
6080
swiftLanguageModes: [.version("6")]
6181
)

‎JAMTests/Sources/JAMTests/BenchSupport.swift‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import Foundation
2+
3+
// Public helpers to expose W3F testcases for external benchmark targets.
4+
public enum JAMBenchSupport {
5+
// Loads W3F testcases from the embedded resources within the JAMTests module.
6+
public static func w3fTestcases(at path: String, ext: String = "bin") throws -> [Testcase] {
7+
try TestLoader.getTestcases(path: path, extension: ext, src: .w3f)
8+
}
9+
10+
// Load a single W3F resource file (e.g. a companion JSON).
11+
public static func w3fFile(at path: String, ext: String) throws -> Data {
12+
try TestLoader.getFile(path: path, extension: ext, src: .w3f)
13+
}
14+
}

‎JAMTests/Sources/JAMTests/JamTestnet.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import Codec
33
import Foundation
44
import Utils
55

6-
public struct JamTestnetTestcase: Codable {
6+
public struct JamTestnetTestcase: Codable, Sendable {
77
public var preState: TestState
88
public var block: Block
99
public var postState: TestState
@@ -18,12 +18,12 @@ extension ProtocolConfig {
1818
}
1919
}
2020

21-
public struct KeyVal: Codable {
21+
public struct KeyVal: Codable, Sendable {
2222
public var key: Data31
2323
public var value: Data
2424
}
2525

26-
public struct TestState: Codable {
26+
public struct TestState: Codable, Sendable {
2727
public var root: Data32
2828
public var keyvals: [KeyVal]
2929

@@ -64,7 +64,7 @@ public enum JamTestnet {
6464
try JamDecoder.decode(JamTestnetTestcase.self, from: input.data, withConfig: config, allowTrailingBytes: true)
6565
}
6666

67-
static func runSTF(
67+
public static func runSTF(
6868
_ testcase: JamTestnetTestcase,
6969
config: ProtocolConfigRef = TestVariants.tiny.config
7070
) async throws -> Result<StateRef, Error> {

0 commit comments

Comments
 (0)