Skip to content

Commit 6a72803

Browse files
AKonnyakuAKonnyaku
committed
feat: refactor JMH benchmarks to isolate initialization overhead from measurements (#496)
1 parent b334732 commit 6a72803

File tree

4 files changed

+425
-172
lines changed

4 files changed

+425
-172
lines changed

‎src/test/java/org/casbin/jcasbin/main/benchmark/CachedEnforcerBenchmarkTest.java‎

Lines changed: 106 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,13 @@
2525
import java.util.List;
2626
import java.util.concurrent.TimeUnit;
2727

28-
@BenchmarkMode(Mode.Throughput)
29-
@Warmup(iterations = 3, time = 1)
28+
@BenchmarkMode({Mode.Throughput, Mode.AverageTime})
29+
@Warmup(iterations = 5, time = 1)
3030
@Measurement(iterations = 5, time = 1)
3131
@Threads(1)
3232
@Fork(1)
3333
@State(value = Scope.Benchmark)
34-
@OutputTimeUnit(TimeUnit.SECONDS)
34+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
3535
public class CachedEnforcerBenchmarkTest {
3636

3737
public static class TestResource {
@@ -71,38 +71,57 @@ public void benchmarkCachedRaw() {
7171
rawEnforce("alice", "data1", "read");
7272
}
7373

74+
private CachedEnforcer cachedBasicModelEnforcer;
75+
76+
@Setup(Level.Trial)
77+
public void setupCachedBasicModel() {
78+
cachedBasicModelEnforcer = new CachedEnforcer("examples/basic_model.conf", "examples/basic_policy.csv");
79+
}
80+
7481
@Benchmark
7582
public void benchmarkCachedBasicModel() {
76-
CachedEnforcer e = new CachedEnforcer("examples/basic_model.conf", "examples/basic_policy.csv");
77-
e.enforce("alice", "data1", "read");
83+
cachedBasicModelEnforcer.enforce("alice", "data1", "read");
84+
}
85+
86+
private CachedEnforcer cachedRBACModelEnforcer;
87+
88+
@Setup(Level.Trial)
89+
public void setupCachedRBACModel() {
90+
cachedRBACModelEnforcer = new CachedEnforcer("examples/rbac_model.conf", "examples/rbac_policy.csv");
7891
}
7992

8093
@Benchmark
8194
public void benchmarkCachedRBACModel() {
82-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "examples/rbac_policy.csv");
83-
e.enforce("alice", "data2", "read");
95+
cachedRBACModelEnforcer.enforce("alice", "data2", "read");
8496
}
8597

86-
@Benchmark
87-
public void benchmarkCachedRBACModelSmall() {
88-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
98+
private CachedEnforcer cachedRBACModelSmallEnforcer;
99+
100+
@Setup(Level.Trial)
101+
public void setupCachedRBACModelSmall() {
102+
cachedRBACModelSmallEnforcer = new CachedEnforcer("examples/rbac_model.conf", "");
89103

90104
// 100 roles, 10 resources.
91105
for (int i = 0; i < 100; i++) {
92-
e.addPolicy(String.format("group%d", i), String.format("data%d", i / 10), "read");
106+
cachedRBACModelSmallEnforcer.addPolicy(String.format("group%d", i), String.format("data%d", i / 10), "read");
93107
}
94108

95109
// 1000 users.
96110
for (int i = 0; i < 1000; i++) {
97-
e.addGroupingPolicy(String.format("user%d", i), String.format("group%d", i / 10));
111+
cachedRBACModelSmallEnforcer.addGroupingPolicy(String.format("user%d", i), String.format("group%d", i / 10));
98112
}
99-
100-
e.enforce("user501", "data9", "read");
101113
}
102114

103115
@Benchmark
104-
public void benchmarkCachedRBACModelMedium() {
105-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
116+
public void benchmarkCachedRBACModelSmall() {
117+
cachedRBACModelSmallEnforcer.enforce("user501", "data9", "read");
118+
}
119+
120+
private CachedEnforcer cachedRBACModelMediumEnforcer;
121+
122+
@Setup(Level.Trial)
123+
public void setupCachedRBACModelMedium() {
124+
cachedRBACModelMediumEnforcer = new CachedEnforcer("examples/rbac_model.conf", "");
106125

107126
// 1000 roles, 100 resources.
108127
List<List<String>> pPolicies = new ArrayList<>();
@@ -113,7 +132,7 @@ public void benchmarkCachedRBACModelMedium() {
113132
policy.add("read");
114133
pPolicies.add(policy);
115134
}
116-
e.addPolicies(pPolicies);
135+
cachedRBACModelMediumEnforcer.addPolicies(pPolicies);
117136

118137
// 10000 users.
119138
List<List<String>> gPolicies = new ArrayList<>();
@@ -123,14 +142,19 @@ public void benchmarkCachedRBACModelMedium() {
123142
policy.add(String.format("group%d", i / 10));
124143
gPolicies.add(policy);
125144
}
126-
e.addGroupingPolicies(gPolicies);
127-
128-
e.enforce("user5001", "data150", "read");
145+
cachedRBACModelMediumEnforcer.addGroupingPolicies(gPolicies);
129146
}
130147

131148
@Benchmark
132-
public void benchmarkCachedRBACModelLarge() {
133-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
149+
public void benchmarkCachedRBACModelMedium() {
150+
cachedRBACModelMediumEnforcer.enforce("user5001", "data150", "read");
151+
}
152+
153+
private CachedEnforcer cachedRBACModelLargeEnforcer;
154+
155+
@Setup(Level.Trial)
156+
public void setupCachedRBACModelLarge() {
157+
cachedRBACModelLargeEnforcer = new CachedEnforcer("examples/rbac_model.conf", "");
134158

135159
// 10000 roles, 1000 resources.
136160
List<List<String>> pPolicies = new ArrayList<>();
@@ -141,7 +165,7 @@ public void benchmarkCachedRBACModelLarge() {
141165
policy.add("read");
142166
pPolicies.add(policy);
143167
}
144-
e.addPolicies(pPolicies);
168+
cachedRBACModelLargeEnforcer.addPolicies(pPolicies);
145169

146170
// 100000 users.
147171
List<List<String>> gPolicies = new ArrayList<>();
@@ -151,46 +175,86 @@ public void benchmarkCachedRBACModelLarge() {
151175
policy.add(String.format("group%d", i / 10));
152176
gPolicies.add(policy);
153177
}
154-
e.addGroupingPolicies(gPolicies);
178+
cachedRBACModelLargeEnforcer.addGroupingPolicies(gPolicies);
179+
}
180+
181+
@Benchmark
182+
public void benchmarkCachedRBACModelLarge() {
183+
cachedRBACModelLargeEnforcer.enforce("user50001", "data1500", "read");
184+
}
185+
186+
private CachedEnforcer cachedRBACModelWithResourceRolesEnforcer;
155187

156-
e.enforce("user50001", "data1500", "read");
188+
@Setup(Level.Trial)
189+
public void setupCachedRBACModelWithResourceRoles() {
190+
cachedRBACModelWithResourceRolesEnforcer = new CachedEnforcer("examples/rbac_with_resource_roles_model.conf", "examples/rbac_with_resource_roles_policy.csv");
157191
}
158192

159193
@Benchmark
160194
public void benchmarkCachedRBACModelWithResourceRoles() {
161-
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_resource_roles_model.conf", "examples/rbac_with_resource_roles_policy.csv");
162-
e.enforce("alice", "data1", "read");
195+
cachedRBACModelWithResourceRolesEnforcer.enforce("alice", "data1", "read");
196+
}
197+
198+
private CachedEnforcer cachedRBACModelWithDomainsEnforcer;
199+
200+
@Setup(Level.Trial)
201+
public void setupCachedRBACModelWithDomains() {
202+
cachedRBACModelWithDomainsEnforcer = new CachedEnforcer("examples/rbac_with_domains_model.conf", "examples/rbac_with_domains_policy.csv");
163203
}
164204

165205
@Benchmark
166206
public void benchmarkCachedRBACModelWithDomains() {
167-
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_domains_model.conf", "examples/rbac_with_domains_policy.csv");
168-
e.enforce("alice", "domain1", "data1", "read");
207+
cachedRBACModelWithDomainsEnforcer.enforce("alice", "domain1", "data1", "read");
208+
}
209+
210+
private CachedEnforcer cachedABACModelEnforcer;
211+
private TestResource cachedABACTestResource;
212+
213+
@Setup(Level.Trial)
214+
public void setupCachedABACModel() {
215+
cachedABACModelEnforcer = new CachedEnforcer("examples/abac_model.conf", "");
216+
cachedABACTestResource = newTestResource("data1", "alice");
169217
}
170218

171219
@Benchmark
172220
public void benchmarkCachedABACModel() {
173-
CachedEnforcer e = new CachedEnforcer("examples/abac_model.conf", "");
174-
TestResource data1 = newTestResource("data1", "alice");
175-
e.enforce("alice", data1, "read");
221+
cachedABACModelEnforcer.enforce("alice", cachedABACTestResource, "read");
222+
}
223+
224+
private CachedEnforcer cachedKeyMatchModelEnforcer;
225+
226+
@Setup(Level.Trial)
227+
public void setupCachedKeyMatchModel() {
228+
cachedKeyMatchModelEnforcer = new CachedEnforcer("examples/keymatch_model.conf", "examples/keymatch_policy.csv");
176229
}
177230

178231
@Benchmark
179232
public void benchmarkCachedKeyMatchModel() {
180-
CachedEnforcer e = new CachedEnforcer("examples/keymatch_model.conf", "examples/keymatch_policy.csv");
181-
e.enforce("alice", "/alice_data/resource1", "GET");
233+
cachedKeyMatchModelEnforcer.enforce("alice", "/alice_data/resource1", "GET");
234+
}
235+
236+
private CachedEnforcer cachedRBACModelWithDenyEnforcer;
237+
238+
@Setup(Level.Trial)
239+
public void setupCachedRBACModelWithDeny() {
240+
cachedRBACModelWithDenyEnforcer = new CachedEnforcer("examples/rbac_with_deny_model.conf", "examples/rbac_with_deny_policy.csv");
182241
}
183242

184243
@Benchmark
185244
public void benchmarkCachedRBACModelWithDeny() {
186-
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_deny_model.conf", "examples/rbac_with_deny_policy.csv");
187-
e.enforce("alice", "data1", "read");
245+
cachedRBACModelWithDenyEnforcer.enforce("alice", "data1", "read");
246+
}
247+
248+
private CachedEnforcer cachedPriorityModelEnforcer;
249+
250+
@Setup(Level.Trial)
251+
public void setupCachedPriorityModel() {
252+
cachedPriorityModelEnforcer = new CachedEnforcer("examples/priority_model.conf", "examples/priority_policy.csv");
188253
}
189254

190255
@Benchmark
191256
public void benchmarkCachedPriorityModel() {
192-
CachedEnforcer e = new CachedEnforcer("examples/priority_model.conf", "examples/priority_policy.csv");
193-
e.enforce("alice", "data1", "read");
257+
cachedPriorityModelEnforcer.enforce("alice", "data1", "read");
194258
}
195259

196260
@Benchmark
@@ -231,7 +295,10 @@ public void setup() {
231295

232296
public static void main(String[] args) throws RunnerException {
233297
Options opt = new OptionsBuilder()
234-
.include(CachedEnforcerBenchmarkTest.class.getName())
298+
.include(CachedEnforcerBenchmarkTest.class.getSimpleName())
299+
.forks(1)
300+
.warmupIterations(5)
301+
.measurementIterations(5)
235302
.build();
236303
new Runner(opt).run();
237304
}

0 commit comments

Comments
 (0)