Skip to content

Commit aa90a03

Browse files
hsluoyzhsluoyz
committed
feat: replace JMH with lightweight timing-based benchmarks (#503)
1 parent eb4476f commit aa90a03

File tree

6 files changed

+603
-538
lines changed

6 files changed

+603
-538
lines changed

‎pom.xml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -207,18 +207,6 @@
207207
<artifactId>commons-io</artifactId>
208208
<version>2.17.0</version>
209209
</dependency>
210-
<dependency>
211-
<groupId>org.openjdk.jmh</groupId>
212-
<artifactId>jmh-core</artifactId>
213-
<version>1.37</version>
214-
<scope>test</scope>
215-
</dependency>
216-
<dependency>
217-
<groupId>org.openjdk.jmh</groupId>
218-
<artifactId>jmh-generator-annprocess</artifactId>
219-
<version>1.37</version>
220-
<scope>test</scope>
221-
</dependency>
222210
<dependency>
223211
<groupId>org.apache.commons</groupId>
224212
<artifactId>commons-csv</artifactId>

‎src/test/java/org/casbin/jcasbin/main/benchmark/BenchmarkUtil.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright 2017 The casbin Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package org.casbin.jcasbin.main.benchmark;
16+
17+
/**
18+
* Utility class for running simple benchmarks without JMH.
19+
* Provides basic timing and throughput measurement functionality.
20+
*/
21+
public class BenchmarkUtil {
22+
23+
/** Default number of warmup iterations */
24+
private static final int DEFAULT_WARMUP_ITERATIONS = 3;
25+
26+
/** Default number of measurement iterations */
27+
private static final int DEFAULT_MEASUREMENT_ITERATIONS = 5;
28+
29+
/**
30+
* Functional interface for benchmark operations.
31+
*/
32+
@FunctionalInterface
33+
public interface BenchmarkOperation {
34+
void execute();
35+
}
36+
37+
/**
38+
* Runs a benchmark operation multiple times and measures throughput.
39+
*
40+
* @param name The name of the benchmark
41+
* @param warmupIterations Number of warmup iterations
42+
* @param measurementIterations Number of measurement iterations
43+
* @param operation The operation to benchmark
44+
*/
45+
public static void runBenchmark(String name, int warmupIterations, int measurementIterations, BenchmarkOperation operation) {
46+
// Warmup phase
47+
for (int i = 0; i < warmupIterations; i++) {
48+
operation.execute();
49+
}
50+
51+
// Measurement phase
52+
long startTime = System.nanoTime();
53+
for (int i = 0; i < measurementIterations; i++) {
54+
operation.execute();
55+
}
56+
long endTime = System.nanoTime();
57+
58+
// Calculate and display results
59+
double durationSeconds = (endTime - startTime) / 1_000_000_000.0;
60+
double throughput = measurementIterations / durationSeconds;
61+
62+
System.out.printf("Benchmark: %s%n", name);
63+
System.out.printf(" Iterations: %d%n", measurementIterations);
64+
System.out.printf(" Duration: %.3f seconds%n", durationSeconds);
65+
System.out.printf(" Throughput: %.2f ops/sec%n", throughput);
66+
System.out.println();
67+
}
68+
69+
/**
70+
* Runs a benchmark operation with default settings (3 warmup iterations, 5 measurement iterations).
71+
*
72+
* @param name The name of the benchmark
73+
* @param operation The operation to benchmark
74+
*/
75+
public static void runBenchmark(String name, BenchmarkOperation operation) {
76+
runBenchmark(name, DEFAULT_WARMUP_ITERATIONS, DEFAULT_MEASUREMENT_ITERATIONS, operation);
77+
}
78+
}

‎src/test/java/org/casbin/jcasbin/main/benchmark/CachedEnforcerBenchmarkTest.java‎

Lines changed: 118 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,11 @@
1515
package org.casbin.jcasbin.main.benchmark;
1616

1717
import org.casbin.jcasbin.main.CachedEnforcer;
18-
import org.openjdk.jmh.annotations.*;
19-
import org.openjdk.jmh.runner.Runner;
20-
import org.openjdk.jmh.runner.RunnerException;
21-
import org.openjdk.jmh.runner.options.Options;
22-
import org.openjdk.jmh.runner.options.OptionsBuilder;
18+
import org.junit.Test;
2319

2420
import java.util.ArrayList;
2521
import java.util.List;
26-
import java.util.concurrent.TimeUnit;
27-
28-
@BenchmarkMode(Mode.Throughput)
29-
@Warmup(iterations = 3, time = 1)
30-
@Measurement(iterations = 5, time = 1)
31-
@Threads(1)
32-
@Fork(1)
33-
@State(value = Scope.Benchmark)
34-
@OutputTimeUnit(TimeUnit.SECONDS)
22+
3523
public class CachedEnforcerBenchmarkTest {
3624

3725
public static class TestResource {
@@ -66,145 +54,161 @@ private static TestResource newTestResource(String name, String owner) {
6654
return new TestResource(name, owner);
6755
}
6856

69-
@Benchmark
57+
@Test
7058
public void benchmarkCachedRaw() {
71-
rawEnforce("alice", "data1", "read");
59+
BenchmarkUtil.runBenchmark("Cached Raw", () -> {
60+
rawEnforce("alice", "data1", "read");
61+
});
7262
}
7363

74-
@Benchmark
64+
@Test
7565
public void benchmarkCachedBasicModel() {
76-
CachedEnforcer e = new CachedEnforcer("examples/basic_model.conf", "examples/basic_policy.csv");
77-
e.enforce("alice", "data1", "read");
66+
BenchmarkUtil.runBenchmark("Cached Basic Model", () -> {
67+
CachedEnforcer e = new CachedEnforcer("examples/basic_model.conf", "examples/basic_policy.csv");
68+
e.enforce("alice", "data1", "read");
69+
});
7870
}
7971

80-
@Benchmark
72+
@Test
8173
public void benchmarkCachedRBACModel() {
82-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "examples/rbac_policy.csv");
83-
e.enforce("alice", "data2", "read");
74+
BenchmarkUtil.runBenchmark("Cached RBAC Model", () -> {
75+
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "examples/rbac_policy.csv");
76+
e.enforce("alice", "data2", "read");
77+
});
8478
}
8579

86-
@Benchmark
80+
@Test
8781
public void benchmarkCachedRBACModelSmall() {
88-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
82+
BenchmarkUtil.runBenchmark("Cached RBAC Model Small", () -> {
83+
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
8984

90-
// 100 roles, 10 resources.
91-
for (int i = 0; i < 100; i++) {
92-
e.addPolicy(String.format("group%d", i), String.format("data%d", i / 10), "read");
93-
}
85+
// 100 roles, 10 resources.
86+
for (int i = 0; i < 100; i++) {
87+
e.addPolicy(String.format("group%d", i), String.format("data%d", i / 10), "read");
88+
}
9489

95-
// 1000 users.
96-
for (int i = 0; i < 1000; i++) {
97-
e.addGroupingPolicy(String.format("user%d", i), String.format("group%d", i / 10));
98-
}
90+
// 1000 users.
91+
for (int i = 0; i < 1000; i++) {
92+
e.addGroupingPolicy(String.format("user%d", i), String.format("group%d", i / 10));
93+
}
9994

100-
e.enforce("user501", "data9", "read");
95+
e.enforce("user501", "data9", "read");
96+
});
10197
}
10298

103-
@Benchmark
99+
@Test
104100
public void benchmarkCachedRBACModelMedium() {
105-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
106-
107-
// 1000 roles, 100 resources.
108-
List<List<String>> pPolicies = new ArrayList<>();
109-
for (int i = 0; i < 1000; i++) {
110-
List<String> policy = new ArrayList<>();
111-
policy.add(String.format("group%d", i));
112-
policy.add(String.format("data%d", i / 10));
113-
policy.add("read");
114-
pPolicies.add(policy);
115-
}
116-
e.addPolicies(pPolicies);
117-
118-
// 10000 users.
119-
List<List<String>> gPolicies = new ArrayList<>();
120-
for (int i = 0; i < 10000; i++) {
121-
List<String> policy = new ArrayList<>();
122-
policy.add(String.format("user%d", i));
123-
policy.add(String.format("group%d", i / 10));
124-
gPolicies.add(policy);
125-
}
126-
e.addGroupingPolicies(gPolicies);
101+
BenchmarkUtil.runBenchmark("Cached RBAC Model Medium", () -> {
102+
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
127103

128-
e.enforce("user5001", "data150", "read");
104+
// 1000 roles, 100 resources.
105+
List<List<String>> pPolicies = new ArrayList<>();
106+
for (int i = 0; i < 1000; i++) {
107+
List<String> policy = new ArrayList<>();
108+
policy.add(String.format("group%d", i));
109+
policy.add(String.format("data%d", i / 10));
110+
policy.add("read");
111+
pPolicies.add(policy);
112+
}
113+
e.addPolicies(pPolicies);
114+
115+
// 10000 users.
116+
List<List<String>> gPolicies = new ArrayList<>();
117+
for (int i = 0; i < 10000; i++) {
118+
List<String> policy = new ArrayList<>();
119+
policy.add(String.format("user%d", i));
120+
policy.add(String.format("group%d", i / 10));
121+
gPolicies.add(policy);
122+
}
123+
e.addGroupingPolicies(gPolicies);
124+
125+
e.enforce("user5001", "data150", "read");
126+
});
129127
}
130128

131-
@Benchmark
129+
@Test
132130
public void benchmarkCachedRBACModelLarge() {
133-
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
134-
135-
// 10000 roles, 1000 resources.
136-
List<List<String>> pPolicies = new ArrayList<>();
137-
for (int i = 0; i < 10000; i++) {
138-
List<String> policy = new ArrayList<>();
139-
policy.add(String.format("group%d", i));
140-
policy.add(String.format("data%d", i / 10));
141-
policy.add("read");
142-
pPolicies.add(policy);
143-
}
144-
e.addPolicies(pPolicies);
145-
146-
// 100000 users.
147-
List<List<String>> gPolicies = new ArrayList<>();
148-
for (int i = 0; i < 100000; i++) {
149-
List<String> policy = new ArrayList<>();
150-
policy.add(String.format("user%d", i));
151-
policy.add(String.format("group%d", i / 10));
152-
gPolicies.add(policy);
153-
}
154-
e.addGroupingPolicies(gPolicies);
131+
BenchmarkUtil.runBenchmark("Cached RBAC Model Large", () -> {
132+
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
155133

156-
e.enforce("user50001", "data1500", "read");
134+
// 10000 roles, 1000 resources.
135+
List<List<String>> pPolicies = new ArrayList<>();
136+
for (int i = 0; i < 10000; i++) {
137+
List<String> policy = new ArrayList<>();
138+
policy.add(String.format("group%d", i));
139+
policy.add(String.format("data%d", i / 10));
140+
policy.add("read");
141+
pPolicies.add(policy);
142+
}
143+
e.addPolicies(pPolicies);
144+
145+
// 100000 users.
146+
List<List<String>> gPolicies = new ArrayList<>();
147+
for (int i = 0; i < 100000; i++) {
148+
List<String> policy = new ArrayList<>();
149+
policy.add(String.format("user%d", i));
150+
policy.add(String.format("group%d", i / 10));
151+
gPolicies.add(policy);
152+
}
153+
e.addGroupingPolicies(gPolicies);
154+
155+
e.enforce("user50001", "data1500", "read");
156+
});
157157
}
158158

159-
@Benchmark
159+
@Test
160160
public void benchmarkCachedRBACModelWithResourceRoles() {
161-
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_resource_roles_model.conf", "examples/rbac_with_resource_roles_policy.csv");
162-
e.enforce("alice", "data1", "read");
161+
BenchmarkUtil.runBenchmark("Cached RBAC Model With Resource Roles", () -> {
162+
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_resource_roles_model.conf", "examples/rbac_with_resource_roles_policy.csv");
163+
e.enforce("alice", "data1", "read");
164+
});
163165
}
164166

165-
@Benchmark
167+
@Test
166168
public void benchmarkCachedRBACModelWithDomains() {
167-
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_domains_model.conf", "examples/rbac_with_domains_policy.csv");
168-
e.enforce("alice", "domain1", "data1", "read");
169+
BenchmarkUtil.runBenchmark("Cached RBAC Model With Domains", () -> {
170+
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_domains_model.conf", "examples/rbac_with_domains_policy.csv");
171+
e.enforce("alice", "domain1", "data1", "read");
172+
});
169173
}
170174

171-
@Benchmark
175+
@Test
172176
public void benchmarkCachedABACModel() {
173-
CachedEnforcer e = new CachedEnforcer("examples/abac_model.conf", "");
174-
TestResource data1 = newTestResource("data1", "alice");
175-
e.enforce("alice", data1, "read");
177+
BenchmarkUtil.runBenchmark("Cached ABAC Model", () -> {
178+
CachedEnforcer e = new CachedEnforcer("examples/abac_model.conf", "");
179+
TestResource data1 = newTestResource("data1", "alice");
180+
e.enforce("alice", data1, "read");
181+
});
176182
}
177183

178-
@Benchmark
184+
@Test
179185
public void benchmarkCachedKeyMatchModel() {
180-
CachedEnforcer e = new CachedEnforcer("examples/keymatch_model.conf", "examples/keymatch_policy.csv");
181-
e.enforce("alice", "/alice_data/resource1", "GET");
186+
BenchmarkUtil.runBenchmark("Cached KeyMatch Model", () -> {
187+
CachedEnforcer e = new CachedEnforcer("examples/keymatch_model.conf", "examples/keymatch_policy.csv");
188+
e.enforce("alice", "/alice_data/resource1", "GET");
189+
});
182190
}
183191

184-
@Benchmark
192+
@Test
185193
public void benchmarkCachedRBACModelWithDeny() {
186-
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_deny_model.conf", "examples/rbac_with_deny_policy.csv");
187-
e.enforce("alice", "data1", "read");
194+
BenchmarkUtil.runBenchmark("Cached RBAC Model With Deny", () -> {
195+
CachedEnforcer e = new CachedEnforcer("examples/rbac_with_deny_model.conf", "examples/rbac_with_deny_policy.csv");
196+
e.enforce("alice", "data1", "read");
197+
});
188198
}
189199

190-
@Benchmark
200+
@Test
191201
public void benchmarkCachedPriorityModel() {
192-
CachedEnforcer e = new CachedEnforcer("examples/priority_model.conf", "examples/priority_policy.csv");
193-
e.enforce("alice", "data1", "read");
194-
}
195-
196-
@Benchmark
197-
public void benchmarkCachedRBACModelMediumParallel(ThreadState state) {
198-
state.e.enforce("user5001", "data150", "read");
202+
BenchmarkUtil.runBenchmark("Cached Priority Model", () -> {
203+
CachedEnforcer e = new CachedEnforcer("examples/priority_model.conf", "examples/priority_policy.csv");
204+
e.enforce("alice", "data1", "read");
205+
});
199206
}
200207

201-
@State(Scope.Thread)
202-
public static class ThreadState {
203-
CachedEnforcer e;
204-
205-
@Setup(Level.Trial)
206-
public void setup() {
207-
e = new CachedEnforcer("examples/rbac_model.conf", "");
208+
@Test
209+
public void benchmarkCachedRBACModelMediumParallel() {
210+
BenchmarkUtil.runBenchmark("Cached RBAC Model Medium Parallel", () -> {
211+
CachedEnforcer e = new CachedEnforcer("examples/rbac_model.conf", "");
208212

209213
// 10000 roles, 1000 resources.
210214
List<List<String>> pPolicies = new ArrayList<>();
@@ -226,13 +230,8 @@ public void setup() {
226230
gPolicies.add(policy);
227231
}
228232
e.addGroupingPolicies(gPolicies);
229-
}
230-
}
231233

232-
public static void main(String[] args) throws RunnerException {
233-
Options opt = new OptionsBuilder()
234-
.include(CachedEnforcerBenchmarkTest.class.getName())
235-
.build();
236-
new Runner(opt).run();
234+
e.enforce("user5001", "data150", "read");
235+
});
237236
}
238237
}

0 commit comments

Comments
 (0)