Skip to content

Commit c4cff0a

Browse files
lhotarilhotari
authored
[improve][broker] Pulsar Rate Limiting Refactoring changes (PIP-322) (#21681)
1 parent 60522c6 commit c4cff0a

File tree

63 files changed

+1995
-2218
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

63 files changed

+1995
-2218
lines changed

Diff for: ‎microbench/README.md

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
<!--
2+
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
20+
-->
21+
22+
# Microbenchmarks for Apache Pulsar
23+
24+
This module contains microbenchmarks for Apache Pulsar.
25+
26+
## Running the benchmarks
27+
28+
The benchmarks are written using [JMH](http://openjdk.java.net/projects/code-tools/jmh/). To compile & run the benchmarks, use the following command:
29+
30+
```bash
31+
# Compile everything for creating the shaded microbenchmarks.jar file
32+
mvn -Pcore-modules,microbench,-main -T 1C clean package
33+
34+
# run the benchmarks using the standalone shaded jar in any environment
35+
java -jar microbench/target/microbenchmarks.jar
36+
```
37+
38+
For fast recompiling of the benchmarks (without compiling Pulsar modules) and creating the shaded jar, you can use the following command:
39+
40+
```bash
41+
mvn -Pmicrobench -pl microbench clean package
42+
```
43+

Diff for: ‎microbench/pom.xml

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
<!--
2+
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
20+
-->
21+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
25+
<parent>
26+
<groupId>org.apache.pulsar</groupId>
27+
<artifactId>pulsar</artifactId>
28+
<version>3.2.0-SNAPSHOT</version>
29+
<relativePath>../pom.xml</relativePath>
30+
</parent>
31+
32+
<artifactId>microbench</artifactId>
33+
<packaging>jar</packaging>
34+
<name>Pulsar Microbenchmarks</name>
35+
36+
<properties>
37+
<jmh.version>1.37</jmh.version>
38+
</properties>
39+
40+
<profiles>
41+
<profile>
42+
<id>microbench</id>
43+
<build>
44+
<plugins>
45+
<plugin>
46+
<groupId>org.apache.maven.plugins</groupId>
47+
<artifactId>maven-shade-plugin</artifactId>
48+
<executions>
49+
<execution>
50+
<phase>package</phase>
51+
<goals>
52+
<goal>shade</goal>
53+
</goals>
54+
<configuration>
55+
<finalName>microbenchmarks</finalName>
56+
<transformers>
57+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
58+
<manifestEntries>
59+
<Main-Class>org.openjdk.jmh.Main</Main-Class>
60+
<Multi-Release>true</Multi-Release>
61+
</manifestEntries>
62+
</transformer>
63+
</transformers>
64+
<filters>
65+
<filter>
66+
<artifact>*:*</artifact>
67+
<excludes>
68+
<exclude>META-INF/*.SF</exclude>
69+
<exclude>META-INF/*.DSA</exclude>
70+
<exclude>META-INF/*.RSA</exclude>
71+
</excludes>
72+
</filter>
73+
</filters>
74+
</configuration>
75+
</execution>
76+
</executions>
77+
</plugin>
78+
</plugins>
79+
</build>
80+
</profile>
81+
</profiles>
82+
83+
<dependencies>
84+
<dependency>
85+
<groupId>org.openjdk.jmh</groupId>
86+
<artifactId>jmh-core</artifactId>
87+
<version>${jmh.version}</version>
88+
</dependency>
89+
<dependency>
90+
<groupId>org.openjdk.jmh</groupId>
91+
<artifactId>jmh-generator-annprocess</artifactId>
92+
<version>${jmh.version}</version>
93+
<scope>provided</scope>
94+
</dependency>
95+
<dependency>
96+
<groupId>${project.groupId}</groupId>
97+
<artifactId>pulsar-broker</artifactId>
98+
<version>${project.version}</version>
99+
</dependency>
100+
</dependencies>
101+
102+
<build>
103+
<plugins>
104+
<plugin>
105+
<groupId>org.apache.maven.plugins</groupId>
106+
<artifactId>maven-deploy-plugin</artifactId>
107+
<configuration>
108+
<skip>true</skip>
109+
</configuration>
110+
</plugin>
111+
<plugin>
112+
<groupId>org.apache.maven.plugins</groupId>
113+
<artifactId>maven-surefire-plugin</artifactId>
114+
<configuration>
115+
<skipTests>true</skipTests>
116+
</configuration>
117+
</plugin>
118+
<plugin>
119+
<groupId>org.apache.maven.plugins</groupId>
120+
<artifactId>maven-compiler-plugin</artifactId>
121+
<configuration>
122+
<annotationProcessorPaths combine.children="append">
123+
<path>
124+
<groupId>org.openjdk.jmh</groupId>
125+
<artifactId>jmh-generator-annprocess</artifactId>
126+
<version>${jmh.version}</version>
127+
</path>
128+
</annotationProcessorPaths>
129+
</configuration>
130+
</plugin>
131+
</plugins>
132+
</build>
133+
</project>

Diff for: ‎microbench/src/main/java/org/apache/pulsar/broker/qos/AsyncTokenBucketBenchmark.java

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.pulsar.broker.qos;
21+
22+
import java.util.concurrent.TimeUnit;
23+
import org.openjdk.jmh.annotations.Benchmark;
24+
import org.openjdk.jmh.annotations.BenchmarkMode;
25+
import org.openjdk.jmh.annotations.Fork;
26+
import org.openjdk.jmh.annotations.Level;
27+
import org.openjdk.jmh.annotations.Measurement;
28+
import org.openjdk.jmh.annotations.Mode;
29+
import org.openjdk.jmh.annotations.OutputTimeUnit;
30+
import org.openjdk.jmh.annotations.Scope;
31+
import org.openjdk.jmh.annotations.Setup;
32+
import org.openjdk.jmh.annotations.State;
33+
import org.openjdk.jmh.annotations.TearDown;
34+
import org.openjdk.jmh.annotations.Threads;
35+
import org.openjdk.jmh.annotations.Warmup;
36+
37+
@Fork(3)
38+
@BenchmarkMode(Mode.Throughput)
39+
@OutputTimeUnit(TimeUnit.SECONDS)
40+
@State(Scope.Thread)
41+
public class AsyncTokenBucketBenchmark {
42+
private AsyncTokenBucket asyncTokenBucket;
43+
private DefaultMonotonicSnapshotClock monotonicSnapshotClock =
44+
new DefaultMonotonicSnapshotClock(TimeUnit.MILLISECONDS.toNanos(8), System::nanoTime);
45+
46+
@Setup(Level.Iteration)
47+
public void setup() {
48+
long ratePerSecond = 100_000_000;
49+
asyncTokenBucket = AsyncTokenBucket.builder().rate(ratePerSecond).clock(monotonicSnapshotClock)
50+
.initialTokens(2 * ratePerSecond).capacity(2 * ratePerSecond).build();
51+
}
52+
53+
@TearDown(Level.Iteration)
54+
public void teardown() {
55+
monotonicSnapshotClock.close();
56+
}
57+
58+
@Threads(1)
59+
@Benchmark
60+
@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
61+
@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
62+
public void consumeTokensBenchmark001Threads() {
63+
asyncTokenBucket.consumeTokens(1);
64+
}
65+
66+
@Threads(10)
67+
@Benchmark
68+
@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
69+
@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
70+
public void consumeTokensBenchmark010Threads() {
71+
asyncTokenBucket.consumeTokens(1);
72+
}
73+
74+
@Threads(100)
75+
@Benchmark
76+
@Measurement(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
77+
@Warmup(time = 10, timeUnit = TimeUnit.SECONDS, iterations = 1)
78+
public void consumeTokensBenchmark100Threads() {
79+
asyncTokenBucket.consumeTokens(1);
80+
}
81+
}

Diff for: ‎pulsar-common/src/main/java/org/apache/pulsar/common/util/RateLimitFunction.java renamed to ‎microbench/src/main/java/org/apache/pulsar/broker/qos/package-info.java

+3-7
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,7 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
package org.apache.pulsar.common.util;
20-
2119
/**
22-
* Function use when rate limiter renew permit.
23-
* */
24-
public interface RateLimitFunction {
25-
void apply();
26-
}
20+
* Benchmarks for Pulsar broker Quality of Service (QoS) related classes.
21+
*/
22+
package org.apache.pulsar.broker.qos;

Diff for: ‎microbench/src/main/resources/log4j2.xml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
21+
-->
22+
<Configuration status="WARN">
23+
<Appenders>
24+
<Console name="Console" target="SYSTEM_OUT">
25+
<PatternLayout pattern="%d{ISO8601_OFFSET_DATE_TIME_HHMM} [%t] %-5level %logger{36} - %msg%n"/>
26+
</Console>
27+
</Appenders>
28+
<Loggers>
29+
<Root level="warn">
30+
<AppenderRef ref="Console"/>
31+
</Root>
32+
</Loggers>
33+
</Configuration>

Diff for: ‎pom.xml

+21
Original file line numberDiff line numberDiff line change
@@ -2247,6 +2247,8 @@ flexible messaging model and an intuitive client API.</description>
22472247
<module>distribution</module>
22482248
<module>docker</module>
22492249
<module>tests</module>
2250+
2251+
<module>microbench</module>
22502252
</modules>
22512253
</profile>
22522254

@@ -2504,6 +2506,25 @@ flexible messaging model and an intuitive client API.</description>
25042506
<module>pulsar-sql</module>
25052507
</modules>
25062508
</profile>
2509+
2510+
<profile>
2511+
<id>microbench</id>
2512+
<modules>
2513+
<module>microbench</module>
2514+
</modules>
2515+
<properties>
2516+
<skipSourceReleaseAssembly>true</skipSourceReleaseAssembly>
2517+
<skipBuildDistribution>true</skipBuildDistribution>
2518+
<spotbugs.skip>true</spotbugs.skip>
2519+
<license.skip>true</license.skip>
2520+
<rat.skip>true</rat.skip>
2521+
<assembly.skipAssembly>true</assembly.skipAssembly>
2522+
<narPluginPhase>none</narPluginPhase>
2523+
<skipDocker>true</skipDocker>
2524+
<skipTests>true</skipTests>
2525+
<checkstyle.skip>true</checkstyle.skip>
2526+
</properties>
2527+
</profile>
25072528
</profiles>
25082529

25092530
<repositories>

0 commit comments

Comments
 (0)