Skip to content

Commit 18bf5bd

Browse files
ikolomiikolomi
committed
feat: Add CI-driven performance comparison graphs for README
Add automated benchmark workflow and graph generation for comparing Java client library performance on GitHub Actions runners. - Reference configs: 5 drivers, 2 workloads (1 and 100 clients) - Java: getDriverVersion(), NDJSON metadata, --commit-id option - Python graph generator with 3-tuple aggregation key - GitHub Actions workflow using Makefile server management - README updates with embedded graphs and documentation Signed-off-by: ikolomi <ikolomin@amazon.com>
1 parent 13ca47a commit 18bf5bd

24 files changed

+990
-2
lines changed

‎.github/benchmark-config/drivers.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"java": [
3+
"configs/drivers/reference/jedis.json",
4+
"configs/drivers/reference/lettuce.json",
5+
"configs/drivers/reference/redisson.json",
6+
"configs/drivers/reference/spring-data-redis-jedis.json",
7+
"configs/drivers/reference/spring-data-redis-lettuce.json"
8+
]
9+
}

‎.github/benchmark-config/workloads.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[
2+
"configs/workloads/reference/basic-standalone-single-client.json",
3+
"configs/workloads/reference/basic-standalone-100-clients.json"
4+
]

‎.github/workflows/benchmark.yml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
name: Benchmark
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
workflow_dispatch:
9+
inputs:
10+
run_benchmark:
11+
description: 'Run benchmark'
12+
required: false
13+
default: 'true'
14+
15+
jobs:
16+
benchmark:
17+
runs-on: ubuntu-latest
18+
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
driver:
23+
- configs/drivers/reference/jedis.json
24+
- configs/drivers/reference/lettuce.json
25+
- configs/drivers/reference/redisson.json
26+
- configs/drivers/reference/spring-data-redis-jedis.json
27+
- configs/drivers/reference/spring-data-redis-lettuce.json
28+
workload:
29+
- configs/workloads/reference/basic-standalone-single-client.json
30+
- configs/workloads/reference/basic-standalone-100-clients.json
31+
32+
steps:
33+
- uses: actions/checkout@v4
34+
35+
- name: Set up JDK 21
36+
uses: actions/setup-java@v4
37+
with:
38+
java-version: '21'
39+
distribution: 'temurin'
40+
cache: maven
41+
42+
- name: Install build dependencies
43+
run: |
44+
sudo apt-get update
45+
sudo apt-get install -y build-essential
46+
47+
- name: Start Valkey server
48+
run: |
49+
# Use Makefile target which builds from source and configures with persistence disabled
50+
make server-standalone-start
51+
# Wait for server to be ready
52+
sleep 2
53+
# Verify server is up and persistence is disabled
54+
work/valkey/bin/valkey-cli ping
55+
work/valkey/bin/valkey-cli CONFIG GET save
56+
57+
- name: Build Java benchmark
58+
run: |
59+
cd java
60+
mvn -B package -DskipTests
61+
62+
- name: Extract names for result file
63+
id: names
64+
run: |
65+
DRIVER_NAME=$(basename ${{ matrix.driver }} .json)
66+
WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json)
67+
echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT
68+
echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT
69+
echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT
70+
71+
- name: Run benchmark
72+
run: |
73+
mkdir -p results/github-runner/reference
74+
java -jar java/target/java-benchmark-1.0-SNAPSHOT.jar \
75+
--server localhost:6379 \
76+
--driver ${{ matrix.driver }} \
77+
--workload ${{ matrix.workload }} \
78+
--metrics ${{ steps.names.outputs.result_file }} \
79+
--commit-id ${{ github.sha }}
80+
81+
- name: Stop Valkey server
82+
if: always()
83+
run: |
84+
make server-standalone-stop || true
85+
86+
- name: Upload results
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: benchmark-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }}
90+
path: ${{ steps.names.outputs.result_file }}
91+
retention-days: 30
92+
93+
generate-graphs:
94+
needs: benchmark
95+
runs-on: ubuntu-latest
96+
97+
steps:
98+
- uses: actions/checkout@v4
99+
100+
- name: Set up Python
101+
uses: actions/setup-python@v5
102+
with:
103+
python-version: '3.11'
104+
105+
- name: Install dependencies
106+
run: |
107+
pip install matplotlib numpy
108+
109+
- name: Download all artifacts
110+
uses: actions/download-artifact@v4
111+
with:
112+
path: results/github-runner/reference
113+
pattern: benchmark-*
114+
merge-multiple: true
115+
116+
- name: List downloaded results
117+
run: |
118+
echo "Downloaded results:"
119+
find results -name "*.ndjson" -type f
120+
121+
- name: Generate graphs - Single Client
122+
run: |
123+
python scripts/generate_graphs.py \
124+
--results "results/github-runner/reference/*-basic-standalone-single-client.ndjson" \
125+
--output graphs/single-client/ \
126+
--phase STEADY \
127+
--workload "Single Client"
128+
129+
- name: Generate graphs - 100 Clients
130+
run: |
131+
python scripts/generate_graphs.py \
132+
--results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \
133+
--output graphs/100-clients/ \
134+
--phase STEADY \
135+
--workload "100 Clients"
136+
137+
- name: Upload graphs
138+
uses: actions/upload-artifact@v4
139+
with:
140+
name: benchmark-graphs
141+
path: graphs/
142+
retention-days: 30
143+
144+
- name: Create PR with updated graphs
145+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
146+
uses: peter-evans/create-pull-request@v6
147+
with:
148+
commit-message: "Update benchmark graphs"
149+
title: "🔄 Update benchmark graphs"
150+
body: |
151+
This PR updates the benchmark comparison graphs from the latest CI run.
152+
153+
Commit: ${{ github.sha }}
154+
Run: ${{ github.run_id }}
155+
branch: update-benchmark-graphs
156+
delete-branch: true
157+
add-paths: |
158+
graphs/

‎.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
work/
33
output/*.ndjson
44

5+
# Results (generated by CI)
6+
results/
7+
8+
# Note: graphs/ directory is tracked in git for README display
9+
# graphs/
10+
511
# Java
612
java/target/
713
java/*.jar

‎README.md‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,26 @@
22

33
A multi-language benchmark suite for RESP protocol (Redis/Valkey) compatible databases and client libraries.
44

5+
## Performance Comparison
6+
7+
The graphs below show performance comparisons of Java client libraries running on GitHub Actions runners. Results are automatically updated via CI.
8+
9+
### Single Client (1 connection)
10+
11+
| Throughput | Latency P99 |
12+
|------------|-------------|
13+
| ![SET RPS - Single Client](graphs/single-client/rps-SET.png) | ![SET Latency P99 - Single Client](graphs/single-client/latency-p99-SET.png) |
14+
| ![GET RPS - Single Client](graphs/single-client/rps-GET.png) | ![GET Latency P99 - Single Client](graphs/single-client/latency-p99-GET.png) |
15+
16+
### 100 Concurrent Clients
17+
18+
| Throughput | Latency P99 |
19+
|------------|-------------|
20+
| ![SET RPS - 100 Clients](graphs/100-clients/rps-SET.png) | ![SET Latency P99 - 100 Clients](graphs/100-clients/latency-p99-SET.png) |
21+
| ![GET RPS - 100 Clients](graphs/100-clients/rps-GET.png) | ![GET Latency P99 - 100 Clients](graphs/100-clients/latency-p99-GET.png) |
22+
23+
> **Note**: These benchmarks run on shared GitHub Actions runners. Results may have variance between runs due to noisy neighbor effects. The graphs show averages across multiple runs (n=count shown in labels).
24+
525
## Overview
626

727
**resp-bench** provides unified benchmark functionality across multiple programming languages, enabling fair comparisons between different client libraries and implementations. All language engines share the same:
@@ -174,6 +194,36 @@ All engines output metrics in NDJSON format (one JSON object per line):
174194
}
175195
```
176196

197+
## Graph Generation
198+
199+
Generate performance comparison graphs from benchmark results:
200+
201+
```bash
202+
# Install Python dependencies
203+
pip install matplotlib numpy
204+
205+
# Generate graphs from all results (auto-detects latest CI run)
206+
python scripts/generate_graphs.py \
207+
--results results/github-runner/reference/*.ndjson \
208+
--output graphs/ \
209+
--phase STEADY \
210+
--workload "My Benchmark"
211+
212+
# Generate graphs for a specific commit
213+
python scripts/generate_graphs.py \
214+
--results results/github-runner/reference/*.ndjson \
215+
--output graphs/ \
216+
--phase STEADY \
217+
--commit-id abc123def456
218+
```
219+
220+
### Aggregation Policy
221+
222+
The script aggregates results using a 3-tuple key: `(commit_id, primary_driver_version, secondary_driver_version)`. This ensures only results from the same CI run are compared together.
223+
224+
- **Auto-detection (default)**: When no explicit filter is provided, the script finds the latest record by timestamp and uses its metadata values as the filter
225+
- **Explicit filter**: Use `--commit-id`, `--primary-driver-version`, or `--secondary-driver-version` to filter specific runs
226+
177227
## Make Targets
178228

179229
### Server Management

‎configs/drivers/reference/jedis.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"schema_version": "1.0",
3+
"description": "Jedis client - reference benchmark configuration",
4+
"driver_id": "jedis",
5+
"mode": "standalone",
6+
"specific_driver_config": {}
7+
}

‎configs/drivers/reference/lettuce.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"schema_version": "1.0",
3+
"description": "Lettuce client - reference benchmark configuration",
4+
"driver_id": "lettuce",
5+
"mode": "standalone",
6+
"specific_driver_config": {}
7+
}

‎configs/drivers/reference/redisson.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"schema_version": "1.0",
3+
"description": "Redisson client - reference benchmark configuration",
4+
"driver_id": "redisson",
5+
"mode": "standalone",
6+
"specific_driver_config": {}
7+
}

‎configs/drivers/reference/spring-data-redis-jedis.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"schema_version": "1.0",
3+
"description": "Spring Data Redis with Jedis driver - reference benchmark configuration",
4+
"driver_id": "spring-data-redis",
5+
"mode": "standalone",
6+
"specific_driver_config": {
7+
"secondary_driver_id": "jedis"
8+
}
9+
}

‎configs/drivers/reference/spring-data-redis-lettuce.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"schema_version": "1.0",
3+
"description": "Spring Data Redis with Lettuce driver - reference benchmark configuration",
4+
"driver_id": "spring-data-redis",
5+
"mode": "standalone",
6+
"specific_driver_config": {
7+
"secondary_driver_id": "lettuce"
8+
}
9+
}

0 commit comments

Comments
 (0)