Benchmark #18
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| benchmark-java: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| driver: | |
| - configs/drivers/reference/jedis.json | |
| - configs/drivers/reference/lettuce.json | |
| - configs/drivers/reference/redisson.json | |
| - configs/drivers/reference/spring-data-redis-jedis.json | |
| - configs/drivers/reference/spring-data-redis-lettuce.json | |
| - configs/drivers/reference/spring-data-valkey-glide.json | |
| - configs/drivers/reference/spring-data-valkey-jedis.json | |
| - configs/drivers/reference/spring-data-valkey-lettuce.json | |
| - configs/drivers/reference/valkey-glide.json | |
| workload: | |
| - configs/workloads/reference/basic-standalone-single-client.json | |
| - configs/workloads/reference/basic-standalone-10-clients.json | |
| - configs/workloads/reference/basic-standalone-100-clients.json | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| cache: maven | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential | |
| - name: Start Valkey server | |
| run: | | |
| # Use Makefile target which builds from source and configures with persistence disabled | |
| make server-standalone-start | |
| # Wait for server to be ready | |
| sleep 2 | |
| # Verify server is up and persistence is disabled | |
| work/valkey/bin/valkey-cli ping | |
| work/valkey/bin/valkey-cli CONFIG GET save | |
| - name: Build Java benchmark | |
| run: | | |
| cd java | |
| mvn -B package -DskipTests | |
| - name: Extract names for result file | |
| id: names | |
| run: | | |
| DRIVER_NAME=$(basename ${{ matrix.driver }} .json) | |
| WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json) | |
| echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT | |
| echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT | |
| echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT | |
| - name: Run benchmark | |
| run: | | |
| mkdir -p results/github-runner/reference | |
| java -jar java/target/resp-bench-java-1.0.0-SNAPSHOT.jar \ | |
| --server localhost:6379 \ | |
| --driver ${{ matrix.driver }} \ | |
| --workload ${{ matrix.workload }} \ | |
| --metrics ${{ steps.names.outputs.result_file }} \ | |
| --commit-id ${{ github.sha }} | |
| - name: Stop Valkey server | |
| if: always() | |
| run: | | |
| make server-standalone-stop || true | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-java-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }} | |
| path: ${{ steps.names.outputs.result_file }} | |
| retention-days: 30 | |
| benchmark-ruby: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| driver: | |
| - configs/drivers/reference/redis-rb.json | |
| - configs/drivers/reference/valkey-glide-ruby.json | |
| workload: | |
| - configs/workloads/reference/basic-standalone-single-client.json | |
| - configs/workloads/reference/basic-standalone-10-clients.json | |
| - configs/workloads/reference/basic-standalone-100-clients.json | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Ruby | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.2' | |
| bundler-cache: true | |
| working-directory: ruby | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y build-essential | |
| - name: Start Valkey server | |
| run: | | |
| # Use Makefile target which builds from source and configures with persistence disabled | |
| make server-standalone-start | |
| # Wait for server to be ready | |
| sleep 2 | |
| # Verify server is up and persistence is disabled | |
| work/valkey/bin/valkey-cli ping | |
| work/valkey/bin/valkey-cli CONFIG GET save | |
| - name: Extract names for result file | |
| id: names | |
| run: | | |
| DRIVER_NAME=$(basename ${{ matrix.driver }} .json) | |
| WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json) | |
| echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT | |
| echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT | |
| echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT | |
| - name: Run benchmark | |
| run: | | |
| mkdir -p results/github-runner/reference | |
| cd ruby && bundle exec ruby bin/resp-bench \ | |
| --server localhost:6379 \ | |
| --driver ../${{ matrix.driver }} \ | |
| --workload ../${{ matrix.workload }} \ | |
| --metrics ../${{ steps.names.outputs.result_file }} \ | |
| --commit-id ${{ github.sha }} | |
| - name: Stop Valkey server | |
| if: always() | |
| run: | | |
| make server-standalone-stop || true | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-ruby-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }} | |
| path: ${{ steps.names.outputs.result_file }} | |
| retention-days: 30 | |
| generate-graphs: | |
| needs: [benchmark-java, benchmark-ruby] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| pip install matplotlib numpy | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: results/github-runner/reference | |
| pattern: benchmark-* | |
| merge-multiple: true | |
| - name: List downloaded results | |
| run: | | |
| echo "Downloaded results:" | |
| find results -name "*.ndjson" -type f | |
| # Java graphs — per concurrency level | |
| - name: Generate Java graphs - 1 Client | |
| run: | | |
| python scripts/generate_graphs.py \ | |
| --results "results/github-runner/reference/*-basic-standalone-single-client.ndjson" \ | |
| --output graphs/java/1-client/ \ | |
| --phase STEADY \ | |
| --language java \ | |
| --workload "Java - 1 Client" \ | |
| --commit-id ${{ github.sha }} | |
| - name: Generate Java graphs - 10 Clients | |
| run: | | |
| python scripts/generate_graphs.py \ | |
| --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ | |
| --output graphs/java/10-clients/ \ | |
| --phase STEADY \ | |
| --language java \ | |
| --workload "Java - 10 Clients" \ | |
| --commit-id ${{ github.sha }} | |
| - name: Generate Java graphs - 100 Clients | |
| run: | | |
| python scripts/generate_graphs.py \ | |
| --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ | |
| --output graphs/java/100-clients/ \ | |
| --phase STEADY \ | |
| --language java \ | |
| --workload "Java - 100 Clients" \ | |
| --commit-id ${{ github.sha }} | |
| # Ruby graphs — per concurrency level | |
| - name: Generate Ruby graphs - 1 Client | |
| run: | | |
| python scripts/generate_graphs.py \ | |
| --results "results/github-runner/reference/*-basic-standalone-single-client.ndjson" \ | |
| --output graphs/ruby/1-client/ \ | |
| --phase STEADY \ | |
| --language ruby \ | |
| --workload "Ruby - 1 Client" \ | |
| --commit-id ${{ github.sha }} | |
| - name: Generate Ruby graphs - 10 Clients | |
| run: | | |
| python scripts/generate_graphs.py \ | |
| --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \ | |
| --output graphs/ruby/10-clients/ \ | |
| --phase STEADY \ | |
| --language ruby \ | |
| --workload "Ruby - 10 Clients" \ | |
| --commit-id ${{ github.sha }} | |
| - name: Generate Ruby graphs - 100 Clients | |
| run: | | |
| python scripts/generate_graphs.py \ | |
| --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \ | |
| --output graphs/ruby/100-clients/ \ | |
| --phase STEADY \ | |
| --language ruby \ | |
| --workload "Ruby - 100 Clients" \ | |
| --commit-id ${{ github.sha }} | |
| - name: Upload graphs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-graphs | |
| path: graphs/ | |
| retention-days: 30 | |
| - name: Create PR with updated graphs | |
| if: github.ref == 'refs/heads/main' | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| commit-message: "Update benchmark graphs" | |
| title: "🔄 Update benchmark graphs" | |
| body: | | |
| This PR updates the benchmark comparison graphs from the latest CI run. | |
| Commit: ${{ github.sha }} | |
| Run: ${{ github.run_id }} | |
| branch: update-benchmark-graphs | |
| delete-branch: true | |
| add-paths: | | |
| graphs/ |