benchmark.yml

name: Benchmark

on:
  workflow_dispatch:

jobs:
  benchmark-java:
    runs-on: ubuntu-latest
    
    strategy:
      fail-fast: false
      matrix:
        driver: 
          - configs/drivers/reference/jedis.json
          - configs/drivers/reference/lettuce.json
          - configs/drivers/reference/redisson.json
          - configs/drivers/reference/spring-data-redis-jedis.json
          - configs/drivers/reference/spring-data-redis-lettuce.json
          - configs/drivers/reference/spring-data-valkey-glide.json
          - configs/drivers/reference/spring-data-valkey-jedis.json
          - configs/drivers/reference/spring-data-valkey-lettuce.json
          - configs/drivers/reference/valkey-glide.json
        workload:
          - configs/workloads/reference/basic-standalone-single-client.json
          - configs/workloads/reference/basic-standalone-10-clients.json
          - configs/workloads/reference/basic-standalone-100-clients.json

    steps:
      - uses: actions/checkout@v4

      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: '21'
          distribution: 'temurin'
          cache: maven

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential

      - name: Start Valkey server
        run: |
          # Use Makefile target which builds from source and configures with persistence disabled
          make server-standalone-start
          # Wait for server to be ready
          sleep 2
          # Verify server is up and persistence is disabled
          work/valkey/bin/valkey-cli ping
          work/valkey/bin/valkey-cli CONFIG GET save

      - name: Build Java benchmark
        run: |
          cd java
          mvn -B package -DskipTests

      - name: Extract names for result file
        id: names
        run: |
          DRIVER_NAME=$(basename ${{ matrix.driver }} .json)
          WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json)
          echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT
          echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT
          echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT

      - name: Run benchmark
        run: |
          mkdir -p results/github-runner/reference
          java -jar java/target/resp-bench-java-1.0.0-SNAPSHOT.jar \
            --server localhost:6379 \
            --driver ${{ matrix.driver }} \
            --workload ${{ matrix.workload }} \
            --metrics ${{ steps.names.outputs.result_file }} \
            --commit-id ${{ github.sha }}

      - name: Stop Valkey server
        if: always()
        run: |
          make server-standalone-stop || true

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-java-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }}
          path: ${{ steps.names.outputs.result_file }}
          retention-days: 30

  benchmark-ruby:
    runs-on: ubuntu-latest
    
    strategy:
      fail-fast: false
      matrix:
        driver: 
          - configs/drivers/reference/redis-rb.json
          - configs/drivers/reference/valkey-glide-ruby.json
        workload:
          - configs/workloads/reference/basic-standalone-single-client.json
          - configs/workloads/reference/basic-standalone-10-clients.json
          - configs/workloads/reference/basic-standalone-100-clients.json

    steps:
      - uses: actions/checkout@v4

      - name: Set up Ruby
        uses: ruby/setup-ruby@v1
        with:
          ruby-version: '3.2'
          bundler-cache: true
          working-directory: ruby

      - name: Install build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y build-essential

      - name: Start Valkey server
        run: |
          # Use Makefile target which builds from source and configures with persistence disabled
          make server-standalone-start
          # Wait for server to be ready
          sleep 2
          # Verify server is up and persistence is disabled
          work/valkey/bin/valkey-cli ping
          work/valkey/bin/valkey-cli CONFIG GET save

      - name: Extract names for result file
        id: names
        run: |
          DRIVER_NAME=$(basename ${{ matrix.driver }} .json)
          WORKLOAD_NAME=$(basename ${{ matrix.workload }} .json)
          echo "driver_name=$DRIVER_NAME" >> $GITHUB_OUTPUT
          echo "workload_name=$WORKLOAD_NAME" >> $GITHUB_OUTPUT
          echo "result_file=results/github-runner/reference/${DRIVER_NAME}-${WORKLOAD_NAME}.ndjson" >> $GITHUB_OUTPUT

      - name: Run benchmark
        run: |
          mkdir -p results/github-runner/reference
          cd ruby && bundle exec ruby bin/resp-bench \
            --server localhost:6379 \
            --driver ../${{ matrix.driver }} \
            --workload ../${{ matrix.workload }} \
            --metrics ../${{ steps.names.outputs.result_file }} \
            --commit-id ${{ github.sha }}

      - name: Stop Valkey server
        if: always()
        run: |
          make server-standalone-stop || true

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-ruby-${{ steps.names.outputs.driver_name }}-${{ steps.names.outputs.workload_name }}
          path: ${{ steps.names.outputs.result_file }}
          retention-days: 30

  generate-graphs:
    needs: [benchmark-java, benchmark-ruby]
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
    
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          pip install matplotlib numpy

      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: results/github-runner/reference
          pattern: benchmark-*
          merge-multiple: true

      - name: List downloaded results
        run: |
          echo "Downloaded results:"
          find results -name "*.ndjson" -type f

      # Java graphs — per concurrency level
      - name: Generate Java graphs - 1 Client
        run: |
          python scripts/generate_graphs.py \
            --results "results/github-runner/reference/*-basic-standalone-single-client.ndjson" \
            --output graphs/java/1-client/ \
            --phase STEADY \
            --language java \
            --workload "Java - 1 Client" \
            --commit-id ${{ github.sha }}

      - name: Generate Java graphs - 10 Clients
        run: |
          python scripts/generate_graphs.py \
            --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \
            --output graphs/java/10-clients/ \
            --phase STEADY \
            --language java \
            --workload "Java - 10 Clients" \
            --commit-id ${{ github.sha }}

      - name: Generate Java graphs - 100 Clients
        run: |
          python scripts/generate_graphs.py \
            --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \
            --output graphs/java/100-clients/ \
            --phase STEADY \
            --language java \
            --workload "Java - 100 Clients" \
            --commit-id ${{ github.sha }}

      # Ruby graphs — per concurrency level
      - name: Generate Ruby graphs - 1 Client
        run: |
          python scripts/generate_graphs.py \
            --results "results/github-runner/reference/*-basic-standalone-single-client.ndjson" \
            --output graphs/ruby/1-client/ \
            --phase STEADY \
            --language ruby \
            --workload "Ruby - 1 Client" \
            --commit-id ${{ github.sha }}

      - name: Generate Ruby graphs - 10 Clients
        run: |
          python scripts/generate_graphs.py \
            --results "results/github-runner/reference/*-basic-standalone-10-clients.ndjson" \
            --output graphs/ruby/10-clients/ \
            --phase STEADY \
            --language ruby \
            --workload "Ruby - 10 Clients" \
            --commit-id ${{ github.sha }}

      - name: Generate Ruby graphs - 100 Clients
        run: |
          python scripts/generate_graphs.py \
            --results "results/github-runner/reference/*-basic-standalone-100-clients.ndjson" \
            --output graphs/ruby/100-clients/ \
            --phase STEADY \
            --language ruby \
            --workload "Ruby - 100 Clients" \
            --commit-id ${{ github.sha }}

      - name: Upload graphs
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-graphs
          path: graphs/
          retention-days: 30

      - name: Create PR with updated graphs
        if: github.ref == 'refs/heads/main'
        uses: peter-evans/create-pull-request@v6
        with:
          commit-message: "Update benchmark graphs"
          title: "🔄 Update benchmark graphs"
          body: |
            This PR updates the benchmark comparison graphs from the latest CI run.
            
            Commit: ${{ github.sha }}
            Run: ${{ github.run_id }}
          branch: update-benchmark-graphs
          delete-branch: true
          add-paths: |
            graphs/