Benchmark #198
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| primary_driver: | |
| description: "Primary driver" | |
| required: true | |
| type: choice | |
| options: | |
| - spring-data-valkey | |
| - spring-data-redis | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - redisson | |
| default: "spring-data-valkey" | |
| secondary_driver: | |
| description: "Secondary driver (for spring-data-* only, ignored for others)" | |
| required: false | |
| type: choice | |
| options: | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - none | |
| default: "valkey-glide" | |
| workload: | |
| description: "Workload config" | |
| required: true | |
| type: choice | |
| options: | |
| - reference-workload-10-client | |
| - reference-workload-1-client | |
| default: "reference-workload-10-client" | |
| primary_version: | |
| description: "Primary driver version (version string or commit ID). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| secondary_version: | |
| description: "Secondary driver version (e.g., valkey-glide commit). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| job_id_prefix: | |
| description: 'Optional job ID prefix (e.g., "nightly", "pr-123")' | |
| required: false | |
| type: string | |
| default: "" | |
| custom_driver_config: | |
| description: "Custom driver config JSON (overrides driver selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| custom_workload_config: | |
| description: "Custom workload config JSON (overrides workload selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay: | |
| description: "Network delay with unit, e.g. '1ms' or '500us' (tc netem). Leave empty to disable." | |
| required: false | |
| type: string | |
| default: "" | |
| network_jitter: | |
| description: "Network jitter with unit, e.g. '1ms' or '100us' (tc netem). Requires network_delay." | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay_distribution: | |
| description: "Jitter distribution (tc netem). Requires jitter. Options: normal, pareto, paretonormal." | |
| required: false | |
| type: choice | |
| options: | |
| - "" | |
| - normal | |
| - pareto | |
| - paretonormal | |
| default: "" | |
| env: | |
| RESP_BENCH_REPO: "https://github.com/ikolomi/resp-bench.git" | |
| jobs: | |
| prepare: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| build_sdv: ${{ steps.analyze.outputs.build_sdv }} | |
| build_glide: ${{ steps.analyze.outputs.build_glide }} | |
| sdv_version: ${{ steps.analyze.outputs.sdv_version }} | |
| sdv_commit: ${{ steps.analyze.outputs.sdv_commit }} | |
| glide_version: ${{ steps.analyze.outputs.glide_version }} | |
| glide_commit: ${{ steps.analyze.outputs.glide_commit }} | |
| driver: ${{ steps.analyze.outputs.driver }} | |
| workload: ${{ steps.analyze.outputs.workload }} | |
| driver_id: ${{ steps.analyze.outputs.driver_id }} | |
| secondary_driver_id: ${{ steps.analyze.outputs.secondary_driver_id }} | |
| primary_version: ${{ steps.analyze.outputs.primary_version }} | |
| secondary_version: ${{ steps.analyze.outputs.secondary_version }} | |
| primary_is_commit: ${{ steps.analyze.outputs.primary_is_commit }} | |
| secondary_is_commit: ${{ steps.analyze.outputs.secondary_is_commit }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Validate inputs and analyze build requirements | |
| id: analyze | |
| run: | | |
| PRIMARY_DRIVER="${{ github.event.inputs.primary_driver }}" | |
| SECONDARY_DRIVER="${{ github.event.inputs.secondary_driver }}" | |
| WORKLOAD="${{ github.event.inputs.workload }}" | |
| PRIMARY_VERSION="${{ github.event.inputs.primary_version }}" | |
| SECONDARY_VERSION="${{ github.event.inputs.secondary_version }}" | |
| NETWORK_DELAY="${{ github.event.inputs.network_delay }}" | |
| NETWORK_JITTER="${{ github.event.inputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ github.event.inputs.network_delay_distribution }}" | |
| CONFIG_DIR=".github/workflows/benchmark_configs" | |
| # --- Validate tc netem input combinations --- | |
| if [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && [ -z "$NETWORK_JITTER" ]; then | |
| echo "ERROR: network_delay_distribution requires network_jitter" | |
| exit 1 | |
| fi | |
| if [ -n "$NETWORK_JITTER" ] && [ -z "$NETWORK_DELAY" ]; then | |
| echo "ERROR: network_jitter requires network_delay" | |
| exit 1 | |
| fi | |
| for val in "$NETWORK_DELAY" "$NETWORK_JITTER"; do | |
| if [ -n "$val" ] && ! [[ "$val" =~ ^[0-9]+(ms|us)$ ]]; then | |
| echo "ERROR: invalid tc netem value '$val' — expected format: <number>ms or <number>us" | |
| exit 1 | |
| fi | |
| done | |
| # --- Resolve and validate driver config --- | |
| CUSTOM_DRIVER_CONFIG_INPUT='${{ github.event.inputs.custom_driver_config }}' | |
| if [ -n "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then | |
| echo "Using custom driver config..." | |
| DRIVER_CONFIG="/tmp/custom-driver.json" | |
| printf '%s\n' "$CUSTOM_DRIVER_CONFIG_INPUT" > "$DRIVER_CONFIG" | |
| if ! jq . "$DRIVER_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom driver config is not valid JSON" | |
| echo "Content written:" | |
| cat "$DRIVER_CONFIG" | |
| exit 1 | |
| fi | |
| DRIVER="custom-driver" | |
| echo "✓ Custom driver config validated" | |
| else | |
| if [[ "$PRIMARY_DRIVER" == spring-data-* ]]; then | |
| # spring-data-* drivers require a secondary driver | |
| if [ "$SECONDARY_DRIVER" = "none" ] || [ -z "$SECONDARY_DRIVER" ]; then | |
| echo "ERROR: $PRIMARY_DRIVER requires a secondary driver (valkey-glide, jedis, or lettuce)" | |
| exit 1 | |
| fi | |
| # spring-data-redis cannot use valkey-glide | |
| if [ "$PRIMARY_DRIVER" = "spring-data-redis" ] && [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| echo "ERROR: spring-data-redis does not support valkey-glide. Use jedis or lettuce instead." | |
| exit 1 | |
| fi | |
| SECONDARY_IN_FILENAME="$SECONDARY_DRIVER" | |
| [ "$SECONDARY_IN_FILENAME" = "valkey-glide" ] && SECONDARY_IN_FILENAME="glide" | |
| DRIVER="${PRIMARY_DRIVER}-${SECONDARY_IN_FILENAME}-standalone" | |
| else | |
| DRIVER="${PRIMARY_DRIVER}-standalone" | |
| if [ "$SECONDARY_DRIVER" != "none" ] && [ -n "$SECONDARY_DRIVER" ]; then | |
| echo "Note: secondary_driver '$SECONDARY_DRIVER' ignored for standalone driver '$PRIMARY_DRIVER'" | |
| fi | |
| fi | |
| DRIVER_CONFIG="${CONFIG_DIR}/drivers/${DRIVER}.json" | |
| if [ ! -f "$DRIVER_CONFIG" ]; then | |
| echo "ERROR: Driver config not found: $DRIVER_CONFIG" | |
| echo "Available driver configs:" | |
| ls -la "${CONFIG_DIR}/drivers/" | |
| exit 1 | |
| fi | |
| echo "✓ Driver config: $DRIVER_CONFIG" | |
| fi | |
| # --- Resolve and validate workload config --- | |
| CUSTOM_WORKLOAD_CONFIG_INPUT='${{ github.event.inputs.custom_workload_config }}' | |
| if [ -n "$CUSTOM_WORKLOAD_CONFIG_INPUT" ]; then | |
| echo "Using custom workload config..." | |
| WORKLOAD_CONFIG="/tmp/custom-workload.json" | |
| printf '%s\n' "$CUSTOM_WORKLOAD_CONFIG_INPUT" > "$WORKLOAD_CONFIG" | |
| if ! jq . "$WORKLOAD_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom workload config is not valid JSON" | |
| echo "Content written:" | |
| cat "$WORKLOAD_CONFIG" | |
| exit 1 | |
| fi | |
| WORKLOAD="custom-workload" | |
| echo "✓ Custom workload config validated" | |
| else | |
| WORKLOAD_CONFIG="${CONFIG_DIR}/workloads/${WORKLOAD}.json" | |
| if [ ! -f "$WORKLOAD_CONFIG" ]; then | |
| echo "ERROR: Workload config not found: $WORKLOAD_CONFIG" | |
| echo "Available workload configs:" | |
| ls -la "${CONFIG_DIR}/workloads/" | |
| exit 1 | |
| fi | |
| echo "✓ Workload config: $WORKLOAD_CONFIG" | |
| fi | |
| echo "Driver config contents:" | |
| cat "$DRIVER_CONFIG" | jq . | |
| echo "Workload phases:" | |
| jq '.phases[] | {id, completion}' "$WORKLOAD_CONFIG" | |
| # --- Analyze driver requirements --- | |
| DRIVER_ID=$(jq -r '.driver_id' "$DRIVER_CONFIG") | |
| SECONDARY_DRIVER_ID=$(jq -r '.specific_driver_config.secondary_driver_id // empty' "$DRIVER_CONFIG") | |
| BUILD_SDV=false | |
| BUILD_GLIDE=false | |
| SDV_COMMIT="" | |
| SDV_VERSION="" | |
| GLIDE_COMMIT="" | |
| GLIDE_VERSION="" | |
| PRIMARY_IS_COMMIT=false | |
| SECONDARY_IS_COMMIT=false | |
| # Check if primary version is a commit ID (7-40 hex chars) | |
| if [ -n "$PRIMARY_VERSION" ] && [[ "$PRIMARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| PRIMARY_IS_COMMIT=true | |
| if [ "$DRIVER_ID" = "spring-data-valkey" ]; then | |
| BUILD_SDV=true | |
| SDV_COMMIT="$PRIMARY_VERSION" | |
| SDV_VERSION="${PRIMARY_VERSION:0:8}-SNAPSHOT" | |
| elif [ "$DRIVER_ID" = "valkey-glide" ]; then | |
| BUILD_GLIDE=true | |
| GLIDE_COMMIT="$PRIMARY_VERSION" | |
| GLIDE_VERSION="${PRIMARY_VERSION:0:8}-SNAPSHOT" | |
| else | |
| echo "ERROR: Commit IDs are only supported for spring-data-valkey and valkey-glide primary drivers" | |
| exit 1 | |
| fi | |
| fi | |
| # Check if secondary version is a commit ID | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ] && [[ "$SECONDARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| if [ "$SECONDARY_DRIVER_ID" = "valkey-glide" ]; then | |
| SECONDARY_IS_COMMIT=true | |
| BUILD_GLIDE=true | |
| GLIDE_COMMIT="$SECONDARY_VERSION" | |
| GLIDE_VERSION="${SECONDARY_VERSION:0:8}-SNAPSHOT" | |
| else | |
| echo "ERROR: Commit IDs are only supported for valkey-glide secondary driver" | |
| exit 1 | |
| fi | |
| fi | |
| # Cannot provide secondary version if driver has no secondary driver | |
| if [ -z "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| echo "ERROR: secondary_version provided but driver has no secondary driver" | |
| exit 1 | |
| fi | |
| # --- Write outputs --- | |
| echo "build_sdv=$BUILD_SDV" >> $GITHUB_OUTPUT | |
| echo "build_glide=$BUILD_GLIDE" >> $GITHUB_OUTPUT | |
| echo "sdv_version=$SDV_VERSION" >> $GITHUB_OUTPUT | |
| echo "sdv_commit=$SDV_COMMIT" >> $GITHUB_OUTPUT | |
| echo "glide_version=$GLIDE_VERSION" >> $GITHUB_OUTPUT | |
| echo "glide_commit=$GLIDE_COMMIT" >> $GITHUB_OUTPUT | |
| echo "driver=$DRIVER" >> $GITHUB_OUTPUT | |
| echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT | |
| echo "driver_id=$DRIVER_ID" >> $GITHUB_OUTPUT | |
| echo "secondary_driver_id=$SECONDARY_DRIVER_ID" >> $GITHUB_OUTPUT | |
| echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "primary_is_commit=$PRIMARY_IS_COMMIT" >> $GITHUB_OUTPUT | |
| echo "secondary_is_commit=$SECONDARY_IS_COMMIT" >> $GITHUB_OUTPUT | |
| echo "" | |
| echo "========================================" | |
| echo "Primary driver: $PRIMARY_DRIVER" | |
| echo "Secondary driver: $SECONDARY_DRIVER" | |
| echo "Workload: $WORKLOAD" | |
| echo "Primary version: '${PRIMARY_VERSION}'" | |
| echo "Secondary version: '${SECONDARY_VERSION}'" | |
| echo "Driver: $DRIVER ($DRIVER_ID)" | |
| echo "Build SDV: $BUILD_SDV (commit: ${SDV_COMMIT:-N/A}, version: ${SDV_VERSION:-N/A})" | |
| echo "Build Glide: $BUILD_GLIDE (commit: ${GLIDE_COMMIT:-N/A}, version: ${GLIDE_VERSION:-N/A})" | |
| echo "========================================" | |
| build-sdv: | |
| needs: prepare | |
| if: needs.prepare.outputs.build_sdv == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Restore SDV build from cache | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository/io/valkey/springframework | |
| key: sdv-build-${{ needs.prepare.outputs.sdv_commit }} | |
| - name: Set up Java 21 | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| - name: Build spring-data-valkey | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| set -euo pipefail | |
| COMMIT_ID="${{ needs.prepare.outputs.sdv_commit }}" | |
| SDV_VERSION="${{ needs.prepare.outputs.sdv_version }}" | |
| echo "Building spring-data-valkey from commit: $COMMIT_ID (version: $SDV_VERSION)" | |
| git clone https://github.com/${{ github.repository }}.git /tmp/spring-data-valkey | |
| cd /tmp/spring-data-valkey | |
| git checkout "$COMMIT_ID" | |
| mvn versions:set -DnewVersion="$SDV_VERSION" -DgenerateBackupPoms=false -q | |
| mvn install -DskipTests -Dmaven.compiler.source=17 -Dmaven.compiler.target=17 -Dgpg.skip=true -q | |
| echo "✓ Built spring-data-valkey version: $SDV_VERSION" | |
| - name: Upload SDV build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdv-build-${{ needs.prepare.outputs.sdv_commit }} | |
| path: ~/.m2/repository/io/valkey/springframework/ | |
| retention-days: 1 | |
| build-glide: | |
| needs: prepare | |
| if: needs.prepare.outputs.build_glide == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Restore glide build from cache | |
| id: cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository/io/valkey/valkey-glide | |
| key: glide-build-${{ needs.prepare.outputs.glide_commit }} | |
| - name: Set up Java 21 | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| - name: Install build dependencies | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| set -euo pipefail | |
| sudo apt-get update | |
| sudo apt-get install -y gcc pkg-config openssl libssl-dev unzip cmake python3-pip | |
| # Install Rust | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| source "$HOME/.cargo/env" | |
| rustc --version | |
| # Install protobuf compiler v29.1 | |
| PROTOC_VERSION="29.1" | |
| curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| unzip -o "protoc-${PROTOC_VERSION}-linux-x86_64.zip" -d "$HOME/.local" | |
| rm "protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| protoc --version | |
| # Install ziglang and cargo-zigbuild | |
| sudo python3 -m pip install ziglang | |
| cargo install --locked cargo-zigbuild | |
| echo "✓ valkey-glide build dependencies installed" | |
| - name: Build valkey-glide | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: | | |
| set -euo pipefail | |
| source "$HOME/.cargo/env" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| COMMIT_ID="${{ needs.prepare.outputs.glide_commit }}" | |
| GLIDE_VERSION="${{ needs.prepare.outputs.glide_version }}" | |
| echo "Building valkey-glide from commit: $COMMIT_ID (version: $GLIDE_VERSION)" | |
| git clone https://github.com/valkey-io/valkey-glide.git /tmp/valkey-glide | |
| cd /tmp/valkey-glide | |
| git checkout "$COMMIT_ID" | |
| cd java | |
| CARGO_PROFILE_RELEASE_DEBUG=true ./gradlew :client:buildAll -x test | |
| GLIDE_RELEASE_VERSION="$GLIDE_VERSION" ./gradlew publishToMavenLocal -x test | |
| echo "✓ Built valkey-glide $GLIDE_VERSION from commit $COMMIT_ID" | |
| - name: Upload glide build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: glide-build-${{ needs.prepare.outputs.glide_commit }} | |
| path: ~/.m2/repository/io/valkey/valkey-glide/ | |
| retention-days: 1 | |
| benchmark: | |
| needs: [prepare] | |
| if: needs.prepare.result == 'success' | |
| runs-on: [self-hosted, Linux, x86, ephemeral, metal] | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Validate EC2 type and CPU model | |
| run: | | |
| # c5.metal: Intel Xeon Platinum 8275CL (Cascade Lake), 96 vCPUs, 2 NUMA nodes | |
| EXPECTED_INSTANCE_TYPE="c5.metal" | |
| EXPECTED_CPU="8275CL" | |
| EXPECTED_STEPPING="7" | |
| EXPECTED_VCPUS="96" | |
| TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:60") | |
| INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-type) | |
| CPU_MODEL=$(grep 'model name' /proc/cpuinfo | head -1 | sed 's/.*: //') | |
| CPU_STEPPING=$(lscpu | grep '^Stepping:' | awk '{print $2}') | |
| CPU_COUNT=$(nproc) | |
| echo "Instance type: $INSTANCE_TYPE" | |
| echo "CPU: $CPU_MODEL" | |
| echo "Stepping: $CPU_STEPPING" | |
| echo "vCPUs: $CPU_COUNT" | |
| lscpu | grep -E 'Model name|Stepping|CPU max MHz|CPU\(s\):|NUMA' | |
| ERRORS="" | |
| if [ "$INSTANCE_TYPE" != "$EXPECTED_INSTANCE_TYPE" ]; then | |
| ERRORS="${ERRORS}Instance type: expected '$EXPECTED_INSTANCE_TYPE', got '$INSTANCE_TYPE'\n" | |
| fi | |
| if ! echo "$CPU_MODEL" | grep -q "$EXPECTED_CPU"; then | |
| ERRORS="${ERRORS}CPU model: expected '$EXPECTED_CPU', got '$CPU_MODEL'\n" | |
| fi | |
| if [ "$CPU_STEPPING" != "$EXPECTED_STEPPING" ]; then | |
| ERRORS="${ERRORS}Stepping: expected '$EXPECTED_STEPPING', got '$CPU_STEPPING'\n" | |
| fi | |
| if [ "$CPU_COUNT" != "$EXPECTED_VCPUS" ]; then | |
| ERRORS="${ERRORS}vCPU count: expected '$EXPECTED_VCPUS', got '$CPU_COUNT'\n" | |
| fi | |
| if [ -n "$ERRORS" ]; then | |
| echo "ERROR: Hardware validation failed:" | |
| echo -e "$ERRORS" | |
| echo "This machine does not match expected c5.metal specs. Failing early to avoid inconsistent benchmark results." | |
| exit 1 | |
| fi | |
| echo "✓ Hardware validated: $CPU_MODEL, stepping $CPU_STEPPING, $CPU_COUNT vCPUs" | |
| - uses: actions/checkout@v4 | |
| - name: Set up Java 21 and Maven | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y openjdk-21-jdk-headless maven | |
| java -version | |
| mvn -version | |
| - name: Clone resp-bench and pre-compile Valkey server | |
| id: resp-bench | |
| run: | | |
| echo "Cloning resp-bench repository..." | |
| git clone --depth 1 ${{ env.RESP_BENCH_REPO }} resp-bench | |
| RESP_BENCH_COMMIT=$(git -C resp-bench rev-parse HEAD) | |
| echo "commit=$RESP_BENCH_COMMIT" >> $GITHUB_OUTPUT | |
| echo "✓ resp-bench cloned (commit: ${RESP_BENCH_COMMIT:0:7})" | |
| # Pre-compile Valkey server binary in the background. | |
| # When the orchestrator later calls `make server-standalone-start`, | |
| # Make will find the binary already exists and just start the servers. | |
| echo "Starting Valkey server compilation in background..." | |
| (cd resp-bench && make work/valkey/bin/valkey-server > /tmp/valkey-build.log 2>&1; echo $? > /tmp/valkey-build.exit) & | |
| - name: Set up config files | |
| id: config | |
| run: | | |
| DRIVER="${{ needs.prepare.outputs.driver }}" | |
| WORKLOAD="${{ needs.prepare.outputs.workload }}" | |
| CONFIG_DIR=".github/workflows/benchmark_configs" | |
| CUSTOM_CONFIG_DIR="/tmp/custom_configs" | |
| mkdir -p "$CUSTOM_CONFIG_DIR" | |
| # Re-create custom config files on this runner (validated in prepare job) | |
| CUSTOM_DRIVER_CONFIG_INPUT='${{ github.event.inputs.custom_driver_config }}' | |
| if [ -n "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then | |
| DRIVER_CONFIG="$CUSTOM_CONFIG_DIR/custom-driver.json" | |
| printf '%s\n' "$CUSTOM_DRIVER_CONFIG_INPUT" > "$DRIVER_CONFIG" | |
| else | |
| DRIVER_CONFIG="${CONFIG_DIR}/drivers/${DRIVER}.json" | |
| fi | |
| CUSTOM_WORKLOAD_CONFIG_INPUT='${{ github.event.inputs.custom_workload_config }}' | |
| if [ -n "$CUSTOM_WORKLOAD_CONFIG_INPUT" ]; then | |
| WORKLOAD_CONFIG="$CUSTOM_CONFIG_DIR/custom-workload.json" | |
| printf '%s\n' "$CUSTOM_WORKLOAD_CONFIG_INPUT" > "$WORKLOAD_CONFIG" | |
| else | |
| WORKLOAD_CONFIG="${CONFIG_DIR}/workloads/${WORKLOAD}.json" | |
| fi | |
| echo "driver_config=$DRIVER_CONFIG" >> $GITHUB_OUTPUT | |
| echo "workload_config=$WORKLOAD_CONFIG" >> $GITHUB_OUTPUT | |
| echo "Driver: $DRIVER" | |
| echo "Driver config: $DRIVER_CONFIG" | |
| echo "Workload: $WORKLOAD" | |
| echo "Workload config: $WORKLOAD_CONFIG" | |
| - name: Wait for build jobs to complete | |
| if: needs.prepare.outputs.build_sdv == 'true' || needs.prepare.outputs.build_glide == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| API_URL="https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" | |
| echo "Waiting for build jobs to complete..." | |
| while true; do | |
| JOBS=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$API_URL?per_page=100") | |
| ALL_DONE=true | |
| for JOB_NAME in "build-sdv" "build-glide"; do | |
| STATUS=$(echo "$JOBS" | jq -r --arg name "$JOB_NAME" '.jobs[] | select(.name==$name) | .status' | head -1) | |
| CONCLUSION=$(echo "$JOBS" | jq -r --arg name "$JOB_NAME" '.jobs[] | select(.name==$name) | .conclusion' | head -1) | |
| if [ -z "$STATUS" ]; then | |
| continue | |
| fi | |
| echo " $JOB_NAME: status=$STATUS conclusion=$CONCLUSION" | |
| if [ "$STATUS" = "completed" ]; then | |
| if [ "$CONCLUSION" = "failure" ] || [ "$CONCLUSION" = "cancelled" ]; then | |
| echo "ERROR: $JOB_NAME failed with conclusion: $CONCLUSION" | |
| exit 1 | |
| fi | |
| else | |
| ALL_DONE=false | |
| fi | |
| done | |
| if [ "$ALL_DONE" = "true" ]; then | |
| echo "✓ All build jobs completed successfully" | |
| break | |
| fi | |
| sleep 10 | |
| done | |
| - name: Download SDV build artifacts | |
| if: needs.prepare.outputs.build_sdv == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdv-build-${{ needs.prepare.outputs.sdv_commit }} | |
| path: ~/.m2/repository/io/valkey/springframework/ | |
| - name: Download glide build artifacts | |
| if: needs.prepare.outputs.build_glide == 'true' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: glide-build-${{ needs.prepare.outputs.glide_commit }} | |
| path: ~/.m2/repository/io/valkey/valkey-glide/ | |
| - name: Update resp-bench driver versions | |
| run: | | |
| cd resp-bench | |
| DRIVER_ID="${{ needs.prepare.outputs.driver_id }}" | |
| SECONDARY_DRIVER_ID="${{ needs.prepare.outputs.secondary_driver_id }}" | |
| SECONDARY_VERSION="${{ needs.prepare.outputs.secondary_version }}" | |
| SECONDARY_IS_COMMIT="${{ needs.prepare.outputs.secondary_is_commit }}" | |
| # Update primary driver version if supplied | |
| PRIMARY_VERSION="${{ needs.prepare.outputs.primary_version }}" | |
| PRIMARY_IS_COMMIT="${{ needs.prepare.outputs.primary_is_commit }}" | |
| if [ -n "$PRIMARY_VERSION" ]; then | |
| case "$DRIVER_ID" in | |
| spring-data-valkey) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| SDV_VERSION="${{ needs.prepare.outputs.sdv_version }}" | |
| echo "Updating spring-data-valkey to locally built $SDV_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$SDV_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| else | |
| echo "Updating spring-data-valkey to release version $PRIMARY_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$PRIMARY_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| fi | |
| ;; | |
| valkey-glide) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| GLIDE_VERSION="${{ needs.prepare.outputs.glide_version }}" | |
| echo "Updating valkey-glide (primary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (primary) to release version $PRIMARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$PRIMARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$PRIMARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$PRIMARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| redisson) | |
| echo "Updating redisson (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<redisson.version>.*</redisson.version>|<redisson.version>$PRIMARY_VERSION</redisson.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| # Update secondary driver version if a version/commit-id for it was supplied. Otherwise use defaults. | |
| # If secondary version is glide and it was built locally, update the pom.xml file to use this built version. | |
| # Otherwise, update the version supplied in the pom.xml. | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| case "$SECONDARY_DRIVER_ID" in | |
| valkey-glide) | |
| if [ "$SECONDARY_IS_COMMIT" = "true" ]; then | |
| # Use the version from the local build | |
| GLIDE_VERSION="${{ needs.prepare.outputs.glide_version }}" | |
| echo "Updating valkey-glide (secondary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (secondary) to release version $SECONDARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$SECONDARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis to $SECONDARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$SECONDARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce to $SECONDARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$SECONDARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| echo "" | |
| echo "=== Updated pom.xml versions ===" | |
| grep -E "(spring-data-valkey|valkey-glide|jedis|lettuce)\.version" java/pom.xml | |
| - name: Build resp-bench Java benchmark | |
| run: | | |
| cd resp-bench | |
| make java-build | |
| echo "✓ Java benchmark built" | |
| ls -la java/target/*.jar | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install -y sysstat python3-pip numactl | |
| sudo python3 -m pip install psycopg2-binary boto3 hdrhistogram | |
| # async-profiler for flame graphs | |
| if [ ! -f "/opt/async-profiler/bin/asprof" ]; then | |
| curl -sL "https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz" | sudo tar xzf - -C /opt | |
| sudo mv /opt/async-profiler-3.0-linux-x64 /opt/async-profiler | |
| fi | |
| /opt/async-profiler/bin/asprof --version | |
| # perf for hardware counters | |
| sudo apt-get install -y linux-tools-$(uname -r) 2>/dev/null || \ | |
| sudo apt-get install -y linux-tools-generic 2>/dev/null || true | |
| command -v perf && perf --version || echo "perf not available" | |
| - name: Wait for Valkey server compilation | |
| run: | | |
| echo "Waiting for Valkey server compilation..." | |
| TIMEOUT=300 | |
| ELAPSED=0 | |
| while [ ! -f "/tmp/valkey-build.exit" ]; do | |
| if [ "$ELAPSED" -ge "$TIMEOUT" ]; then | |
| echo "ERROR: Valkey server compilation timed out after ${TIMEOUT}s" | |
| echo "--- build log ---" | |
| cat /tmp/valkey-build.log 2>/dev/null || echo "(no log)" | |
| exit 1 | |
| fi | |
| sleep 2 | |
| ELAPSED=$((ELAPSED + 2)) | |
| done | |
| EXIT_CODE=$(cat /tmp/valkey-build.exit) | |
| if [ "$EXIT_CODE" -ne 0 ]; then | |
| echo "ERROR: Valkey server compilation failed (exit $EXIT_CODE)" | |
| echo "--- build log ---" | |
| cat /tmp/valkey-build.log 2>/dev/null || echo "(no log)" | |
| exit 1 | |
| fi | |
| echo "✓ Valkey server compiled (${ELAPSED}s)" | |
| resp-bench/work/valkey/bin/valkey-server --version | |
| - name: Run benchmark and publish results to DB | |
| run: | | |
| DRIVER="${{ needs.prepare.outputs.driver }}" | |
| WORKLOAD="${{ needs.prepare.outputs.workload }}" | |
| DRIVER_CONFIG="${{ steps.config.outputs.driver_config }}" | |
| WORKLOAD_CONFIG="${{ steps.config.outputs.workload_config }}" | |
| JOB_ID_PREFIX="${{ github.event.inputs.job_id_prefix }}" | |
| PREFIX_ARG="" | |
| [ -n "$JOB_ID_PREFIX" ] && PREFIX_ARG="--job-id-prefix $JOB_ID_PREFIX" | |
| NETWORK_DELAY="${{ github.event.inputs.network_delay }}" | |
| NETWORK_JITTER="${{ github.event.inputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ github.event.inputs.network_delay_distribution }}" | |
| TC_ARGS="" | |
| [ -n "$NETWORK_DELAY" ] && TC_ARGS="$TC_ARGS --network-delay $NETWORK_DELAY" | |
| [ -n "$NETWORK_JITTER" ] && TC_ARGS="$TC_ARGS --network-jitter $NETWORK_JITTER" | |
| [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && TC_ARGS="$TC_ARGS --network-delay-distribution $NETWORK_DELAY_DISTRIBUTION" | |
| python3 -u .github/workflows/benchmark_orchestrator.py \ | |
| --output "benchmark_results_${DRIVER}_${WORKLOAD}.json" \ | |
| --workload-config "$WORKLOAD_CONFIG" \ | |
| --driver-config "$DRIVER_CONFIG" \ | |
| --resp-bench-dir "${{ github.workspace }}/resp-bench" \ | |
| --resp-bench-commit "${{ steps.resp-bench.outputs.commit }}" \ | |
| --s3-bucket "${{ secrets.BENCHMARK_S3_BUCKET }}" \ | |
| --pg-host "${{ secrets.BENCHMARK_PG_HOST }}" \ | |
| --pg-secret-name "${{ secrets.BENCHMARK_PG_SECRET_NAME }}" \ | |
| $PREFIX_ARG $TC_ARGS | |
| - name: Display results summary | |
| if: always() | |
| run: | | |
| DRIVER="${{ needs.prepare.outputs.driver }}" | |
| WORKLOAD="${{ needs.prepare.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| if [ -f "$RESULT_FILE" ]; then | |
| echo "=== Benchmark Results Summary ===" | |
| python3 -c " | |
| import json | |
| with open('$RESULT_FILE') as f: | |
| data = json.load(f) | |
| print(f\"Job ID: {data['job_id']}\") | |
| print(f\"Driver: {data['config']['driver']['driver_id']}\") | |
| print(f\"Workload: {data['config']['workload']['benchmark_profile']['name']}\") | |
| print(f\"Elapsed: {data['results']['elapsed_ms']}ms\") | |
| print() | |
| versions = data.get('versions', {}) | |
| print(f\"Primary: {versions.get('primary_driver_id')} @ {versions.get('primary_driver_version', 'N/A')}\") | |
| print(f\"Secondary: {versions.get('secondary_driver_id', 'N/A')} @ {versions.get('secondary_driver_version', 'N/A')}\") | |
| if versions.get('commit_id'): | |
| print(f\"resp-bench: {versions.get('commit_id')}\") | |
| print() | |
| steady = data['results']['phases'].get('STEADY', {}) | |
| if steady: | |
| totals = steady.get('totals', {}) | |
| print(f\"STEADY totals: {totals.get('requests', 0):,} requests, {totals.get('errors', 0)} errors\") | |
| print() | |
| print('Per-command metrics:') | |
| for cmd, m in steady.get('metrics', {}).items(): | |
| reqs = m['requests'] | |
| errs = m['errors'] | |
| lat = m.get('latency', {}) | |
| summary = lat.get('summary', {}) | |
| print(f\" {cmd}:\") | |
| print(f\" Requests: {reqs:,} (errors: {errs})\") | |
| print(f\" Latency: min={summary.get('min')} p50={summary.get('p50')} p95={summary.get('p95')} p99={summary.get('p99')} p999={summary.get('p999')} max={summary.get('max')} us\") | |
| print() | |
| print('Perf:') | |
| perf = data['results']['perf']['counters'] | |
| if perf.get('ipc'): | |
| print(f\" IPC: {perf['ipc']}\") | |
| if perf.get('cache_miss_rate'): | |
| print(f\" Cache miss rate: {perf['cache_miss_rate']}%\") | |
| if perf.get('branch_miss_rate'): | |
| print(f\" Branch miss rate: {perf['branch_miss_rate']}%\") | |
| " | |
| else | |
| echo "⚠ Result file not found: $RESULT_FILE" | |
| fi | |
| - name: Generate summary report | |
| if: always() | |
| run: | | |
| DRIVER="${{ needs.prepare.outputs.driver }}" | |
| WORKLOAD="${{ needs.prepare.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| echo "# Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "$RESULT_FILE" ]; then | |
| python3 << EOF >> $GITHUB_STEP_SUMMARY | |
| import json | |
| with open("$RESULT_FILE") as f: | |
| data = json.load(f) | |
| job_id = data["job_id"] | |
| versions = data.get("versions", {}) | |
| driver = data["config"]["driver"]["driver_id"] | |
| workload_profile = data["config"]["workload"]["benchmark_profile"]["name"] | |
| elapsed = data["results"]["elapsed_ms"] | |
| network_delay = data["results"].get("network_delay", "") | |
| network_jitter = data["results"].get("network_jitter", "") | |
| network_distribution = data["results"].get("network_delay_distribution", "") | |
| pv = versions.get("primary_driver_version") or "N/A" | |
| sv = versions.get("secondary_driver_version") | |
| sid = versions.get("secondary_driver_id") | |
| commit_id = versions.get("commit_id") | |
| print(f"**Job ID:** \`{job_id}\`") | |
| print(f"**Driver:** {driver} @ \`{pv}\`") | |
| if sid and sv: | |
| print(f"**Secondary:** {sid} @ \`{sv}\`") | |
| if commit_id: | |
| print(f"**resp-bench:** \`{commit_id}\`") | |
| print(f"**Workload config:** \`${WORKLOAD}\` ({workload_profile})") | |
| print(f"**Driver config:** \`${DRIVER}\`") | |
| delay_str = network_delay or "disabled" | |
| if network_jitter: | |
| delay_str += f" jitter {network_jitter}" | |
| if network_distribution: | |
| delay_str += f" distribution {network_distribution}" | |
| print(f"**Simulated network delay:** {delay_str}") | |
| print(f"**Elapsed:** {elapsed}ms") | |
| print() | |
| print("| Versions | Driver | Workload | Status | Elapsed |") | |
| print("|----------|--------|----------|--------|---------|") | |
| version_str = pv | |
| if sv: | |
| version_str += f" ({sid}: {sv})" | |
| print(f"| {version_str} | {driver} | \`${WORKLOAD}\` | ✅ | {elapsed}ms |") | |
| print() | |
| steady = data["results"]["phases"].get("STEADY", {}) | |
| if steady: | |
| print("### Steady State Metrics") | |
| print("| Command | Requests | Errors | p50 (μs) | p95 (μs) | p99 (μs) | p999 (μs) |") | |
| print("|---------|----------|--------|----------|----------|----------|-----------|") | |
| for cmd, m in steady.get("metrics", {}).items(): | |
| s = m.get("latency", {}).get("summary", {}) | |
| print(f"| {cmd} | {m['requests']:,} | {m['errors']} | {s.get('p50','-')} | {s.get('p95','-')} | {s.get('p99','-')} | {s.get('p999','-')} |") | |
| EOF | |
| else | |
| echo "❌ No results found" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ needs.prepare.outputs.driver }}-${{ needs.prepare.outputs.workload }} | |
| path: | | |
| benchmark_results_${{ needs.prepare.outputs.driver }}_${{ needs.prepare.outputs.workload }}.json | |
| benchmark_results_${{ needs.prepare.outputs.driver }}_${{ needs.prepare.outputs.workload }}.ndjson | |
| benchmark_results_${{ needs.prepare.outputs.driver }}_${{ needs.prepare.outputs.workload }}_collapsed.txt |