Benchmark #238
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| primary_driver: | |
| description: "Primary driver" | |
| required: true | |
| type: choice | |
| options: | |
| - spring-data-valkey | |
| - spring-data-redis | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - redisson | |
| default: "spring-data-valkey" | |
| secondary_driver: | |
| description: "Secondary driver (for spring-data-* only, ignored for others)" | |
| required: false | |
| type: choice | |
| options: | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - none | |
| default: "valkey-glide" | |
| mode: | |
| description: "Server topology mode" | |
| required: false | |
| type: choice | |
| options: | |
| - standalone | |
| - cluster | |
| default: "standalone" | |
| workload: | |
| description: "Workload config" | |
| required: true | |
| type: choice | |
| options: | |
| - reference-workload-10-client | |
| - reference-workload-1-client | |
| default: "reference-workload-10-client" | |
| primary_version: | |
| description: "Primary driver version (version string or commit ID). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| secondary_version: | |
| description: "Secondary driver version (e.g., valkey-glide commit). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| job_id_prefix: | |
| description: 'Optional job ID prefix (e.g., "nightly", "pr-123")' | |
| required: false | |
| type: string | |
| default: "" | |
| custom_driver_config: | |
| description: "Custom driver config JSON (overrides driver selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| custom_workload_config: | |
| description: "Custom workload config JSON (overrides workload selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay: | |
| description: "Network delay with unit, e.g. '1ms' or '500us' (tc netem). Leave empty to disable." | |
| required: false | |
| type: string | |
| default: "" | |
| network_jitter: | |
| description: "Network jitter with unit, e.g. '1ms' or '100us' (tc netem). Requires network_delay." | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay_distribution: | |
| description: "Jitter distribution (tc netem). Requires jitter. Options: normal, pareto, paretonormal." | |
| required: false | |
| type: choice | |
| options: | |
| - "" | |
| - normal | |
| - pareto | |
| - paretonormal | |
| default: "" | |
| env: | |
| RESP_BENCH_REPO: "https://github.com/ikolomi/resp-bench.git" | |
| jobs: | |
| benchmark: | |
| runs-on: [self-hosted, Linux, x86, ephemeral, metal] | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Validate EC2 type and CPU model | |
| run: | | |
| # c5.metal: Intel Xeon Platinum 8275CL (Cascade Lake), 96 vCPUs, 2 NUMA nodes | |
| EXPECTED_INSTANCE_TYPE="c5.metal" | |
| EXPECTED_CPU="8275CL" | |
| EXPECTED_STEPPING="7" | |
| EXPECTED_VCPUS="96" | |
| TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:60") | |
| INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-type) | |
| CPU_MODEL=$(grep 'model name' /proc/cpuinfo | head -1 | sed 's/.*: //') | |
| CPU_STEPPING=$(lscpu | grep '^Stepping:' | awk '{print $2}') | |
| CPU_COUNT=$(nproc) | |
| echo "Instance type: $INSTANCE_TYPE" | |
| echo "CPU: $CPU_MODEL" | |
| echo "Stepping: $CPU_STEPPING" | |
| echo "vCPUs: $CPU_COUNT" | |
| lscpu | grep -E 'Model name|Stepping|CPU max MHz|CPU\(s\):|NUMA' | |
| ERRORS="" | |
| if [ "$INSTANCE_TYPE" != "$EXPECTED_INSTANCE_TYPE" ]; then | |
| ERRORS="${ERRORS}Instance type: expected '$EXPECTED_INSTANCE_TYPE', got '$INSTANCE_TYPE'\n" | |
| fi | |
| if ! echo "$CPU_MODEL" | grep -q "$EXPECTED_CPU"; then | |
| ERRORS="${ERRORS}CPU model: expected '$EXPECTED_CPU', got '$CPU_MODEL'\n" | |
| fi | |
| if [ "$CPU_STEPPING" != "$EXPECTED_STEPPING" ]; then | |
| ERRORS="${ERRORS}Stepping: expected '$EXPECTED_STEPPING', got '$CPU_STEPPING'\n" | |
| fi | |
| if [ "$CPU_COUNT" != "$EXPECTED_VCPUS" ]; then | |
| ERRORS="${ERRORS}vCPU count: expected '$EXPECTED_VCPUS', got '$CPU_COUNT'\n" | |
| fi | |
| if [ -n "$ERRORS" ]; then | |
| echo "ERROR: Hardware validation failed:" | |
| echo -e "$ERRORS" | |
| echo "This machine does not match expected c5.metal specs. Failing early to avoid inconsistent benchmark results." | |
| exit 1 | |
| fi | |
| echo "✓ Hardware validated: $CPU_MODEL, stepping $CPU_STEPPING, $CPU_COUNT vCPUs" | |
| - uses: actions/checkout@v4 | |
| - name: Set up Java 21 and Maven | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y openjdk-21-jdk-headless maven | |
| java -version | |
| mvn -version | |
| - name: Clone resp-bench | |
| id: resp-bench | |
| run: | | |
| echo "Cloning resp-bench repository..." | |
| git clone --depth 1 ${{ env.RESP_BENCH_REPO }} resp-bench | |
| RESP_BENCH_COMMIT=$(git -C resp-bench rev-parse HEAD) | |
| echo "commit=$RESP_BENCH_COMMIT" >> $GITHUB_OUTPUT | |
| echo "✓ resp-bench cloned (commit: ${RESP_BENCH_COMMIT:0:7})" | |
| - name: Resolve inputs | |
| id: inputs | |
| run: | | |
| PRIMARY_DRIVER="${{ github.event.inputs.primary_driver }}" | |
| SECONDARY_DRIVER="${{ github.event.inputs.secondary_driver }}" | |
| MODE="${{ github.event.inputs.mode || 'standalone' }}" | |
| WORKLOAD="${{ github.event.inputs.workload }}" | |
| PRIMARY_VERSION="${{ github.event.inputs.primary_version }}" | |
| SECONDARY_VERSION="${{ github.event.inputs.secondary_version }}" | |
| JOB_ID_PREFIX="${{ github.event.inputs.job_id_prefix }}" | |
| NETWORK_DELAY="${{ github.event.inputs.network_delay }}" | |
| NETWORK_JITTER="${{ github.event.inputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ github.event.inputs.network_delay_distribution }}" | |
| # Validate tc netem input combinations | |
| if [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && [ -z "$NETWORK_JITTER" ]; then | |
| echo "ERROR: network_delay_distribution requires network_jitter" | |
| exit 1 | |
| fi | |
| if [ -n "$NETWORK_JITTER" ] && [ -z "$NETWORK_DELAY" ]; then | |
| echo "ERROR: network_jitter requires network_delay" | |
| exit 1 | |
| fi | |
| # Validate unit format (number followed by ms or us) | |
| for val in "$NETWORK_DELAY" "$NETWORK_JITTER"; do | |
| if [ -n "$val" ] && ! [[ "$val" =~ ^[0-9]+(ms|us)$ ]]; then | |
| echo "ERROR: invalid tc netem value '$val' — expected format: <number>ms or <number>us" | |
| exit 1 | |
| fi | |
| done | |
| echo "primary_driver=$PRIMARY_DRIVER" >> $GITHUB_OUTPUT | |
| echo "secondary_driver=$SECONDARY_DRIVER" >> $GITHUB_OUTPUT | |
| echo "mode=$MODE" >> $GITHUB_OUTPUT | |
| echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT | |
| echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "job_id_prefix=$JOB_ID_PREFIX" >> $GITHUB_OUTPUT | |
| echo "network_delay=$NETWORK_DELAY" >> $GITHUB_OUTPUT | |
| echo "network_jitter=$NETWORK_JITTER" >> $GITHUB_OUTPUT | |
| echo "network_delay_distribution=$NETWORK_DELAY_DISTRIBUTION" >> $GITHUB_OUTPUT | |
| echo "========================================" | |
| echo "Primary driver: $PRIMARY_DRIVER" | |
| echo "Secondary driver: $SECONDARY_DRIVER" | |
| echo "Mode: $MODE" | |
| echo "Workload: $WORKLOAD" | |
| echo "Primary version: '${PRIMARY_VERSION}'" | |
| echo "Secondary version: '${SECONDARY_VERSION}'" | |
| echo "Job ID prefix: '${JOB_ID_PREFIX}'" | |
| echo "Network delay: '${NETWORK_DELAY}' jitter='${NETWORK_JITTER}' distribution='${NETWORK_DELAY_DISTRIBUTION}'" | |
| echo "========================================" | |
| - name: Validate configs exist | |
| id: validate | |
| run: | | |
| PRIMARY_DRIVER="${{ steps.inputs.outputs.primary_driver }}" | |
| SECONDARY_DRIVER="${{ steps.inputs.outputs.secondary_driver }}" | |
| MODE="${{ steps.inputs.outputs.mode }}" | |
| WORKLOAD="${{ steps.inputs.outputs.workload }}" | |
| CONFIG_DIR=".github/workflows/benchmark_configs" | |
| CUSTOM_CONFIG_DIR="/tmp/custom_configs" | |
| mkdir -p "$CUSTOM_CONFIG_DIR" | |
| # Handle custom driver config | |
| CUSTOM_DRIVER_CONFIG_INPUT='${{ github.event.inputs.custom_driver_config }}' | |
| if [ -n "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then | |
| echo "Using custom driver config..." | |
| DRIVER_CONFIG="$CUSTOM_CONFIG_DIR/custom-driver.json" | |
| printf '%s\n' "$CUSTOM_DRIVER_CONFIG_INPUT" > "$DRIVER_CONFIG" | |
| # Validate JSON | |
| if ! jq . "$DRIVER_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom driver config is not valid JSON" | |
| echo "Content written:" | |
| cat "$DRIVER_CONFIG" | |
| exit 1 | |
| fi | |
| DRIVER="custom-driver" | |
| echo "✓ Custom driver config validated" | |
| else | |
| # Resolve driver config filename based on primary and secondary driver | |
| if [[ "$PRIMARY_DRIVER" == spring-data-* ]]; then | |
| # spring-data-* drivers require a secondary driver | |
| if [ "$SECONDARY_DRIVER" = "none" ] || [ -z "$SECONDARY_DRIVER" ]; then | |
| echo "ERROR: $PRIMARY_DRIVER requires a secondary driver (valkey-glide, jedis, or lettuce)" | |
| exit 1 | |
| fi | |
| # spring-data-redis cannot use valkey-glide | |
| if [ "$PRIMARY_DRIVER" = "spring-data-redis" ] && [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| echo "ERROR: spring-data-redis does not support valkey-glide. Use jedis or lettuce instead." | |
| exit 1 | |
| fi | |
| # Map secondary driver name to filename convention (valkey-glide -> glide) | |
| SECONDARY_IN_FILENAME="$SECONDARY_DRIVER" | |
| if [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| SECONDARY_IN_FILENAME="glide" | |
| fi | |
| DRIVER="${PRIMARY_DRIVER}-${SECONDARY_IN_FILENAME}" | |
| else | |
| # Standalone drivers (valkey-glide, jedis, lettuce, redisson) | |
| DRIVER="${PRIMARY_DRIVER}" | |
| if [ "$SECONDARY_DRIVER" != "none" ] && [ -n "$SECONDARY_DRIVER" ]; then | |
| echo "Note: secondary_driver '$SECONDARY_DRIVER' ignored for standalone driver '$PRIMARY_DRIVER'" | |
| fi | |
| fi | |
| DRIVER_CONFIG="${CONFIG_DIR}/drivers/${DRIVER}.json" | |
| if [ ! -f "$DRIVER_CONFIG" ]; then | |
| echo "ERROR: Driver config not found: $DRIVER_CONFIG" | |
| echo "Available driver configs:" | |
| ls -la "${CONFIG_DIR}/drivers/" | |
| exit 1 | |
| fi | |
| echo "✓ Driver config: $DRIVER_CONFIG" | |
| fi | |
| # Handle selected driver mode (server topology) | |
| if [ -z "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then | |
| DRIVER_CONFIG_WITH_MODE="$CUSTOM_CONFIG_DIR/driver-config.json" | |
| jq --arg mode "$MODE" '.mode = $mode' "$DRIVER_CONFIG" > "$DRIVER_CONFIG_WITH_MODE" | |
| DRIVER_CONFIG="$DRIVER_CONFIG_WITH_MODE" | |
| echo "✓ Mode set to: $MODE" | |
| fi | |
| # Handle custom workload config | |
| CUSTOM_WORKLOAD_CONFIG_INPUT='${{ github.event.inputs.custom_workload_config }}' | |
| if [ -n "$CUSTOM_WORKLOAD_CONFIG_INPUT" ]; then | |
| echo "Using custom workload config..." | |
| WORKLOAD_CONFIG="$CUSTOM_CONFIG_DIR/custom-workload.json" | |
| printf '%s\n' "$CUSTOM_WORKLOAD_CONFIG_INPUT" > "$WORKLOAD_CONFIG" | |
| # Validate JSON | |
| if ! jq . "$WORKLOAD_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom workload config is not valid JSON" | |
| echo "Content written:" | |
| cat "$WORKLOAD_CONFIG" | |
| exit 1 | |
| fi | |
| WORKLOAD="custom-workload" | |
| echo "✓ Custom workload config validated" | |
| else | |
| WORKLOAD_CONFIG="${CONFIG_DIR}/workloads/${WORKLOAD}.json" | |
| if [ ! -f "$WORKLOAD_CONFIG" ]; then | |
| echo "ERROR: Workload config not found: $WORKLOAD_CONFIG" | |
| echo "Available workload configs:" | |
| ls -la "${CONFIG_DIR}/workloads/" | |
| exit 1 | |
| fi | |
| echo "✓ Workload config: $WORKLOAD_CONFIG" | |
| fi | |
| echo "driver=$DRIVER" >> $GITHUB_OUTPUT | |
| echo "driver_config=$DRIVER_CONFIG" >> $GITHUB_OUTPUT | |
| echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT | |
| echo "workload_config=$WORKLOAD_CONFIG" >> $GITHUB_OUTPUT | |
| echo "Driver config contents:" | |
| cat "$DRIVER_CONFIG" | jq . | |
| echo "Workload phases:" | |
| jq '.phases[] | {id, completion}' "$WORKLOAD_CONFIG" | |
| - name: Analyze driver requirements | |
| id: driver-info | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| PRIMARY_VERSION="${{ steps.inputs.outputs.primary_version }}" | |
| SECONDARY_VERSION="${{ steps.inputs.outputs.secondary_version }}" | |
| DRIVER_CONFIG="${{ steps.validate.outputs.driver_config }}" | |
| DRIVER_ID=$(jq -r '.driver_id' "$DRIVER_CONFIG") | |
| SECONDARY_DRIVER_ID=$(jq -r '.specific_driver_config.secondary_driver_id // empty' "$DRIVER_CONFIG") | |
| echo "driver_id=$DRIVER_ID" >> $GITHUB_OUTPUT | |
| echo "secondary_driver_id=$SECONDARY_DRIVER_ID" >> $GITHUB_OUTPUT | |
| # Check if primary version is a commit ID (7-40 hex chars) or version number | |
| PRIMARY_VERSION="${{ steps.inputs.outputs.primary_version }}" | |
| if [ -n "$PRIMARY_VERSION" ] && [[ "$PRIMARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| echo "primary_is_commit=true" >> $GITHUB_OUTPUT | |
| echo "Detected: primary_version '$PRIMARY_VERSION' is a commit ID" | |
| # Set build flags based on driver type | |
| if [ "$DRIVER_ID" = "spring-data-valkey" ]; then | |
| echo "build_sdv=true" >> $GITHUB_OUTPUT | |
| elif [ "$DRIVER_ID" = "valkey-glide" ]; then | |
| echo "build_glide_primary=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "ERROR: Commit IDs are only supported for spring-data-valkey and valkey-glide primary drivers" | |
| exit 1 | |
| fi | |
| else | |
| echo "primary_is_commit=false" >> $GITHUB_OUTPUT | |
| if [ "$DRIVER_ID" = "spring-data-valkey" ]; then | |
| echo "build_sdv=false" >> $GITHUB_OUTPUT | |
| elif [ "$DRIVER_ID" = "valkey-glide" ]; then | |
| echo "build_glide_primary=false" >> $GITHUB_OUTPUT | |
| fi | |
| if [ -n "$PRIMARY_VERSION" ]; then | |
| echo "Detected: primary_version '$PRIMARY_VERSION' is a release version" | |
| fi | |
| fi | |
| # Check if secondary version is a commit ID (7-40 hex chars) or version number | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ] && [[ "$SECONDARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| if [ "$SECONDARY_DRIVER_ID" = "valkey-glide" ]; then | |
| echo "secondary_is_commit=true" >> $GITHUB_OUTPUT | |
| echo "Detected: secondary_version '$SECONDARY_VERSION' is a commit ID" | |
| else | |
| echo "ERROR: Commit IDs are only supported for valkey-glide secondary driver" | |
| exit 1 | |
| fi | |
| else | |
| echo "secondary_is_commit=false" >> $GITHUB_OUTPUT | |
| if [ -n "$SECONDARY_VERSION" ]; then | |
| echo "Detected: secondary_version '$SECONDARY_VERSION' is a release version" | |
| fi | |
| fi | |
| echo "" | |
| echo "=== Driver Analysis ===" | |
| echo "Primary driver: $DRIVER_ID" | |
| echo "Primary version: ${PRIMARY_VERSION:-<default>}" | |
| echo "Secondary driver: ${SECONDARY_DRIVER_ID:-<none>}" | |
| echo "Secondary version: ${SECONDARY_VERSION:-<default>}" | |
| - name: Validate and resolve versions | |
| id: versions | |
| run: | | |
| PRIMARY_VERSION="${{ steps.inputs.outputs.primary_version }}" | |
| SECONDARY_VERSION="${{ steps.inputs.outputs.secondary_version }}" | |
| DRIVER_ID="${{ steps.driver-info.outputs.driver_id }}" | |
| SECONDARY_DRIVER_ID="${{ steps.driver-info.outputs.secondary_driver_id }}" | |
| # Cannot provide secondary version if driver has no secondary driver | |
| if [ -z "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| echo "ERROR: secondary_version provided but driver has no secondary driver" | |
| exit 1 | |
| fi | |
| echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT | |
| - name: Build spring-data-valkey from source | |
| id: build-sdv | |
| if: steps.driver-info.outputs.build_sdv == 'true' | |
| run: | | |
| COMMIT_ID="${{ steps.versions.outputs.primary_version }}" | |
| echo "Building spring-data-valkey from commit: $COMMIT_ID" | |
| # Clone to a separate directory so the main workspace (with orchestrator, | |
| # configs, etc.) is preserved — allows building commits older than the benchmark infra | |
| git clone ${{ github.server_url }}/${{ github.repository }}.git /tmp/spring-data-valkey | |
| cd /tmp/spring-data-valkey | |
| git checkout "$COMMIT_ID" | |
| # Use commit ID as version for local Maven | |
| SDV_VERSION="${COMMIT_ID:0:8}-SNAPSHOT" | |
| echo "Build version: $SDV_VERSION" | |
| # Set version to include commit ID | |
| mvn versions:set -DnewVersion="$SDV_VERSION" -DgenerateBackupPoms=false -q | |
| # Build and install to local Maven repository | |
| mvn install -DskipTests -Dmaven.compiler.source=17 -Dmaven.compiler.target=17 -Dgpg.skip=true -q | |
| echo "✓ Built spring-data-valkey version: $SDV_VERSION" | |
| echo "version=$SDV_VERSION" >> $GITHUB_OUTPUT | |
| - name: Install valkey-glide build dependencies | |
| if: steps.driver-info.outputs.build_glide_primary == 'true' || (steps.driver-info.outputs.secondary_driver_id == 'valkey-glide' && steps.driver-info.outputs.secondary_is_commit == 'true') | |
| run: | | |
| echo "Installing dependencies for building valkey-glide from source" | |
| # Install system dependencies | |
| sudo apt-get update | |
| sudo apt-get install -y git gcc pkg-config openssl libssl-dev unzip cmake python3-pip | |
| # Install JDK 11 (required by valkey-glide Gradle toolchain) | |
| sudo apt-get install -y openjdk-11-jdk-headless | |
| # Install Rust | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| source "$HOME/.cargo/env" | |
| rustc --version | |
| # Install protobuf compiler v29.1 | |
| PROTOC_VERSION="29.1" | |
| curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| unzip -o "protoc-${PROTOC_VERSION}-linux-x86_64.zip" -d "$HOME/.local" | |
| rm "protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| protoc --version | |
| # Install ziglang and cargo-zigbuild (required for Linux builds) | |
| sudo python3 -m pip install ziglang | |
| cargo install --locked cargo-zigbuild | |
| echo "✓ valkey-glide build dependencies installed" | |
| - name: Build valkey-glide from source (primary) | |
| id: build-glide-primary | |
| if: steps.driver-info.outputs.build_glide_primary == 'true' | |
| run: | | |
| source "$HOME/.cargo/env" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 | |
| COMMIT_ID="${{ steps.inputs.outputs.primary_version }}" | |
| echo "Building valkey-glide from commit: $COMMIT_ID (primary driver)" | |
| # Clone valkey-glide at specific commit | |
| git clone https://github.com/valkey-io/valkey-glide.git /tmp/valkey-glide-primary | |
| cd /tmp/valkey-glide-primary | |
| git checkout "$COMMIT_ID" | |
| # Use commit ID as version for local Maven | |
| GLIDE_VERSION="${COMMIT_ID:0:8}-SNAPSHOT" | |
| echo "Building valkey-glide version: $GLIDE_VERSION" | |
| # build with debug symbols for flamegraph visibility | |
| cd java | |
| CARGO_PROFILE_RELEASE_DEBUG=true ./gradlew :client:buildAll -x test | |
| GLIDE_RELEASE_VERSION="$GLIDE_VERSION" ./gradlew publishToMavenLocal -x test | |
| echo "✓ Built valkey-glide $GLIDE_VERSION from commit $COMMIT_ID (primary driver)" | |
| echo "version=$GLIDE_VERSION" >> $GITHUB_OUTPUT | |
| - name: Build valkey-glide from source (secondary) | |
| id: build-glide-secondary | |
| if: steps.driver-info.outputs.secondary_driver_id == 'valkey-glide' && steps.driver-info.outputs.secondary_is_commit == 'true' | |
| run: | | |
| source "$HOME/.cargo/env" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 | |
| COMMIT_ID="${{ steps.versions.outputs.secondary_version }}" | |
| echo "Building valkey-glide from commit: $COMMIT_ID (secondary driver)" | |
| # Clone valkey-glide at specific commit | |
| git clone https://github.com/valkey-io/valkey-glide.git /tmp/valkey-glide-secondary | |
| cd /tmp/valkey-glide-secondary | |
| git checkout "$COMMIT_ID" | |
| # Use commit ID as version for local Maven | |
| GLIDE_VERSION="${COMMIT_ID:0:8}-SNAPSHOT" | |
| echo "Building valkey-glide version: $GLIDE_VERSION" | |
| # build with debug symbols for flamegraph visibility | |
| cd java | |
| CARGO_PROFILE_RELEASE_DEBUG=true ./gradlew :client:buildAll -x test | |
| GLIDE_RELEASE_VERSION="$GLIDE_VERSION" ./gradlew publishToMavenLocal -x test | |
| echo "✓ Built valkey-glide $GLIDE_VERSION from commit $COMMIT_ID (secondary driver)" | |
| echo "version=$GLIDE_VERSION" >> $GITHUB_OUTPUT | |
| - name: Update resp-bench driver versions | |
| run: | | |
| cd resp-bench | |
| DRIVER_ID="${{ steps.driver-info.outputs.driver_id }}" | |
| SECONDARY_DRIVER_ID="${{ steps.driver-info.outputs.secondary_driver_id }}" | |
| SECONDARY_VERSION="${{ steps.versions.outputs.secondary_version }}" | |
| SECONDARY_IS_COMMIT="${{ steps.driver-info.outputs.secondary_is_commit }}" | |
| # Update primary driver version if supplied | |
| PRIMARY_VERSION="${{ steps.versions.outputs.primary_version }}" | |
| PRIMARY_IS_COMMIT="${{ steps.driver-info.outputs.primary_is_commit }}" | |
| if [ -n "$PRIMARY_VERSION" ]; then | |
| case "$DRIVER_ID" in | |
| spring-data-valkey) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| SDV_VERSION="${{ steps.build-sdv.outputs.version }}" | |
| echo "Updating spring-data-valkey to locally built $SDV_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$SDV_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| else | |
| echo "Updating spring-data-valkey to release version $PRIMARY_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$PRIMARY_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| fi | |
| ;; | |
| valkey-glide) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| GLIDE_VERSION="${{ steps.build-glide-primary.outputs.version }}" | |
| echo "Updating valkey-glide (primary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (primary) to release version $PRIMARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$PRIMARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$PRIMARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$PRIMARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| redisson) | |
| echo "Updating redisson (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<redisson.version>.*</redisson.version>|<redisson.version>$PRIMARY_VERSION</redisson.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| # Update secondary driver version if a version/commit-id for it was supplied. Otherwise use defaults. | |
| # If secondary version is glide and it was built locally, update the pom.xml file to use this built version. | |
| # Otherwise, update the version supplied in the pom.xml. | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| case "$SECONDARY_DRIVER_ID" in | |
| valkey-glide) | |
| if [ "$SECONDARY_IS_COMMIT" = "true" ]; then | |
| # Use the version from the local build | |
| GLIDE_VERSION="${{ steps.build-glide-secondary.outputs.version }}" | |
| echo "Updating valkey-glide (secondary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (secondary) to release version $SECONDARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$SECONDARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis to $SECONDARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$SECONDARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce to $SECONDARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$SECONDARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| echo "" | |
| echo "=== Updated pom.xml versions ===" | |
| grep -E "(spring-data-valkey|valkey-glide|jedis|lettuce)\.version" java/pom.xml | |
| - name: Build resp-bench Java benchmark | |
| run: | | |
| cd resp-bench | |
| make java-build | |
| echo "✓ Java benchmark built" | |
| ls -la java/target/*.jar | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get install -y sysstat python3-pip numactl | |
| sudo python3 -m pip install psycopg2-binary boto3 hdrhistogram | |
| # async-profiler for flame graphs | |
| if [ ! -f "/opt/async-profiler/bin/asprof" ]; then | |
| curl -sL "https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz" | sudo tar xzf - -C /opt | |
| sudo mv /opt/async-profiler-3.0-linux-x64 /opt/async-profiler | |
| fi | |
| /opt/async-profiler/bin/asprof --version | |
| # perf for hardware counters | |
| sudo apt-get install -y linux-tools-$(uname -r) 2>/dev/null || \ | |
| sudo apt-get install -y linux-tools-generic 2>/dev/null || true | |
| command -v perf && perf --version || echo "perf not available" | |
| - name: Run benchmark and publish results to DB | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| WORKLOAD="${{ steps.validate.outputs.workload }}" | |
| DRIVER_CONFIG="${{ steps.validate.outputs.driver_config }}" | |
| WORKLOAD_CONFIG="${{ steps.validate.outputs.workload_config }}" | |
| JOB_ID_PREFIX="${{ steps.inputs.outputs.job_id_prefix }}" | |
| PREFIX_ARG="" | |
| [ -n "$JOB_ID_PREFIX" ] && PREFIX_ARG="--job-id-prefix $JOB_ID_PREFIX" | |
| NETWORK_DELAY="${{ steps.inputs.outputs.network_delay }}" | |
| NETWORK_JITTER="${{ steps.inputs.outputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ steps.inputs.outputs.network_delay_distribution }}" | |
| TC_ARGS="" | |
| [ -n "$NETWORK_DELAY" ] && TC_ARGS="$TC_ARGS --network-delay $NETWORK_DELAY" | |
| [ -n "$NETWORK_JITTER" ] && TC_ARGS="$TC_ARGS --network-jitter $NETWORK_JITTER" | |
| [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && TC_ARGS="$TC_ARGS --network-delay-distribution $NETWORK_DELAY_DISTRIBUTION" | |
| python3 -u .github/workflows/benchmark_orchestrator.py \ | |
| --output "benchmark_results_${DRIVER}_${WORKLOAD}.json" \ | |
| --workload-config "$WORKLOAD_CONFIG" \ | |
| --driver-config "$DRIVER_CONFIG" \ | |
| --resp-bench-dir "${{ github.workspace }}/resp-bench" \ | |
| --resp-bench-commit "${{ steps.resp-bench.outputs.commit }}" \ | |
| --s3-bucket "${{ secrets.BENCHMARK_S3_BUCKET }}" \ | |
| --pg-host "${{ secrets.BENCHMARK_PG_HOST }}" \ | |
| --pg-secret-name "${{ secrets.BENCHMARK_PG_SECRET_NAME }}" \ | |
| $PREFIX_ARG $TC_ARGS | |
| - name: Display results summary | |
| if: always() | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| WORKLOAD="${{ steps.validate.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| if [ -f "$RESULT_FILE" ]; then | |
| echo "=== Benchmark Results Summary ===" | |
| python3 -c " | |
| import json | |
| with open('$RESULT_FILE') as f: | |
| data = json.load(f) | |
| print(f\"Job ID: {data['job_id']}\") | |
| print(f\"Driver: {data['config']['driver']['driver_id']}\") | |
| print(f\"Workload: {data['config']['workload']['benchmark_profile']['name']}\") | |
| print(f\"Elapsed: {data['results']['elapsed_ms']}ms\") | |
| print() | |
| versions = data.get('versions', {}) | |
| print(f\"Primary: {versions.get('primary_driver_id')} @ {versions.get('primary_driver_version', 'N/A')}\") | |
| print(f\"Secondary: {versions.get('secondary_driver_id', 'N/A')} @ {versions.get('secondary_driver_version', 'N/A')}\") | |
| if versions.get('commit_id'): | |
| print(f\"resp-bench: {versions.get('commit_id')}\") | |
| print() | |
| steady = data['results']['phases'].get('STEADY', {}) | |
| if steady: | |
| totals = steady.get('totals', {}) | |
| print(f\"STEADY totals: {totals.get('requests', 0):,} requests, {totals.get('errors', 0)} errors\") | |
| print() | |
| print('Per-command metrics:') | |
| for cmd, m in steady.get('metrics', {}).items(): | |
| reqs = m['requests'] | |
| errs = m['errors'] | |
| lat = m.get('latency', {}) | |
| summary = lat.get('summary', {}) | |
| print(f\" {cmd}:\") | |
| print(f\" Requests: {reqs:,} (errors: {errs})\") | |
| print(f\" Latency: min={summary.get('min')} p50={summary.get('p50')} p95={summary.get('p95')} p99={summary.get('p99')} p999={summary.get('p999')} max={summary.get('max')} us\") | |
| print() | |
| print('Perf:') | |
| perf = data['results']['perf']['counters'] | |
| if perf.get('ipc'): | |
| print(f\" IPC: {perf['ipc']}\") | |
| if perf.get('cache_miss_rate'): | |
| print(f\" Cache miss rate: {perf['cache_miss_rate']}%\") | |
| if perf.get('branch_miss_rate'): | |
| print(f\" Branch miss rate: {perf['branch_miss_rate']}%\") | |
| " | |
| else | |
| echo "⚠ Result file not found: $RESULT_FILE" | |
| fi | |
| - name: Generate summary report | |
| if: always() | |
| run: | | |
| DRIVER="${{ steps.validate.outputs.driver }}" | |
| WORKLOAD="${{ steps.validate.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| echo "# Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "$RESULT_FILE" ]; then | |
| python3 << EOF >> $GITHUB_STEP_SUMMARY | |
| import json | |
| with open("$RESULT_FILE") as f: | |
| data = json.load(f) | |
| job_id = data["job_id"] | |
| versions = data.get("versions", {}) | |
| driver = data["config"]["driver"]["driver_id"] | |
| workload_profile = data["config"]["workload"]["benchmark_profile"]["name"] | |
| elapsed = data["results"]["elapsed_ms"] | |
| network_delay = data["results"].get("network_delay", "") | |
| network_jitter = data["results"].get("network_jitter", "") | |
| network_distribution = data["results"].get("network_delay_distribution", "") | |
| pv = versions.get("primary_driver_version") or "N/A" | |
| sv = versions.get("secondary_driver_version") | |
| sid = versions.get("secondary_driver_id") | |
| commit_id = versions.get("commit_id") | |
| print(f"**Job ID:** \`{job_id}\`") | |
| print(f"**Driver:** {driver} @ \`{pv}\`") | |
| if sid and sv: | |
| print(f"**Secondary:** {sid} @ \`{sv}\`") | |
| if commit_id: | |
| print(f"**resp-bench:** \`{commit_id}\`") | |
| print(f"**Workload config:** \`${WORKLOAD}\` ({workload_profile})") | |
| print(f"**Driver config:** \`${DRIVER}\`") | |
| delay_str = network_delay or "disabled" | |
| if network_jitter: | |
| delay_str += f" jitter {network_jitter}" | |
| if network_distribution: | |
| delay_str += f" distribution {network_distribution}" | |
| print(f"**Simulated network delay:** {delay_str}") | |
| print(f"**Elapsed:** {elapsed}ms") | |
| print() | |
| print("| Versions | Driver | Workload | Status | Elapsed |") | |
| print("|----------|--------|----------|--------|---------|") | |
| version_str = pv | |
| if sv: | |
| version_str += f" ({sid}: {sv})" | |
| print(f"| {version_str} | {driver} | \`${WORKLOAD}\` | ✅ | {elapsed}ms |") | |
| print() | |
| steady = data["results"]["phases"].get("STEADY", {}) | |
| if steady: | |
| print("### Steady State Metrics") | |
| print("| Command | Requests | Errors | p50 (μs) | p95 (μs) | p99 (μs) | p999 (μs) |") | |
| print("|---------|----------|--------|----------|----------|----------|-----------|") | |
| for cmd, m in steady.get("metrics", {}).items(): | |
| s = m.get("latency", {}).get("summary", {}) | |
| print(f"| {cmd} | {m['requests']:,} | {m['errors']} | {s.get('p50','-')} | {s.get('p95','-')} | {s.get('p99','-')} | {s.get('p999','-')} |") | |
| EOF | |
| else | |
| echo "❌ No results found" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ steps.validate.outputs.driver }}-${{ steps.validate.outputs.workload }} | |
| path: | | |
| benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}.json | |
| benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}.ndjson | |
| benchmark_results_${{ steps.validate.outputs.driver }}_${{ steps.validate.outputs.workload }}_collapsed.txt |