Benchmark #209
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| primary_driver: | |
| description: "Primary driver" | |
| required: true | |
| type: choice | |
| options: | |
| - spring-data-valkey | |
| - spring-data-redis | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - redisson | |
| default: "spring-data-valkey" | |
| secondary_driver: | |
| description: "Secondary driver (for spring-data-* only, ignored for others)" | |
| required: false | |
| type: choice | |
| options: | |
| - valkey-glide | |
| - jedis | |
| - lettuce | |
| - none | |
| default: "valkey-glide" | |
| workload: | |
| description: "Workload config" | |
| required: true | |
| type: choice | |
| options: | |
| - reference-workload-10-client | |
| - reference-workload-1-client | |
| default: "reference-workload-10-client" | |
| primary_version: | |
| description: "Primary driver version (version string or commit ID). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| secondary_version: | |
| description: "Secondary driver version (e.g., valkey-glide commit). Leave empty to use default." | |
| required: false | |
| type: string | |
| default: "" | |
| job_id_prefix: | |
| description: 'Optional job ID prefix (e.g., "nightly", "pr-123")' | |
| required: false | |
| type: string | |
| default: "" | |
| custom_driver_config: | |
| description: "Custom driver config JSON (overrides driver selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| custom_workload_config: | |
| description: "Custom workload config JSON (overrides workload selection)" | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay: | |
| description: "Network delay with unit, e.g. '1ms' or '500us' (tc netem). Leave empty to disable." | |
| required: false | |
| type: string | |
| default: "" | |
| network_jitter: | |
| description: "Network jitter with unit, e.g. '1ms' or '100us' (tc netem). Requires network_delay." | |
| required: false | |
| type: string | |
| default: "" | |
| network_delay_distribution: | |
| description: "Jitter distribution (tc netem). Requires jitter. Options: normal, pareto, paretonormal." | |
| required: false | |
| type: choice | |
| options: | |
| - "" | |
| - normal | |
| - pareto | |
| - paretonormal | |
| default: "" | |
| env: | |
| RESP_BENCH_REPO: "https://github.com/ikolomi/resp-bench.git" | |
| jobs: | |
| benchmark: | |
| runs-on: [self-hosted, Linux, x86, ephemeral, metal] | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Validate EC2 type and CPU model | |
| run: | | |
| # c5.metal: Intel Xeon Platinum 8275CL (Cascade Lake), 96 vCPUs, 2 NUMA nodes | |
| EXPECTED_INSTANCE_TYPE="c5.metal" | |
| EXPECTED_CPU="8275CL" | |
| EXPECTED_STEPPING="7" | |
| EXPECTED_VCPUS="96" | |
| TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds:60") | |
| INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-type) | |
| CPU_MODEL=$(grep 'model name' /proc/cpuinfo | head -1 | sed 's/.*: //') | |
| CPU_STEPPING=$(lscpu | grep '^Stepping:' | awk '{print $2}') | |
| CPU_COUNT=$(nproc) | |
| echo "Instance type: $INSTANCE_TYPE" | |
| echo "CPU: $CPU_MODEL" | |
| echo "Stepping: $CPU_STEPPING" | |
| echo "vCPUs: $CPU_COUNT" | |
| lscpu | grep -E 'Model name|Stepping|CPU max MHz|CPU\(s\):|NUMA' | |
| ERRORS="" | |
| if [ "$INSTANCE_TYPE" != "$EXPECTED_INSTANCE_TYPE" ]; then | |
| ERRORS="${ERRORS}Instance type: expected '$EXPECTED_INSTANCE_TYPE', got '$INSTANCE_TYPE'\n" | |
| fi | |
| if ! echo "$CPU_MODEL" | grep -q "$EXPECTED_CPU"; then | |
| ERRORS="${ERRORS}CPU model: expected '$EXPECTED_CPU', got '$CPU_MODEL'\n" | |
| fi | |
| if [ "$CPU_STEPPING" != "$EXPECTED_STEPPING" ]; then | |
| ERRORS="${ERRORS}Stepping: expected '$EXPECTED_STEPPING', got '$CPU_STEPPING'\n" | |
| fi | |
| if [ "$CPU_COUNT" != "$EXPECTED_VCPUS" ]; then | |
| ERRORS="${ERRORS}vCPU count: expected '$EXPECTED_VCPUS', got '$CPU_COUNT'\n" | |
| fi | |
| if [ -n "$ERRORS" ]; then | |
| echo "ERROR: Hardware validation failed:" | |
| echo -e "$ERRORS" | |
| echo "This machine does not match expected c5.metal specs. Failing early to avoid inconsistent benchmark results." | |
| exit 1 | |
| fi | |
| echo "✓ Hardware validated: $CPU_MODEL, stepping $CPU_STEPPING, $CPU_COUNT vCPUs" | |
| - uses: actions/checkout@v4 | |
| - name: Set up Java 21 and Maven | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y openjdk-21-jdk-headless maven python3-pip \ | |
| sysstat numactl \ | |
| gcc pkg-config openssl libssl-dev unzip cmake | |
| sudo apt-get install -y linux-tools-$(uname -r) 2>/dev/null || \ | |
| sudo apt-get install -y linux-tools-generic 2>/dev/null || true | |
| java -version | |
| mvn -version | |
| - name: Clone resp-bench | |
| id: resp-bench | |
| run: | | |
| echo "Cloning resp-bench repository..." | |
| git clone --depth 1 ${{ env.RESP_BENCH_REPO }} resp-bench | |
| RESP_BENCH_COMMIT=$(git -C resp-bench rev-parse HEAD) | |
| echo "commit=$RESP_BENCH_COMMIT" >> $GITHUB_OUTPUT | |
| echo "✓ resp-bench cloned (commit: ${RESP_BENCH_COMMIT:0:7})" | |
| - name: Validate inputs and resolve configs | |
| id: config | |
| run: | | |
| PRIMARY_DRIVER="${{ github.event.inputs.primary_driver }}" | |
| SECONDARY_DRIVER="${{ github.event.inputs.secondary_driver }}" | |
| WORKLOAD="${{ github.event.inputs.workload }}" | |
| PRIMARY_VERSION="${{ github.event.inputs.primary_version }}" | |
| SECONDARY_VERSION="${{ github.event.inputs.secondary_version }}" | |
| NETWORK_DELAY="${{ github.event.inputs.network_delay }}" | |
| NETWORK_JITTER="${{ github.event.inputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ github.event.inputs.network_delay_distribution }}" | |
| CONFIG_DIR=".github/workflows/benchmark_configs" | |
| CUSTOM_CONFIG_DIR="/tmp/custom_configs" | |
| mkdir -p "$CUSTOM_CONFIG_DIR" | |
| # --- Validate tc netem input combinations --- | |
| if [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && [ -z "$NETWORK_JITTER" ]; then | |
| echo "ERROR: network_delay_distribution requires network_jitter" | |
| exit 1 | |
| fi | |
| if [ -n "$NETWORK_JITTER" ] && [ -z "$NETWORK_DELAY" ]; then | |
| echo "ERROR: network_jitter requires network_delay" | |
| exit 1 | |
| fi | |
| for val in "$NETWORK_DELAY" "$NETWORK_JITTER"; do | |
| if [ -n "$val" ] && ! [[ "$val" =~ ^[0-9]+(ms|us)$ ]]; then | |
| echo "ERROR: invalid tc netem value '$val' — expected format: <number>ms or <number>us" | |
| exit 1 | |
| fi | |
| done | |
| # --- Resolve and validate driver config --- | |
| CUSTOM_DRIVER_CONFIG_INPUT='${{ github.event.inputs.custom_driver_config }}' | |
| if [ -n "$CUSTOM_DRIVER_CONFIG_INPUT" ]; then | |
| echo "Using custom driver config..." | |
| DRIVER_CONFIG="$CUSTOM_CONFIG_DIR/custom-driver.json" | |
| printf '%s\n' "$CUSTOM_DRIVER_CONFIG_INPUT" > "$DRIVER_CONFIG" | |
| if ! jq . "$DRIVER_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom driver config is not valid JSON" | |
| echo "Content written:" | |
| cat "$DRIVER_CONFIG" | |
| exit 1 | |
| fi | |
| DRIVER="custom-driver" | |
| echo "✓ Custom driver config validated" | |
| else | |
| if [[ "$PRIMARY_DRIVER" == spring-data-* ]]; then | |
| if [ "$SECONDARY_DRIVER" = "none" ] || [ -z "$SECONDARY_DRIVER" ]; then | |
| echo "ERROR: $PRIMARY_DRIVER requires a secondary driver (valkey-glide, jedis, or lettuce)" | |
| exit 1 | |
| fi | |
| if [ "$PRIMARY_DRIVER" = "spring-data-redis" ] && [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| echo "ERROR: spring-data-redis does not support valkey-glide. Use jedis or lettuce instead." | |
| exit 1 | |
| fi | |
| SECONDARY_IN_FILENAME="$SECONDARY_DRIVER" | |
| if [ "$SECONDARY_DRIVER" = "valkey-glide" ]; then | |
| SECONDARY_IN_FILENAME="glide" | |
| fi | |
| DRIVER="${PRIMARY_DRIVER}-${SECONDARY_IN_FILENAME}-standalone" | |
| else | |
| DRIVER="${PRIMARY_DRIVER}-standalone" | |
| if [ "$SECONDARY_DRIVER" != "none" ] && [ -n "$SECONDARY_DRIVER" ]; then | |
| echo "Note: secondary_driver '$SECONDARY_DRIVER' ignored for standalone driver '$PRIMARY_DRIVER'" | |
| fi | |
| fi | |
| DRIVER_CONFIG="${CONFIG_DIR}/drivers/${DRIVER}.json" | |
| if [ ! -f "$DRIVER_CONFIG" ]; then | |
| echo "ERROR: Driver config not found: $DRIVER_CONFIG" | |
| echo "Available driver configs:" | |
| ls -la "${CONFIG_DIR}/drivers/" | |
| exit 1 | |
| fi | |
| echo "✓ Driver config: $DRIVER_CONFIG" | |
| fi | |
| # --- Resolve and validate workload config --- | |
| CUSTOM_WORKLOAD_CONFIG_INPUT='${{ github.event.inputs.custom_workload_config }}' | |
| if [ -n "$CUSTOM_WORKLOAD_CONFIG_INPUT" ]; then | |
| echo "Using custom workload config..." | |
| WORKLOAD_CONFIG="$CUSTOM_CONFIG_DIR/custom-workload.json" | |
| printf '%s\n' "$CUSTOM_WORKLOAD_CONFIG_INPUT" > "$WORKLOAD_CONFIG" | |
| if ! jq . "$WORKLOAD_CONFIG" > /dev/null 2>&1; then | |
| echo "ERROR: Custom workload config is not valid JSON" | |
| echo "Content written:" | |
| cat "$WORKLOAD_CONFIG" | |
| exit 1 | |
| fi | |
| WORKLOAD="custom-workload" | |
| echo "✓ Custom workload config validated" | |
| else | |
| WORKLOAD_CONFIG="${CONFIG_DIR}/workloads/${WORKLOAD}.json" | |
| if [ ! -f "$WORKLOAD_CONFIG" ]; then | |
| echo "ERROR: Workload config not found: $WORKLOAD_CONFIG" | |
| echo "Available workload configs:" | |
| ls -la "${CONFIG_DIR}/workloads/" | |
| exit 1 | |
| fi | |
| echo "✓ Workload config: $WORKLOAD_CONFIG" | |
| fi | |
| echo "Driver config contents:" | |
| cat "$DRIVER_CONFIG" | jq . | |
| echo "Workload phases:" | |
| jq '.phases[] | {id, completion}' "$WORKLOAD_CONFIG" | |
| # --- Analyze driver requirements --- | |
| DRIVER_ID=$(jq -r '.driver_id' "$DRIVER_CONFIG") | |
| SECONDARY_DRIVER_ID=$(jq -r '.specific_driver_config.secondary_driver_id // empty' "$DRIVER_CONFIG") | |
| BUILD_SDV=false | |
| BUILD_GLIDE=false | |
| SDV_COMMIT="" | |
| SDV_VERSION="" | |
| GLIDE_COMMIT="" | |
| GLIDE_VERSION="" | |
| PRIMARY_IS_COMMIT=false | |
| SECONDARY_IS_COMMIT=false | |
| if [ -n "$PRIMARY_VERSION" ] && [[ "$PRIMARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| PRIMARY_IS_COMMIT=true | |
| if [ "$DRIVER_ID" = "spring-data-valkey" ]; then | |
| BUILD_SDV=true | |
| SDV_COMMIT="$PRIMARY_VERSION" | |
| SDV_VERSION="${PRIMARY_VERSION:0:8}-SNAPSHOT" | |
| elif [ "$DRIVER_ID" = "valkey-glide" ]; then | |
| BUILD_GLIDE=true | |
| GLIDE_COMMIT="$PRIMARY_VERSION" | |
| GLIDE_VERSION="${PRIMARY_VERSION:0:8}-SNAPSHOT" | |
| else | |
| echo "ERROR: Commit IDs are only supported for spring-data-valkey and valkey-glide primary drivers" | |
| exit 1 | |
| fi | |
| fi | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ] && [[ "$SECONDARY_VERSION" =~ ^[0-9a-fA-F]{7,40}$ ]]; then | |
| if [ "$SECONDARY_DRIVER_ID" = "valkey-glide" ]; then | |
| SECONDARY_IS_COMMIT=true | |
| BUILD_GLIDE=true | |
| GLIDE_COMMIT="$SECONDARY_VERSION" | |
| GLIDE_VERSION="${SECONDARY_VERSION:0:8}-SNAPSHOT" | |
| else | |
| echo "ERROR: Commit IDs are only supported for valkey-glide secondary driver" | |
| exit 1 | |
| fi | |
| fi | |
| if [ -z "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| echo "ERROR: secondary_version provided but driver has no secondary driver" | |
| exit 1 | |
| fi | |
| # --- Write outputs --- | |
| echo "driver=$DRIVER" >> $GITHUB_OUTPUT | |
| echo "driver_config=$DRIVER_CONFIG" >> $GITHUB_OUTPUT | |
| echo "workload=$WORKLOAD" >> $GITHUB_OUTPUT | |
| echo "workload_config=$WORKLOAD_CONFIG" >> $GITHUB_OUTPUT | |
| echo "driver_id=$DRIVER_ID" >> $GITHUB_OUTPUT | |
| echo "secondary_driver_id=$SECONDARY_DRIVER_ID" >> $GITHUB_OUTPUT | |
| echo "primary_version=$PRIMARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "secondary_version=$SECONDARY_VERSION" >> $GITHUB_OUTPUT | |
| echo "primary_is_commit=$PRIMARY_IS_COMMIT" >> $GITHUB_OUTPUT | |
| echo "secondary_is_commit=$SECONDARY_IS_COMMIT" >> $GITHUB_OUTPUT | |
| echo "build_sdv=$BUILD_SDV" >> $GITHUB_OUTPUT | |
| echo "build_glide=$BUILD_GLIDE" >> $GITHUB_OUTPUT | |
| echo "sdv_version=$SDV_VERSION" >> $GITHUB_OUTPUT | |
| echo "sdv_commit=$SDV_COMMIT" >> $GITHUB_OUTPUT | |
| echo "glide_version=$GLIDE_VERSION" >> $GITHUB_OUTPUT | |
| echo "glide_commit=$GLIDE_COMMIT" >> $GITHUB_OUTPUT | |
| echo "" | |
| echo "========================================" | |
| echo "Driver: $DRIVER ($DRIVER_ID)" | |
| echo "Workload: $WORKLOAD" | |
| echo "Primary version: '${PRIMARY_VERSION}' (commit: $PRIMARY_IS_COMMIT)" | |
| echo "Secondary: ${SECONDARY_DRIVER_ID:-none} @ '${SECONDARY_VERSION}' (commit: $SECONDARY_IS_COMMIT)" | |
| echo "Build SDV: $BUILD_SDV (${SDV_VERSION:-N/A})" | |
| echo "Build Glide: $BUILD_GLIDE (${GLIDE_VERSION:-N/A})" | |
| echo "========================================" | |
| # --- Cache restore for source builds (same runner type = no version hash mismatch) --- | |
| - name: Restore SDV build from cache | |
| if: steps.config.outputs.build_sdv == 'true' | |
| id: cache-sdv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository/io/valkey/springframework | |
| key: sdv-build-${{ steps.config.outputs.sdv_commit }} | |
| - name: Restore glide build from cache | |
| if: steps.config.outputs.build_glide == 'true' | |
| id: cache-glide | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.m2/repository/io/valkey/valkey-glide | |
| key: glide-build-${{ steps.config.outputs.glide_commit }} | |
| # --- Parallel background builds + wait (must be in same step for `wait` to work) --- | |
| - name: Parallel builds and installs | |
| run: | | |
| set +e | |
| LOG_DIR="/tmp/build-logs" | |
| mkdir -p "$LOG_DIR" | |
| PIDS=() | |
| NAMES=() | |
| # 1. Pre-compile Valkey server (needed by orchestrator) | |
| echo "Starting Valkey server compilation..." | |
| ( | |
| set -euo pipefail | |
| cd resp-bench && make "$(pwd)/work/valkey/bin/valkey-server" | |
| ) > "$LOG_DIR/valkey.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("valkey") | |
| # 2. Build SDV from source (if needed and cache miss) | |
| if [ "${{ steps.config.outputs.build_sdv }}" = "true" ] && [ "${{ steps.cache-sdv.outputs.cache-hit }}" != "true" ]; then | |
| echo "Starting SDV build from source..." | |
| ( | |
| set -euo pipefail | |
| COMMIT_ID="${{ steps.config.outputs.sdv_commit }}" | |
| SDV_VERSION="${{ steps.config.outputs.sdv_version }}" | |
| git clone ${{ github.server_url }}/${{ github.repository }}.git /tmp/spring-data-valkey | |
| cd /tmp/spring-data-valkey | |
| git checkout "$COMMIT_ID" | |
| mvn versions:set -DnewVersion="$SDV_VERSION" -DgenerateBackupPoms=false -q | |
| mvn install -DskipTests -Dmaven.compiler.source=17 -Dmaven.compiler.target=17 -Dgpg.skip=true -q | |
| echo "Built spring-data-valkey $SDV_VERSION" | |
| ) > "$LOG_DIR/sdv.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("sdv") | |
| elif [ "${{ steps.config.outputs.build_sdv }}" = "true" ]; then | |
| echo "SDV build restored from cache" | |
| fi | |
| # 3. Build glide from source (if needed and cache miss) | |
| if [ "${{ steps.config.outputs.build_glide }}" = "true" ] && [ "${{ steps.cache-glide.outputs.cache-hit }}" != "true" ]; then | |
| echo "Starting glide build from source..." | |
| ( | |
| set -euo pipefail | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y | |
| source "$HOME/.cargo/env" | |
| PROTOC_VERSION="29.1" | |
| curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| unzip -o "protoc-${PROTOC_VERSION}-linux-x86_64.zip" -d "$HOME/.local" | |
| rm "protoc-${PROTOC_VERSION}-linux-x86_64.zip" | |
| export PATH="$PATH:$HOME/.local/bin" | |
| sudo python3 -m pip install ziglang | |
| cargo install --locked cargo-zigbuild | |
| COMMIT_ID="${{ steps.config.outputs.glide_commit }}" | |
| GLIDE_VERSION="${{ steps.config.outputs.glide_version }}" | |
| git clone https://github.com/valkey-io/valkey-glide.git /tmp/valkey-glide | |
| cd /tmp/valkey-glide | |
| git checkout "$COMMIT_ID" | |
| cd java | |
| CARGO_PROFILE_RELEASE_DEBUG=true ./gradlew :client:buildAll -x test | |
| GLIDE_RELEASE_VERSION="$GLIDE_VERSION" ./gradlew publishToMavenLocal -x test | |
| echo "Built valkey-glide $GLIDE_VERSION" | |
| ) > "$LOG_DIR/glide.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("glide") | |
| elif [ "${{ steps.config.outputs.build_glide }}" = "true" ]; then | |
| echo "Glide build restored from cache" | |
| fi | |
| # 4. Install Python deps | |
| echo "Starting pip install..." | |
| ( | |
| set -euo pipefail | |
| sudo python3 -m pip install psycopg2-binary boto3 hdrhistogram | |
| ) > "$LOG_DIR/pip.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("pip") | |
| # 5. Install async-profiler | |
| if [ ! -f "/opt/async-profiler/bin/asprof" ]; then | |
| echo "Downloading async-profiler..." | |
| ( | |
| set -euo pipefail | |
| curl -sL "https://github.com/async-profiler/async-profiler/releases/download/v3.0/async-profiler-3.0-linux-x64.tar.gz" | sudo tar xzf - -C /opt | |
| sudo mv /opt/async-profiler-3.0-linux-x64 /opt/async-profiler | |
| ) > "$LOG_DIR/ap.log" 2>&1 & | |
| PIDS+=($!); NAMES+=("ap") | |
| else | |
| echo "async-profiler already installed" | |
| fi | |
| # Wait for all background processes | |
| echo "" | |
| echo "=== Waiting for background builds ===" | |
| FAILED=0 | |
| for i in "${!PIDS[@]}"; do | |
| echo -n " ${NAMES[$i]}: " | |
| if wait "${PIDS[$i]}"; then | |
| echo "OK" | |
| else | |
| EXIT_CODE=$? | |
| echo "FAILED (exit $EXIT_CODE)" | |
| echo "::error::${NAMES[$i]} build failed" | |
| echo "--- ${NAMES[$i]} log ---" | |
| cat "$LOG_DIR/${NAMES[$i]}.log" 2>/dev/null || echo "(no log)" | |
| echo "--- end ---" | |
| FAILED=1 | |
| fi | |
| done | |
| if [ "$FAILED" -ne 0 ]; then | |
| exit 1 | |
| fi | |
| echo "" | |
| echo "=== All builds completed ===" | |
| resp-bench/work/valkey/bin/valkey-server --version | |
| /opt/async-profiler/bin/asprof --version | |
| command -v perf && perf --version || echo "perf not available" | |
| - name: Update resp-bench driver versions | |
| run: | | |
| cd resp-bench | |
| DRIVER_ID="${{ steps.config.outputs.driver_id }}" | |
| SECONDARY_DRIVER_ID="${{ steps.config.outputs.secondary_driver_id }}" | |
| SECONDARY_VERSION="${{ steps.config.outputs.secondary_version }}" | |
| SECONDARY_IS_COMMIT="${{ steps.config.outputs.secondary_is_commit }}" | |
| PRIMARY_VERSION="${{ steps.config.outputs.primary_version }}" | |
| PRIMARY_IS_COMMIT="${{ steps.config.outputs.primary_is_commit }}" | |
| if [ -n "$PRIMARY_VERSION" ]; then | |
| case "$DRIVER_ID" in | |
| spring-data-valkey) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| SDV_VERSION="${{ steps.config.outputs.sdv_version }}" | |
| echo "Updating spring-data-valkey to locally built $SDV_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$SDV_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| else | |
| echo "Updating spring-data-valkey to release version $PRIMARY_VERSION" | |
| sed -i "s|<spring-data-valkey.version>.*</spring-data-valkey.version>|<spring-data-valkey.version>$PRIMARY_VERSION</spring-data-valkey.version>|" java/pom.xml | |
| fi | |
| ;; | |
| valkey-glide) | |
| if [ "$PRIMARY_IS_COMMIT" = "true" ]; then | |
| GLIDE_VERSION="${{ steps.config.outputs.glide_version }}" | |
| echo "Updating valkey-glide (primary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (primary) to release version $PRIMARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$PRIMARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$PRIMARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$PRIMARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| redisson) | |
| echo "Updating redisson (primary) to $PRIMARY_VERSION" | |
| sed -i "s|<redisson.version>.*</redisson.version>|<redisson.version>$PRIMARY_VERSION</redisson.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| # Update secondary driver version if supplied | |
| if [ -n "$SECONDARY_DRIVER_ID" ] && [ -n "$SECONDARY_VERSION" ]; then | |
| case "$SECONDARY_DRIVER_ID" in | |
| valkey-glide) | |
| if [ "$SECONDARY_IS_COMMIT" = "true" ]; then | |
| GLIDE_VERSION="${{ steps.config.outputs.glide_version }}" | |
| echo "Updating valkey-glide (secondary) to locally built $GLIDE_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$GLIDE_VERSION</valkey-glide.version>|" java/pom.xml | |
| else | |
| echo "Updating valkey-glide (secondary) to release version $SECONDARY_VERSION" | |
| sed -i "s|<valkey-glide.version>.*</valkey-glide.version>|<valkey-glide.version>$SECONDARY_VERSION</valkey-glide.version>|" java/pom.xml | |
| fi | |
| ;; | |
| jedis) | |
| echo "Updating jedis to $SECONDARY_VERSION" | |
| sed -i "s|<jedis.version>.*</jedis.version>|<jedis.version>$SECONDARY_VERSION</jedis.version>|" java/pom.xml | |
| ;; | |
| lettuce) | |
| echo "Updating lettuce to $SECONDARY_VERSION" | |
| sed -i "s|<lettuce.version>.*</lettuce.version>|<lettuce.version>$SECONDARY_VERSION</lettuce.version>|" java/pom.xml | |
| ;; | |
| esac | |
| fi | |
| echo "" | |
| echo "=== Updated pom.xml versions ===" | |
| grep -E "(spring-data-valkey|valkey-glide|jedis|lettuce)\.version" java/pom.xml | |
| - name: Build resp-bench Java benchmark | |
| run: | | |
| cd resp-bench | |
| make java-build | |
| echo "✓ Java benchmark built" | |
| ls -la java/target/*.jar | |
| - name: Run benchmark and publish results to DB | |
| run: | | |
| DRIVER="${{ steps.config.outputs.driver }}" | |
| WORKLOAD="${{ steps.config.outputs.workload }}" | |
| DRIVER_CONFIG="${{ steps.config.outputs.driver_config }}" | |
| WORKLOAD_CONFIG="${{ steps.config.outputs.workload_config }}" | |
| JOB_ID_PREFIX="${{ github.event.inputs.job_id_prefix }}" | |
| PREFIX_ARG="" | |
| [ -n "$JOB_ID_PREFIX" ] && PREFIX_ARG="--job-id-prefix $JOB_ID_PREFIX" | |
| NETWORK_DELAY="${{ github.event.inputs.network_delay }}" | |
| NETWORK_JITTER="${{ github.event.inputs.network_jitter }}" | |
| NETWORK_DELAY_DISTRIBUTION="${{ github.event.inputs.network_delay_distribution }}" | |
| TC_ARGS="" | |
| [ -n "$NETWORK_DELAY" ] && TC_ARGS="$TC_ARGS --network-delay $NETWORK_DELAY" | |
| [ -n "$NETWORK_JITTER" ] && TC_ARGS="$TC_ARGS --network-jitter $NETWORK_JITTER" | |
| [ -n "$NETWORK_DELAY_DISTRIBUTION" ] && TC_ARGS="$TC_ARGS --network-delay-distribution $NETWORK_DELAY_DISTRIBUTION" | |
| python3 -u .github/workflows/benchmark_orchestrator.py \ | |
| --output "benchmark_results_${DRIVER}_${WORKLOAD}.json" \ | |
| --workload-config "$WORKLOAD_CONFIG" \ | |
| --driver-config "$DRIVER_CONFIG" \ | |
| --resp-bench-dir "${{ github.workspace }}/resp-bench" \ | |
| --resp-bench-commit "${{ steps.resp-bench.outputs.commit }}" \ | |
| --s3-bucket "${{ secrets.BENCHMARK_S3_BUCKET }}" \ | |
| --pg-host "${{ secrets.BENCHMARK_PG_HOST }}" \ | |
| --pg-secret-name "${{ secrets.BENCHMARK_PG_SECRET_NAME }}" \ | |
| $PREFIX_ARG $TC_ARGS | |
| - name: Display results summary | |
| if: always() | |
| run: | | |
| DRIVER="${{ steps.config.outputs.driver }}" | |
| WORKLOAD="${{ steps.config.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| if [ -f "$RESULT_FILE" ]; then | |
| echo "=== Benchmark Results Summary ===" | |
| python3 -c " | |
| import json | |
| with open('$RESULT_FILE') as f: | |
| data = json.load(f) | |
| print(f\"Job ID: {data['job_id']}\") | |
| print(f\"Driver: {data['config']['driver']['driver_id']}\") | |
| print(f\"Workload: {data['config']['workload']['benchmark_profile']['name']}\") | |
| print(f\"Elapsed: {data['results']['elapsed_ms']}ms\") | |
| print() | |
| versions = data.get('versions', {}) | |
| print(f\"Primary: {versions.get('primary_driver_id')} @ {versions.get('primary_driver_version', 'N/A')}\") | |
| print(f\"Secondary: {versions.get('secondary_driver_id', 'N/A')} @ {versions.get('secondary_driver_version', 'N/A')}\") | |
| if versions.get('commit_id'): | |
| print(f\"resp-bench: {versions.get('commit_id')}\") | |
| print() | |
| steady = data['results']['phases'].get('STEADY', {}) | |
| if steady: | |
| totals = steady.get('totals', {}) | |
| print(f\"STEADY totals: {totals.get('requests', 0):,} requests, {totals.get('errors', 0)} errors\") | |
| print() | |
| print('Per-command metrics:') | |
| for cmd, m in steady.get('metrics', {}).items(): | |
| reqs = m['requests'] | |
| errs = m['errors'] | |
| lat = m.get('latency', {}) | |
| summary = lat.get('summary', {}) | |
| print(f\" {cmd}:\") | |
| print(f\" Requests: {reqs:,} (errors: {errs})\") | |
| print(f\" Latency: min={summary.get('min')} p50={summary.get('p50')} p95={summary.get('p95')} p99={summary.get('p99')} p999={summary.get('p999')} max={summary.get('max')} us\") | |
| print() | |
| print('Perf:') | |
| perf = data['results']['perf']['counters'] | |
| if perf.get('ipc'): | |
| print(f\" IPC: {perf['ipc']}\") | |
| if perf.get('cache_miss_rate'): | |
| print(f\" Cache miss rate: {perf['cache_miss_rate']}%\") | |
| if perf.get('branch_miss_rate'): | |
| print(f\" Branch miss rate: {perf['branch_miss_rate']}%\") | |
| " | |
| else | |
| echo "⚠ Result file not found: $RESULT_FILE" | |
| fi | |
| - name: Generate summary report | |
| if: always() | |
| run: | | |
| DRIVER="${{ steps.config.outputs.driver }}" | |
| WORKLOAD="${{ steps.config.outputs.workload }}" | |
| RESULT_FILE="benchmark_results_${DRIVER}_${WORKLOAD}.json" | |
| echo "# Benchmark Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ -f "$RESULT_FILE" ]; then | |
| python3 << EOF >> $GITHUB_STEP_SUMMARY | |
| import json | |
| with open("$RESULT_FILE") as f: | |
| data = json.load(f) | |
| job_id = data["job_id"] | |
| versions = data.get("versions", {}) | |
| driver = data["config"]["driver"]["driver_id"] | |
| workload_profile = data["config"]["workload"]["benchmark_profile"]["name"] | |
| elapsed = data["results"]["elapsed_ms"] | |
| network_delay = data["results"].get("network_delay", "") | |
| network_jitter = data["results"].get("network_jitter", "") | |
| network_distribution = data["results"].get("network_delay_distribution", "") | |
| pv = versions.get("primary_driver_version") or "N/A" | |
| sv = versions.get("secondary_driver_version") | |
| sid = versions.get("secondary_driver_id") | |
| commit_id = versions.get("commit_id") | |
| print(f"**Job ID:** \`{job_id}\`") | |
| print(f"**Driver:** {driver} @ \`{pv}\`") | |
| if sid and sv: | |
| print(f"**Secondary:** {sid} @ \`{sv}\`") | |
| if commit_id: | |
| print(f"**resp-bench:** \`{commit_id}\`") | |
| print(f"**Workload config:** \`${WORKLOAD}\` ({workload_profile})") | |
| print(f"**Driver config:** \`${DRIVER}\`") | |
| delay_str = network_delay or "disabled" | |
| if network_jitter: | |
| delay_str += f" jitter {network_jitter}" | |
| if network_distribution: | |
| delay_str += f" distribution {network_distribution}" | |
| print(f"**Simulated network delay:** {delay_str}") | |
| print(f"**Elapsed:** {elapsed}ms") | |
| print() | |
| print("| Versions | Driver | Workload | Status | Elapsed |") | |
| print("|----------|--------|----------|--------|---------|") | |
| version_str = pv | |
| if sv: | |
| version_str += f" ({sid}: {sv})" | |
| print(f"| {version_str} | {driver} | \`${WORKLOAD}\` | ✅ | {elapsed}ms |") | |
| print() | |
| steady = data["results"]["phases"].get("STEADY", {}) | |
| if steady: | |
| print("### Steady State Metrics") | |
| print("| Command | Requests | Errors | p50 (μs) | p95 (μs) | p99 (μs) | p999 (μs) |") | |
| print("|---------|----------|--------|----------|----------|----------|-----------|") | |
| for cmd, m in steady.get("metrics", {}).items(): | |
| s = m.get("latency", {}).get("summary", {}) | |
| print(f"| {cmd} | {m['requests']:,} | {m['errors']} | {s.get('p50','-')} | {s.get('p95','-')} | {s.get('p99','-')} | {s.get('p999','-')} |") | |
| EOF | |
| else | |
| echo "❌ No results found" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmark-results-${{ steps.config.outputs.driver }}-${{ steps.config.outputs.workload }} | |
| path: | | |
| benchmark_results_${{ steps.config.outputs.driver }}_${{ steps.config.outputs.workload }}.json | |
| benchmark_results_${{ steps.config.outputs.driver }}_${{ steps.config.outputs.workload }}.ndjson | |
| benchmark_results_${{ steps.config.outputs.driver }}_${{ steps.config.outputs.workload }}_collapsed.txt |